Source code

001/**
002 * Copyright (C) 2006-2024 Talend Inc. - www.talend.com
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.talend.sdk.component.runtime.manager.chain.internal;
017
018import static java.util.Collections.singletonList;
019import static java.util.stream.Collectors.toList;
020import static java.util.stream.Collectors.toMap;
021import static java.util.stream.Collectors.toSet;
022
023import java.io.BufferedReader;
024import java.io.IOException;
025import java.io.InputStream;
026import java.io.InputStreamReader;
027import java.lang.reflect.InvocationTargetException;
028import java.util.AbstractMap;
029import java.util.ArrayList;
030import java.util.Collection;
031import java.util.HashMap;
032import java.util.HashSet;
033import java.util.Iterator;
034import java.util.List;
035import java.util.Map;
036import java.util.ServiceLoader;
037import java.util.Set;
038import java.util.TreeMap;
039import java.util.concurrent.atomic.AtomicBoolean;
040import java.util.concurrent.atomic.AtomicInteger;
041import java.util.concurrent.atomic.AtomicLong;
042import java.util.concurrent.atomic.AtomicReference;
043import java.util.function.Function;
044import java.util.function.Supplier;
045
046import javax.json.bind.Jsonb;
047import javax.json.bind.JsonbBuilder;
048import javax.json.bind.JsonbConfig;
049
050import org.talend.sdk.component.api.processor.OutputEmitter;
051import org.talend.sdk.component.api.record.Record;
052import org.talend.sdk.component.api.service.record.RecordBuilderFactory;
053import org.talend.sdk.component.runtime.base.Lifecycle;
054import org.talend.sdk.component.runtime.input.Input;
055import org.talend.sdk.component.runtime.input.Mapper;
056import org.talend.sdk.component.runtime.manager.ComponentManager;
057import org.talend.sdk.component.runtime.manager.chain.AutoChunkProcessor;
058import org.talend.sdk.component.runtime.manager.chain.ChainedMapper;
059import org.talend.sdk.component.runtime.manager.chain.GroupKeyProvider;
060import org.talend.sdk.component.runtime.manager.chain.Job;
061import org.talend.sdk.component.runtime.output.InputFactory;
062import org.talend.sdk.component.runtime.output.OutputFactory;
063import org.talend.sdk.component.runtime.output.Processor;
064import org.talend.sdk.component.runtime.output.ProcessorImpl;
065import org.talend.sdk.component.runtime.record.RecordBuilderFactoryImpl;
066import org.talend.sdk.component.runtime.record.RecordConverters;
067
068import lombok.AllArgsConstructor;
069import lombok.Data;
070import lombok.Getter;
071import lombok.RequiredArgsConstructor;
072import lombok.extern.slf4j.Slf4j;
073
074public class JobImpl implements Job {
075
076    public static class NodeBuilderImpl implements NodeBuilder {
077
078        private final List<Component> nodes = new ArrayList<>();
079
080        private final Map<String, Map<String, Object>> properties = new HashMap<>();
081
082        @Override
083        public NodeBuilder property(final String name, final Object value) {
084            final Component lastComponent = nodes.get(nodes.size() - 1);
085            properties.computeIfAbsent(lastComponent.getId(), s -> new HashMap<>());
086            properties.get(lastComponent.getId()).put(name, value);
087            return this;
088        }
089
090        @Override
091        public NodeBuilder component(final String id, final String uri) {
092            nodes.add(new Component(id, DSLParser.parse(uri)));
093            return this;
094        }
095
096        @Override
097        public LinkBuilder connections() {
098            return new LinkBuilder(nodes, properties);
099        }
100
101    }
102
103    @Slf4j
104    @RequiredArgsConstructor
105    public static class LinkBuilder implements Job.FromBuilder, Builder {
106
107        private final List<Component> nodes;
108
109        private final Map<String, Map<String, Object>> properties;
110
111        private final List<Edge> edges = new ArrayList<>();
112
113        private final Map<Integer, Set<Component>> levels = new TreeMap<>();
114
115        @Override
116        public ToBuilder from(final String id, final String branch) {
117            final Component from = nodes
118                    .stream()
119                    .filter(node -> node.getId().equals(id))
120                    .findFirst()
121                    .orElseThrow(
122                            () -> new IllegalStateException("No component with id '" + id + "' in created components"));
123
124            edges
125                    .stream()
126                    .filter(edge -> edge.getFrom().getNode().getId().equals(id)
127                            && edge.getFrom().getBranch().equals(branch))
128                    .findFirst()
129                    .ifPresent(edge -> {
130                        throw new IllegalStateException(
131                                "(" + id + "," + branch + ") node is already connected : " + edge);
132                    });
133
134            return new To(nodes, edges, new Connection(from, branch), this);
135        }
136
137        public void doBuild() {
138            final List<Component> orphans = nodes
139                    .stream()
140                    .filter(n -> edges
141                            .stream()
142                            .noneMatch(l -> l.getFrom().getNode().equals(n) || l.getTo().getNode().equals(n)))
143                    .collect(toList());
144            orphans.forEach(o -> log.warn("component '" + o + "' is orphan in this graph. it will be ignored."));
145            nodes.removeAll(orphans);
146
147            // set up sources
148            nodes
149                    .stream()
150                    .filter(node -> edges.stream().noneMatch(l -> l.getTo().getNode().equals(node)))
151                    .forEach(component -> component.setSource(true));
152            calculateGraphOrder(0, new HashSet<>(nodes), new ArrayList<>(edges), levels);
153        }
154
155        private void calculateGraphOrder(final int order, final Set<Component> nodes, final List<Edge> edges,
156                final Map<Integer, Set<Component>> orderedGraph) {
157            if (edges.isEmpty()) {
158                orderedGraph.put(order, nodes); // last nodes
159                return;
160            }
161            final Set<Component> startingNodes = nodes
162                    .stream()
163                    .filter(node -> edges.stream().noneMatch(l -> l.getTo().getNode().equals(node)))
164                    .collect(toSet());
165            if (order == 0 && startingNodes.isEmpty()) {
166                throw new IllegalStateException("There is no starting component in this graph.");
167            }
168            final List<Edge> level = edges
169                    .stream()
170                    .filter(edge -> startingNodes.contains(edge.getFrom().getNode()))
171                    .filter(edge -> edges
172                            .stream()
173                            .filter(others -> edge.getTo().getNode().equals(others.getTo().getNode()))
174                            .map(others -> others.getFrom().getNode())
175                            .allMatch(startingNodes::contains))
176                    .collect(toList());
177            if (level.isEmpty()) {
178                throw new IllegalStateException("the job pipeline has cyclic connection");
179            }
180            final Set<Component> components = level.stream().map(edge -> edge.getFrom().getNode()).collect(toSet());
181            orderedGraph.put(order, components);
182            edges.removeAll(level);
183            nodes.removeAll(components);
184            calculateGraphOrder(order + 1, nodes, edges, orderedGraph);
185        }
186
187        @Override
188        public JobExecutor build() {
189            doBuild();
190            return new JobExecutor(levels, edges, properties);
191        }
192    }
193
194    @RequiredArgsConstructor
195    private static class To implements ToBuilder {
196
197        private final List<Component> nodes;
198
199        private final List<Edge> edges;
200
201        private final Connection from;
202
203        private final Builder builder;
204
205        @Override
206        public Builder to(final String id, final String branch) {
207            final Component to = nodes
208                    .stream()
209                    .filter(node -> node.getId().equals(id))
210                    .findFirst()
211                    .orElseThrow(() -> new IllegalStateException("No component with id '" + id + "' in created nodes"));
212
213            edges
214                    .stream()
215                    .filter(edge -> edge.getTo().getNode().getId().equals(id)
216                            && edge.getTo().getBranch().equals(branch))
217                    .findFirst()
218                    .ifPresent(edge -> {
219                        throw new IllegalStateException(
220                                "(" + id + "," + branch + ") node is already connected : " + edge);
221                    });
222            edges.add(new Edge(from, new Connection(to, branch)));
223            return builder;
224        }
225    }
226
227    @Getter
228    @Slf4j
229    @RequiredArgsConstructor
230    public static class JobExecutor implements Job.ExecutorBuilder {
231
232        private final Map<Integer, Set<Component>> levels;
233
234        private final List<Edge> edges;
235
236        private final Map<String, Map<String, Object>> componentProperties;
237
238        private final Map<String, Object> jobProperties = new HashMap<>();
239
240        private final ComponentManager manager = ComponentManager.instance();
241
242        @Override
243        public ExecutorBuilder property(final String name, final Object value) {
244            jobProperties.put(name, value);
245            return this;
246        }
247
248        @Override
249        public void run() {
250            ExecutorBuilder runner = this;
251            final Object o = jobProperties.get(ExecutorBuilder.class.getName());
252            if (ExecutorBuilder.class.isInstance(o)) {
253                runner = ExecutorBuilder.class.cast(o);
254            } else if (Class.class.isInstance(o)) {
255                runner = newRunner(Class.class.cast(o));
256            } else if (String.class.isInstance(o)) {
257                final String name = String.class.cast(o).trim();
258                if (!"standalone".equalsIgnoreCase(name) && !"default".equalsIgnoreCase(name)
259                        && !"local".equalsIgnoreCase(name)) {
260                    if ("beam".equalsIgnoreCase(name)) {
261                        try {
262                            runner = newRunner(Thread.currentThread().getContextClassLoader(),
263                                    "org.talend.sdk.component.runtime.beam.chain.impl.BeamExecutor");
264                        } catch (final RuntimeException re) {
265                            log
266                                    .error("Can't instantiate beam job integration, "
267                                            + "did you add org.talend.sdk.component:component-runtime-beam in your dependencies",
268                                            re);
269                        }
270                    } else {
271                        runner = newRunner(Thread.currentThread().getContextClassLoader(), name);
272                    }
273                }
274            } else if (o != null) {
275                throw new IllegalArgumentException(o + " is not an ExecutionBuilder");
276            } else {
277                final ClassLoader loader = Thread.currentThread().getContextClassLoader();
278                try (final InputStream stream =
279                        loader.getResourceAsStream("META-INF/services/" + ExecutorBuilder.class.getName())) {
280                    if (stream != null) {
281                        runner = new BufferedReader(new InputStreamReader(stream))
282                                .lines()
283                                .map(String::trim)
284                                .filter(s -> !s.startsWith("#") && !s.isEmpty())
285                                .findFirst()
286                                .map(clazz -> newRunner(loader, clazz))
287                                .orElse(this);
288                    }
289                } catch (final IOException e) {
290                    log.debug(e.getMessage(), e);
291                }
292            }
293
294            if (runner == this) {
295                JobExecutor.class.cast(runner).localRun();
296            } else {
297                runner.run();
298            }
299        }
300
301        private ExecutorBuilder newRunner(final ClassLoader loader, final String clazz) {
302            try {
303                final Class<? extends ExecutorBuilder> aClass =
304                        (Class<? extends ExecutorBuilder>) loader.loadClass(clazz);
305                return newRunner(aClass);
306            } catch (final ClassNotFoundException e) {
307                throw new IllegalArgumentException(e);
308            }
309        }
310
311        private ExecutorBuilder newRunner(final Class<? extends ExecutorBuilder> runnerType) {
312            try {
313                try {
314                    return runnerType.getConstructor(JobExecutor.class).newInstance(JobExecutor.this);
315                } catch (final NoSuchMethodException e) {
316                    return runnerType.getConstructor().newInstance();
317                }
318            } catch (final NoSuchMethodException | InstantiationException | IllegalAccessException e1) {
319                throw new IllegalArgumentException(e1);
320            } catch (InvocationTargetException e1) {
321                throw new IllegalArgumentException(e1.getTargetException());
322            }
323        }
324
325        private void localRun() {
326            final long maxRecords =
327                    Long.parseLong(String.valueOf(getJobProperties().getOrDefault("streaming.maxRecords", "-1")));
328            final Map<String, InputRunner> inputs =
329                    levels.values().stream().flatMap(Collection::stream).filter(Component::isSource).map(n -> {
330                        final Mapper mapper = manager
331                                .findMapper(n.getNode().getFamily(), n.getNode().getComponent(),
332                                        n.getNode().getVersion(), n.getNode().getConfiguration())
333                                .orElseThrow(() -> new IllegalStateException("No mapper found for: " + n.getNode()));
334                        return new AbstractMap.SimpleEntry<>(n.getId(), new InputRunner(mapper, maxRecords));
335                    }).collect(toMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue));
336
337            final Map<String, AutoChunkProcessor> processors = levels
338                    .values()
339                    .stream()
340                    .flatMap(Collection::stream)
341                    .filter(component -> !component.isSource())
342                    .map(component -> {
343                        final Processor processor = manager
344                                .findProcessor(component.getNode().getFamily(), component.getNode().getComponent(),
345                                        component.getNode().getVersion(), component.getNode().getConfiguration())
346                                .orElseThrow(() -> new IllegalStateException(
347                                        "No processor found for:" + component.getNode()));
348                        final AtomicInteger maxBatchSize = new AtomicInteger(1);
349                        if (ProcessorImpl.class.isInstance(processor)) {
350                            ProcessorImpl.class
351                                    .cast(processor)
352                                    .getInternalConfiguration()
353                                    .entrySet()
354                                    .stream()
355                                    .filter(it -> it.getKey().endsWith("$maxBatchSize") && it.getValue() != null
356                                            && !it.getValue().trim().isEmpty())
357                                    .findFirst()
358                                    .ifPresent(val -> {
359                                        try {
360                                            maxBatchSize.set(Integer.parseInt(val.getValue().trim()));
361                                        } catch (final NumberFormatException nfe) {
362                                            throw new IllegalArgumentException("Invalid configuratoin: " + val);
363                                        }
364                                    });
365                        }
366                        return new AbstractMap.SimpleEntry<>(component.getId(),
367                                new AutoChunkProcessor(maxBatchSize.get(), processor));
368                    })
369                    .collect(toMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue));
370
371            final RecordConverters.MappingMetaRegistry registry = new RecordConverters.MappingMetaRegistry();
372            final AtomicReference<DataOutputFactory> outs = new AtomicReference<>();
373            try {
374                final Map<String, AtomicBoolean> sourcesWithData = levels
375                        .values()
376                        .stream()
377                        .flatMap(Collection::stream)
378                        .filter(Component::isSource)
379                        .map(component -> new AbstractMap.SimpleEntry<>(component.getId(), new AtomicBoolean(true)))
380                        .collect(toMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue));
381                processors.values().forEach(Lifecycle::start); // start processor
382                final Map<String, Map<String, Map<String, Collection<Record>>>> flowData = new HashMap<>();
383                final AtomicBoolean running = new AtomicBoolean(true);
384                do {
385                    levels.forEach((level, components) -> components.forEach((Component component) -> {
386                        if (component.isSource()) {
387                            final InputRunner source = inputs.get(component.getId());
388                            final Record data = source.next();
389                            if (data == null) {
390                                sourcesWithData.get(component.getId()).set(false);
391                                return;
392                            }
393                            final String key = getKeyProvider(component.getId())
394                                    .apply(new GroupContextImpl(data, component.getId(), "__default__"));
395                            flowData.computeIfAbsent(component.getId(), s -> new HashMap<>());
396                            flowData.get(component.getId()).computeIfAbsent("__default__", s -> new TreeMap<>());
397                            flowData
398                                    .get(component.getId())
399                                    .get("__default__")
400                                    .computeIfAbsent(key, k -> new ArrayList<>())
401                                    .add(data);
402                        } else {
403                            final List<Edge> connections =
404                                    getConnections(getEdges(), component, e -> e.getTo().getNode());
405                            final DataInputFactory dataInputFactory = new DataInputFactory();
406                            if (connections.size() == 1) {
407                                final Edge edge = connections.get(0);
408                                final String fromId = edge.getFrom().getNode().getId();
409                                final String fromBranch = edge.getFrom().getBranch();
410                                final String toBranch = edge.getTo().getBranch();
411
412                                final Map<String, Map<String, Collection<Record>>> idData = flowData.get(fromId);
413                                final Record data = idData == null ? null : pollFirst(idData.get(fromBranch));
414                                if (data != null) {
415                                    dataInputFactory.withInput(toBranch, singletonList(data));
416                                }
417                            } else { // need grouping
418                                final Map<String, Map<String, Collection<Record>>> availableDataForStep =
419                                        new HashMap<>();
420                                connections.forEach(edge -> {
421                                    final String fromId = edge.getFrom().getNode().getId();
422                                    final String fromBranch = edge.getFrom().getBranch();
423                                    final String toBranch = edge.getTo().getBranch();
424                                    final Map<String, Collection<Record>> data =
425                                            flowData.get(fromId) == null ? null : flowData.get(fromId).get(fromBranch);
426                                    if (data != null && !data.isEmpty()) {
427                                        availableDataForStep.put(toBranch, data);
428                                    }
429                                });
430
431                                final Map<String, String> joined = joinWithFusionSort(availableDataForStep);
432                                if (!joined.isEmpty() && connections.size() == joined.size()) {
433                                    joined.forEach((k, v) -> {
434                                        final Collection data = availableDataForStep.get(k).remove(v);
435                                        dataInputFactory.withInput(k, data);
436                                    });
437                                }
438                            }
439                            if (dataInputFactory.inputs.isEmpty()) {
440                                if (level.equals(levels.size() - 1)
441                                        && sourcesWithData.entrySet().stream().noneMatch(e -> e.getValue().get())) {
442                                    running.set(false);
443                                }
444                                return;
445                            }
446                            final AutoChunkProcessor processor = processors.get(component.getId());
447
448                            final DataOutputFactory dataOutputFactory = new DataOutputFactory(getManager()
449                                    .findPlugin(processor.plugin())
450                                    .get()
451                                    .get(ComponentManager.AllServices.class)
452                                    .getServices(), registry);
453                            processor.onElement(dataInputFactory, dataOutputFactory);
454                            dataOutputFactory.getOutputs().forEach((branch, data) -> data.forEach(item -> {
455                                final String key = getKeyProvider(component.getId())
456                                        .apply(new GroupContextImpl(item, component.getId(), branch));
457                                flowData.computeIfAbsent(component.getId(), s -> new HashMap<>());
458                                flowData.get(component.getId()).computeIfAbsent(branch, s -> new TreeMap<>());
459                                flowData
460                                        .get(component.getId())
461                                        .get(branch)
462                                        .computeIfAbsent(key, k -> new ArrayList<>())
463                                        .add(item);
464                            }));
465                            outs.set(dataOutputFactory);
466                        }
467                    }));
468                } while (running.get());
469            } finally {
470                if (outs.get() != null) {
471                    processors.values().forEach(p -> p.flush(outs.get()));
472                }
473                processors.values().forEach(Lifecycle::stop);
474                inputs.values().forEach(InputRunner::stop);
475                levels
476                        .values()
477                        .stream()
478                        .flatMap(Collection::stream)
479                        .map(Component::getId)
480                        .forEach(LocalSequenceHolder::clean);
481            }
482        }
483
484        private Map<String, String>
485                joinWithFusionSort(final Map<String, Map<String, Collection<Record>>> dataByBranch) {
486            final Map<String, String> join = new HashMap<>();
487            dataByBranch.forEach((branch1, records1) -> {
488                dataByBranch.forEach((branch2, records2) -> {
489                    if (!branch1.equals(branch2)) {
490                        for (final String key1 : records1.keySet()) {
491                            for (final String key2 : records2.keySet()) {
492                                if (key1.equals(key2)) {
493                                    join.putIfAbsent(branch1, key1);
494                                    join.putIfAbsent(branch2, key2);
495                                } else if (key1.compareTo(key2) < 0) {
496                                    break;// see fusion sort
497                                }
498                            }
499                        }
500                    }
501                });
502            });
503            return join;
504        }
505
506        private Record pollFirst(final Map<String, Collection<Record>> data) {
507            if (data == null || data.isEmpty()) {
508                return null;
509            }
510            while (!data.isEmpty()) {
511                final String key = data.keySet().iterator().next();
512                final Collection<Record> items = data.get(key);
513                if (!items.isEmpty()) {
514                    final Iterator<Record> iterator = items.iterator();
515                    final Record item = iterator.next();
516                    iterator.remove();
517                    return item;
518                } else {
519                    data.remove(key);
520                }
521            }
522            return null;
523        }
524
525        private List<Job.Edge> getConnections(final List<Job.Edge> edges, final Job.Component step,
526                final Function<Edge, Component> direction) {
527            return edges.stream().filter(edge -> direction.apply(edge).equals(step)).collect(toList());
528        }
529
530        public GroupKeyProvider getKeyProvider(final String componentId) {
531            if (componentProperties.get(componentId) != null) {
532                final Object o = componentProperties.get(componentId).get(GroupKeyProvider.class.getName());
533                if (GroupKeyProvider.class.isInstance(o)) {
534                    return new GroupKeyProviderImpl(GroupKeyProvider.class.cast(o));
535                }
536            }
537
538            final Object o = jobProperties.get(GroupKeyProvider.class.getName());
539            if (GroupKeyProvider.class.isInstance(o)) {
540                return new GroupKeyProviderImpl(GroupKeyProvider.class.cast(o));
541            }
542
543            final ServiceLoader<GroupKeyProvider> services = ServiceLoader.load(GroupKeyProvider.class);
544            if (services.iterator().hasNext()) {
545                return services.iterator().next();
546            }
547
548            return LocalSequenceHolder.cleanAndGet(componentId);
549        }
550    }
551
552    @Data
553    private static class GroupContextImpl implements GroupKeyProvider.GroupContext {
554
555        private final Record data;
556
557        private final String componentId;
558
559        private final String branchName;
560    }
561
562    public static class LocalSequenceHolder {
563
564        private static final Map<String, AtomicLong> GENERATORS = new HashMap<>();
565
566        public static GroupKeyProvider cleanAndGet(final String name) {
567            GENERATORS.put(name, new AtomicLong(0));
568            return c -> Long.toString(GENERATORS.get(name).incrementAndGet());
569        }
570
571        public static void clean(final String name) {
572            GENERATORS.remove(name);
573        }
574    }
575
576    @Slf4j
577    private static class InputRunner {
578
579        private final Mapper chainedMapper;
580
581        private final Input input;
582
583        private final long maxRecords;
584
585        private long currentRecords;
586
587        private InputRunner(final Mapper mapper, final long maxRecords) {
588            this.maxRecords = maxRecords;
589            RuntimeException error = null;
590            try {
591                mapper.start();
592                chainedMapper = new ChainedMapper(mapper, mapper.split(mapper.assess()).iterator());
593                chainedMapper.start();
594                input = chainedMapper.create();
595                input.start();
596            } catch (final RuntimeException re) {
597                error = re;
598                throw re;
599            } finally {
600                try {
601                    mapper.stop();
602                } catch (final RuntimeException re) {
603                    if (error == null) {
604                        throw re;
605                    }
606                    log.error(re.getMessage(), re);
607                }
608            }
609        }
610
611        public Record next() {
612            if (maxRecords > 0 && currentRecords >= maxRecords) {
613                return null;
614            }
615            final Object next = input.next();
616            if (next == null) {
617                return null;
618            }
619            currentRecords++;
620            return Record.class.cast(next);
621        }
622
623        public void stop() {
624            RuntimeException error = null;
625            try {
626                if (input != null) {
627                    input.stop();
628                }
629            } catch (final RuntimeException re) {
630                error = re;
631                throw re;
632            } finally {
633                try {
634                    if (chainedMapper != null) {
635                        chainedMapper.stop();
636                    }
637                } catch (final RuntimeException re) {
638                    if (error == null) {
639                        throw re;
640                    }
641                    log.error(re.getMessage(), re);
642                }
643            }
644        }
645    }
646
647    @Data
648    private static class DataOutputFactory implements OutputFactory {
649
650        private final Map<Class<?>, Object> services;
651
652        private final RecordConverters.MappingMetaRegistry registry;
653
654        private final Map<String, Collection<Record>> outputs = new HashMap<>();
655
656        @Override
657        public OutputEmitter create(final String name) {
658            return new OutputEmitterImpl(name, registry);
659        }
660
661        @AllArgsConstructor
662        private class OutputEmitterImpl implements OutputEmitter {
663
664            private final String name;
665
666            private final RecordConverters.MappingMetaRegistry registry;
667
668            @Override
669            public void emit(final Object value) {
670                outputs
671                        .computeIfAbsent(name, k -> new ArrayList<>())
672                        .add(new RecordConverters()
673                                .toRecord(registry, value, () -> Jsonb.class.cast(services.get(Jsonb.class)),
674                                        () -> RecordBuilderFactory.class
675                                                .cast(services.get(RecordBuilderFactory.class))));
676            }
677        }
678    }
679
680    private static class DataInputFactory implements InputFactory {
681
682        private final Map<String, Iterator<Object>> inputs = new HashMap<>();
683
684        private volatile Jsonb jsonb;
685
686        private volatile RecordBuilderFactory factory;
687
688        private volatile RecordConverters.MappingMetaRegistry registry;
689
690        private DataInputFactory withInput(final String branch, final Collection<Object> branchData) {
691            inputs.put(branch, branchData.iterator());
692            return this;
693        }
694
695        @Override
696        public Object read(final String name) {
697            final Iterator<?> iterator = inputs.get(name);
698            if (iterator != null && iterator.hasNext()) {
699                return map(iterator.next());
700            }
701            return null;
702        }
703
704        private Object map(final Object next) {
705            if (next == null || Record.class.isInstance(next)) {
706                return next;
707            }
708
709            final String str = jsonb().get().toJson(next);
710            // primitives mainly, not that accurate in main code but for now not forbidden
711            if (str.equals(next.toString())) {
712                return next;
713            }
714            if (registry == null) {
715                synchronized (this) {
716                    if (registry == null) {
717                        registry = new RecordConverters.MappingMetaRegistry();
718                    }
719                }
720            }
721            // pojo
722            return new RecordConverters().toRecord(registry, next, jsonb(), () -> {
723                if (factory == null) {
724                    synchronized (this) {
725                        if (factory == null) {
726                            factory = new RecordBuilderFactoryImpl("test");
727                        }
728                    }
729                }
730                return factory;
731            });
732        }
733
734        private Supplier<Jsonb> jsonb() {
735            return () -> {
736                if (jsonb == null) {
737                    synchronized (this) {
738                        if (jsonb == null) {
739                            jsonb = JsonbBuilder.create(new JsonbConfig().setProperty("johnzon.cdi.activated", false));
740                        }
741                    }
742                }
743                return jsonb;
744            };
745        }
746    }
747
748    @AllArgsConstructor
749    protected static class GroupKeyProviderImpl implements GroupKeyProvider {
750
751        private final GroupKeyProvider delegate;
752
753        @Override
754        public String apply(final GroupKeyProvider.GroupContext context) {
755            return delegate.apply(context);
756        }
757    }
758}