Source code

001/**
002 * Copyright (C) 2006-2022 Talend Inc. - www.talend.com
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.talend.sdk.component.runtime.record;
017
018import static java.util.stream.Collectors.toList;
019import static java.util.stream.Collectors.toSet;
020
021import java.io.ObjectStreamException;
022import java.io.Serializable;
023import java.lang.reflect.Constructor;
024import java.lang.reflect.InvocationTargetException;
025import java.lang.reflect.Method;
026import java.math.BigDecimal;
027import java.time.ZonedDateTime;
028import java.time.format.DateTimeFormatter;
029import java.util.Base64;
030import java.util.Collection;
031import java.util.Collections;
032import java.util.Date;
033import java.util.List;
034import java.util.Map;
035import java.util.Set;
036import java.util.concurrent.ConcurrentHashMap;
037import java.util.function.Function;
038import java.util.function.Supplier;
039import java.util.stream.Collector;
040import java.util.stream.Stream;
041
042import javax.json.JsonArray;
043import javax.json.JsonArrayBuilder;
044import javax.json.JsonBuilderFactory;
045import javax.json.JsonNumber;
046import javax.json.JsonObject;
047import javax.json.JsonObjectBuilder;
048import javax.json.JsonString;
049import javax.json.JsonValue;
050import javax.json.bind.Jsonb;
051import javax.json.spi.JsonProvider;
052
053import org.apache.johnzon.core.JsonLongImpl;
054import org.apache.johnzon.jsonb.extension.JsonValueReader;
055import org.talend.sdk.component.api.record.Record;
056import org.talend.sdk.component.api.record.Schema;
057import org.talend.sdk.component.api.service.record.RecordBuilderFactory;
058import org.talend.sdk.component.runtime.record.json.OutputRecordHolder;
059import org.talend.sdk.component.runtime.record.json.PojoJsonbProvider;
060
061import lombok.Data;
062
063public class RecordConverters implements Serializable {
064
065    public <T> Record toRecord(final MappingMetaRegistry registry, final T data, final Supplier<Jsonb> jsonbProvider,
066            final Supplier<RecordBuilderFactory> recordBuilderProvider) {
067        if (data == null) {
068            return null;
069        }
070        if (Record.class.isInstance(data)) {
071            return Record.class.cast(data);
072        }
073        if (JsonObject.class.isInstance(data)) {
074            return json2Record(recordBuilderProvider.get(), JsonObject.class.cast(data));
075        }
076
077        final MappingMeta meta = registry.find(data.getClass());
078        if (meta.isLinearMapping()) {
079            return meta.newRecord(data, recordBuilderProvider.get());
080        }
081
082        final Jsonb jsonb = jsonbProvider.get();
083        if (!String.class.isInstance(data) && !data.getClass().isPrimitive()
084                && PojoJsonbProvider.class.isInstance(jsonb)) {
085            final Jsonb pojoMapper = PojoJsonbProvider.class.cast(jsonb).get();
086            final OutputRecordHolder holder = new OutputRecordHolder(data);
087            try (final OutputRecordHolder stream = holder) {
088                pojoMapper.toJson(data, stream);
089            }
090            return holder.getRecord();
091        }
092        return json2Record(recordBuilderProvider.get(), jsonb.fromJson(jsonb.toJson(data), JsonObject.class));
093    }
094
095    private Record json2Record(final RecordBuilderFactory factory, final JsonObject object) {
096        final Record.Builder builder = factory.newRecordBuilder();
097        object.forEach((key, value) -> {
098            switch (value.getValueType()) {
099            case ARRAY: {
100                final List<Object> items =
101                        value.asJsonArray().stream().map(it -> mapJson(factory, it)).collect(toList());
102                builder
103                        .withArray(factory
104                                .newEntryBuilder()
105                                .withName(key)
106                                .withType(Schema.Type.ARRAY)
107                                .withElementSchema(getArrayElementSchema(factory, items))
108                                .build(), items);
109                break;
110            }
111            case OBJECT: {
112                final Record record = json2Record(factory, value.asJsonObject());
113                builder
114                        .withRecord(factory
115                                .newEntryBuilder()
116                                .withName(key)
117                                .withType(Schema.Type.RECORD)
118                                .withElementSchema(record.getSchema())
119                                .build(), record);
120                break;
121            }
122            case TRUE:
123            case FALSE:
124                builder.withBoolean(key, JsonValue.TRUE.equals(value));
125                break;
126            case STRING:
127                builder.withString(key, JsonString.class.cast(value).getString());
128                break;
129            case NUMBER:
130                final JsonNumber number = JsonNumber.class.cast(value);
131                builder.withDouble(key, number.doubleValue());
132                break;
133            case NULL:
134                break;
135            default:
136                throw new IllegalArgumentException("Unsupported value type: " + value);
137            }
138        });
139        return builder.build();
140    }
141
142    private Schema getArrayElementSchema(final RecordBuilderFactory factory, final List<Object> items) {
143        if (items.isEmpty()) {
144            return factory.newSchemaBuilder(Schema.Type.STRING).build();
145        }
146        final Schema firstSchema = toSchema(factory, items.iterator().next());
147        switch (firstSchema.getType()) {
148        case RECORD:
149            return items.stream().map(it -> toSchema(factory, it)).reduce(null, (s1, s2) -> {
150                if (s1 == null) {
151                    return s2;
152                }
153                if (s2 == null) { // unlikely
154                    return s1;
155                }
156                final Set<String> names1 = s1.getAllEntries().map(Schema.Entry::getName).collect(toSet());
157                final Set<String> names2 = s2.getAllEntries().map(Schema.Entry::getName).collect(toSet());
158                if (!names1.equals(names2)) {
159                    // here we are not good since values will not be right anymore,
160                    // forbidden for current version anyway but potentially supported later
161                    final Schema.Builder builder = factory.newSchemaBuilder(Schema.Type.RECORD);
162                    s1.getAllEntries().forEach(builder::withEntry);
163                    s2.getAllEntries().filter(it -> !(names1.contains(it.getName()))).forEach(builder::withEntry);
164                    return builder.build();
165                }
166                return s1;
167            });
168        default:
169            return firstSchema;
170        }
171    }
172
173    private Object mapJson(final RecordBuilderFactory factory, final JsonValue it) {
174        if (JsonObject.class.isInstance(it)) {
175            return json2Record(factory, JsonObject.class.cast(it));
176        }
177        if (JsonArray.class.isInstance(it)) {
178            return JsonArray.class.cast(it).stream().map(i -> mapJson(factory, i)).collect(toList());
179        }
180        if (JsonString.class.isInstance(it)) {
181            return JsonString.class.cast(it).getString();
182        }
183        if (JsonNumber.class.isInstance(it)) {
184            return JsonNumber.class.cast(it).numberValue();
185        }
186        if (JsonValue.FALSE.equals(it)) {
187            return false;
188        }
189        if (JsonValue.TRUE.equals(it)) {
190            return true;
191        }
192        if (JsonValue.NULL.equals(it)) {
193            return null;
194        }
195        return it;
196    }
197
198    public static Schema toSchema(final RecordBuilderFactory factory, final Object next) {
199        if (String.class.isInstance(next) || JsonString.class.isInstance(next)) {
200            return factory.newSchemaBuilder(Schema.Type.STRING).build();
201        }
202        if (Integer.class.isInstance(next)) {
203            return factory.newSchemaBuilder(Schema.Type.INT).build();
204        }
205        if (Long.class.isInstance(next) || JsonLongImpl.class.isInstance(next)) {
206            return factory.newSchemaBuilder(Schema.Type.LONG).build();
207        }
208        if (Float.class.isInstance(next)) {
209            return factory.newSchemaBuilder(Schema.Type.FLOAT).build();
210        }
211        if (JsonNumber.class.isInstance(next)) {
212            return factory.newSchemaBuilder(Schema.Type.DOUBLE).build();
213        }
214        if (Double.class.isInstance(next) || JsonNumber.class.isInstance(next)) {
215            return factory.newSchemaBuilder(Schema.Type.DOUBLE).build();
216        }
217        if (Boolean.class.isInstance(next) || JsonValue.TRUE.equals(next) || JsonValue.FALSE.equals(next)) {
218            return factory.newSchemaBuilder(Schema.Type.BOOLEAN).build();
219        }
220        if (Date.class.isInstance(next) || ZonedDateTime.class.isInstance(next)) {
221            return factory.newSchemaBuilder(Schema.Type.DATETIME).build();
222        }
223        if (BigDecimal.class.isInstance(next)) {
224            return factory.newSchemaBuilder(Schema.Type.DECIMAL).build();
225        }
226        if (byte[].class.isInstance(next)) {
227            return factory.newSchemaBuilder(Schema.Type.BYTES).build();
228        }
229        if (Collection.class.isInstance(next) || JsonArray.class.isInstance(next)) {
230            final Collection collection = Collection.class.cast(next);
231            if (collection.isEmpty()) {
232                return factory.newSchemaBuilder(Schema.Type.STRING).build();
233            }
234            return factory
235                    .newSchemaBuilder(Schema.Type.ARRAY)
236                    .withElementSchema(toSchema(factory, collection.iterator().next()))
237                    .build();
238        }
239        if (Record.class.isInstance(next)) {
240            return Record.class.cast(next).getSchema();
241        }
242        throw new IllegalArgumentException("unsupported type for " + next);
243    }
244
245    public Object toType(final MappingMetaRegistry registry, final Object data, final Class<?> parameterType,
246            final Supplier<JsonBuilderFactory> factorySupplier, final Supplier<JsonProvider> providerSupplier,
247            final Supplier<Jsonb> jsonbProvider, final Supplier<RecordBuilderFactory> recordBuilderProvider) {
248        return toType(registry, data, parameterType, factorySupplier, providerSupplier, jsonbProvider,
249                recordBuilderProvider, Collections.emptyMap());
250    }
251
252    public Object toType(final MappingMetaRegistry registry, final Object data, final Class<?> parameterType,
253            final Supplier<JsonBuilderFactory> factorySupplier, final Supplier<JsonProvider> providerSupplier,
254            final Supplier<Jsonb> jsonbProvider, final Supplier<RecordBuilderFactory> recordBuilderProvider,
255            final java.util.Map<String, String> metadata) {
256        if (parameterType.isInstance(data)) {
257            return data;
258        }
259
260        final JsonObject inputAsJson;
261        if (JsonObject.class.isInstance(data)) {
262            if (JsonObject.class == parameterType) {
263                return data;
264            }
265            inputAsJson = JsonObject.class.cast(data);
266        } else if (Record.class.isInstance(data)) {
267            final Record record = Record.class.cast(data);
268            if (!JsonObject.class.isAssignableFrom(parameterType)) {
269                final MappingMeta mappingMeta = registry.find(parameterType);
270                if (mappingMeta.isLinearMapping()) {
271                    return mappingMeta.newInstance(record, metadata);
272                }
273            }
274            final JsonObject asJson = toJson(factorySupplier, providerSupplier, record);
275            if (JsonObject.class == parameterType) {
276                return asJson;
277            }
278            inputAsJson = asJson;
279        } else {
280            if (parameterType == Record.class) {
281                return toRecord(registry, data, jsonbProvider, recordBuilderProvider);
282            }
283            final Jsonb jsonb = jsonbProvider.get();
284            inputAsJson = jsonb.fromJson(jsonb.toJson(data), JsonObject.class);
285        }
286        return jsonbProvider.get().fromJson(new JsonValueReader<>(inputAsJson), parameterType);
287    }
288
289    private JsonObject toJson(final Supplier<JsonBuilderFactory> factorySupplier,
290            final Supplier<JsonProvider> providerSupplier, final Record record) {
291        return buildRecord(factorySupplier.get(), providerSupplier, record).build();
292    }
293
294    private JsonObjectBuilder buildRecord(final JsonBuilderFactory factory,
295            final Supplier<JsonProvider> providerSupplier, final Record record) {
296        final Schema schema = record.getSchema();
297        final JsonObjectBuilder builder = factory.createObjectBuilder();
298        schema.getEntries().forEach(entry -> {
299            final String name = entry.getName();
300            switch (entry.getType()) {
301            case STRING: {
302                final String value = record.get(String.class, name);
303                if (value != null) {
304                    builder.add(name, value);
305                }
306                break;
307            }
308            case INT: {
309                final Integer value = record.get(Integer.class, name);
310                if (value != null) {
311                    builder.add(name, value);
312                }
313                break;
314            }
315            case LONG: {
316                final Long value = record.get(Long.class, name);
317                if (value != null) {
318                    builder.add(name, value);
319                }
320                break;
321            }
322            case FLOAT: {
323                final Float value = record.get(Float.class, name);
324                if (value != null) {
325                    builder.add(name, value);
326                }
327                break;
328            }
329            case DOUBLE: {
330                final Double value = record.get(Double.class, name);
331                if (value != null) {
332                    builder.add(name, value);
333                }
334                break;
335            }
336            case BOOLEAN: {
337                final Boolean value = record.get(Boolean.class, name);
338                if (value != null) {
339                    builder.add(name, value);
340                }
341                break;
342            }
343            case BYTES: {
344                final byte[] value = record.get(byte[].class, name);
345                if (value != null) {
346                    builder.add(name, Base64.getEncoder().encodeToString(value));
347                }
348                break;
349            }
350            case DATETIME: {
351                final ZonedDateTime value = record.get(ZonedDateTime.class, name);
352                if (value != null) {
353                    builder.add(name, value.format(DateTimeFormatter.ISO_ZONED_DATE_TIME));
354                }
355                break;
356            }
357            case DECIMAL: {
358                final BigDecimal value = record.get(BigDecimal.class, name);
359                if (value != null) {
360                    builder.add(name, value.toString());
361                }
362                break;
363            }
364            case RECORD: {
365                final Record value = record.get(Record.class, name);
366                if (value != null) {
367                    builder.add(name, buildRecord(factory, providerSupplier, value));
368                }
369                break;
370            }
371            case ARRAY:
372                final Collection<?> collection = record.get(Collection.class, name);
373                if (collection == null) {
374                    break;
375                }
376                if (collection.isEmpty()) {
377                    builder.add(name, factory.createArrayBuilder().build());
378                } else { // only homogeneous collections
379                    final Object item = collection.iterator().next();
380                    if (String.class.isInstance(item)) {
381                        final JsonProvider jsonProvider = providerSupplier.get();
382                        builder
383                                .add(name, toArray(factory, v -> jsonProvider.createValue(String.class.cast(v)),
384                                        collection));
385                    } else if (Double.class.isInstance(item)) {
386                        final JsonProvider jsonProvider = providerSupplier.get();
387                        builder
388                                .add(name, toArray(factory, v -> jsonProvider.createValue(Double.class.cast(v)),
389                                        collection));
390                    } else if (Float.class.isInstance(item)) {
391                        final JsonProvider jsonProvider = providerSupplier.get();
392                        builder
393                                .add(name, toArray(factory, v -> jsonProvider.createValue(Float.class.cast(v)),
394                                        collection));
395                    } else if (Integer.class.isInstance(item)) {
396                        final JsonProvider jsonProvider = providerSupplier.get();
397                        builder
398                                .add(name, toArray(factory, v -> jsonProvider.createValue(Integer.class.cast(v)),
399                                        collection));
400                    } else if (Long.class.isInstance(item)) {
401                        final JsonProvider jsonProvider = providerSupplier.get();
402                        builder
403                                .add(name, toArray(factory, v -> jsonProvider.createValue(Long.class.cast(v)),
404                                        collection));
405                    } else if (Boolean.class.isInstance(item)) {
406                        builder
407                                .add(name, toArray(factory,
408                                        v -> Boolean.class.cast(v) ? JsonValue.TRUE : JsonValue.FALSE, collection));
409                    } else if (ZonedDateTime.class.isInstance(item)) {
410                        final JsonProvider jsonProvider = providerSupplier.get();
411                        builder
412                                .add(name,
413                                        toArray(factory, v -> jsonProvider
414                                                .createValue(ZonedDateTime.class.cast(v).toInstant().toEpochMilli()),
415                                                collection));
416                    } else if (Date.class.isInstance(item)) {
417                        final JsonProvider jsonProvider = providerSupplier.get();
418                        builder
419                                .add(name, toArray(factory, v -> jsonProvider.createValue(Date.class.cast(v).getTime()),
420                                        collection));
421                    } else if (Record.class.isInstance(item)) {
422                        builder
423                                .add(name, toArray(factory,
424                                        v -> buildRecord(factory, providerSupplier, Record.class.cast(v)).build(),
425                                        collection));
426                    } else if (JsonValue.class.isInstance(item)) {
427                        builder.add(name, toArray(factory, JsonValue.class::cast, collection));
428                    } // else throw?
429                }
430                break;
431            default:
432                throw new IllegalArgumentException("Unsupported type: " + entry.getType() + " for '" + name + "'");
433            }
434        });
435        return builder;
436    }
437
438    private JsonArray toArray(final JsonBuilderFactory factory, final Function<Object, JsonValue> valueFactory,
439            final Collection<?> collection) {
440        final Collector<JsonValue, JsonArrayBuilder, JsonArray> collector = Collector
441                .of(factory::createArrayBuilder, JsonArrayBuilder::add, JsonArrayBuilder::addAll,
442                        JsonArrayBuilder::build);
443        return collection.stream().map(valueFactory).collect(collector);
444    }
445
446    public <T> T coerce(final Class<T> expectedType, final Object value, final String name) {
447        if (value == null) {
448            return null;
449        }
450
451        // here mean get(Object.class, name) return origin store type, like DATETIME return long, is expected?
452        if (!expectedType.isInstance(value)) {
453            return expectedType.cast(MappingUtils.coerce(expectedType, value, name));
454        }
455
456        return expectedType.cast(value);
457    }
458
459    @Data
460    public static class MappingMeta {
461
462        private final boolean linearMapping;
463
464        private final Class<?> rowStruct;
465
466        private Object recordVisitor;
467
468        private Method visitRecord;
469
470        private Object rowStructVisitor;
471
472        private Method visitRowStruct;
473
474        public MappingMeta(final Class<?> type, final MappingMetaRegistry registry) {
475            linearMapping = Stream.of(type.getInterfaces()).anyMatch(it -> it.getName().startsWith("routines.system."));
476            rowStruct = type;
477        }
478
479        public Object newInstance(final Record record) {
480            return newInstance(record, Collections.emptyMap());
481        }
482
483        public Object newInstance(final Record record, final java.util.Map<String, String> metadata) {
484            if (recordVisitor == null) {
485                try {
486                    final String className = "org.talend.sdk.component.runtime.di.record.DiRecordVisitor";
487                    final Class<?> visitorClass = getClass().getClassLoader().loadClass(className);
488                    final Constructor<?> constructor = visitorClass.getDeclaredConstructors()[0];
489                    constructor.setAccessible(true);
490                    recordVisitor = constructor.newInstance(rowStruct, metadata);
491                    visitRecord = visitorClass.getDeclaredMethod("visit", Record.class);
492                } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException
493                        | InvocationTargetException | NoSuchMethodException e) {
494                    throw new IllegalStateException(e);
495                }
496            }
497            try {
498                return visitRecord.invoke(recordVisitor, record);
499            } catch (final IllegalAccessException | InvocationTargetException e) {
500                throw new IllegalStateException(e);
501            }
502        }
503
504        public <T> Record newRecord(final T data, final RecordBuilderFactory factory) {
505            if (rowStructVisitor == null) {
506                try {
507                    final String className = "org.talend.sdk.component.runtime.di.record.DiRowStructVisitor";
508                    final Class<?> visitorClass = getClass().getClassLoader().loadClass(className);
509                    final Constructor<?> constructor = visitorClass.getConstructors()[0];
510                    constructor.setAccessible(true);
511                    rowStructVisitor = constructor.newInstance();
512                    visitRowStruct = visitorClass.getDeclaredMethod("get", Object.class, RecordBuilderFactory.class);
513                } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException
514                        | InvocationTargetException | NoSuchMethodException e) {
515                    throw new IllegalStateException(e);
516                }
517            }
518            try {
519                return Record.class.cast(visitRowStruct.invoke(rowStructVisitor, data, factory));
520            } catch (final IllegalAccessException | InvocationTargetException e) {
521                throw new IllegalStateException(e);
522            }
523        }
524
525    }
526
527    @Data
528    public static class MappingMetaRegistry implements Serializable {
529
530        protected final Map<Class<?>, MappingMeta> registry = new ConcurrentHashMap<>();
531
532        private Object writeReplace() throws ObjectStreamException {
533            return new Factory(); // don't serialize the mapping, recalculate it lazily
534        }
535
536        public MappingMeta find(final Class<?> parameterType) {
537            final MappingMeta meta = registry.get(parameterType);
538            if (meta != null) {
539                return meta;
540            }
541            final MappingMeta mappingMeta = new MappingMeta(parameterType, this);
542            final MappingMeta existing = registry.putIfAbsent(parameterType, mappingMeta);
543            if (existing != null) {
544                return existing;
545            }
546            return mappingMeta;
547        }
548
549        public static class Factory implements Serializable {
550
551            private Object readResolve() throws ObjectStreamException {
552                return new MappingMetaRegistry();
553            }
554        }
555    }
556}