Source code

001/**
002 * Copyright (C) 2006-2024 Talend Inc. - www.talend.com
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.talend.sdk.component.runtime.record;
017
018import static java.util.stream.Collectors.toList;
019import static java.util.stream.Collectors.toSet;
020
021import java.io.ObjectStreamException;
022import java.io.Serializable;
023import java.lang.reflect.Constructor;
024import java.lang.reflect.InvocationTargetException;
025import java.lang.reflect.Method;
026import java.math.BigDecimal;
027import java.time.Instant;
028import java.time.ZonedDateTime;
029import java.time.format.DateTimeFormatter;
030import java.util.Base64;
031import java.util.Collection;
032import java.util.Collections;
033import java.util.Date;
034import java.util.List;
035import java.util.Map;
036import java.util.Set;
037import java.util.concurrent.ConcurrentHashMap;
038import java.util.function.Function;
039import java.util.function.Supplier;
040import java.util.stream.Collector;
041import java.util.stream.Stream;
042
043import javax.json.JsonArray;
044import javax.json.JsonArrayBuilder;
045import javax.json.JsonBuilderFactory;
046import javax.json.JsonNumber;
047import javax.json.JsonObject;
048import javax.json.JsonObjectBuilder;
049import javax.json.JsonString;
050import javax.json.JsonValue;
051import javax.json.bind.Jsonb;
052import javax.json.spi.JsonProvider;
053
054import org.apache.johnzon.core.JsonLongImpl;
055import org.apache.johnzon.jsonb.extension.JsonValueReader;
056import org.talend.sdk.component.api.record.Record;
057import org.talend.sdk.component.api.record.Schema;
058import org.talend.sdk.component.api.service.record.RecordBuilderFactory;
059import org.talend.sdk.component.runtime.record.json.OutputRecordHolder;
060import org.talend.sdk.component.runtime.record.json.PojoJsonbProvider;
061
062import lombok.Data;
063
064public class RecordConverters implements Serializable {
065
066    public <T> Record toRecord(final MappingMetaRegistry registry, final T data, final Supplier<Jsonb> jsonbProvider,
067            final Supplier<RecordBuilderFactory> recordBuilderProvider) {
068        if (data == null) {
069            return null;
070        }
071        if (Record.class.isInstance(data)) {
072            return Record.class.cast(data);
073        }
074        if (JsonObject.class.isInstance(data)) {
075            return json2Record(recordBuilderProvider.get(), JsonObject.class.cast(data));
076        }
077
078        final MappingMeta meta = registry.find(data.getClass());
079        if (meta.isLinearMapping()) {
080            return meta.newRecord(data, recordBuilderProvider.get());
081        }
082
083        final Jsonb jsonb = jsonbProvider.get();
084        if (!String.class.isInstance(data) && !data.getClass().isPrimitive()
085                && PojoJsonbProvider.class.isInstance(jsonb)) {
086            final Jsonb pojoMapper = PojoJsonbProvider.class.cast(jsonb).get();
087            final OutputRecordHolder holder = new OutputRecordHolder(data);
088            try (final OutputRecordHolder stream = holder) {
089                pojoMapper.toJson(data, stream);
090            }
091            return holder.getRecord();
092        }
093        return json2Record(recordBuilderProvider.get(), jsonb.fromJson(jsonb.toJson(data), JsonObject.class));
094    }
095
096    private Record json2Record(final RecordBuilderFactory factory, final JsonObject object) {
097        final Record.Builder builder = factory.newRecordBuilder();
098        object.forEach((key, value) -> {
099            switch (value.getValueType()) {
100            case ARRAY: {
101                final List<Object> items =
102                        value.asJsonArray().stream().map(it -> mapJson(factory, it)).collect(toList());
103                builder
104                        .withArray(factory
105                                .newEntryBuilder()
106                                .withName(key)
107                                .withType(Schema.Type.ARRAY)
108                                .withElementSchema(getArrayElementSchema(factory, items))
109                                .build(), items);
110                break;
111            }
112            case OBJECT: {
113                final Record record = json2Record(factory, value.asJsonObject());
114                builder
115                        .withRecord(factory
116                                .newEntryBuilder()
117                                .withName(key)
118                                .withType(Schema.Type.RECORD)
119                                .withElementSchema(record.getSchema())
120                                .build(), record);
121                break;
122            }
123            case TRUE:
124            case FALSE:
125                builder.withBoolean(key, JsonValue.TRUE.equals(value));
126                break;
127            case STRING:
128                builder.withString(key, JsonString.class.cast(value).getString());
129                break;
130            case NUMBER:
131                final JsonNumber number = JsonNumber.class.cast(value);
132                builder.withDouble(key, number.doubleValue());
133                break;
134            case NULL:
135                break;
136            default:
137                throw new IllegalArgumentException("Unsupported value type: " + value);
138            }
139        });
140        return builder.build();
141    }
142
143    private Schema getArrayElementSchema(final RecordBuilderFactory factory, final List<Object> items) {
144        if (items.isEmpty()) {
145            return factory.newSchemaBuilder(Schema.Type.STRING).build();
146        }
147        final Schema firstSchema = toSchema(factory, items.iterator().next());
148        switch (firstSchema.getType()) {
149        case RECORD:
150            return items.stream().map(it -> toSchema(factory, it)).reduce(null, (s1, s2) -> {
151                if (s1 == null) {
152                    return s2;
153                }
154                if (s2 == null) { // unlikely
155                    return s1;
156                }
157                final Set<String> names1 = s1.getAllEntries().map(Schema.Entry::getName).collect(toSet());
158                final Set<String> names2 = s2.getAllEntries().map(Schema.Entry::getName).collect(toSet());
159                if (!names1.equals(names2)) {
160                    // here we are not good since values will not be right anymore,
161                    // forbidden for current version anyway but potentially supported later
162                    final Schema.Builder builder = factory.newSchemaBuilder(Schema.Type.RECORD);
163                    s1.getAllEntries().forEach(builder::withEntry);
164                    s2.getAllEntries().filter(it -> !(names1.contains(it.getName()))).forEach(builder::withEntry);
165                    return builder.build();
166                }
167                return s1;
168            });
169        default:
170            return firstSchema;
171        }
172    }
173
174    private Object mapJson(final RecordBuilderFactory factory, final JsonValue it) {
175        if (JsonObject.class.isInstance(it)) {
176            return json2Record(factory, JsonObject.class.cast(it));
177        }
178        if (JsonArray.class.isInstance(it)) {
179            return JsonArray.class.cast(it).stream().map(i -> mapJson(factory, i)).collect(toList());
180        }
181        if (JsonString.class.isInstance(it)) {
182            return JsonString.class.cast(it).getString();
183        }
184        if (JsonNumber.class.isInstance(it)) {
185            return JsonNumber.class.cast(it).numberValue();
186        }
187        if (JsonValue.FALSE.equals(it)) {
188            return false;
189        }
190        if (JsonValue.TRUE.equals(it)) {
191            return true;
192        }
193        if (JsonValue.NULL.equals(it)) {
194            return null;
195        }
196        return it;
197    }
198
199    public static Schema toSchema(final RecordBuilderFactory factory, final Object next) {
200        if (String.class.isInstance(next) || JsonString.class.isInstance(next)) {
201            return factory.newSchemaBuilder(Schema.Type.STRING).build();
202        }
203        if (Integer.class.isInstance(next)) {
204            return factory.newSchemaBuilder(Schema.Type.INT).build();
205        }
206        if (Long.class.isInstance(next) || JsonLongImpl.class.isInstance(next)) {
207            return factory.newSchemaBuilder(Schema.Type.LONG).build();
208        }
209        if (Float.class.isInstance(next)) {
210            return factory.newSchemaBuilder(Schema.Type.FLOAT).build();
211        }
212        if (JsonNumber.class.isInstance(next)) {
213            return factory.newSchemaBuilder(Schema.Type.DOUBLE).build();
214        }
215        if (Double.class.isInstance(next) || JsonNumber.class.isInstance(next)) {
216            return factory.newSchemaBuilder(Schema.Type.DOUBLE).build();
217        }
218        if (Boolean.class.isInstance(next) || JsonValue.TRUE.equals(next) || JsonValue.FALSE.equals(next)) {
219            return factory.newSchemaBuilder(Schema.Type.BOOLEAN).build();
220        }
221        if (Date.class.isInstance(next) || ZonedDateTime.class.isInstance(next) || Instant.class.isInstance(next)) {
222            return factory.newSchemaBuilder(Schema.Type.DATETIME).build();
223        }
224        if (BigDecimal.class.isInstance(next)) {
225            return factory.newSchemaBuilder(Schema.Type.DECIMAL).build();
226        }
227        if (byte[].class.isInstance(next)) {
228            return factory.newSchemaBuilder(Schema.Type.BYTES).build();
229        }
230        if (Collection.class.isInstance(next) || JsonArray.class.isInstance(next)) {
231            final Collection collection = Collection.class.cast(next);
232            if (collection.isEmpty()) {
233                return factory.newSchemaBuilder(Schema.Type.STRING).build();
234            }
235            return factory
236                    .newSchemaBuilder(Schema.Type.ARRAY)
237                    .withElementSchema(toSchema(factory, collection.iterator().next()))
238                    .build();
239        }
240        if (Record.class.isInstance(next)) {
241            return Record.class.cast(next).getSchema();
242        }
243        throw new IllegalArgumentException("unsupported type for " + next);
244    }
245
246    public Object toType(final MappingMetaRegistry registry, final Object data, final Class<?> parameterType,
247            final Supplier<JsonBuilderFactory> factorySupplier, final Supplier<JsonProvider> providerSupplier,
248            final Supplier<Jsonb> jsonbProvider, final Supplier<RecordBuilderFactory> recordBuilderProvider) {
249        return toType(registry, data, parameterType, factorySupplier, providerSupplier, jsonbProvider,
250                recordBuilderProvider, Collections.emptyMap());
251    }
252
253    public Object toType(final MappingMetaRegistry registry, final Object data, final Class<?> parameterType,
254            final Supplier<JsonBuilderFactory> factorySupplier, final Supplier<JsonProvider> providerSupplier,
255            final Supplier<Jsonb> jsonbProvider, final Supplier<RecordBuilderFactory> recordBuilderProvider,
256            final java.util.Map<String, String> metadata) {
257        if (parameterType.isInstance(data)) {
258            return data;
259        }
260
261        final JsonObject inputAsJson;
262        if (JsonObject.class.isInstance(data)) {
263            if (JsonObject.class == parameterType) {
264                return data;
265            }
266            inputAsJson = JsonObject.class.cast(data);
267        } else if (Record.class.isInstance(data)) {
268            final Record record = Record.class.cast(data);
269            if (!JsonObject.class.isAssignableFrom(parameterType)) {
270                final MappingMeta mappingMeta = registry.find(parameterType);
271                if (mappingMeta.isLinearMapping()) {
272                    return mappingMeta.newInstance(record, metadata);
273                }
274            }
275            final JsonObject asJson = toJson(factorySupplier, providerSupplier, record);
276            if (JsonObject.class == parameterType) {
277                return asJson;
278            }
279            inputAsJson = asJson;
280        } else {
281            if (parameterType == Record.class) {
282                return toRecord(registry, data, jsonbProvider, recordBuilderProvider);
283            }
284            final Jsonb jsonb = jsonbProvider.get();
285            inputAsJson = jsonb.fromJson(jsonb.toJson(data), JsonObject.class);
286        }
287        return jsonbProvider.get().fromJson(new JsonValueReader<>(inputAsJson), parameterType);
288    }
289
290    private JsonObject toJson(final Supplier<JsonBuilderFactory> factorySupplier,
291            final Supplier<JsonProvider> providerSupplier, final Record record) {
292        return buildRecord(factorySupplier.get(), providerSupplier, record).build();
293    }
294
295    private JsonObjectBuilder buildRecord(final JsonBuilderFactory factory,
296            final Supplier<JsonProvider> providerSupplier, final Record record) {
297        final Schema schema = record.getSchema();
298        final JsonObjectBuilder builder = factory.createObjectBuilder();
299        schema.getEntries().forEach(entry -> {
300            final String name = entry.getName();
301            switch (entry.getType()) {
302            case STRING: {
303                final String value = record.get(String.class, name);
304                if (value != null) {
305                    builder.add(name, value);
306                }
307                break;
308            }
309            case INT: {
310                final Integer value = record.get(Integer.class, name);
311                if (value != null) {
312                    builder.add(name, value);
313                }
314                break;
315            }
316            case LONG: {
317                final Long value = record.get(Long.class, name);
318                if (value != null) {
319                    builder.add(name, value);
320                }
321                break;
322            }
323            case FLOAT: {
324                final Float value = record.get(Float.class, name);
325                if (value != null) {
326                    builder.add(name, value);
327                }
328                break;
329            }
330            case DOUBLE: {
331                final Double value = record.get(Double.class, name);
332                if (value != null) {
333                    builder.add(name, value);
334                }
335                break;
336            }
337            case BOOLEAN: {
338                final Boolean value = record.get(Boolean.class, name);
339                if (value != null) {
340                    builder.add(name, value);
341                }
342                break;
343            }
344            case BYTES: {
345                final byte[] value = record.get(byte[].class, name);
346                if (value != null) {
347                    builder.add(name, Base64.getEncoder().encodeToString(value));
348                }
349                break;
350            }
351            case DATETIME: {
352                final ZonedDateTime value = record.get(ZonedDateTime.class, name);
353                if (value != null) {
354                    builder.add(name, value.format(DateTimeFormatter.ISO_ZONED_DATE_TIME));
355                }
356                break;
357            }
358            case DECIMAL: {
359                final BigDecimal value = record.get(BigDecimal.class, name);
360                if (value != null) {
361                    builder.add(name, value.toString());
362                }
363                break;
364            }
365            case RECORD: {
366                final Record value = record.get(Record.class, name);
367                if (value != null) {
368                    builder.add(name, buildRecord(factory, providerSupplier, value));
369                }
370                break;
371            }
372            case ARRAY:
373                final Collection<?> collection = record.get(Collection.class, name);
374                if (collection == null) {
375                    break;
376                }
377                if (collection.isEmpty()) {
378                    builder.add(name, factory.createArrayBuilder().build());
379                } else { // only homogeneous collections
380                    final Object item = collection.iterator().next();
381                    if (String.class.isInstance(item)) {
382                        final JsonProvider jsonProvider = providerSupplier.get();
383                        builder
384                                .add(name, toArray(factory, v -> jsonProvider.createValue(String.class.cast(v)),
385                                        collection));
386                    } else if (Double.class.isInstance(item)) {
387                        final JsonProvider jsonProvider = providerSupplier.get();
388                        builder
389                                .add(name, toArray(factory, v -> jsonProvider.createValue(Double.class.cast(v)),
390                                        collection));
391                    } else if (Float.class.isInstance(item)) {
392                        final JsonProvider jsonProvider = providerSupplier.get();
393                        builder
394                                .add(name, toArray(factory, v -> jsonProvider.createValue(Float.class.cast(v)),
395                                        collection));
396                    } else if (Integer.class.isInstance(item)) {
397                        final JsonProvider jsonProvider = providerSupplier.get();
398                        builder
399                                .add(name, toArray(factory, v -> jsonProvider.createValue(Integer.class.cast(v)),
400                                        collection));
401                    } else if (Long.class.isInstance(item)) {
402                        final JsonProvider jsonProvider = providerSupplier.get();
403                        builder
404                                .add(name, toArray(factory, v -> jsonProvider.createValue(Long.class.cast(v)),
405                                        collection));
406                    } else if (Boolean.class.isInstance(item)) {
407                        builder
408                                .add(name, toArray(factory,
409                                        v -> Boolean.class.cast(v) ? JsonValue.TRUE : JsonValue.FALSE, collection));
410                    } else if (ZonedDateTime.class.isInstance(item)) {
411                        final JsonProvider jsonProvider = providerSupplier.get();
412                        builder
413                                .add(name,
414                                        toArray(factory, v -> jsonProvider
415                                                .createValue(ZonedDateTime.class.cast(v).toInstant().toEpochMilli()),
416                                                collection));
417                    } else if (Date.class.isInstance(item)) {
418                        final JsonProvider jsonProvider = providerSupplier.get();
419                        builder
420                                .add(name, toArray(factory, v -> jsonProvider.createValue(Date.class.cast(v).getTime()),
421                                        collection));
422                    } else if (Record.class.isInstance(item)) {
423                        builder
424                                .add(name, toArray(factory,
425                                        v -> buildRecord(factory, providerSupplier, Record.class.cast(v)).build(),
426                                        collection));
427                    } else if (JsonValue.class.isInstance(item)) {
428                        builder.add(name, toArray(factory, JsonValue.class::cast, collection));
429                    } // else throw?
430                }
431                break;
432            default:
433                throw new IllegalArgumentException("Unsupported type: " + entry.getType() + " for '" + name + "'");
434            }
435        });
436        return builder;
437    }
438
439    private JsonArray toArray(final JsonBuilderFactory factory, final Function<Object, JsonValue> valueFactory,
440            final Collection<?> collection) {
441        final Collector<JsonValue, JsonArrayBuilder, JsonArray> collector = Collector
442                .of(factory::createArrayBuilder, JsonArrayBuilder::add, JsonArrayBuilder::addAll,
443                        JsonArrayBuilder::build);
444        return collection.stream().map(valueFactory).collect(collector);
445    }
446
447    public <T> T coerce(final Class<T> expectedType, final Object value, final String name) {
448        if (value == null) {
449            return null;
450        }
451
452        // here mean get(Object.class, name) return origin store type, like DATETIME return long, is expected?
453        if (!expectedType.isInstance(value)) {
454            return expectedType.cast(MappingUtils.coerce(expectedType, value, name));
455        }
456
457        return expectedType.cast(value);
458    }
459
460    @Data
461    public static class MappingMeta {
462
463        private final boolean linearMapping;
464
465        private final Class<?> rowStruct;
466
467        private Object recordVisitor;
468
469        private Method visitRecord;
470
471        private Object rowStructVisitor;
472
473        private Method visitRowStruct;
474
475        public MappingMeta(final Class<?> type, final MappingMetaRegistry registry) {
476            linearMapping = Stream.of(type.getInterfaces()).anyMatch(it -> it.getName().startsWith("routines.system."));
477            rowStruct = type;
478        }
479
480        public Object newInstance(final Record record) {
481            return newInstance(record, Collections.emptyMap());
482        }
483
484        public Object newInstance(final Record record, final java.util.Map<String, String> metadata) {
485            if (recordVisitor == null) {
486                try {
487                    final String className = "org.talend.sdk.component.runtime.di.record.DiRecordVisitor";
488                    final Class<?> visitorClass = getClass().getClassLoader().loadClass(className);
489                    final Constructor<?> constructor = visitorClass.getDeclaredConstructors()[0];
490                    constructor.setAccessible(true);
491                    recordVisitor = constructor.newInstance(rowStruct, metadata);
492                    visitRecord = visitorClass.getDeclaredMethod("visit", Record.class);
493                } catch (final NoClassDefFoundError | ClassNotFoundException | InstantiationException
494                        | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
495                    if (e.getMessage().matches(".*routines.system.Dynamic.*")) {
496                        throw new IllegalStateException("TOS does not support dynamic type", e);
497                    }
498                    throw new IllegalStateException(e);
499                }
500            }
501            try {
502                return visitRecord.invoke(recordVisitor, record);
503            } catch (final IllegalAccessException | InvocationTargetException e) {
504                throw new IllegalStateException(e);
505            }
506        }
507
508        public <T> Record newRecord(final T data, final RecordBuilderFactory factory) {
509            if (rowStructVisitor == null) {
510                try {
511                    final String className = "org.talend.sdk.component.runtime.di.record.DiRowStructVisitor";
512                    final Class<?> visitorClass = getClass().getClassLoader().loadClass(className);
513                    final Constructor<?> constructor = visitorClass.getConstructors()[0];
514                    constructor.setAccessible(true);
515                    rowStructVisitor = constructor.newInstance();
516                    visitRowStruct = visitorClass.getMethod("get", Object.class, RecordBuilderFactory.class);
517                } catch (final NoClassDefFoundError | ClassNotFoundException | InstantiationException
518                        | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
519                    if (e.getMessage().matches(".*routines.system.Dynamic.*")) {
520                        throw new IllegalStateException("TOS does not support dynamic type", e);
521                    }
522                    throw new IllegalStateException(e);
523                }
524            }
525            try {
526                return Record.class.cast(visitRowStruct.invoke(rowStructVisitor, data, factory));
527            } catch (final IllegalAccessException | InvocationTargetException e) {
528                throw new IllegalStateException(e);
529            }
530        }
531
532    }
533
534    @Data
535    public static class MappingMetaRegistry implements Serializable {
536
537        protected final Map<Class<?>, MappingMeta> registry = new ConcurrentHashMap<>();
538
539        private Object writeReplace() throws ObjectStreamException {
540            return new Factory(); // don't serialize the mapping, recalculate it lazily
541        }
542
543        public MappingMeta find(final Class<?> parameterType) {
544            final MappingMeta meta = registry.get(parameterType);
545            if (meta != null) {
546                return meta;
547            }
548            final MappingMeta mappingMeta = new MappingMeta(parameterType, this);
549            final MappingMeta existing = registry.putIfAbsent(parameterType, mappingMeta);
550            if (existing != null) {
551                return existing;
552            }
553            return mappingMeta;
554        }
555
556        public static class Factory implements Serializable {
557
558            private Object readResolve() throws ObjectStreamException {
559                return new MappingMetaRegistry();
560            }
561        }
562    }
563}