Source code

001/**
002 * Copyright (C) 2006-2022 Talend Inc. - www.talend.com
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.talend.sdk.component.api.record;
017
018import static java.util.Collections.emptyList;
019
020import java.io.StringReader;
021import java.nio.charset.Charset;
022import java.nio.charset.CharsetEncoder;
023import java.nio.charset.StandardCharsets;
024import java.time.temporal.Temporal;
025import java.util.Arrays;
026import java.util.Base64;
027import java.util.Collection;
028import java.util.Collections;
029import java.util.Comparator;
030import java.util.Date;
031import java.util.List;
032import java.util.Map;
033import java.util.Objects;
034import java.util.Optional;
035import java.util.Set;
036import java.util.function.BiConsumer;
037import java.util.function.Supplier;
038import java.util.stream.Collectors;
039import java.util.stream.Stream;
040
041import javax.json.Json;
042import javax.json.JsonValue;
043import javax.json.bind.annotation.JsonbTransient;
044
045import lombok.AllArgsConstructor;
046import lombok.EqualsAndHashCode;
047import lombok.Getter;
048import lombok.ToString;
049
050public interface Schema {
051
052    /**
053     * @return the type of this schema.
054     */
055    Type getType();
056
057    /**
058     * @return the nested element schema for arrays.
059     */
060    Schema getElementSchema();
061
062    /**
063     * @return the data entries for records (not contains meta data entries).
064     */
065    List<Entry> getEntries();
066
067    /**
068     * @return the metadata entries for records (not contains ordinary data entries).
069     */
070    List<Entry> getMetadata();
071
072    /**
073     * @return All entries, including data and metadata, of this schema.
074     */
075    Stream<Entry> getAllEntries();
076
077    /**
078     * Get a Builder from the current schema.
079     *
080     * @return a {@link Schema.Builder}
081     */
082    default Schema.Builder toBuilder() {
083        throw new UnsupportedOperationException("#toBuilder is not implemented");
084    }
085
086    /**
087     * Get all entries sorted by schema designed order.
088     *
089     * @return all entries ordered
090     */
091    default List<Entry> getEntriesOrdered() {
092        return getEntriesOrdered(naturalOrder());
093    }
094
095    /**
096     * Get all entries sorted using a custom comparator.
097     *
098     * @param comparator the comparator
099     *
100     * @return all entries ordered with provided comparator
101     */
102    @JsonbTransient
103    default List<Entry> getEntriesOrdered(final Comparator<Entry> comparator) {
104        return getAllEntries().sorted(comparator).collect(Collectors.toList());
105    }
106
107    /**
108     * Get the EntriesOrder defined with Builder.
109     *
110     * @return the EntriesOrder
111     */
112
113    default EntriesOrder naturalOrder() {
114        throw new UnsupportedOperationException("#naturalOrder is not implemented");
115    }
116
117    default Entry getEntry(final String name) {
118        return getAllEntries() //
119                .filter((Entry e) -> Objects.equals(e.getName(), name)) //
120                .findFirst() //
121                .orElse(null);
122    }
123
124    /**
125     * @return the metadata props
126     */
127    Map<String, String> getProps();
128
129    /**
130     * @param property : property name.
131     *
132     * @return the requested metadata prop
133     */
134    String getProp(String property);
135
136    /**
137     * Get a property values from schema with its name.
138     *
139     * @param name : property's name.
140     *
141     * @return property's value.
142     */
143    default JsonValue getJsonProp(final String name) {
144        final String prop = this.getProp(name);
145        if (prop == null) {
146            return null;
147        }
148        try {
149            return Json.createParser(new StringReader(prop)).getValue();
150        } catch (RuntimeException ex) {
151            return Json.createValue(prop);
152        }
153    }
154
155    enum Type {
156
157        RECORD(new Class<?>[] { Record.class }),
158        ARRAY(new Class<?>[] { Collection.class }),
159        STRING(new Class<?>[] { String.class }),
160        BYTES(new Class<?>[] { byte[].class, Byte[].class }),
161        INT(new Class<?>[] { Integer.class }),
162        LONG(new Class<?>[] { Long.class }),
163        FLOAT(new Class<?>[] { Float.class }),
164        DOUBLE(new Class<?>[] { Double.class }),
165        BOOLEAN(new Class<?>[] { Boolean.class }),
166        DATETIME(new Class<?>[] { Long.class, Date.class, Temporal.class });
167
168        /**
169         * All compatibles Java classes
170         */
171        private final Class<?>[] classes;
172
173        Type(final Class<?>[] classes) {
174            this.classes = classes;
175        }
176
177        /**
178         * Check if input can be affected to an entry of this type.
179         *
180         * @param input : object.
181         *
182         * @return true if input is null or ok.
183         */
184        public boolean isCompatible(final Object input) {
185            if (input == null) {
186                return true;
187            }
188            for (final Class<?> clazz : classes) {
189                if (clazz.isInstance(input)) {
190                    return true;
191                }
192            }
193            return false;
194        }
195    }
196
197    interface Entry {
198
199        /**
200         * @return The name of this entry.
201         */
202        String getName();
203
204        /**
205         * @return The raw name of this entry.
206         */
207        String getRawName();
208
209        /**
210         * @return the raw name of this entry if exists, else return name.
211         */
212        String getOriginalFieldName();
213
214        /**
215         * @return Type of the entry, this determine which other fields are populated.
216         */
217        Type getType();
218
219        /**
220         * @return Is this entry nullable or always valued.
221         */
222        boolean isNullable();
223
224        /**
225         * @return true if this entry is for metadata; false for ordinary data.
226         */
227        boolean isMetadata();
228
229        /**
230         * @param <T> the default value type.
231         *
232         * @return Default value for this entry.
233         */
234        <T> T getDefaultValue();
235
236        /**
237         * @return For type == record, the element type.
238         */
239        Schema getElementSchema();
240
241        /**
242         * @return Allows to associate to this field a comment - for doc purposes, no use in the runtime.
243         */
244        String getComment();
245
246        /**
247         * @return the metadata props
248         */
249        Map<String, String> getProps();
250
251        /**
252         * @param property : property name.
253         *
254         * @return the requested metadata prop
255         */
256        String getProp(String property);
257
258        /**
259         * Get a property values from entry with its name.
260         *
261         * @param name : property's name.
262         *
263         * @return property's value.
264         */
265        default JsonValue getJsonProp(final String name) {
266            final String prop = this.getProp(name);
267            if (prop == null) {
268                return null;
269            }
270            try {
271                return Json.createParser(new StringReader(prop)).getValue();
272            } catch (RuntimeException ex) {
273                return Json.createValue(prop);
274            }
275        }
276
277        /**
278         * @return an {@link Entry.Builder} from this entry.
279         */
280        default Entry.Builder toBuilder() {
281            throw new UnsupportedOperationException("#toBuilder is not implemented");
282        }
283
284        /**
285         * Plain builder matching {@link Entry} structure.
286         */
287        interface Builder {
288
289            Builder withName(String name);
290
291            Builder withRawName(String rawName);
292
293            Builder withType(Type type);
294
295            Builder withNullable(boolean nullable);
296
297            Builder withMetadata(boolean metadata);
298
299            <T> Builder withDefaultValue(T value);
300
301            Builder withElementSchema(Schema schema);
302
303            Builder withComment(String comment);
304
305            Builder withProps(Map<String, String> props);
306
307            Builder withProp(String key, String value);
308
309            Entry build();
310        }
311    }
312
313    /**
314     * Allows to build a {@link Schema}.
315     */
316    interface Builder {
317
318        /**
319         * @param type schema type.
320         *
321         * @return this builder.
322         */
323        Builder withType(Type type);
324
325        /**
326         * @param entry element for either an array or record type.
327         *
328         * @return this builder.
329         */
330        Builder withEntry(Entry entry);
331
332        /**
333         * Insert the entry after the specified entry.
334         *
335         * @param after the entry name reference
336         * @param entry the entry name
337         *
338         * @return this builder
339         */
340        default Builder withEntryAfter(String after, Entry entry) {
341            throw new UnsupportedOperationException("#withEntryAfter is not implemented");
342        }
343
344        /**
345         * Insert the entry before the specified entry.
346         *
347         * @param before the entry name reference
348         * @param entry the entry name
349         *
350         * @return this builder
351         */
352        default Builder withEntryBefore(String before, Entry entry) {
353            throw new UnsupportedOperationException("#withEntryBefore is not implemented");
354        }
355
356        /**
357         * Remove entry from builder.
358         *
359         * @param name the entry name
360         *
361         * @return this builder
362         */
363        default Builder remove(String name) {
364            throw new UnsupportedOperationException("#remove is not implemented");
365        }
366
367        /**
368         * Remove entry from builder.
369         *
370         * @param entry the entry
371         *
372         * @return this builder
373         */
374        default Builder remove(Entry entry) {
375            throw new UnsupportedOperationException("#remove is not implemented");
376        }
377
378        /**
379         * Move an entry after another one.
380         *
381         * @param after the entry name reference
382         * @param name the entry name
383         */
384        default Builder moveAfter(final String after, final String name) {
385            throw new UnsupportedOperationException("#moveAfter is not implemented");
386        }
387
388        /**
389         * Move an entry before another one.
390         *
391         * @param before the entry name reference
392         * @param name the entry name
393         */
394        default Builder moveBefore(final String before, final String name) {
395            throw new UnsupportedOperationException("#moveBefore is not implemented");
396        }
397
398        /**
399         * Swap two entries.
400         *
401         * @param name the entry name
402         * @param with the other entry name
403         */
404        default Builder swap(final String name, final String with) {
405            throw new UnsupportedOperationException("#swap is not implemented");
406        }
407
408        /**
409         * @param schema nested element schema.
410         *
411         * @return this builder.
412         */
413        Builder withElementSchema(Schema schema);
414
415        /**
416         * @param props schema properties
417         *
418         * @return this builder
419         */
420        Builder withProps(Map<String, String> props);
421
422        /**
423         * @param key the prop key name
424         * @param value the prop value
425         *
426         * @return this builder
427         */
428        Builder withProp(String key, String value);
429
430        /**
431         * @return the described schema.
432         */
433        Schema build();
434
435        /**
436         * Same as {@link Builder#build()} but entries order is specified by {@code order}. This takes precedence on any
437         * previous defined order with builder and may void it.
438         *
439         * @param order the wanted order for entries.
440         * @return the described schema.
441         */
442        default Schema build(Comparator<Entry> order) {
443            throw new UnsupportedOperationException("#build(EntriesOrder) is not implemented");
444        }
445    }
446
447    /**
448     * Sanitize name to be avro compatible.
449     *
450     * @param name : original name.
451     *
452     * @return avro compatible name.
453     */
454    static String sanitizeConnectionName(final String name) {
455        if (name == null || name.isEmpty()) {
456            return name;
457        }
458
459        char current = name.charAt(0);
460        final CharsetEncoder ascii = Charset.forName(StandardCharsets.US_ASCII.name()).newEncoder();
461        final boolean skipFirstChar = ((!ascii.canEncode(current)) || (!Character.isLetter(current) && current != '_'))
462                && name.length() > 1 && (!Character.isDigit(name.charAt(1)));
463
464        final StringBuilder sanitizedBuilder = new StringBuilder();
465
466        if (!skipFirstChar) {
467            if (((!Character.isLetter(current)) && current != '_') || (!ascii.canEncode(current))) {
468                sanitizedBuilder.append('_');
469            } else {
470                sanitizedBuilder.append(current);
471            }
472        }
473        for (int i = 1; i < name.length(); i++) {
474            current = name.charAt(i);
475            if (!ascii.canEncode(current)) {
476                if (Character.isLowerCase(current) || Character.isUpperCase(current)) {
477                    sanitizedBuilder.append('_');
478                } else {
479                    final byte[] encoded =
480                            Base64.getEncoder().encode(name.substring(i, i + 1).getBytes(StandardCharsets.UTF_8));
481                    final String enc = new String(encoded);
482                    if (sanitizedBuilder.length() == 0 && Character.isDigit(enc.charAt(0))) {
483                        sanitizedBuilder.append('_');
484                    }
485                    for (int iter = 0; iter < enc.length(); iter++) {
486                        if (Character.isLetterOrDigit(enc.charAt(iter))) {
487                            sanitizedBuilder.append(enc.charAt(iter));
488                        } else {
489                            sanitizedBuilder.append('_');
490                        }
491                    }
492                }
493            } else if (Character.isLetterOrDigit(current)) {
494                sanitizedBuilder.append(current);
495            } else {
496                sanitizedBuilder.append('_');
497            }
498
499        }
500        return sanitizedBuilder.toString();
501    }
502
503    @AllArgsConstructor
504    @ToString
505    @EqualsAndHashCode
506    class EntriesOrder implements Comparator<Entry> {
507
508        @Getter
509        private final List<String> fieldsOrder;
510
511        /**
512         * Build an EntriesOrder according fields.
513         *
514         * @param fields the fields ordering. Each field should be {@code ,} separated.
515         *
516         * @return the order EntriesOrder
517         */
518        public static EntriesOrder of(final String fields) {
519            return new EntriesOrder(fields);
520        }
521
522        /**
523         * Build an EntriesOrder according fields.
524         *
525         * @param fields the fields ordering.
526         *
527         * @return the order EntriesOrder
528         */
529        public static EntriesOrder of(final List<String> fields) {
530            return new EntriesOrder(fields);
531        }
532
533        public EntriesOrder(final String fields) {
534            if (fields == null) {
535                fieldsOrder = emptyList();
536            } else {
537                fieldsOrder = Arrays.stream(fields.split(",")).collect(Collectors.toList());
538            }
539        }
540
541        /**
542         * Move a field after another one.
543         *
544         * @param after the field name reference
545         * @param name the field name
546         *
547         * @return this EntriesOrder
548         */
549        public EntriesOrder moveAfter(final String after, final String name) {
550            if (getFieldsOrder().indexOf(after) == -1) {
551                throw new IllegalArgumentException(String.format("%s not in schema", after));
552            }
553            getFieldsOrder().remove(name);
554            final int destination = getFieldsOrder().indexOf(after) + 1;
555            if (destination < getFieldsOrder().size()) {
556                getFieldsOrder().add(destination, name);
557            } else {
558                getFieldsOrder().add(name);
559            }
560            return this;
561        }
562
563        /**
564         * Move a field before another one.
565         *
566         * @param before the field name reference
567         * @param name the field name
568         *
569         * @return this EntriesOrder
570         */
571        public EntriesOrder moveBefore(final String before, final String name) {
572            if (getFieldsOrder().indexOf(before) == -1) {
573                throw new IllegalArgumentException(String.format("%s not in schema", before));
574            }
575            getFieldsOrder().remove(name);
576            getFieldsOrder().add(getFieldsOrder().indexOf(before), name);
577            return this;
578        }
579
580        /**
581         * Swap two fields.
582         *
583         * @param name the field name
584         * @param with the other field
585         *
586         * @return this EntriesOrder
587         */
588        public EntriesOrder swap(final String name, final String with) {
589            Collections.swap(getFieldsOrder(), getFieldsOrder().indexOf(name), getFieldsOrder().indexOf(with));
590            return this;
591        }
592
593        public String toFields() {
594            return getFieldsOrder().stream().collect(Collectors.joining(","));
595        }
596
597        @Override
598        public int compare(final Entry e1, final Entry e2) {
599            final int index1 = getFieldsOrder().indexOf(e1.getName());
600            final int index2 = getFieldsOrder().indexOf(e2.getName());
601            if (index1 >= 0 && index2 >= 0) {
602                return index1 - index2;
603            }
604            if (index1 >= 0) {
605                return -1;
606            }
607            if (index2 >= 0) {
608                return 1;
609            }
610            return 0;
611        }
612    }
613
614    static Schema.Entry avoidCollision(final Schema.Entry newEntry,
615            final Supplier<Stream<Schema.Entry>> allEntriesSupplier, final BiConsumer<String, Entry> replaceFunction) {
616        final Optional<Entry> collisionedEntry = allEntriesSupplier //
617                .get() //
618                .filter((final Entry field) -> field.getName().equals(newEntry.getName())
619                        && !Objects.equals(field, newEntry)) //
620                .findFirst();
621        if (!collisionedEntry.isPresent()) {
622            // No collision, return new entry.
623            return newEntry;
624        }
625        final Entry matchedEntry = collisionedEntry.get();
626        final boolean matchedToChange = matchedEntry.getRawName() != null && !(matchedEntry.getRawName().isEmpty());
627        if (matchedToChange) {
628            // the rename has to be applied on entry already inside schema, so replace.
629            replaceFunction.accept(matchedEntry.getName(), newEntry);
630        } else if (newEntry.getRawName() == null || newEntry.getRawName().isEmpty()) {
631            // try to add exactly same raw, skip the add here.
632            return null;
633        }
634        final Entry fieldToChange = matchedToChange ? matchedEntry : newEntry;
635        int indexForAnticollision = 1;
636        final String baseName = Schema.sanitizeConnectionName(fieldToChange.getRawName()); // recalc primiti name.
637
638        String newName = baseName + "_" + indexForAnticollision;
639        final Set<String> existingNames = allEntriesSupplier //
640                .get() //
641                .map(Entry::getName) //
642                .collect(Collectors.toSet());
643        while (existingNames.contains(newName)) {
644            indexForAnticollision++;
645            newName = baseName + "_" + indexForAnticollision;
646        }
647        final Entry newFieldToAdd = fieldToChange.toBuilder().withName(newName).build();
648
649        return newFieldToAdd; // matchedToChange ? newFieldToAdd : newEntry;
650    }
651}