001/**
002 * Copyright (C) 2006-2022 Talend Inc. - www.talend.com
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.talend.sdk.component.runtime.input;
017
018import static java.util.Collections.emptySet;
019import static java.util.Optional.ofNullable;
020
021import java.io.Serializable;
022import java.util.Set;
023import java.util.concurrent.TimeUnit;
024import java.util.function.Supplier;
025
026import org.talend.sdk.component.api.service.configuration.LocalConfiguration;
027import org.talend.sdk.component.runtime.serialization.ContainerFinder;
028
029import lombok.AllArgsConstructor;
030import lombok.Data;
031import lombok.NoArgsConstructor;
032import lombok.extern.slf4j.Slf4j;
033
034@Slf4j
035public class Streaming {
036
037    private static Supplier<LocalConfiguration> defaultLocalConfiguration = () -> new LocalConfiguration() {
038
039        @Override
040        public String get(final String key) {
041            return null;
042        }
043
044        @Override
045        public Set<String> keys() {
046            return emptySet();
047        }
048    };
049
050    public static RetryConfiguration loadRetryConfiguration(final String plugin) {
051        // note: this configuratoin could be read on the mapper too and distributed
052        final LocalConfiguration configuration = ofNullable(ContainerFinder.Instance.get().find(plugin))
053                .map(it -> it.findService(LocalConfiguration.class))
054                .orElseGet(defaultLocalConfiguration);
055        final int maxRetries = ofNullable(configuration.get("talend.input.streaming.retry.maxRetries"))
056                .map(Integer::parseInt)
057                .orElse(Integer.MAX_VALUE);
058        return new RetryConfiguration(maxRetries, getStrategy(configuration));
059    }
060
061    public static RetryStrategy getStrategy(final LocalConfiguration configuration) {
062        switch (ofNullable(configuration.get("talend.input.streaming.retry.strategy")).orElse("constant")) {
063        case "exponential":
064            return new RetryConfiguration.ExponentialBackoff(
065                    ofNullable(configuration.get("talend.input.streaming.retry.exponential.exponent"))
066                            .map(Double::parseDouble)
067                            .orElse(1.5),
068                    ofNullable(configuration.get("talend.input.streaming.retry.exponential.randomizationFactor"))
069                            .map(Double::parseDouble)
070                            .orElse(.5),
071                    ofNullable(configuration.get("talend.input.streaming.retry.exponential.maxDuration"))
072                            .map(Long::parseLong)
073                            .orElse(TimeUnit.MINUTES.toMillis(5)),
074                    ofNullable(configuration.get("talend.input.streaming.retry.exponential.initialBackOff"))
075                            .map(Long::parseLong)
076                            .orElse(TimeUnit.SECONDS.toMillis(1)),
077                    0);
078        case "constant":
079        default:
080            return new RetryConfiguration.Constant(
081                    ofNullable(configuration.get("talend.input.streaming.retry.constant.timeout"))
082                            .map(Long::parseLong)
083                            .orElse(500L));
084        }
085    }
086
087    public static StopStrategy loadStopStrategy(final String plugin,
088            final java.util.Map<String, String> internalConfiguration) {
089        final LocalConfiguration configuration = ofNullable(ContainerFinder.Instance.get().find(plugin))
090                .map(it -> it.findService(LocalConfiguration.class))
091                .orElseGet(defaultLocalConfiguration);
092        final Long maxReadRecords = ofNullable(internalConfiguration.entrySet()
093                .stream()
094                .filter(e -> e.getKey().startsWith("$maxRecords") || e.getKey().contains(".$maxRecords"))
095                .findFirst()
096                .map(e -> e.getValue())
097                .map(Long::parseLong)).get()
098                        .orElseGet(() -> ofNullable(System.getProperty(
099                                String.format("%s.talend.input.streaming.maxRecords", plugin)))
100                                        .map(Long::parseLong)
101                                        .orElseGet(() -> ofNullable(
102                                                configuration.get("talend.input.streaming.maxRecords"))
103                                                        .map(Long::parseLong)
104                                                        .orElseGet(() -> null)));
105        Long maxActiveTime = ofNullable(internalConfiguration.entrySet()
106                .stream()
107                .filter(e -> e.getKey().startsWith("$maxDurationMs") || e.getKey().contains(".$maxDurationMs"))
108                .findFirst()
109                .map(e -> e.getValue())
110                .map(Long::parseLong)).get()
111                        .orElseGet(() -> ofNullable(System.getProperty(
112                                String.format("%s.talend.input.streaming.maxDurationMs", plugin)))
113                                        .map(Long::parseLong)
114                                        .orElseGet(() -> ofNullable(
115                                                configuration.get("talend.input.streaming.maxDurationMs"))
116                                                        .map(Long::parseLong)
117                                                        .orElseGet(() -> null)));
118        log.debug("[loadStopStrategy] Records: {}; Duration: {}.", maxReadRecords, maxActiveTime);
119        return new StopConfiguration(maxReadRecords, maxActiveTime, null);
120    }
121
122    public interface RetryStrategy {
123
124        long nextPauseDuration();
125
126        void reset();
127    }
128
129    @Data
130    @NoArgsConstructor
131    @AllArgsConstructor
132    public static class RetryConfiguration implements Serializable {
133
134        private int maxRetries;
135
136        private RetryStrategy strategy;
137
138        @Data
139        @NoArgsConstructor
140        @AllArgsConstructor
141        public static class Constant implements Serializable, RetryStrategy {
142
143            private long timeout;
144
145            @Override
146            public long nextPauseDuration() {
147                return timeout;
148            }
149
150            @Override
151            public void reset() {
152                // no-op
153            }
154        }
155
156        @Data
157        @NoArgsConstructor
158        @AllArgsConstructor
159        public static class ExponentialBackoff implements Serializable, RetryStrategy {
160
161            private double exponent;
162
163            private double randomizationFactor;
164
165            private long max;
166
167            private long initialBackOff;
168
169            // state
170            private int iteration;
171
172            @Override
173            public long nextPauseDuration() {
174                final double currentIntervalMillis = Math.min(initialBackOff * Math.pow(exponent, iteration), max);
175                final double randomOffset = (Math.random() * 2 - 1) * randomizationFactor * currentIntervalMillis;
176                final long nextBackoffMillis = Math.min(max, Math.round(currentIntervalMillis + randomOffset));
177                iteration += 1;
178                return nextBackoffMillis;
179            }
180
181            @Override
182            public void reset() {
183                iteration = 0;
184            }
185        }
186    }
187
188    public interface StopStrategy {
189
190        /**
191         * Check if the stop strategy is active according specified conditions.
192         * 
193         * @return true if strategy is active, false otherwise.
194         */
195        boolean isActive();
196
197        /**
198         * Check the stop strategy conditions.
199         * 
200         * @param read - already read records.
201         * @return true if the lifecycle should be stopped, false otherwise.
202         */
203        boolean shouldStop(long read);
204
205        /**
206         * Maximum records to read.
207         * 
208         * @return max number of records to read.
209         */
210        long getMaxReadRecords();
211
212        /**
213         * Maximum duration the lifecycle should run in ms.
214         * 
215         * @return max activity duration.
216         */
217        long getMaxActiveTime();
218
219        /**
220         * The system time in millis when the lifecycle started.
221         * 
222         * @return started time.
223         */
224        long getStartedAtTime();
225
226    }
227
228    @Data
229    public static class StopConfiguration implements StopStrategy, Serializable {
230
231        private long maxReadRecords;
232
233        private long maxActiveTime;
234
235        private long startedAtTime;
236
237        public StopConfiguration() {
238            maxReadRecords = -1L;
239            maxActiveTime = -1L;
240            startedAtTime = System.currentTimeMillis();
241        }
242
243        public StopConfiguration(final Long maxRecords, final Long maxTime, final Long start) {
244            maxReadRecords = maxRecords == null ? -1L : maxRecords;
245            maxActiveTime = maxTime == null ? -1L : maxTime;
246            startedAtTime = start == null ? System.currentTimeMillis() : start;
247        }
248
249        @Override
250        public boolean isActive() {
251            return (maxReadRecords > -1) || (maxActiveTime > -1);
252        }
253
254        private boolean hasEnoughRecords(final long read) {
255            return maxReadRecords != -1 && read >= maxReadRecords;
256        }
257
258        private boolean isTimePassed() {
259            return maxActiveTime != -1 && System.currentTimeMillis() - startedAtTime >= maxActiveTime;
260        }
261
262        @Override
263        public boolean shouldStop(final long readRecords) {
264            return hasEnoughRecords(readRecords) || isTimePassed();
265        }
266    }
267
268}