001/**
002 * Copyright (C) 2006-2022 Talend Inc. - www.talend.com
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.talend.sdk.component.runtime.input;
017
018import static java.lang.Thread.sleep;
019import static java.util.concurrent.TimeUnit.MILLISECONDS;
020import static org.talend.sdk.component.runtime.input.Streaming.RetryStrategy;
021
022import java.io.IOException;
023import java.io.InvalidObjectException;
024import java.io.ObjectStreamException;
025import java.io.Serializable;
026import java.util.concurrent.ExecutorService;
027import java.util.concurrent.Executors;
028import java.util.concurrent.Future;
029import java.util.concurrent.Semaphore;
030import java.util.concurrent.TimeoutException;
031import java.util.concurrent.atomic.AtomicBoolean;
032
033import org.talend.sdk.component.runtime.input.Streaming.RetryConfiguration;
034import org.talend.sdk.component.runtime.input.Streaming.StopStrategy;
035
036import lombok.extern.slf4j.Slf4j;
037
038@Slf4j
039public class StreamingInputImpl extends InputImpl {
040
041    private RetryConfiguration retryConfiguration;
042
043    private transient Thread shutdownHook;
044
045    private final AtomicBoolean running = new AtomicBoolean();
046
047    private transient Semaphore semaphore;
048
049    private StopStrategy stopStrategy;
050
051    private transient long readRecords = 0L;
052
053    public StreamingInputImpl(final String rootName, final String name, final String plugin,
054            final Serializable instance, final RetryConfiguration retryConfiguration, final StopStrategy stopStrategy) {
055        super(rootName, name, plugin, instance);
056        shutdownHook = new Thread(() -> running.compareAndSet(true, false),
057                getClass().getName() + "_" + rootName() + "-" + name() + "_" + hashCode());
058        this.retryConfiguration = retryConfiguration;
059        this.stopStrategy = stopStrategy;
060        log.debug("[StreamingInputImpl] Created with retryStrategy: {}, stopStrategy: {}.", this.retryConfiguration,
061                this.stopStrategy);
062    }
063
064    protected StreamingInputImpl() {
065        // no-op
066    }
067
068    @Override
069    protected Object readNext() {
070        if (!running.get()) {
071            return null;
072        }
073        if (stopStrategy.isActive() && stopStrategy.shouldStop(readRecords)) {
074            log.debug("[readNext] stopStrategy condition validated.");
075            return null;
076        }
077        try {
078            semaphore.acquire();
079        } catch (final InterruptedException e) {
080            Thread.currentThread().interrupt();
081            return null;
082        }
083        try {
084            final RetryStrategy strategy = retryConfiguration.getStrategy();
085            int retries = retryConfiguration.getMaxRetries();
086            while (running.get() && retries > 0) {
087                Object next = null;
088                if (stopStrategy.isActive() && stopStrategy.getMaxActiveTime() > -1) {
089                    // Some connectors do not block input and return null (rabbitmq for instance). Thus, the future
090                    // timeout is never reached and retryStrategy is run then. So, need to check timeout in the loop.
091                    if (stopStrategy.shouldStop(readRecords)) {
092                        log.debug("[readNext] shouldStop now! Duration {}ms",
093                                System.currentTimeMillis() - stopStrategy.getStartedAtTime());
094                        return null;
095                    }
096                    final ExecutorService executor = Executors.newSingleThreadExecutor();
097                    final Future<Object> reader = executor.submit(super::readNext);
098                    // manage job latency...
099                    final long estimatedTimeout = stopStrategy.getMaxActiveTime()
100                            - (System.currentTimeMillis() - stopStrategy.getStartedAtTime());
101                    final long timeout = estimatedTimeout < -1 ? 10 : estimatedTimeout;
102                    log.debug(
103                            "[readNext] Applying duration strategy for reading record: will interrupt in {}ms (estimated:{}ms Duration:{}ms).",
104                            timeout, estimatedTimeout, stopStrategy.getMaxActiveTime());
105                    try {
106                        next = reader.get(timeout, MILLISECONDS);
107                    } catch (TimeoutException e) {
108                        log.debug("[readNext] Read record: timeout received.");
109                        reader.cancel(true);
110                        return next;
111                    } catch (Exception e) {
112                        // nop
113                    } finally {
114                        executor.shutdownNow();
115                    }
116                } else {
117                    next = super.readNext();
118                }
119                if (next != null) {
120                    strategy.reset();
121                    readRecords++;
122                    return next;
123                }
124
125                retries--;
126                try {
127                    final long millis = strategy.nextPauseDuration();
128                    if (millis < 0) { // assume it means "give up"
129                        prepareStop();
130                    } else if (millis > 0) { // we can wait 1s but not minutes to quit
131                        if (millis < 1000) {
132                            sleep(millis);
133                        } else {
134                            long remaining = millis;
135                            while (running.get() && remaining > 0) {
136                                final long current = Math.min(remaining, 250);
137                                remaining -= current;
138                                sleep(current);
139                            }
140                        }
141                    } // else if millis == 0 no need to call any method
142                } catch (final InterruptedException e) {
143                    prepareStop(); // stop the stream
144                }
145            }
146            return null;
147        } finally {
148            semaphore.release();
149        }
150    }
151
152    @Override
153    protected void init() {
154        super.init();
155        semaphore = new Semaphore(1);
156    }
157
158    @Override
159    public void start() {
160        super.start();
161        running.compareAndSet(false, true);
162        Runtime.getRuntime().addShutdownHook(shutdownHook);
163    }
164
165    @Override
166    public void stop() {
167        prepareStop();
168        super.stop();
169    }
170
171    private void prepareStop() {
172        running.compareAndSet(true, false);
173        if (shutdownHook != null) {
174            try {
175                Runtime.getRuntime().removeShutdownHook(shutdownHook);
176            } catch (final IllegalStateException itse) {
177                // ok to ignore
178            }
179        }
180        try {
181            semaphore.acquire();
182        } catch (final InterruptedException e) {
183            Thread.currentThread().interrupt();
184        }
185    }
186
187    @Override
188    protected Object writeReplace() throws ObjectStreamException {
189        return new StreamSerializationReplacer(plugin(), rootName(), name(), serializeDelegate(), retryConfiguration,
190                stopStrategy);
191    }
192
193    private static class StreamSerializationReplacer extends SerializationReplacer {
194
195        private final RetryConfiguration retryConfiguration;
196
197        private final StopStrategy stopStrategy;
198
199        StreamSerializationReplacer(final String plugin, final String component, final String name, final byte[] value,
200                final RetryConfiguration retryConfiguration, final StopStrategy stopStrategy) {
201            super(plugin, component, name, value);
202            this.retryConfiguration = retryConfiguration;
203            this.stopStrategy = stopStrategy;
204        }
205
206        protected Object readResolve() throws ObjectStreamException {
207            try {
208                return new StreamingInputImpl(component, name, plugin, loadDelegate(), retryConfiguration,
209                        stopStrategy);
210            } catch (final IOException | ClassNotFoundException e) {
211                final InvalidObjectException invalidObjectException = new InvalidObjectException(e.getMessage());
212                invalidObjectException.initCause(e);
213                throw invalidObjectException;
214            }
215        }
216    }
217}