001/** 002 * Copyright (C) 2006-2022 Talend Inc. - www.talend.com 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.talend.sdk.component.runtime.input; 017 018import static java.lang.Thread.sleep; 019import static java.util.concurrent.TimeUnit.MILLISECONDS; 020import static org.talend.sdk.component.runtime.input.Streaming.RetryStrategy; 021 022import java.io.IOException; 023import java.io.InvalidObjectException; 024import java.io.ObjectStreamException; 025import java.io.Serializable; 026import java.util.concurrent.ExecutorService; 027import java.util.concurrent.Executors; 028import java.util.concurrent.Future; 029import java.util.concurrent.Semaphore; 030import java.util.concurrent.TimeoutException; 031import java.util.concurrent.atomic.AtomicBoolean; 032 033import org.talend.sdk.component.runtime.input.Streaming.RetryConfiguration; 034import org.talend.sdk.component.runtime.input.Streaming.StopStrategy; 035 036import lombok.extern.slf4j.Slf4j; 037 038@Slf4j 039public class StreamingInputImpl extends InputImpl { 040 041 private RetryConfiguration retryConfiguration; 042 043 private transient Thread shutdownHook; 044 045 private final AtomicBoolean running = new AtomicBoolean(); 046 047 private transient Semaphore semaphore; 048 049 private StopStrategy stopStrategy; 050 051 private transient long readRecords = 0L; 052 053 public StreamingInputImpl(final String rootName, final String name, final String plugin, 054 final Serializable instance, final RetryConfiguration retryConfiguration, final StopStrategy stopStrategy) { 055 super(rootName, name, plugin, instance); 056 shutdownHook = new Thread(() -> running.compareAndSet(true, false), 057 getClass().getName() + "_" + rootName() + "-" + name() + "_" + hashCode()); 058 this.retryConfiguration = retryConfiguration; 059 this.stopStrategy = stopStrategy; 060 log.debug("[StreamingInputImpl] Created with retryStrategy: {}, stopStrategy: {}.", this.retryConfiguration, 061 this.stopStrategy); 062 } 063 064 protected StreamingInputImpl() { 065 // no-op 066 } 067 068 @Override 069 protected Object readNext() { 070 if (!running.get()) { 071 return null; 072 } 073 if (stopStrategy.isActive() && stopStrategy.shouldStop(readRecords)) { 074 log.debug("[readNext] stopStrategy condition validated."); 075 return null; 076 } 077 try { 078 semaphore.acquire(); 079 } catch (final InterruptedException e) { 080 Thread.currentThread().interrupt(); 081 return null; 082 } 083 try { 084 final RetryStrategy strategy = retryConfiguration.getStrategy(); 085 int retries = retryConfiguration.getMaxRetries(); 086 while (running.get() && retries > 0) { 087 Object next = null; 088 if (stopStrategy.isActive() && stopStrategy.getMaxActiveTime() > -1) { 089 // Some connectors do not block input and return null (rabbitmq for instance). Thus, the future 090 // timeout is never reached and retryStrategy is run then. So, need to check timeout in the loop. 091 if (stopStrategy.shouldStop(readRecords)) { 092 log.debug("[readNext] shouldStop now! Duration {}ms", 093 System.currentTimeMillis() - stopStrategy.getStartedAtTime()); 094 return null; 095 } 096 final ExecutorService executor = Executors.newSingleThreadExecutor(); 097 final Future<Object> reader = executor.submit(super::readNext); 098 // manage job latency... 099 final long estimatedTimeout = stopStrategy.getMaxActiveTime() 100 - (System.currentTimeMillis() - stopStrategy.getStartedAtTime()); 101 final long timeout = estimatedTimeout < -1 ? 10 : estimatedTimeout; 102 log.debug( 103 "[readNext] Applying duration strategy for reading record: will interrupt in {}ms (estimated:{}ms Duration:{}ms).", 104 timeout, estimatedTimeout, stopStrategy.getMaxActiveTime()); 105 try { 106 next = reader.get(timeout, MILLISECONDS); 107 } catch (TimeoutException e) { 108 log.debug("[readNext] Read record: timeout received."); 109 reader.cancel(true); 110 return next; 111 } catch (Exception e) { 112 // nop 113 } finally { 114 executor.shutdownNow(); 115 } 116 } else { 117 next = super.readNext(); 118 } 119 if (next != null) { 120 strategy.reset(); 121 readRecords++; 122 return next; 123 } 124 125 retries--; 126 try { 127 final long millis = strategy.nextPauseDuration(); 128 if (millis < 0) { // assume it means "give up" 129 prepareStop(); 130 } else if (millis > 0) { // we can wait 1s but not minutes to quit 131 if (millis < 1000) { 132 sleep(millis); 133 } else { 134 long remaining = millis; 135 while (running.get() && remaining > 0) { 136 final long current = Math.min(remaining, 250); 137 remaining -= current; 138 sleep(current); 139 } 140 } 141 } // else if millis == 0 no need to call any method 142 } catch (final InterruptedException e) { 143 prepareStop(); // stop the stream 144 } 145 } 146 return null; 147 } finally { 148 semaphore.release(); 149 } 150 } 151 152 @Override 153 protected void init() { 154 super.init(); 155 semaphore = new Semaphore(1); 156 } 157 158 @Override 159 public void start() { 160 super.start(); 161 running.compareAndSet(false, true); 162 Runtime.getRuntime().addShutdownHook(shutdownHook); 163 } 164 165 @Override 166 public void stop() { 167 prepareStop(); 168 super.stop(); 169 } 170 171 private void prepareStop() { 172 running.compareAndSet(true, false); 173 if (shutdownHook != null) { 174 try { 175 Runtime.getRuntime().removeShutdownHook(shutdownHook); 176 } catch (final IllegalStateException itse) { 177 // ok to ignore 178 } 179 } 180 try { 181 semaphore.acquire(); 182 } catch (final InterruptedException e) { 183 Thread.currentThread().interrupt(); 184 } 185 } 186 187 @Override 188 protected Object writeReplace() throws ObjectStreamException { 189 return new StreamSerializationReplacer(plugin(), rootName(), name(), serializeDelegate(), retryConfiguration, 190 stopStrategy); 191 } 192 193 private static class StreamSerializationReplacer extends SerializationReplacer { 194 195 private final RetryConfiguration retryConfiguration; 196 197 private final StopStrategy stopStrategy; 198 199 StreamSerializationReplacer(final String plugin, final String component, final String name, final byte[] value, 200 final RetryConfiguration retryConfiguration, final StopStrategy stopStrategy) { 201 super(plugin, component, name, value); 202 this.retryConfiguration = retryConfiguration; 203 this.stopStrategy = stopStrategy; 204 } 205 206 protected Object readResolve() throws ObjectStreamException { 207 try { 208 return new StreamingInputImpl(component, name, plugin, loadDelegate(), retryConfiguration, 209 stopStrategy); 210 } catch (final IOException | ClassNotFoundException e) { 211 final InvalidObjectException invalidObjectException = new InvalidObjectException(e.getMessage()); 212 invalidObjectException.initCause(e); 213 throw invalidObjectException; 214 } 215 } 216 } 217}