001/** 002 * Copyright (C) 2006-2022 Talend Inc. - www.talend.com 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.talend.sdk.component.runtime.input; 017 018import static java.util.Collections.emptySet; 019import static java.util.Optional.ofNullable; 020 021import java.io.Serializable; 022import java.util.Set; 023import java.util.concurrent.TimeUnit; 024import java.util.function.Supplier; 025 026import org.talend.sdk.component.api.service.configuration.LocalConfiguration; 027import org.talend.sdk.component.runtime.serialization.ContainerFinder; 028 029import lombok.AllArgsConstructor; 030import lombok.Data; 031import lombok.NoArgsConstructor; 032import lombok.extern.slf4j.Slf4j; 033 034@Slf4j 035public class Streaming { 036 037 private static Supplier<LocalConfiguration> defaultLocalConfiguration = () -> new LocalConfiguration() { 038 039 @Override 040 public String get(final String key) { 041 return null; 042 } 043 044 @Override 045 public Set<String> keys() { 046 return emptySet(); 047 } 048 }; 049 050 public static RetryConfiguration loadRetryConfiguration(final String plugin) { 051 // note: this configuratoin could be read on the mapper too and distributed 052 final LocalConfiguration configuration = ofNullable(ContainerFinder.Instance.get().find(plugin)) 053 .map(it -> it.findService(LocalConfiguration.class)) 054 .orElseGet(defaultLocalConfiguration); 055 final int maxRetries = ofNullable(configuration.get("talend.input.streaming.retry.maxRetries")) 056 .map(Integer::parseInt) 057 .orElse(Integer.MAX_VALUE); 058 return new RetryConfiguration(maxRetries, getStrategy(configuration)); 059 } 060 061 public static RetryStrategy getStrategy(final LocalConfiguration configuration) { 062 switch (ofNullable(configuration.get("talend.input.streaming.retry.strategy")).orElse("constant")) { 063 case "exponential": 064 return new RetryConfiguration.ExponentialBackoff( 065 ofNullable(configuration.get("talend.input.streaming.retry.exponential.exponent")) 066 .map(Double::parseDouble) 067 .orElse(1.5), 068 ofNullable(configuration.get("talend.input.streaming.retry.exponential.randomizationFactor")) 069 .map(Double::parseDouble) 070 .orElse(.5), 071 ofNullable(configuration.get("talend.input.streaming.retry.exponential.maxDuration")) 072 .map(Long::parseLong) 073 .orElse(TimeUnit.MINUTES.toMillis(5)), 074 ofNullable(configuration.get("talend.input.streaming.retry.exponential.initialBackOff")) 075 .map(Long::parseLong) 076 .orElse(TimeUnit.SECONDS.toMillis(1)), 077 0); 078 case "constant": 079 default: 080 return new RetryConfiguration.Constant( 081 ofNullable(configuration.get("talend.input.streaming.retry.constant.timeout")) 082 .map(Long::parseLong) 083 .orElse(500L)); 084 } 085 } 086 087 public static StopStrategy loadStopStrategy(final String plugin, 088 final java.util.Map<String, String> internalConfiguration) { 089 final LocalConfiguration configuration = ofNullable(ContainerFinder.Instance.get().find(plugin)) 090 .map(it -> it.findService(LocalConfiguration.class)) 091 .orElseGet(defaultLocalConfiguration); 092 final Long maxReadRecords = ofNullable(internalConfiguration.entrySet() 093 .stream() 094 .filter(e -> e.getKey().startsWith("$maxRecords") || e.getKey().contains(".$maxRecords")) 095 .findFirst() 096 .map(e -> e.getValue()) 097 .map(Long::parseLong)).get() 098 .orElseGet(() -> ofNullable(System.getProperty( 099 String.format("%s.talend.input.streaming.maxRecords", plugin))) 100 .map(Long::parseLong) 101 .orElseGet(() -> ofNullable( 102 configuration.get("talend.input.streaming.maxRecords")) 103 .map(Long::parseLong) 104 .orElseGet(() -> null))); 105 Long maxActiveTime = ofNullable(internalConfiguration.entrySet() 106 .stream() 107 .filter(e -> e.getKey().startsWith("$maxDurationMs") || e.getKey().contains(".$maxDurationMs")) 108 .findFirst() 109 .map(e -> e.getValue()) 110 .map(Long::parseLong)).get() 111 .orElseGet(() -> ofNullable(System.getProperty( 112 String.format("%s.talend.input.streaming.maxDurationMs", plugin))) 113 .map(Long::parseLong) 114 .orElseGet(() -> ofNullable( 115 configuration.get("talend.input.streaming.maxDurationMs")) 116 .map(Long::parseLong) 117 .orElseGet(() -> null))); 118 log.debug("[loadStopStrategy] Records: {}; Duration: {}.", maxReadRecords, maxActiveTime); 119 return new StopConfiguration(maxReadRecords, maxActiveTime, null); 120 } 121 122 public interface RetryStrategy { 123 124 long nextPauseDuration(); 125 126 void reset(); 127 } 128 129 @Data 130 @NoArgsConstructor 131 @AllArgsConstructor 132 public static class RetryConfiguration implements Serializable { 133 134 private int maxRetries; 135 136 private RetryStrategy strategy; 137 138 @Data 139 @NoArgsConstructor 140 @AllArgsConstructor 141 public static class Constant implements Serializable, RetryStrategy { 142 143 private long timeout; 144 145 @Override 146 public long nextPauseDuration() { 147 return timeout; 148 } 149 150 @Override 151 public void reset() { 152 // no-op 153 } 154 } 155 156 @Data 157 @NoArgsConstructor 158 @AllArgsConstructor 159 public static class ExponentialBackoff implements Serializable, RetryStrategy { 160 161 private double exponent; 162 163 private double randomizationFactor; 164 165 private long max; 166 167 private long initialBackOff; 168 169 // state 170 private int iteration; 171 172 @Override 173 public long nextPauseDuration() { 174 final double currentIntervalMillis = Math.min(initialBackOff * Math.pow(exponent, iteration), max); 175 final double randomOffset = (Math.random() * 2 - 1) * randomizationFactor * currentIntervalMillis; 176 final long nextBackoffMillis = Math.min(max, Math.round(currentIntervalMillis + randomOffset)); 177 iteration += 1; 178 return nextBackoffMillis; 179 } 180 181 @Override 182 public void reset() { 183 iteration = 0; 184 } 185 } 186 } 187 188 public interface StopStrategy { 189 190 /** 191 * Check if the stop strategy is active according specified conditions. 192 * 193 * @return true if strategy is active, false otherwise. 194 */ 195 boolean isActive(); 196 197 /** 198 * Check the stop strategy conditions. 199 * 200 * @param read - already read records. 201 * @return true if the lifecycle should be stopped, false otherwise. 202 */ 203 boolean shouldStop(long read); 204 205 /** 206 * Maximum records to read. 207 * 208 * @return max number of records to read. 209 */ 210 long getMaxReadRecords(); 211 212 /** 213 * Maximum duration the lifecycle should run in ms. 214 * 215 * @return max activity duration. 216 */ 217 long getMaxActiveTime(); 218 219 /** 220 * The system time in millis when the lifecycle started. 221 * 222 * @return started time. 223 */ 224 long getStartedAtTime(); 225 226 } 227 228 @Data 229 public static class StopConfiguration implements StopStrategy, Serializable { 230 231 private long maxReadRecords; 232 233 private long maxActiveTime; 234 235 private long startedAtTime; 236 237 public StopConfiguration() { 238 maxReadRecords = -1L; 239 maxActiveTime = -1L; 240 startedAtTime = System.currentTimeMillis(); 241 } 242 243 public StopConfiguration(final Long maxRecords, final Long maxTime, final Long start) { 244 maxReadRecords = maxRecords == null ? -1L : maxRecords; 245 maxActiveTime = maxTime == null ? -1L : maxTime; 246 startedAtTime = start == null ? System.currentTimeMillis() : start; 247 } 248 249 @Override 250 public boolean isActive() { 251 return (maxReadRecords > -1) || (maxActiveTime > -1); 252 } 253 254 private boolean hasEnoughRecords(final long read) { 255 return maxReadRecords != -1 && read >= maxReadRecords; 256 } 257 258 private boolean isTimePassed() { 259 return maxActiveTime != -1 && System.currentTimeMillis() - startedAtTime >= maxActiveTime; 260 } 261 262 @Override 263 public boolean shouldStop(final long readRecords) { 264 return hasEnoughRecords(readRecords) || isTimePassed(); 265 } 266 } 267 268}