001/** 002 * Copyright (C) 2006-2022 Talend Inc. - www.talend.com 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.talend.sdk.component.server.service; 017 018import static java.util.Collections.emptyList; 019import static java.util.Optional.ofNullable; 020import static java.util.concurrent.TimeUnit.SECONDS; 021import static java.util.stream.Collectors.toList; 022import static java.util.stream.Collectors.toSet; 023import static java.util.stream.Stream.empty; 024 025import java.io.IOException; 026import java.io.InputStream; 027import java.nio.file.Files; 028import java.nio.file.Path; 029import java.nio.file.Paths; 030import java.nio.file.attribute.FileTime; 031import java.time.Instant; 032import java.util.Collection; 033import java.util.Date; 034import java.util.List; 035import java.util.Locale; 036import java.util.Objects; 037import java.util.Optional; 038import java.util.Properties; 039import java.util.concurrent.CompletionStage; 040import java.util.concurrent.Executors; 041import java.util.concurrent.ScheduledExecutorService; 042import java.util.function.Supplier; 043import java.util.stream.Stream; 044 045import javax.annotation.PostConstruct; 046import javax.annotation.PreDestroy; 047import javax.enterprise.context.ApplicationScoped; 048import javax.enterprise.context.Initialized; 049import javax.enterprise.event.Event; 050import javax.enterprise.event.Observes; 051import javax.enterprise.inject.Produces; 052import javax.inject.Inject; 053import javax.ws.rs.core.Context; 054import javax.ws.rs.core.UriInfo; 055 056import org.talend.sdk.component.container.Container; 057import org.talend.sdk.component.container.ContainerListener; 058import org.talend.sdk.component.dependencies.maven.Artifact; 059import org.talend.sdk.component.dependencies.maven.MvnCoordinateToFileConverter; 060import org.talend.sdk.component.design.extension.RepositoryModel; 061import org.talend.sdk.component.design.extension.repository.Config; 062import org.talend.sdk.component.path.PathFactory; 063import org.talend.sdk.component.runtime.manager.ComponentFamilyMeta; 064import org.talend.sdk.component.runtime.manager.ComponentManager; 065import org.talend.sdk.component.runtime.manager.ContainerComponentRegistry; 066import org.talend.sdk.component.server.configuration.ComponentServerConfiguration; 067import org.talend.sdk.component.server.dao.ComponentActionDao; 068import org.talend.sdk.component.server.dao.ComponentDao; 069import org.talend.sdk.component.server.dao.ComponentFamilyDao; 070import org.talend.sdk.component.server.dao.ConfigurationDao; 071import org.talend.sdk.component.server.front.model.Connectors; 072import org.talend.sdk.component.server.service.event.DeployedComponent; 073 074import lombok.AllArgsConstructor; 075import lombok.Data; 076import lombok.extern.slf4j.Slf4j; 077 078@Slf4j 079@ApplicationScoped 080public class ComponentManagerService { 081 082 @Inject 083 private ComponentServerConfiguration configuration; 084 085 @Inject 086 private ComponentDao componentDao; 087 088 @Inject 089 private ComponentFamilyDao componentFamilyDao; 090 091 @Inject 092 private ComponentActionDao actionDao; 093 094 @Inject 095 private ConfigurationDao configurationDao; 096 097 @Inject 098 private VirtualDependenciesService virtualDependenciesService; 099 100 @Inject 101 private GlobService globService; 102 103 @Inject 104 private Event<DeployedComponent> deployedComponentEvent; 105 106 @Inject 107 @Context 108 private UriInfo uriInfo; 109 110 @Inject 111 private LocaleMapper localeMapper; 112 113 private ComponentManager instance; 114 115 private MvnCoordinateToFileConverter mvnCoordinateToFileConverter; 116 117 private DeploymentListener deploymentListener; 118 119 private volatile Date lastUpdated = new Date(); 120 121 private Connectors connectors; 122 123 private boolean started; 124 125 private Path m2; 126 127 private Long latestPluginUpdate; 128 129 private ScheduledExecutorService scheduledExecutorService; 130 131 public void startupLoad(@Observes @Initialized(ApplicationScoped.class) final Object start) { 132 // no-op 133 } 134 135 @PostConstruct 136 private void init() { 137 if (log.isWarnEnabled()) { 138 final String filter = System.getProperty("jdk.serialFilter"); 139 if (filter == null) { 140 log.warn("No system property 'jdk.serialFilter', ensure it is intended"); 141 } 142 } 143 144 mvnCoordinateToFileConverter = new MvnCoordinateToFileConverter(); 145 m2 = configuration 146 .getMavenRepository() 147 .map(PathFactory::get) 148 .filter(Files::exists) 149 .orElseGet(ComponentManager::findM2); 150 log.info("Using maven repository: '{}'", m2); 151 instance = new ComponentManager(m2) { 152 153 @Override 154 protected Supplier<Locale> getLocalSupplier() { 155 return ComponentManagerService.this::readCurrentLocale; 156 } 157 }; 158 deploymentListener = new DeploymentListener(componentDao, componentFamilyDao, actionDao, configurationDao, 159 virtualDependenciesService); 160 instance.getContainer().registerListener(deploymentListener); 161 // deploy plugins 162 deployPlugins(); 163 // check if we find a connectors version information file on top of the m2 164 synchronizeConnectors(); 165 // auto-reload plugins executor service 166 if (configuration.getPluginsReloadActive()) { 167 final boolean useTimestamp = "timestamp".equals(configuration.getPluginsReloadMethod()); 168 if (useTimestamp) { 169 latestPluginUpdate = readPluginsTimestamp(); 170 } 171 scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); 172 scheduledExecutorService 173 .scheduleWithFixedDelay(() -> checkPlugins(), configuration.getPluginsReloadInterval(), 174 configuration.getPluginsReloadInterval(), SECONDS); 175 log 176 .info("Plugin reloading enabled with {} method and interval check of {}s.", 177 useTimestamp ? "timestamp" : "connectors version", 178 configuration.getPluginsReloadInterval()); 179 } 180 181 started = true; 182 } 183 184 @PreDestroy 185 private void destroy() { 186 started = false; 187 instance.getContainer().unregisterListener(deploymentListener); 188 instance.close(); 189 // shutdown auto-reload service 190 if (scheduledExecutorService != null) { 191 scheduledExecutorService.shutdownNow(); 192 try { 193 scheduledExecutorService.awaitTermination(10, SECONDS); 194 } catch (final InterruptedException e) { 195 Thread.currentThread().interrupt(); 196 } 197 } 198 } 199 200 private Locale readCurrentLocale() { 201 try { 202 return ofNullable(uriInfo.getQueryParameters().getFirst("lang")) 203 .map(localeMapper::mapLocale) 204 .orElseGet(Locale::getDefault); 205 } catch (final RuntimeException ex) { 206 log.debug("Can't get the locale from current request in thread '{}'", Thread.currentThread().getName(), ex); 207 return Locale.getDefault(); 208 } 209 } 210 211 private void synchronizeConnectors() { 212 connectors = new Connectors(readConnectorsVersion(), manager().getContainer().getPluginsHash(), 213 manager().getContainer().getPluginsList()); 214 } 215 216 private CompletionStage<Void> checkPlugins() { 217 boolean reload; 218 if ("timestamp".equals(configuration.getPluginsReloadMethod())) { 219 final long checked = readPluginsTimestamp(); 220 reload = checked > latestPluginUpdate; 221 log 222 .info("checkPlugins w/ timestamp {} vs {}. Reloading: {}.", 223 Instant.ofEpochMilli(latestPluginUpdate), Instant.ofEpochMilli(checked), reload); 224 latestPluginUpdate = checked; 225 } else { 226 // connectors version used 227 final String cv = readConnectorsVersion(); 228 reload = !cv.equals(getConnectors().getVersion()); 229 log.info("checkPlugins w/ connectors {} vs {}. Reloading: {}.", connectors.getVersion(), cv, reload); 230 } 231 if (!reload) { 232 return null; 233 } 234 // undeploy plugins 235 log.info("Un-deploying plugins..."); 236 manager().getContainer().findAll().forEach(container -> container.close()); 237 // redeploy plugins 238 log.info("Re-deploying plugins..."); 239 deployPlugins(); 240 log.info("Plugins deployed."); 241 // reset connectors' version 242 synchronizeConnectors(); 243 244 return null; 245 } 246 247 private synchronized String readConnectorsVersion() { 248 // check if we find a connectors version information file on top of the m2 249 final String version = Optional.of(m2.resolve("CONNECTORS_VERSION")).filter(Files::exists).map(p -> { 250 try { 251 return Files.lines(p).findFirst().get(); 252 } catch (IOException e) { 253 log.warn("Failed reading connectors version {}", e.getMessage()); 254 return "unknown"; 255 } 256 }).orElse("unknown"); 257 log.debug("Using connectors version: '{}'", version); 258 259 return version; 260 } 261 262 private synchronized Long readPluginsTimestamp() { 263 final Long last = Stream 264 .concat(Stream.of(configuration.getPluginsReloadFileMarker().orElse("")), 265 configuration.getComponentRegistry().map(Collection::stream).orElse(Stream.empty())) 266 .filter(s -> !s.isEmpty()) 267 .map(cr -> Paths.get(cr)) 268 .filter(Files::exists) 269 .peek(f -> log.debug("[readPluginsTimestamp] getting {} timestamp.", f)) 270 .map(f -> { 271 try { 272 return Files.getAttribute(f, "lastModifiedTime"); 273 } catch (IOException e) { 274 return null; 275 } 276 }) 277 .filter(Objects::nonNull) 278 .findFirst() 279 .map(ts -> FileTime.class.cast(ts).toMillis()) 280 .orElse(0L); 281 log.debug("[readPluginsTimestamp] Latest: {}.", Instant.ofEpochMilli(last)); 282 return last; 283 } 284 285 private synchronized void deployPlugins() { 286 // note: we don't want to download anything from the manager, if we need to download any artifact we need 287 // to ensure it is controlled (secured) and allowed so don't make it implicit but enforce a first phase 288 // where it is cached locally (provisioning solution) 289 final List<String> coords = configuration 290 .getComponentCoordinates() 291 .map(it -> Stream.of(it.split(",")).map(String::trim).filter(i -> !i.isEmpty()).collect(toList())) 292 .orElse(emptyList()); 293 coords.forEach(this::deploy); 294 configuration 295 .getComponentRegistry() 296 .map(Collection::stream) 297 .orElseGet(Stream::empty) 298 .flatMap(globService::toFiles) 299 .forEach(registry -> { 300 final Properties properties = new Properties(); 301 try (final InputStream is = Files.newInputStream(registry)) { 302 properties.load(is); 303 } catch (final IOException e) { 304 throw new IllegalArgumentException(e); 305 } 306 properties 307 .stringPropertyNames() 308 .stream() 309 .map(properties::getProperty) 310 .filter(gav -> !coords.contains(gav)) 311 .forEach(this::deploy); 312 }); 313 } 314 315 public String deploy(final String pluginGAV) { 316 final String pluginPath = ofNullable(pluginGAV) 317 .map(gav -> mvnCoordinateToFileConverter.toArtifact(gav)) 318 .map(Artifact::toPath) 319 .orElseThrow(() -> new IllegalArgumentException("Plugin GAV can't be empty")); 320 321 final String plugin = 322 instance.addWithLocationPlugin(pluginGAV, m2.resolve(pluginPath).toAbsolutePath().toString()); 323 lastUpdated = new Date(); 324 synchronizeConnectors(); 325 if (started) { 326 deployedComponentEvent.fire(new DeployedComponent()); 327 } 328 return plugin; 329 } 330 331 public synchronized void undeploy(final String pluginGAV) { 332 if (pluginGAV == null || pluginGAV.isEmpty()) { 333 throw new IllegalArgumentException("plugin maven GAV are required to undeploy a plugin"); 334 } 335 336 final String pluginID = instance 337 .find(c -> pluginGAV.equals(c.get(ComponentManager.OriginalId.class).getValue()) ? Stream.of(c.getId()) 338 : empty()) 339 .findFirst() 340 .orElseThrow(() -> new IllegalArgumentException("No plugin found using maven GAV: " + pluginGAV)); 341 342 instance.removePlugin(pluginID); 343 lastUpdated = new Date(); 344 synchronizeConnectors(); 345 } 346 347 public Date findLastUpdated() { 348 return lastUpdated; 349 } 350 351 public Connectors getConnectors() { 352 return connectors; 353 } 354 355 @AllArgsConstructor 356 private static class DeploymentListener implements ContainerListener { 357 358 private final ComponentDao componentDao; 359 360 private final ComponentFamilyDao componentFamilyDao; 361 362 private final ComponentActionDao actionDao; 363 364 private final ConfigurationDao configurationDao; 365 366 private final VirtualDependenciesService virtualDependenciesService; 367 368 @Override 369 public void onCreate(final Container container) { 370 container.set(CleanupTask.class, new CleanupTask(postDeploy(container))); 371 } 372 373 @Override 374 public void onClose(final Container container) { 375 if (container.getState() == Container.State.ON_ERROR) { 376 // means it was not deployed so don't drop old state 377 return; 378 } 379 ofNullable(container.get(CleanupTask.class)).ifPresent(c -> c.getCleanup().run()); 380 } 381 382 private Runnable postDeploy(final Container plugin) { 383 final Collection<String> componentIds = plugin 384 .get(ContainerComponentRegistry.class) 385 .getComponents() 386 .values() 387 .stream() 388 .flatMap(c -> Stream 389 .of(c.getPartitionMappers().values().stream(), c.getProcessors().values().stream(), 390 c.getDriverRunners().values().stream()) 391 .flatMap(t -> t)) 392 .peek(componentDao::createOrUpdate) 393 .map(ComponentFamilyMeta.BaseMeta::getId) 394 .collect(toSet()); 395 396 final Collection<ComponentActionDao.ActionKey> actions = plugin 397 .get(ContainerComponentRegistry.class) 398 .getServices() 399 .stream() 400 .flatMap(c -> c.getActions().stream()) 401 .map(actionDao::createOrUpdate) 402 .collect(toList()); 403 404 final Collection<String> families = plugin 405 .get(ContainerComponentRegistry.class) 406 .getComponents() 407 .values() 408 .stream() 409 .map(componentFamilyDao::createOrUpdate) 410 .collect(toList()); 411 412 final Collection<String> configs = ofNullable(plugin.get(RepositoryModel.class)) 413 .map(r -> r 414 .getFamilies() 415 .stream() 416 .flatMap(f -> configAsStream(f.getConfigs().get().stream())) 417 .collect(toList())) 418 .orElse(emptyList()) 419 .stream() 420 .map(configurationDao::createOrUpdate) 421 .collect(toList()); 422 423 return () -> { 424 virtualDependenciesService.onUnDeploy(plugin); 425 componentIds.forEach(componentDao::removeById); 426 actions.forEach(actionDao::removeById); 427 families.forEach(componentFamilyDao::removeById); 428 configs.forEach(configurationDao::removeById); 429 }; 430 } 431 432 private Stream<Config> configAsStream(final Stream<Config> stream) { 433 return stream.flatMap(s -> Stream.concat(Stream.of(s), s.getChildConfigs().stream())); 434 } 435 } 436 437 @Data 438 private static class CleanupTask { 439 440 private final Runnable cleanup; 441 } 442 443 @Produces 444 public ComponentManager manager() { 445 return instance; 446 } 447 448}