001/** 002 * Copyright (C) 2006-2023 Talend Inc. - www.talend.com 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.talend.sdk.component.server.service; 017 018import static java.util.Collections.emptyList; 019import static java.util.Optional.ofNullable; 020import static java.util.concurrent.TimeUnit.SECONDS; 021import static java.util.stream.Collectors.toList; 022import static java.util.stream.Collectors.toSet; 023import static java.util.stream.Stream.empty; 024 025import java.io.IOException; 026import java.io.InputStream; 027import java.nio.file.Files; 028import java.nio.file.Path; 029import java.nio.file.Paths; 030import java.nio.file.attribute.FileTime; 031import java.time.Instant; 032import java.util.Collection; 033import java.util.Date; 034import java.util.List; 035import java.util.Locale; 036import java.util.Objects; 037import java.util.Optional; 038import java.util.Properties; 039import java.util.concurrent.CompletionStage; 040import java.util.concurrent.Executors; 041import java.util.concurrent.ScheduledExecutorService; 042import java.util.function.Supplier; 043import java.util.stream.Stream; 044 045import javax.annotation.PostConstruct; 046import javax.annotation.PreDestroy; 047import javax.enterprise.context.ApplicationScoped; 048import javax.enterprise.context.Initialized; 049import javax.enterprise.event.Event; 050import javax.enterprise.event.Observes; 051import javax.enterprise.inject.Produces; 052import javax.inject.Inject; 053import javax.ws.rs.core.Context; 054import javax.ws.rs.core.UriInfo; 055 056import org.talend.sdk.component.container.Container; 057import org.talend.sdk.component.container.ContainerListener; 058import org.talend.sdk.component.dependencies.maven.Artifact; 059import org.talend.sdk.component.dependencies.maven.MvnCoordinateToFileConverter; 060import org.talend.sdk.component.design.extension.RepositoryModel; 061import org.talend.sdk.component.design.extension.repository.Config; 062import org.talend.sdk.component.path.PathFactory; 063import org.talend.sdk.component.runtime.manager.ComponentFamilyMeta; 064import org.talend.sdk.component.runtime.manager.ComponentManager; 065import org.talend.sdk.component.runtime.manager.ContainerComponentRegistry; 066import org.talend.sdk.component.server.configuration.ComponentServerConfiguration; 067import org.talend.sdk.component.server.dao.ComponentActionDao; 068import org.talend.sdk.component.server.dao.ComponentDao; 069import org.talend.sdk.component.server.dao.ComponentFamilyDao; 070import org.talend.sdk.component.server.dao.ConfigurationDao; 071import org.talend.sdk.component.server.front.model.Connectors; 072import org.talend.sdk.component.server.service.event.DeployedComponent; 073import org.talend.sdk.component.server.service.jcache.FrontCacheResolver; 074 075import lombok.AllArgsConstructor; 076import lombok.Data; 077import lombok.extern.slf4j.Slf4j; 078 079@Slf4j 080@ApplicationScoped 081public class ComponentManagerService { 082 083 @Inject 084 private ComponentServerConfiguration configuration; 085 086 @Inject 087 private ComponentDao componentDao; 088 089 @Inject 090 private ComponentFamilyDao componentFamilyDao; 091 092 @Inject 093 private ComponentActionDao actionDao; 094 095 @Inject 096 private ConfigurationDao configurationDao; 097 098 @Inject 099 private VirtualDependenciesService virtualDependenciesService; 100 101 @Inject 102 private GlobService globService; 103 104 @Inject 105 private Event<DeployedComponent> deployedComponentEvent; 106 107 @Inject 108 @Context 109 private UriInfo uriInfo; 110 111 @Inject 112 private LocaleMapper localeMapper; 113 114 private ComponentManager instance; 115 116 private MvnCoordinateToFileConverter mvnCoordinateToFileConverter; 117 118 private DeploymentListener deploymentListener; 119 120 private volatile Date lastUpdated = new Date(); 121 122 private Connectors connectors; 123 124 private boolean started; 125 126 private Path m2; 127 128 private Long latestPluginUpdate; 129 130 private ScheduledExecutorService scheduledExecutorService; 131 132 @Inject 133 private FrontCacheResolver cacheResolver; 134 135 public void startupLoad(@Observes @Initialized(ApplicationScoped.class) final Object start) { 136 // no-op 137 } 138 139 @PostConstruct 140 private void init() { 141 if (log.isWarnEnabled()) { 142 final String filter = System.getProperty("jdk.serialFilter"); 143 if (filter == null) { 144 log.warn("No system property 'jdk.serialFilter', ensure it is intended"); 145 } 146 } 147 148 mvnCoordinateToFileConverter = new MvnCoordinateToFileConverter(); 149 m2 = configuration 150 .getMavenRepository() 151 .map(PathFactory::get) 152 .filter(Files::exists) 153 .orElseGet(ComponentManager::findM2); 154 log.info("Using maven repository: '{}'", m2); 155 instance = new ComponentManager(m2) { 156 157 @Override 158 protected Supplier<Locale> getLocalSupplier() { 159 return ComponentManagerService.this::readCurrentLocale; 160 } 161 }; 162 deploymentListener = new DeploymentListener(componentDao, componentFamilyDao, actionDao, configurationDao, 163 virtualDependenciesService); 164 instance.getContainer().registerListener(deploymentListener); 165 // deploy plugins 166 deployPlugins(); 167 // check if we find a connectors version information file on top of the m2 168 synchronizeConnectors(); 169 // auto-reload plugins executor service 170 if (configuration.getPluginsReloadActive()) { 171 final boolean useTimestamp = "timestamp".equals(configuration.getPluginsReloadMethod()); 172 if (useTimestamp) { 173 latestPluginUpdate = readPluginsTimestamp(); 174 } 175 scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); 176 scheduledExecutorService 177 .scheduleWithFixedDelay(() -> checkPlugins(), configuration.getPluginsReloadInterval(), 178 configuration.getPluginsReloadInterval(), SECONDS); 179 log 180 .info("Plugin reloading enabled with {} method and interval check of {}s.", 181 useTimestamp ? "timestamp" : "connectors version", 182 configuration.getPluginsReloadInterval()); 183 } 184 185 started = true; 186 } 187 188 @PreDestroy 189 private void destroy() { 190 started = false; 191 instance.getContainer().unregisterListener(deploymentListener); 192 instance.close(); 193 // shutdown auto-reload service 194 if (scheduledExecutorService != null) { 195 scheduledExecutorService.shutdownNow(); 196 try { 197 scheduledExecutorService.awaitTermination(10, SECONDS); 198 } catch (final InterruptedException e) { 199 Thread.currentThread().interrupt(); 200 } 201 } 202 } 203 204 private Locale readCurrentLocale() { 205 try { 206 return ofNullable(uriInfo.getQueryParameters().getFirst("lang")) 207 .map(localeMapper::mapLocale) 208 .orElseGet(Locale::getDefault); 209 } catch (final RuntimeException ex) { 210 log.debug("Can't get the locale from current request in thread '{}'", Thread.currentThread().getName(), ex); 211 return Locale.getDefault(); 212 } 213 } 214 215 private void synchronizeConnectors() { 216 connectors = new Connectors(readConnectorsVersion(), manager().getContainer().getPluginsHash(), 217 manager().getContainer().getPluginsList()); 218 } 219 220 private CompletionStage<Void> checkPlugins() { 221 boolean reload; 222 if ("timestamp".equals(configuration.getPluginsReloadMethod())) { 223 final long checked = readPluginsTimestamp(); 224 reload = checked > latestPluginUpdate; 225 log 226 .info("checkPlugins w/ timestamp {} vs {}. Reloading: {}.", 227 Instant.ofEpochMilli(latestPluginUpdate), Instant.ofEpochMilli(checked), reload); 228 latestPluginUpdate = checked; 229 } else { 230 // connectors version used 231 final String cv = readConnectorsVersion(); 232 reload = !cv.equals(getConnectors().getVersion()); 233 log.info("checkPlugins w/ connectors {} vs {}. Reloading: {}.", connectors.getVersion(), cv, reload); 234 } 235 if (!reload) { 236 return null; 237 } 238 // undeploy plugins 239 log.info("Un-deploying plugins..."); 240 manager().getContainer().findAll().forEach(container -> container.close()); 241 // redeploy plugins 242 log.info("Re-deploying plugins..."); 243 deployPlugins(); 244 log.info("Plugins deployed."); 245 // reset connectors' version 246 synchronizeConnectors(); 247 // reset caches 248 cacheResolver.clearCaches(); 249 250 return null; 251 } 252 253 private synchronized String readConnectorsVersion() { 254 // check if we find a connectors version information file on top of the m2 255 final String version = Optional.of(m2.resolve("CONNECTORS_VERSION")).filter(Files::exists).map(p -> { 256 try { 257 return Files.lines(p).findFirst().get(); 258 } catch (IOException e) { 259 log.warn("Failed reading connectors version {}", e.getMessage()); 260 return "unknown"; 261 } 262 }).orElse("unknown"); 263 log.debug("Using connectors version: '{}'", version); 264 265 return version; 266 } 267 268 private synchronized Long readPluginsTimestamp() { 269 final Long last = Stream 270 .concat(Stream.of(configuration.getPluginsReloadFileMarker().orElse("")), 271 configuration.getComponentRegistry().map(Collection::stream).orElse(Stream.empty())) 272 .filter(s -> !s.isEmpty()) 273 .map(cr -> Paths.get(cr)) 274 .filter(Files::exists) 275 .peek(f -> log.debug("[readPluginsTimestamp] getting {} timestamp.", f)) 276 .map(f -> { 277 try { 278 return Files.getAttribute(f, "lastModifiedTime"); 279 } catch (IOException e) { 280 return null; 281 } 282 }) 283 .filter(Objects::nonNull) 284 .findFirst() 285 .map(ts -> FileTime.class.cast(ts).toMillis()) 286 .orElse(0L); 287 log.debug("[readPluginsTimestamp] Latest: {}.", Instant.ofEpochMilli(last)); 288 return last; 289 } 290 291 private synchronized void deployPlugins() { 292 // note: we don't want to download anything from the manager, if we need to download any artifact we need 293 // to ensure it is controlled (secured) and allowed so don't make it implicit but enforce a first phase 294 // where it is cached locally (provisioning solution) 295 final List<String> coords = configuration 296 .getComponentCoordinates() 297 .map(it -> Stream.of(it.split(",")).map(String::trim).filter(i -> !i.isEmpty()).collect(toList())) 298 .orElse(emptyList()); 299 coords.forEach(this::deploy); 300 configuration 301 .getComponentRegistry() 302 .map(Collection::stream) 303 .orElseGet(Stream::empty) 304 .flatMap(globService::toFiles) 305 .forEach(registry -> { 306 final Properties properties = new Properties(); 307 try (final InputStream is = Files.newInputStream(registry)) { 308 properties.load(is); 309 } catch (final IOException e) { 310 // when using plugin reloading, main registry file may be not available at startup. 311 if (configuration.getPluginsReloadActive()) { 312 log.warn("[deployPlugins] registry file {} is unavailable.", registry); 313 } else { 314 throw new IllegalArgumentException(e); 315 } 316 } 317 properties 318 .stringPropertyNames() 319 .stream() 320 .map(properties::getProperty) 321 .filter(gav -> !coords.contains(gav)) 322 .forEach(this::deploy); 323 }); 324 } 325 326 public String deploy(final String pluginGAV) { 327 final String pluginPath = ofNullable(pluginGAV) 328 .map(gav -> mvnCoordinateToFileConverter.toArtifact(gav)) 329 .map(Artifact::toPath) 330 .orElseThrow(() -> new IllegalArgumentException("Plugin GAV can't be empty")); 331 332 final String plugin = 333 instance.addWithLocationPlugin(pluginGAV, m2.resolve(pluginPath).toAbsolutePath().toString()); 334 lastUpdated = new Date(); 335 synchronizeConnectors(); 336 if (started) { 337 deployedComponentEvent.fire(new DeployedComponent()); 338 } 339 return plugin; 340 } 341 342 public synchronized void undeploy(final String pluginGAV) { 343 if (pluginGAV == null || pluginGAV.isEmpty()) { 344 throw new IllegalArgumentException("plugin maven GAV are required to undeploy a plugin"); 345 } 346 347 final String pluginID = instance 348 .find(c -> pluginGAV.equals(c.get(ComponentManager.OriginalId.class).getValue()) ? Stream.of(c.getId()) 349 : empty()) 350 .findFirst() 351 .orElseThrow(() -> new IllegalArgumentException("No plugin found using maven GAV: " + pluginGAV)); 352 353 instance.removePlugin(pluginID); 354 lastUpdated = new Date(); 355 synchronizeConnectors(); 356 } 357 358 public Date findLastUpdated() { 359 return lastUpdated; 360 } 361 362 public Connectors getConnectors() { 363 return connectors; 364 } 365 366 @AllArgsConstructor 367 private static class DeploymentListener implements ContainerListener { 368 369 private final ComponentDao componentDao; 370 371 private final ComponentFamilyDao componentFamilyDao; 372 373 private final ComponentActionDao actionDao; 374 375 private final ConfigurationDao configurationDao; 376 377 private final VirtualDependenciesService virtualDependenciesService; 378 379 @Override 380 public void onCreate(final Container container) { 381 container.set(CleanupTask.class, new CleanupTask(postDeploy(container))); 382 } 383 384 @Override 385 public void onClose(final Container container) { 386 if (container.getState() == Container.State.ON_ERROR) { 387 // means it was not deployed so don't drop old state 388 return; 389 } 390 ofNullable(container.get(CleanupTask.class)).ifPresent(c -> c.getCleanup().run()); 391 } 392 393 private Runnable postDeploy(final Container plugin) { 394 final Collection<String> componentIds = plugin 395 .get(ContainerComponentRegistry.class) 396 .getComponents() 397 .values() 398 .stream() 399 .flatMap(c -> Stream 400 .of(c.getPartitionMappers().values().stream(), c.getProcessors().values().stream(), 401 c.getDriverRunners().values().stream()) 402 .flatMap(t -> t)) 403 .peek(componentDao::createOrUpdate) 404 .map(ComponentFamilyMeta.BaseMeta::getId) 405 .collect(toSet()); 406 407 final Collection<ComponentActionDao.ActionKey> actions = plugin 408 .get(ContainerComponentRegistry.class) 409 .getServices() 410 .stream() 411 .flatMap(c -> c.getActions().stream()) 412 .map(actionDao::createOrUpdate) 413 .collect(toList()); 414 415 final Collection<String> families = plugin 416 .get(ContainerComponentRegistry.class) 417 .getComponents() 418 .values() 419 .stream() 420 .map(componentFamilyDao::createOrUpdate) 421 .collect(toList()); 422 423 final Collection<String> configs = ofNullable(plugin.get(RepositoryModel.class)) 424 .map(r -> r 425 .getFamilies() 426 .stream() 427 .flatMap(f -> configAsStream(f.getConfigs().get().stream())) 428 .collect(toList())) 429 .orElse(emptyList()) 430 .stream() 431 .map(configurationDao::createOrUpdate) 432 .collect(toList()); 433 434 return () -> { 435 virtualDependenciesService.onUnDeploy(plugin); 436 componentIds.forEach(componentDao::removeById); 437 actions.forEach(actionDao::removeById); 438 families.forEach(componentFamilyDao::removeById); 439 configs.forEach(configurationDao::removeById); 440 }; 441 } 442 443 private Stream<Config> configAsStream(final Stream<Config> stream) { 444 return stream.flatMap(s -> Stream.concat(Stream.of(s), s.getChildConfigs().stream())); 445 } 446 } 447 448 @Data 449 private static class CleanupTask { 450 451 private final Runnable cleanup; 452 } 453 454 @Produces 455 public ComponentManager manager() { 456 return instance; 457 } 458 459}