001/**
002 * Copyright (C) 2006-2025 Talend Inc. - www.talend.com
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.talend.sdk.component.server.service;
017
018import static java.util.Collections.emptyList;
019import static java.util.Optional.ofNullable;
020import static java.util.concurrent.TimeUnit.SECONDS;
021import static java.util.stream.Collectors.toList;
022import static java.util.stream.Collectors.toSet;
023import static java.util.stream.Stream.empty;
024
025import java.io.IOException;
026import java.io.InputStream;
027import java.nio.file.Files;
028import java.nio.file.Path;
029import java.nio.file.Paths;
030import java.nio.file.attribute.FileTime;
031import java.time.Instant;
032import java.util.Collection;
033import java.util.Date;
034import java.util.List;
035import java.util.Locale;
036import java.util.Objects;
037import java.util.Optional;
038import java.util.Properties;
039import java.util.concurrent.CompletionStage;
040import java.util.concurrent.Executors;
041import java.util.concurrent.ScheduledExecutorService;
042import java.util.function.Supplier;
043import java.util.stream.Stream;
044
045import javax.annotation.PostConstruct;
046import javax.annotation.PreDestroy;
047import javax.enterprise.context.ApplicationScoped;
048import javax.enterprise.context.Initialized;
049import javax.enterprise.event.Event;
050import javax.enterprise.event.Observes;
051import javax.enterprise.inject.Produces;
052import javax.inject.Inject;
053import javax.ws.rs.core.Context;
054import javax.ws.rs.core.UriInfo;
055
056import org.talend.sdk.component.container.Container;
057import org.talend.sdk.component.container.ContainerListener;
058import org.talend.sdk.component.dependencies.maven.Artifact;
059import org.talend.sdk.component.dependencies.maven.MvnCoordinateToFileConverter;
060import org.talend.sdk.component.design.extension.RepositoryModel;
061import org.talend.sdk.component.design.extension.repository.Config;
062import org.talend.sdk.component.path.PathFactory;
063import org.talend.sdk.component.runtime.manager.ComponentFamilyMeta;
064import org.talend.sdk.component.runtime.manager.ComponentManager;
065import org.talend.sdk.component.runtime.manager.ContainerComponentRegistry;
066import org.talend.sdk.component.server.configuration.ComponentServerConfiguration;
067import org.talend.sdk.component.server.dao.ComponentActionDao;
068import org.talend.sdk.component.server.dao.ComponentDao;
069import org.talend.sdk.component.server.dao.ComponentFamilyDao;
070import org.talend.sdk.component.server.dao.ConfigurationDao;
071import org.talend.sdk.component.server.front.model.Connectors;
072import org.talend.sdk.component.server.service.event.DeployedComponent;
073import org.talend.sdk.component.server.service.jcache.FrontCacheResolver;
074
075import lombok.AllArgsConstructor;
076import lombok.Data;
077import lombok.extern.slf4j.Slf4j;
078
079@Slf4j
080@ApplicationScoped
081public class ComponentManagerService {
082
083    @Inject
084    private ComponentServerConfiguration configuration;
085
086    @Inject
087    private ComponentDao componentDao;
088
089    @Inject
090    private ComponentFamilyDao componentFamilyDao;
091
092    @Inject
093    private ComponentActionDao actionDao;
094
095    @Inject
096    private ConfigurationDao configurationDao;
097
098    @Inject
099    private VirtualDependenciesService virtualDependenciesService;
100
101    @Inject
102    private GlobService globService;
103
104    @Inject
105    private Event<DeployedComponent> deployedComponentEvent;
106
107    @Inject
108    @Context
109    private UriInfo uriInfo;
110
111    @Inject
112    private LocaleMapper localeMapper;
113
114    private ComponentManager instance;
115
116    private MvnCoordinateToFileConverter mvnCoordinateToFileConverter;
117
118    private DeploymentListener deploymentListener;
119
120    private volatile Date lastUpdated = new Date();
121
122    private Connectors connectors;
123
124    private boolean started;
125
126    private Path m2;
127
128    private Long latestPluginUpdate;
129
130    private ScheduledExecutorService scheduledExecutorService;
131
132    @Inject
133    private FrontCacheResolver cacheResolver;
134
135    public void startupLoad(@Observes @Initialized(ApplicationScoped.class) final Object start) {
136        // no-op
137    }
138
139    @PostConstruct
140    private void init() {
141        if (log.isWarnEnabled()) {
142            final String filter = System.getProperty("jdk.serialFilter");
143            if (filter == null) {
144                log.warn("No system property 'jdk.serialFilter', ensure it is intended");
145            }
146        }
147
148        mvnCoordinateToFileConverter = new MvnCoordinateToFileConverter();
149        m2 = configuration
150                .getMavenRepository()
151                .map(PathFactory::get)
152                .filter(Files::exists)
153                .orElseGet(ComponentManager::findM2);
154        log.info("Using maven repository: '{}'", m2);
155        instance = new ComponentManager(m2) {
156
157            @Override
158            protected Supplier<Locale> getLocalSupplier() {
159                return ComponentManagerService.this::readCurrentLocale;
160            }
161        };
162        deploymentListener = new DeploymentListener(componentDao, componentFamilyDao, actionDao, configurationDao,
163                virtualDependenciesService);
164        instance.getContainer().registerListener(deploymentListener);
165        // deploy plugins
166        deployPlugins();
167        // check if we find a connectors version information file on top of the m2
168        synchronizeConnectors();
169        // auto-reload plugins executor service
170        if (configuration.getPluginsReloadActive()) {
171            final boolean useTimestamp = "timestamp".equals(configuration.getPluginsReloadMethod());
172            if (useTimestamp) {
173                latestPluginUpdate = readPluginsTimestamp();
174            }
175            scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
176            scheduledExecutorService
177                    .scheduleWithFixedDelay(() -> checkPlugins(), configuration.getPluginsReloadInterval(),
178                            configuration.getPluginsReloadInterval(), SECONDS);
179            log
180                    .info("Plugin reloading enabled with {} method and interval check of {}s.",
181                            useTimestamp ? "timestamp" : "connectors version",
182                            configuration.getPluginsReloadInterval());
183        }
184
185        started = true;
186    }
187
188    @PreDestroy
189    private void destroy() {
190        started = false;
191        instance.getContainer().unregisterListener(deploymentListener);
192        instance.close();
193        // shutdown auto-reload service
194        if (scheduledExecutorService != null) {
195            scheduledExecutorService.shutdownNow();
196            try {
197                scheduledExecutorService.awaitTermination(10, SECONDS);
198            } catch (final InterruptedException e) {
199                Thread.currentThread().interrupt();
200            }
201        }
202    }
203
204    private Locale readCurrentLocale() {
205        try {
206            return ofNullable(uriInfo.getQueryParameters().getFirst("lang"))
207                    .map(localeMapper::mapLocale)
208                    .orElseGet(Locale::getDefault);
209        } catch (final RuntimeException ex) {
210            log.debug("Can't get the locale from current request in thread '{}'", Thread.currentThread().getName(), ex);
211            return Locale.getDefault();
212        }
213    }
214
215    private void synchronizeConnectors() {
216        connectors = new Connectors(readConnectorsVersion(), manager().getContainer().getPluginsHash(),
217                manager().getContainer().getPluginsList());
218    }
219
220    private CompletionStage<Void> checkPlugins() {
221        boolean reload;
222        if ("timestamp".equals(configuration.getPluginsReloadMethod())) {
223            final long checked = readPluginsTimestamp();
224            reload = checked > latestPluginUpdate;
225            log
226                    .info("checkPlugins w/ timestamp {} vs {}. Reloading: {}.",
227                            Instant.ofEpochMilli(latestPluginUpdate), Instant.ofEpochMilli(checked), reload);
228            latestPluginUpdate = checked;
229        } else {
230            // connectors version used
231            final String cv = readConnectorsVersion();
232            reload = !cv.equals(getConnectors().getVersion());
233            log.info("checkPlugins w/ connectors {} vs {}. Reloading: {}.", connectors.getVersion(), cv, reload);
234        }
235        if (!reload) {
236            return null;
237        }
238        redeployPlugins();
239
240        return null;
241    }
242
243    private synchronized String readConnectorsVersion() {
244        // check if we find a connectors version information file on top of the m2
245        final String version = Optional.of(m2.resolve("CONNECTORS_VERSION")).filter(Files::exists).map(p -> {
246            try {
247                return Files.lines(p).findFirst().get();
248            } catch (IOException e) {
249                log.warn("Failed reading connectors version {}", e.getMessage());
250                return "unknown";
251            }
252        }).orElse("unknown");
253        log.debug("Using connectors version: '{}'", version);
254
255        return version;
256    }
257
258    private synchronized Long readPluginsTimestamp() {
259        final Long last = Stream
260                .concat(Stream.of(configuration.getPluginsReloadFileMarker().orElse("")),
261                        configuration.getComponentRegistry().map(Collection::stream).orElse(Stream.empty()))
262                .filter(s -> !s.isEmpty())
263                .map(cr -> Paths.get(cr))
264                .filter(Files::exists)
265                .peek(f -> log.debug("[readPluginsTimestamp] getting {} timestamp.", f))
266                .map(f -> {
267                    try {
268                        return Files.getAttribute(f, "lastModifiedTime");
269                    } catch (IOException e) {
270                        return null;
271                    }
272                })
273                .filter(Objects::nonNull)
274                .findFirst()
275                .map(ts -> FileTime.class.cast(ts).toMillis())
276                .orElse(0L);
277        log.debug("[readPluginsTimestamp] Latest: {}.", Instant.ofEpochMilli(last));
278        return last;
279    }
280
281    private synchronized void deployPlugins() {
282        // note: we don't want to download anything from the manager, if we need to download any artifact we need
283        // to ensure it is controlled (secured) and allowed so don't make it implicit but enforce a first phase
284        // where it is cached locally (provisioning solution)
285        final List<String> coords = configuration
286                .getComponentCoordinates()
287                .map(it -> Stream.of(it.split(",")).map(String::trim).filter(i -> !i.isEmpty()).collect(toList()))
288                .orElse(emptyList());
289        coords.forEach(this::deploy);
290        configuration
291                .getComponentRegistry()
292                .map(Collection::stream)
293                .orElseGet(Stream::empty)
294                .flatMap(globService::toFiles)
295                .forEach(registry -> {
296                    final Properties properties = new Properties();
297                    try (final InputStream is = Files.newInputStream(registry)) {
298                        properties.load(is);
299                    } catch (final IOException e) {
300                        // when using plugin reloading, main registry file may be not available at startup.
301                        if (configuration.getPluginsReloadActive()) {
302                            log.warn("[deployPlugins] registry file {} is unavailable.", registry);
303                        } else {
304                            throw new IllegalArgumentException(e);
305                        }
306                    }
307                    properties
308                            .stringPropertyNames()
309                            .stream()
310                            .map(properties::getProperty)
311                            .filter(gav -> !coords.contains(gav))
312                            .forEach(this::deploy);
313                });
314    }
315
316    public String deploy(final String pluginGAV) {
317        final String pluginPath = ofNullable(pluginGAV)
318                .map(gav -> mvnCoordinateToFileConverter.toArtifact(gav))
319                .map(Artifact::toPath)
320                .orElseThrow(() -> new IllegalArgumentException("Plugin GAV can't be empty"));
321
322        final String plugin =
323                instance.addWithLocationPlugin(pluginGAV, m2.resolve(pluginPath).toAbsolutePath().toString());
324        lastUpdated = new Date();
325        synchronizeConnectors();
326        if (started) {
327            deployedComponentEvent.fire(new DeployedComponent());
328        }
329        return plugin;
330    }
331
332    public synchronized void undeploy(final String pluginGAV) {
333        if (pluginGAV == null || pluginGAV.isEmpty()) {
334            throw new IllegalArgumentException("plugin maven GAV are required to undeploy a plugin");
335        }
336
337        final String pluginID = instance
338                .find(c -> pluginGAV.equals(c.get(ComponentManager.OriginalId.class).getValue()) ? Stream.of(c.getId())
339                        : empty())
340                .findFirst()
341                .orElseThrow(() -> new IllegalArgumentException("No plugin found using maven GAV: " + pluginGAV));
342
343        instance.removePlugin(pluginID);
344        lastUpdated = new Date();
345        synchronizeConnectors();
346    }
347
348    public void redeployPlugins() {
349        // undeploy plugins
350        log.info("Un-deploying plugins...");
351        manager().getContainer().findAll().forEach(Container::close);
352        // redeploy plugins
353        log.info("Re-deploying plugins...");
354        deployPlugins();
355        log.info("Plugins deployed.");
356        // reset connectors' version
357        synchronizeConnectors();
358        // reset caches
359        cacheResolver.cleanupCaches();
360    }
361
362    public Date findLastUpdated() {
363        return lastUpdated;
364    }
365
366    public Connectors getConnectors() {
367        return connectors;
368    }
369
370    @AllArgsConstructor
371    private static class DeploymentListener implements ContainerListener {
372
373        private final ComponentDao componentDao;
374
375        private final ComponentFamilyDao componentFamilyDao;
376
377        private final ComponentActionDao actionDao;
378
379        private final ConfigurationDao configurationDao;
380
381        private final VirtualDependenciesService virtualDependenciesService;
382
383        @Override
384        public void onCreate(final Container container) {
385            container.set(CleanupTask.class, new CleanupTask(postDeploy(container)));
386        }
387
388        @Override
389        public void onClose(final Container container) {
390            if (container.getState() == Container.State.ON_ERROR) {
391                // means it was not deployed so don't drop old state
392                return;
393            }
394            ofNullable(container.get(CleanupTask.class)).ifPresent(c -> c.getCleanup().run());
395        }
396
397        private Runnable postDeploy(final Container plugin) {
398            final Collection<String> componentIds = plugin
399                    .get(ContainerComponentRegistry.class)
400                    .getComponents()
401                    .values()
402                    .stream()
403                    .flatMap(c -> Stream
404                            .of(c.getPartitionMappers().values().stream(), c.getProcessors().values().stream(),
405                                    c.getDriverRunners().values().stream())
406                            .flatMap(t -> t))
407                    .peek(componentDao::createOrUpdate)
408                    .map(ComponentFamilyMeta.BaseMeta::getId)
409                    .collect(toSet());
410
411            final Collection<ComponentActionDao.ActionKey> actions = plugin
412                    .get(ContainerComponentRegistry.class)
413                    .getServices()
414                    .stream()
415                    .flatMap(c -> c.getActions().stream())
416                    .map(actionDao::createOrUpdate)
417                    .collect(toList());
418
419            final Collection<String> families = plugin
420                    .get(ContainerComponentRegistry.class)
421                    .getComponents()
422                    .values()
423                    .stream()
424                    .map(componentFamilyDao::createOrUpdate)
425                    .collect(toList());
426
427            final Collection<String> configs = ofNullable(plugin.get(RepositoryModel.class))
428                    .map(r -> r
429                            .getFamilies()
430                            .stream()
431                            .flatMap(f -> configAsStream(f.getConfigs().get().stream()))
432                            .collect(toList()))
433                    .orElse(emptyList())
434                    .stream()
435                    .map(configurationDao::createOrUpdate)
436                    .collect(toList());
437
438            return () -> {
439                virtualDependenciesService.onUnDeploy(plugin);
440                componentIds.forEach(componentDao::removeById);
441                actions.forEach(actionDao::removeById);
442                families.forEach(componentFamilyDao::removeById);
443                configs.forEach(configurationDao::removeById);
444            };
445        }
446
447        private Stream<Config> configAsStream(final Stream<Config> stream) {
448            return stream.flatMap(s -> Stream.concat(Stream.of(s), s.getChildConfigs().stream()));
449        }
450    }
451
452    @Data
453    private static class CleanupTask {
454
455        private final Runnable cleanup;
456    }
457
458    @Produces
459    public ComponentManager manager() {
460        return instance;
461    }
462
463}