001/**
002 * Copyright (C) 2006-2022 Talend Inc. - www.talend.com
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.talend.sdk.component.server.service;
017
018import static java.util.Collections.emptyList;
019import static java.util.Optional.ofNullable;
020import static java.util.concurrent.TimeUnit.SECONDS;
021import static java.util.stream.Collectors.toList;
022import static java.util.stream.Collectors.toSet;
023import static java.util.stream.Stream.empty;
024
025import java.io.IOException;
026import java.io.InputStream;
027import java.nio.file.Files;
028import java.nio.file.Path;
029import java.nio.file.Paths;
030import java.nio.file.attribute.FileTime;
031import java.time.Instant;
032import java.util.Collection;
033import java.util.Date;
034import java.util.List;
035import java.util.Locale;
036import java.util.Objects;
037import java.util.Optional;
038import java.util.Properties;
039import java.util.concurrent.CompletionStage;
040import java.util.concurrent.Executors;
041import java.util.concurrent.ScheduledExecutorService;
042import java.util.function.Supplier;
043import java.util.stream.Stream;
044
045import javax.annotation.PostConstruct;
046import javax.annotation.PreDestroy;
047import javax.enterprise.context.ApplicationScoped;
048import javax.enterprise.context.Initialized;
049import javax.enterprise.event.Event;
050import javax.enterprise.event.Observes;
051import javax.enterprise.inject.Produces;
052import javax.inject.Inject;
053import javax.ws.rs.core.Context;
054import javax.ws.rs.core.UriInfo;
055
056import org.talend.sdk.component.container.Container;
057import org.talend.sdk.component.container.ContainerListener;
058import org.talend.sdk.component.dependencies.maven.Artifact;
059import org.talend.sdk.component.dependencies.maven.MvnCoordinateToFileConverter;
060import org.talend.sdk.component.design.extension.RepositoryModel;
061import org.talend.sdk.component.design.extension.repository.Config;
062import org.talend.sdk.component.path.PathFactory;
063import org.talend.sdk.component.runtime.manager.ComponentFamilyMeta;
064import org.talend.sdk.component.runtime.manager.ComponentManager;
065import org.talend.sdk.component.runtime.manager.ContainerComponentRegistry;
066import org.talend.sdk.component.server.configuration.ComponentServerConfiguration;
067import org.talend.sdk.component.server.dao.ComponentActionDao;
068import org.talend.sdk.component.server.dao.ComponentDao;
069import org.talend.sdk.component.server.dao.ComponentFamilyDao;
070import org.talend.sdk.component.server.dao.ConfigurationDao;
071import org.talend.sdk.component.server.front.model.Connectors;
072import org.talend.sdk.component.server.service.event.DeployedComponent;
073
074import lombok.AllArgsConstructor;
075import lombok.Data;
076import lombok.extern.slf4j.Slf4j;
077
078@Slf4j
079@ApplicationScoped
080public class ComponentManagerService {
081
082    @Inject
083    private ComponentServerConfiguration configuration;
084
085    @Inject
086    private ComponentDao componentDao;
087
088    @Inject
089    private ComponentFamilyDao componentFamilyDao;
090
091    @Inject
092    private ComponentActionDao actionDao;
093
094    @Inject
095    private ConfigurationDao configurationDao;
096
097    @Inject
098    private VirtualDependenciesService virtualDependenciesService;
099
100    @Inject
101    private GlobService globService;
102
103    @Inject
104    private Event<DeployedComponent> deployedComponentEvent;
105
106    @Inject
107    @Context
108    private UriInfo uriInfo;
109
110    @Inject
111    private LocaleMapper localeMapper;
112
113    private ComponentManager instance;
114
115    private MvnCoordinateToFileConverter mvnCoordinateToFileConverter;
116
117    private DeploymentListener deploymentListener;
118
119    private volatile Date lastUpdated = new Date();
120
121    private Connectors connectors;
122
123    private boolean started;
124
125    private Path m2;
126
127    private Long latestPluginUpdate;
128
129    private ScheduledExecutorService scheduledExecutorService;
130
131    public void startupLoad(@Observes @Initialized(ApplicationScoped.class) final Object start) {
132        // no-op
133    }
134
135    @PostConstruct
136    private void init() {
137        if (log.isWarnEnabled()) {
138            final String filter = System.getProperty("jdk.serialFilter");
139            if (filter == null) {
140                log.warn("No system property 'jdk.serialFilter', ensure it is intended");
141            }
142        }
143
144        mvnCoordinateToFileConverter = new MvnCoordinateToFileConverter();
145        m2 = configuration
146                .getMavenRepository()
147                .map(PathFactory::get)
148                .filter(Files::exists)
149                .orElseGet(ComponentManager::findM2);
150        log.info("Using maven repository: '{}'", m2);
151        instance = new ComponentManager(m2) {
152
153            @Override
154            protected Supplier<Locale> getLocalSupplier() {
155                return ComponentManagerService.this::readCurrentLocale;
156            }
157        };
158        deploymentListener = new DeploymentListener(componentDao, componentFamilyDao, actionDao, configurationDao,
159                virtualDependenciesService);
160        instance.getContainer().registerListener(deploymentListener);
161        // deploy plugins
162        deployPlugins();
163        // check if we find a connectors version information file on top of the m2
164        synchronizeConnectors();
165        // auto-reload plugins executor service
166        if (configuration.getPluginsReloadActive()) {
167            final boolean useTimestamp = "timestamp".equals(configuration.getPluginsReloadMethod());
168            if (useTimestamp) {
169                latestPluginUpdate = readPluginsTimestamp();
170            }
171            scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
172            scheduledExecutorService
173                    .scheduleWithFixedDelay(() -> checkPlugins(), configuration.getPluginsReloadInterval(),
174                            configuration.getPluginsReloadInterval(), SECONDS);
175            log
176                    .info("Plugin reloading enabled with {} method and interval check of {}s.",
177                            useTimestamp ? "timestamp" : "connectors version",
178                            configuration.getPluginsReloadInterval());
179        }
180
181        started = true;
182    }
183
184    @PreDestroy
185    private void destroy() {
186        started = false;
187        instance.getContainer().unregisterListener(deploymentListener);
188        instance.close();
189        // shutdown auto-reload service
190        if (scheduledExecutorService != null) {
191            scheduledExecutorService.shutdownNow();
192            try {
193                scheduledExecutorService.awaitTermination(10, SECONDS);
194            } catch (final InterruptedException e) {
195                Thread.currentThread().interrupt();
196            }
197        }
198    }
199
200    private Locale readCurrentLocale() {
201        try {
202            return ofNullable(uriInfo.getQueryParameters().getFirst("lang"))
203                    .map(localeMapper::mapLocale)
204                    .orElseGet(Locale::getDefault);
205        } catch (final RuntimeException ex) {
206            log.debug("Can't get the locale from current request in thread '{}'", Thread.currentThread().getName(), ex);
207            return Locale.getDefault();
208        }
209    }
210
211    private void synchronizeConnectors() {
212        connectors = new Connectors(readConnectorsVersion(), manager().getContainer().getPluginsHash(),
213                manager().getContainer().getPluginsList());
214    }
215
216    private CompletionStage<Void> checkPlugins() {
217        boolean reload;
218        if ("timestamp".equals(configuration.getPluginsReloadMethod())) {
219            final long checked = readPluginsTimestamp();
220            reload = checked > latestPluginUpdate;
221            log
222                    .info("checkPlugins w/ timestamp {} vs {}. Reloading: {}.",
223                            Instant.ofEpochMilli(latestPluginUpdate), Instant.ofEpochMilli(checked), reload);
224            latestPluginUpdate = checked;
225        } else {
226            // connectors version used
227            final String cv = readConnectorsVersion();
228            reload = !cv.equals(getConnectors().getVersion());
229            log.info("checkPlugins w/ connectors {} vs {}. Reloading: {}.", connectors.getVersion(), cv, reload);
230        }
231        if (!reload) {
232            return null;
233        }
234        // undeploy plugins
235        log.info("Un-deploying plugins...");
236        manager().getContainer().findAll().forEach(container -> container.close());
237        // redeploy plugins
238        log.info("Re-deploying plugins...");
239        deployPlugins();
240        log.info("Plugins deployed.");
241        // reset connectors' version
242        synchronizeConnectors();
243
244        return null;
245    }
246
247    private synchronized String readConnectorsVersion() {
248        // check if we find a connectors version information file on top of the m2
249        final String version = Optional.of(m2.resolve("CONNECTORS_VERSION")).filter(Files::exists).map(p -> {
250            try {
251                return Files.lines(p).findFirst().get();
252            } catch (IOException e) {
253                log.warn("Failed reading connectors version {}", e.getMessage());
254                return "unknown";
255            }
256        }).orElse("unknown");
257        log.debug("Using connectors version: '{}'", version);
258
259        return version;
260    }
261
262    private synchronized Long readPluginsTimestamp() {
263        final Long last = Stream
264                .concat(Stream.of(configuration.getPluginsReloadFileMarker().orElse("")),
265                        configuration.getComponentRegistry().map(Collection::stream).orElse(Stream.empty()))
266                .filter(s -> !s.isEmpty())
267                .map(cr -> Paths.get(cr))
268                .filter(Files::exists)
269                .peek(f -> log.debug("[readPluginsTimestamp] getting {} timestamp.", f))
270                .map(f -> {
271                    try {
272                        return Files.getAttribute(f, "lastModifiedTime");
273                    } catch (IOException e) {
274                        return null;
275                    }
276                })
277                .filter(Objects::nonNull)
278                .findFirst()
279                .map(ts -> FileTime.class.cast(ts).toMillis())
280                .orElse(0L);
281        log.debug("[readPluginsTimestamp] Latest: {}.", Instant.ofEpochMilli(last));
282        return last;
283    }
284
285    private synchronized void deployPlugins() {
286        // note: we don't want to download anything from the manager, if we need to download any artifact we need
287        // to ensure it is controlled (secured) and allowed so don't make it implicit but enforce a first phase
288        // where it is cached locally (provisioning solution)
289        final List<String> coords = configuration
290                .getComponentCoordinates()
291                .map(it -> Stream.of(it.split(",")).map(String::trim).filter(i -> !i.isEmpty()).collect(toList()))
292                .orElse(emptyList());
293        coords.forEach(this::deploy);
294        configuration
295                .getComponentRegistry()
296                .map(Collection::stream)
297                .orElseGet(Stream::empty)
298                .flatMap(globService::toFiles)
299                .forEach(registry -> {
300                    final Properties properties = new Properties();
301                    try (final InputStream is = Files.newInputStream(registry)) {
302                        properties.load(is);
303                    } catch (final IOException e) {
304                        throw new IllegalArgumentException(e);
305                    }
306                    properties
307                            .stringPropertyNames()
308                            .stream()
309                            .map(properties::getProperty)
310                            .filter(gav -> !coords.contains(gav))
311                            .forEach(this::deploy);
312                });
313    }
314
315    public String deploy(final String pluginGAV) {
316        final String pluginPath = ofNullable(pluginGAV)
317                .map(gav -> mvnCoordinateToFileConverter.toArtifact(gav))
318                .map(Artifact::toPath)
319                .orElseThrow(() -> new IllegalArgumentException("Plugin GAV can't be empty"));
320
321        final String plugin =
322                instance.addWithLocationPlugin(pluginGAV, m2.resolve(pluginPath).toAbsolutePath().toString());
323        lastUpdated = new Date();
324        synchronizeConnectors();
325        if (started) {
326            deployedComponentEvent.fire(new DeployedComponent());
327        }
328        return plugin;
329    }
330
331    public synchronized void undeploy(final String pluginGAV) {
332        if (pluginGAV == null || pluginGAV.isEmpty()) {
333            throw new IllegalArgumentException("plugin maven GAV are required to undeploy a plugin");
334        }
335
336        final String pluginID = instance
337                .find(c -> pluginGAV.equals(c.get(ComponentManager.OriginalId.class).getValue()) ? Stream.of(c.getId())
338                        : empty())
339                .findFirst()
340                .orElseThrow(() -> new IllegalArgumentException("No plugin found using maven GAV: " + pluginGAV));
341
342        instance.removePlugin(pluginID);
343        lastUpdated = new Date();
344        synchronizeConnectors();
345    }
346
347    public Date findLastUpdated() {
348        return lastUpdated;
349    }
350
351    public Connectors getConnectors() {
352        return connectors;
353    }
354
355    @AllArgsConstructor
356    private static class DeploymentListener implements ContainerListener {
357
358        private final ComponentDao componentDao;
359
360        private final ComponentFamilyDao componentFamilyDao;
361
362        private final ComponentActionDao actionDao;
363
364        private final ConfigurationDao configurationDao;
365
366        private final VirtualDependenciesService virtualDependenciesService;
367
368        @Override
369        public void onCreate(final Container container) {
370            container.set(CleanupTask.class, new CleanupTask(postDeploy(container)));
371        }
372
373        @Override
374        public void onClose(final Container container) {
375            if (container.getState() == Container.State.ON_ERROR) {
376                // means it was not deployed so don't drop old state
377                return;
378            }
379            ofNullable(container.get(CleanupTask.class)).ifPresent(c -> c.getCleanup().run());
380        }
381
382        private Runnable postDeploy(final Container plugin) {
383            final Collection<String> componentIds = plugin
384                    .get(ContainerComponentRegistry.class)
385                    .getComponents()
386                    .values()
387                    .stream()
388                    .flatMap(c -> Stream
389                            .of(c.getPartitionMappers().values().stream(), c.getProcessors().values().stream(),
390                                    c.getDriverRunners().values().stream())
391                            .flatMap(t -> t))
392                    .peek(componentDao::createOrUpdate)
393                    .map(ComponentFamilyMeta.BaseMeta::getId)
394                    .collect(toSet());
395
396            final Collection<ComponentActionDao.ActionKey> actions = plugin
397                    .get(ContainerComponentRegistry.class)
398                    .getServices()
399                    .stream()
400                    .flatMap(c -> c.getActions().stream())
401                    .map(actionDao::createOrUpdate)
402                    .collect(toList());
403
404            final Collection<String> families = plugin
405                    .get(ContainerComponentRegistry.class)
406                    .getComponents()
407                    .values()
408                    .stream()
409                    .map(componentFamilyDao::createOrUpdate)
410                    .collect(toList());
411
412            final Collection<String> configs = ofNullable(plugin.get(RepositoryModel.class))
413                    .map(r -> r
414                            .getFamilies()
415                            .stream()
416                            .flatMap(f -> configAsStream(f.getConfigs().get().stream()))
417                            .collect(toList()))
418                    .orElse(emptyList())
419                    .stream()
420                    .map(configurationDao::createOrUpdate)
421                    .collect(toList());
422
423            return () -> {
424                virtualDependenciesService.onUnDeploy(plugin);
425                componentIds.forEach(componentDao::removeById);
426                actions.forEach(actionDao::removeById);
427                families.forEach(componentFamilyDao::removeById);
428                configs.forEach(configurationDao::removeById);
429            };
430        }
431
432        private Stream<Config> configAsStream(final Stream<Config> stream) {
433            return stream.flatMap(s -> Stream.concat(Stream.of(s), s.getChildConfigs().stream()));
434        }
435    }
436
437    @Data
438    private static class CleanupTask {
439
440        private final Runnable cleanup;
441    }
442
443    @Produces
444    public ComponentManager manager() {
445        return instance;
446    }
447
448}