Pre-loads and cache Jenkins CI plugins list Getting the list of plugins available on Jenkins CI may take quite a long time (over 10-20 sec). We do need to pre-load that information at plugin start-up and cache it in memory. The list of plugins is typically quite small and does not change often. Having a 24h TTL is more than acceptable. Change-Id: Iea38f70bdc68f1712a298270af7906a32ec9c9bd
diff --git a/BUCK b/BUCK index 8499ba4..51378d9 100644 --- a/BUCK +++ b/BUCK
@@ -8,6 +8,7 @@ 'Gerrit-PluginName: plugin-manager', 'Gerrit-ApiType: plugin', 'Gerrit-HttpModule: com.googlesource.gerrit.plugins.manager.WebModule', + 'Gerrit-Module: com.googlesource.gerrit.plugins.manager.Module', 'Implementation-Title: Plugin manager', 'Implementation-URL: https://gerrit-review.googlesource.com/#/admin/projects/plugins/plugin-manager', ],
diff --git a/src/main/java/com/googlesource/gerrit/plugins/manager/ListAvailablePlugins.java b/src/main/java/com/googlesource/gerrit/plugins/manager/ListAvailablePlugins.java index d45c16f..d3d8316 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/manager/ListAvailablePlugins.java +++ b/src/main/java/com/googlesource/gerrit/plugins/manager/ListAvailablePlugins.java
@@ -27,20 +27,20 @@ import com.googlesource.gerrit.plugins.manager.repository.PluginInfo; -import java.io.IOException; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; /** List plugins available for installation. */ @RequiresCapability(GlobalCapability.VIEW_PLUGINS) public class ListAvailablePlugins implements RestReadView<TopLevelResource> { - private final PluginsCentralLoader loader; + private final PluginsCentralCache pluginsCache; @Inject - public ListAvailablePlugins(PluginsCentralLoader loader) { - this.loader = loader; + public ListAvailablePlugins(PluginsCentralCache pluginsCache) { + this.pluginsCache = pluginsCache; } @Override @@ -52,8 +52,8 @@ Map<String, PluginInfo> output = Maps.newTreeMap(); List<PluginInfo> plugins; try { - plugins = loader.availablePlugins(); - } catch (IOException e) { + plugins = pluginsCache.availablePlugins(); + } catch (ExecutionException e) { throw new RestApiException( "Unable to load the list of available plugins", e); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/manager/Module.java b/src/main/java/com/googlesource/gerrit/plugins/manager/Module.java new file mode 100644 index 0000000..1c44dcd --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/manager/Module.java
@@ -0,0 +1,36 @@ +// Copyright (C) 2015 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.manager; + +import com.google.gerrit.extensions.events.LifecycleListener; +import com.google.inject.AbstractModule; +import com.google.inject.internal.UniqueAnnotations; + +import com.googlesource.gerrit.plugins.manager.repository.JenkinsCiPluginsRepository; +import com.googlesource.gerrit.plugins.manager.repository.PluginsRepository; + +public class Module extends AbstractModule { + + @Override + protected void configure() { + bind(PluginsRepository.class).to(JenkinsCiPluginsRepository.class); + + install(PluginsCentralCache.module()); + + bind(LifecycleListener.class).annotatedWith(UniqueAnnotations.create()).to( + OnStartStop.class); + + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/manager/OnStartStop.java b/src/main/java/com/googlesource/gerrit/plugins/manager/OnStartStop.java new file mode 100644 index 0000000..8156016 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/manager/OnStartStop.java
@@ -0,0 +1,73 @@ +// Copyright (C) 2015 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.manager; + +import com.google.gerrit.common.Version; +import com.google.gerrit.extensions.annotations.PluginName; +import com.google.gerrit.extensions.events.LifecycleListener; +import com.google.inject.Inject; + +import com.googlesource.gerrit.plugins.manager.repository.PluginInfo; +import com.googlesource.gerrit.plugins.manager.repository.PluginsRepository; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Collection; + +public class OnStartStop implements LifecycleListener { + private static final Logger log = LoggerFactory.getLogger(OnStartStop.class); + + private final PluginsRepository pluginsRepo; + + private final String pluginName; + + private final PluginManagerConfig config; + + @Inject + public OnStartStop(PluginsRepository pluginsRepo, + @PluginName String pluginName, PluginManagerConfig config) { + this.pluginsRepo = pluginsRepo; + this.pluginName = pluginName; + this.config = config; + } + + @Override + public void start() { + if (config.isCachePreloadEnabled()) { + Thread preloader = new Thread(new Runnable() { + + @Override + public void run() { + log.info("Start-up: pre-loading list of plugins from registry"); + try { + Collection<PluginInfo> plugins = + pluginsRepo.list(Version.getVersion()); + log.info("{} plugins successfully pre-loaded", plugins.size()); + } catch (IOException e) { + log.error("Cannot access plugins list at this time", e); + } + } + }); + preloader.setName(pluginName + "-preloader"); + preloader.start(); + } + } + + @Override + public void stop() { + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/manager/PluginManagerConfig.java b/src/main/java/com/googlesource/gerrit/plugins/manager/PluginManagerConfig.java new file mode 100644 index 0000000..b7dfb63 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/manager/PluginManagerConfig.java
@@ -0,0 +1,41 @@ +// Copyright (C) 2015 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.manager; + +import com.google.gerrit.extensions.annotations.PluginName; +import com.google.gerrit.server.config.PluginConfig; +import com.google.gerrit.server.config.PluginConfigFactory; +import com.google.inject.Inject; + +public class PluginManagerConfig { + private static final String DEFAULT_GERRIT_CI_URL = + "https://gerrit-ci.gerritforge.com"; + + private final PluginConfig config; + + @Inject + public PluginManagerConfig(PluginConfigFactory configFactory, + @PluginName String pluginName) { + this.config = configFactory.getFromGerritConfig(pluginName); + } + + public String getJenkinsUrl() { + return config.getString("jenkinsUrl", DEFAULT_GERRIT_CI_URL); + } + + public boolean isCachePreloadEnabled() { + return config.getBoolean("preload", true); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/manager/PluginsCentralCache.java b/src/main/java/com/googlesource/gerrit/plugins/manager/PluginsCentralCache.java new file mode 100644 index 0000000..56cb82f --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/manager/PluginsCentralCache.java
@@ -0,0 +1,57 @@ +// Copyright (C) 2015 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.manager; + +import com.google.common.cache.LoadingCache; +import com.google.gerrit.server.cache.CacheModule; +import com.google.inject.Inject; +import com.google.inject.TypeLiteral; +import com.google.inject.name.Named; + +import com.googlesource.gerrit.plugins.manager.PluginsCentralLoader.ListKey; +import com.googlesource.gerrit.plugins.manager.repository.PluginInfo; + +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +public class PluginsCentralCache { + + private final LoadingCache<ListKey, List<PluginInfo>> pluginsCache; + + public static final String PLUGINS_LIST_CACHE_NAME = "plugins_list"; + + @Inject + public PluginsCentralCache(@Named(PLUGINS_LIST_CACHE_NAME) LoadingCache<ListKey, List<PluginInfo>> pluginsCache) { + this.pluginsCache = pluginsCache; + } + + public List<PluginInfo> availablePlugins() throws ExecutionException { + return pluginsCache.get(ListKey.ALL); + } + + static CacheModule module() { + return new CacheModule() { + @Override + protected void configure() { + cache(PluginsCentralCache.PLUGINS_LIST_CACHE_NAME, ListKey.class, + new TypeLiteral<List<PluginInfo>>() {}).expireAfterWrite(1, + TimeUnit.DAYS).loader(PluginsCentralLoader.class); + + bind(PluginsCentralCache.class); + } + }; + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/manager/PluginsCentralLoader.java b/src/main/java/com/googlesource/gerrit/plugins/manager/PluginsCentralLoader.java index 33943af..d0385c9 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/manager/PluginsCentralLoader.java +++ b/src/main/java/com/googlesource/gerrit/plugins/manager/PluginsCentralLoader.java
@@ -14,18 +14,26 @@ package com.googlesource.gerrit.plugins.manager; +import com.google.common.cache.CacheLoader; import com.google.gerrit.common.Version; import com.google.inject.Inject; import com.google.inject.Singleton; +import com.googlesource.gerrit.plugins.manager.PluginsCentralLoader.ListKey; import com.googlesource.gerrit.plugins.manager.repository.PluginInfo; import com.googlesource.gerrit.plugins.manager.repository.PluginsRepository; -import java.io.IOException; import java.util.List; @Singleton -public class PluginsCentralLoader { +public class PluginsCentralLoader extends + CacheLoader<ListKey, List<PluginInfo>> { + + public static class ListKey { + static final ListKey ALL = new ListKey(); + + private ListKey() {} + } private final PluginsRepository repository; @@ -34,7 +42,8 @@ this.repository = repository; } - public List<PluginInfo> availablePlugins() throws IOException { + @Override + public List<PluginInfo> load(ListKey all) throws Exception { return repository.list(Version.getVersion()); } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/manager/WebModule.java b/src/main/java/com/googlesource/gerrit/plugins/manager/WebModule.java index c9994f1..a80ee9a 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/manager/WebModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/manager/WebModule.java
@@ -15,15 +15,11 @@ import com.google.inject.servlet.ServletModule; -import com.googlesource.gerrit.plugins.manager.repository.JenkinsCiPluginsRepository; -import com.googlesource.gerrit.plugins.manager.repository.PluginsRepository; - public class WebModule extends ServletModule { @Override protected void configureServlets() { bind(AvailablePluginsCollection.class); - bind(PluginsRepository.class).to(JenkinsCiPluginsRepository.class); serve("/available*").with(PluginManagerRestApiServlet.class);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/manager/repository/JenkinsCiPluginsRepository.java b/src/main/java/com/googlesource/gerrit/plugins/manager/repository/JenkinsCiPluginsRepository.java index d1620b3..6148407 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/manager/repository/JenkinsCiPluginsRepository.java +++ b/src/main/java/com/googlesource/gerrit/plugins/manager/repository/JenkinsCiPluginsRepository.java
@@ -16,9 +16,6 @@ import com.google.common.base.Function; import com.google.common.base.Optional; -import com.google.gerrit.extensions.annotations.PluginName; -import com.google.gerrit.server.config.PluginConfig; -import com.google.gerrit.server.config.PluginConfigFactory; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.inject.Inject; @@ -26,6 +23,7 @@ import com.google.inject.Singleton; import com.googlesource.gerrit.plugins.manager.GerritVersionBranch; +import com.googlesource.gerrit.plugins.manager.PluginManagerConfig; import com.googlesource.gerrit.plugins.manager.gson.SmartGson; import com.googlesource.gerrit.plugins.manager.gson.SmartJson; @@ -35,6 +33,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; @Singleton @@ -43,12 +42,11 @@ private static final Logger log = LoggerFactory .getLogger(JenkinsCiPluginsRepository.class); - private static final String DEFAULT_GERRIT_CI_URL = - "https://gerrit-ci.gerritforge.com"; - private static final Optional<PluginInfo> noPluginInfo = Optional.absent(); - private final PluginConfig config; + private final PluginManagerConfig config; + + private HashMap<String, List<PluginInfo>> cache = new HashMap<>(); static class View { String name; @@ -65,20 +63,29 @@ @Inject public JenkinsCiPluginsRepository(Provider<SmartGson> gsonProvider, - PluginConfigFactory configFactory, @PluginName String pluginName) { + PluginManagerConfig config) { this.gsonProvider = gsonProvider; - this.config = configFactory.getFromGerritConfig(pluginName); + this.config = config; } @Override public List<PluginInfo> list(String gerritVersion) throws IOException { + List<PluginInfo> list = cache.get(gerritVersion); + if(list == null) { + list = getList(gerritVersion); + cache.put(gerritVersion, list); + } + return list; + } + + private List<PluginInfo> getList(String gerritVersion) throws IOException { SmartGson gson = gsonProvider.get(); String viewName = "Plugins-" + GerritVersionBranch.getBranch(gerritVersion); List<PluginInfo> plugins = new ArrayList<>(); try { Job[] jobs = - gson.get(getJenkinsUrl() + "/view/" + viewName + "/api/json", + gson.get(config.getJenkinsUrl() + "/view/" + viewName + "/api/json", View.class).jobs; for (Job job : jobs) { @@ -96,10 +103,6 @@ return plugins; } - private String getJenkinsUrl() { - return config.getString("jenkinsUrl", DEFAULT_GERRIT_CI_URL); - } - private Optional<PluginInfo> getPluginInfo(final SmartGson gson, String url) throws IOException { SmartJson jobDetails = gson.get(url + "/api/json");