Allow active read-write traffic to all nodes
Currently, multiple Gerrit servers in HA configuration with the
high-availability plugin could not serve read and writes concurrently,
because of the underlying NFS and JGit cache.
Using global-refdb allows to have a coordinator between different Gerrit
instances for updating refs without risking the repository corruption
and loss of commits, because of concurrent update of the same ref.
Use of Java 11 because Gerrit plugin API and global-refdb dependency
are available only with Java 11.
Feature: Issue 13429
Change-Id: Ia9633a6afdf6607a0795974a44c3ea39de68d328
diff --git a/BUILD b/BUILD
index 3b896ff..5975b31 100644
--- a/BUILD
+++ b/BUILD
@@ -19,7 +19,10 @@
"Implementation-URL: https://gerrit-review.googlesource.com/#/admin/projects/plugins/high-availability",
],
resources = glob(["src/main/resources/**/*"]),
- deps = ["@jgroups//jar"],
+ deps = [
+ "@jgroups//jar",
+ "@global-refdb//jar",
+ ],
)
junit_tests(
@@ -42,6 +45,7 @@
visibility = ["//visibility:public"],
exports = PLUGIN_DEPS + PLUGIN_TEST_DEPS + [
":high-availability__plugin",
+ "@global-refdb//jar",
"@wiremock//jar",
],
)
diff --git a/external_plugin_deps.bzl b/external_plugin_deps.bzl
index 0f8b43b..cb3f043 100644
--- a/external_plugin_deps.bzl
+++ b/external_plugin_deps.bzl
@@ -12,3 +12,9 @@
artifact = "org.jgroups:jgroups:3.6.15.Final",
sha1 = "755afcfc6c8a8ea1e15ef0073417c0b6e8c6d6e4",
)
+
+ maven_jar(
+ name = "global-refdb",
+ artifact = "com.gerritforge:global-refdb:3.3.0-rc1",
+ sha1 = "1b005b31c27a30ff10de97f903fa2834051bcadf",
+ )
diff --git a/src/main/java/com/ericsson/gerrit/plugins/highavailability/Configuration.java b/src/main/java/com/ericsson/gerrit/plugins/highavailability/Configuration.java
index 3bd58a3..9af312d 100644
--- a/src/main/java/com/ericsson/gerrit/plugins/highavailability/Configuration.java
+++ b/src/main/java/com/ericsson/gerrit/plugins/highavailability/Configuration.java
@@ -17,18 +17,19 @@
import static java.util.concurrent.TimeUnit.HOURS;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDbConfiguration;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.CharMatcher;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.common.Nullable;
-import com.google.gerrit.extensions.annotations.PluginName;
import com.google.gerrit.server.config.ConfigUtil;
-import com.google.gerrit.server.config.PluginConfigFactory;
import com.google.gerrit.server.config.SitePaths;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
@@ -39,7 +40,10 @@
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
+import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
@Singleton
public class Configuration {
@@ -47,6 +51,7 @@
public static final int DEFAULT_NUM_STRIPED_LOCKS = 10;
public static final int DEFAULT_TIMEOUT_MS = 5000;
+ public static final String PLUGIN_CONFIG_FILE = "high-availability.config";
// common parameter to peerInfo section
static final String PEER_INFO_SECTION = "peerInfo";
@@ -70,6 +75,7 @@
private PeerInfoStatic peerInfoStatic;
private PeerInfoJGroups peerInfoJGroups;
private HealthCheck healthCheck;
+ private final SharedRefDbConfiguration sharedRefDb;
public enum PeerInfoStrategy {
JGROUPS,
@@ -77,9 +83,12 @@
}
@Inject
- Configuration(
- PluginConfigFactory pluginConfigFactory, @PluginName String pluginName, SitePaths site) {
- Config cfg = pluginConfigFactory.getGlobalPluginConfig(pluginName);
+ Configuration(SitePaths sitePaths) {
+ this(getConfigFile(sitePaths, PLUGIN_CONFIG_FILE), sitePaths);
+ }
+
+ @VisibleForTesting
+ Configuration(Config cfg, SitePaths site) {
main = new Main(site, cfg);
autoReindex = new AutoReindex(cfg);
peerInfo = new PeerInfo(cfg);
@@ -100,6 +109,20 @@
index = new Index(cfg);
websession = new Websession(cfg);
healthCheck = new HealthCheck(cfg);
+ sharedRefDb = new SharedRefDbConfiguration(cfg);
+ }
+
+ private static FileBasedConfig getConfigFile(SitePaths sitePaths, String configFileName) {
+ FileBasedConfig cfg =
+ new FileBasedConfig(sitePaths.etc_dir.resolve(configFileName).toFile(), FS.DETECTED);
+ String fileConfigFileName = cfg.getFile().getPath();
+ try {
+ log.atInfo().log("Loading configuration from {}", fileConfigFileName);
+ cfg.load();
+ } catch (IOException | ConfigInvalidException e) {
+ log.atSevere().withCause(e).log("Unable to load configuration from " + fileConfigFileName);
+ }
+ return cfg;
}
public Main main() {
@@ -150,6 +173,10 @@
return healthCheck;
}
+ public SharedRefDbConfiguration sharedRefDb() {
+ return sharedRefDb;
+ }
+
private static int getInt(Config cfg, String section, String name, int defaultValue) {
try {
return cfg.getInt(section, name, defaultValue);
diff --git a/src/main/java/com/ericsson/gerrit/plugins/highavailability/Module.java b/src/main/java/com/ericsson/gerrit/plugins/highavailability/Module.java
index 744fac9..ddc294b 100644
--- a/src/main/java/com/ericsson/gerrit/plugins/highavailability/Module.java
+++ b/src/main/java/com/ericsson/gerrit/plugins/highavailability/Module.java
@@ -21,7 +21,7 @@
import com.ericsson.gerrit.plugins.highavailability.forwarder.rest.RestForwarderModule;
import com.ericsson.gerrit.plugins.highavailability.index.IndexModule;
import com.ericsson.gerrit.plugins.highavailability.peers.PeerInfoModule;
-import com.google.inject.AbstractModule;
+import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.inject.Inject;
import com.google.inject.Provides;
import com.google.inject.Singleton;
@@ -29,7 +29,7 @@
import java.nio.file.Files;
import java.nio.file.Path;
-class Module extends AbstractModule {
+class Module extends LifecycleModule {
private final Configuration config;
@Inject
@@ -55,6 +55,10 @@
install(new AutoReindexModule());
}
install(new PeerInfoModule(config.peerInfo().strategy()));
+
+ if (config.sharedRefDb().getSharedRefDb().isEnabled()) {
+ listener().to(PluginStartup.class);
+ }
}
@Provides
diff --git a/src/main/java/com/ericsson/gerrit/plugins/highavailability/PluginStartup.java b/src/main/java/com/ericsson/gerrit/plugins/highavailability/PluginStartup.java
new file mode 100644
index 0000000..5c19eec
--- /dev/null
+++ b/src/main/java/com/ericsson/gerrit/plugins/highavailability/PluginStartup.java
@@ -0,0 +1,45 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.ericsson.gerrit.plugins.highavailability;
+
+import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDatabaseWrapper;
+import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+
+/**
+ * Purpose of this class is to copy SharedRefDatabaseWrapper instance from DB injector to plugin
+ * injector. This allows to bind different global ref-db implementations.
+ */
+public class PluginStartup implements LifecycleListener {
+ private SharedRefDatabaseWrapper sharedRefDb;
+ private Injector injector;
+
+ @Inject
+ public PluginStartup(SharedRefDatabaseWrapper sharedRefDb, Injector injector) {
+ this.sharedRefDb = sharedRefDb;
+ this.injector = injector;
+ }
+
+ @Override
+ public void start() {
+ injector.injectMembers(sharedRefDb);
+ }
+
+ @Override
+ public void stop() {
+ // Nothing to do
+ }
+}
diff --git a/src/main/java/com/ericsson/gerrit/plugins/highavailability/ValidationModule.java b/src/main/java/com/ericsson/gerrit/plugins/highavailability/ValidationModule.java
new file mode 100644
index 0000000..74db7d2
--- /dev/null
+++ b/src/main/java/com/ericsson/gerrit/plugins/highavailability/ValidationModule.java
@@ -0,0 +1,70 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.ericsson.gerrit.plugins.highavailability;
+
+import com.gerritforge.gerrit.globalrefdb.validation.BatchRefUpdateValidator;
+import com.gerritforge.gerrit.globalrefdb.validation.LockWrapper;
+import com.gerritforge.gerrit.globalrefdb.validation.Log4jSharedRefLogger;
+import com.gerritforge.gerrit.globalrefdb.validation.RefUpdateValidator;
+import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDatabaseWrapper;
+import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDbBatchRefUpdate;
+import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDbConfiguration;
+import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDbGitRepositoryManager;
+import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDbRefDatabase;
+import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDbRefUpdate;
+import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDbRepository;
+import com.gerritforge.gerrit.globalrefdb.validation.SharedRefLogger;
+import com.gerritforge.gerrit.globalrefdb.validation.dfsrefdb.CustomSharedRefEnforcementByProject;
+import com.gerritforge.gerrit.globalrefdb.validation.dfsrefdb.DefaultSharedRefEnforcement;
+import com.gerritforge.gerrit.globalrefdb.validation.dfsrefdb.SharedRefEnforcement;
+import com.google.gerrit.extensions.config.FactoryModule;
+import com.google.gerrit.server.git.GitRepositoryManager;
+import com.google.inject.Inject;
+import com.google.inject.Scopes;
+
+public class ValidationModule extends FactoryModule {
+ final Configuration configuration;
+
+ @Inject
+ public ValidationModule(Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ @Override
+ protected void configure() {
+ factory(SharedRefDbRepository.Factory.class);
+ factory(SharedRefDbRefDatabase.Factory.class);
+ factory(SharedRefDbRefUpdate.Factory.class);
+ factory(SharedRefDbBatchRefUpdate.Factory.class);
+ factory(RefUpdateValidator.Factory.class);
+ factory(BatchRefUpdateValidator.Factory.class);
+
+ bind(SharedRefDbConfiguration.class).toInstance(configuration.sharedRefDb());
+
+ bind(SharedRefDatabaseWrapper.class).in(Scopes.SINGLETON);
+ bind(SharedRefLogger.class).to(Log4jSharedRefLogger.class);
+ factory(LockWrapper.Factory.class);
+
+ bind(GitRepositoryManager.class).to(SharedRefDbGitRepositoryManager.class);
+
+ if (configuration.sharedRefDb().getSharedRefDb().getEnforcementRules().isEmpty()) {
+ bind(SharedRefEnforcement.class).to(DefaultSharedRefEnforcement.class).in(Scopes.SINGLETON);
+ } else {
+ bind(SharedRefEnforcement.class)
+ .to(CustomSharedRefEnforcementByProject.class)
+ .in(Scopes.SINGLETON);
+ }
+ }
+}
diff --git a/src/main/resources/Documentation/about.md b/src/main/resources/Documentation/about.md
index fd0b56e..851f584 100644
--- a/src/main/resources/Documentation/about.md
+++ b/src/main/resources/Documentation/about.md
@@ -7,10 +7,10 @@
* sharing the git repositories using a shared file system (e.g. NFS)
* behind a load balancer (e.g. HAProxy)
-Currently, the mode supported is one active instance and multiple backup
-(passive) instances but eventually the plan is to support `n` active instances.
-In the active/passive mode, the active instance is handling all traffic while
-the passives are kept updated to be always ready to take over.
+Default mode is one active instance and multiple backup (passive) instances
+but `n` active instances can be configured. In the active/passive mode, the active
+instance is handling all traffic while the passives are kept updated to be always
+ready to take over.
Even if git repositories are shared by the instances, there are a few areas
of concern in order to be able to switch traffic between instances in a
@@ -46,7 +46,7 @@
The built-in Gerrit H2 based web session cache is replaced with a file based
implementation that is shared amongst the instances.
-## Setup
+## Active/passive setup
Prerequisites:
@@ -103,3 +103,72 @@
For further information and supported options, refer to [config](config.md)
documentation.
+
+## Active/active setup
+
+Prerequisites:
+
+* Git repositories must be located on a shared file system
+* A directory on a shared file system must be available for @PLUGIN@ to use
+* An implementation of global-refdb (e.g. Zookeeper) must be accessible from all the active
+instances
+
+For the instances:
+
+* Configure gerrit.basePath in gerrit.config to the shared repositories location
+* Configure gerrit.serverId in gerrit.config based on [config](config.md)'s introduction
+* Install and configure this @PLUGIN@ plugin [further](config.md) or based on example
+configuration
+* Install @PLUGIN@ plugin as a database module in $GERRIT_SITE/lib(please note that
+@PLUGIN plugin must be installed as a plugin and as a database module) and add
+`installDbModule = com.ericsson.gerrit.plugins.highavailability.ValidationModule`
+to the gerrit section in gerrit.config
+* Install [global-refdb library](https://mvnrepository.com/artifact/com.gerritforge/global-refdb) as a library module in $GERRIT_SITE/lib and add
+`installModule = com.gerritforge.gerrit.globalrefdb.validation.LibModule` to the gerrit
+section in gerrit.config
+* Install and configure [zookeeper-refdb plugin](https://gerrit-ci.gerritforge.com/view/Plugins-master/job/plugin-zookeeper-refdb-bazel-master) based on [config.md](https://gerrit.googlesource.com/plugins/zookeeper-refdb/+/refs/heads/master/src/main/resources/Documentation/config.md)
+* Configure ref-database.enabled = true in @PLUGIN@.config to enable validation with
+global-refdb.
+
+Here is an example of the minimal @PLUGIN@.config:
+
+Active instance one
+
+```
+[main]
+ sharedDirectory = /directory/accessible/from/both/instances
+
+[peerInfo "static"]
+ url = http://backupNodeHost1:8081/
+
+[http]
+ user = username
+ password = password
+
+[ref-database]
+ enabled = true
+```
+
+Active instance two
+
+```
+[main]
+ sharedDirectory = /directory/accessible/from/both/instances
+
+[peerInfo "static"]
+ url = http://primaryNodeHost:8080/
+
+[http]
+ user = username
+ password = password
+
+[ref-database]
+ enabled = true
+```
+
+Minimal zookeeper-refdb.config for both active instances:
+
+```
+[ref-database "zookeeper"]
+ connectString = zookeeperhost:2181
+```
\ No newline at end of file
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index cdb7b47..217a5b2 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -239,6 +239,32 @@
```healthcheck.enable```
: Whether to enable the health check endpoint. Defaults to 'true'.
+```ref-database.enabled```
+: Enable the use of a global ref-database. Defaults to 'false'.
+
+```ref-database.enforcementRules.<policy>```
+: Level of consistency enforcement across sites on a project:refs basis.
+ Supports two values for enforcing the policy on multiple projects or refs.
+ If the project or ref is omitted, apply the policy to all projects or all refs.
+
+ The <policy> can be one of the following values:
+
+ 1. REQUIRED - Throw an exception if a git ref-update is processed against
+ a local ref not yet in sync with the global ref-database.
+ The user transaction is cancelled. LOCK_FAILURE is reported upstream.
+
+ 2. IGNORED - Do not validate against the global ref-database.
+
+ *Example:*
+ ```
+ [ref-database "enforcementRules"]
+ IGNORED = AProject:/refs/heads/feature
+ ```
+
+ Ignore the alignment with the global ref-db for AProject on refs/heads/feature.
+
+ Defaults to no rule. All projects are REQUIRED to be consistent on all refs.
+
File 'gerrit.config'
--------------------
diff --git a/src/test/java/com/ericsson/gerrit/plugins/highavailability/ConfigurationTest.java b/src/test/java/com/ericsson/gerrit/plugins/highavailability/ConfigurationTest.java
index a9537bd..4bfac28 100644
--- a/src/test/java/com/ericsson/gerrit/plugins/highavailability/ConfigurationTest.java
+++ b/src/test/java/com/ericsson/gerrit/plugins/highavailability/ConfigurationTest.java
@@ -62,7 +62,6 @@
import static com.google.common.truth.Truth8.assertThat;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.when;
import com.ericsson.gerrit.plugins.highavailability.Configuration.PeerInfoStrategy;
import com.google.common.collect.ImmutableList;
@@ -104,12 +103,11 @@
@Before
public void setUp() throws IOException {
globalPluginConfig = new Config();
- when(pluginConfigFactoryMock.getGlobalPluginConfig(PLUGIN_NAME)).thenReturn(globalPluginConfig);
sitePaths = new SitePaths(SITE_PATH);
}
private Configuration getConfiguration() {
- return new Configuration(pluginConfigFactoryMock, PLUGIN_NAME, sitePaths);
+ return new Configuration(globalPluginConfig, sitePaths);
}
@Test
diff --git a/src/test/java/com/ericsson/gerrit/plugins/highavailability/index/AbstractIndexForwardingIT.java b/src/test/java/com/ericsson/gerrit/plugins/highavailability/index/AbstractIndexForwardingIT.java
index 5843908..83f86d0 100644
--- a/src/test/java/com/ericsson/gerrit/plugins/highavailability/index/AbstractIndexForwardingIT.java
+++ b/src/test/java/com/ericsson/gerrit/plugins/highavailability/index/AbstractIndexForwardingIT.java
@@ -25,15 +25,19 @@
import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options;
import static com.google.common.truth.Truth.assertThat;
+import com.ericsson.gerrit.plugins.highavailability.Configuration;
import com.github.tomakehurst.wiremock.junit.WireMockRule;
import com.google.gerrit.acceptance.LightweightPluginDaemonTest;
import com.google.gerrit.acceptance.NoHttpd;
import com.google.gerrit.acceptance.TestPlugin;
import com.google.gerrit.acceptance.UseLocalDisk;
-import com.google.gerrit.acceptance.config.GlobalPluginConfig;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.inject.Inject;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.apache.http.HttpStatus;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
@@ -50,17 +54,23 @@
@Rule public WireMockRule wireMockRule = new WireMockRule(options().port(PORT));
+ @Inject SitePaths sitePaths;
+
@Override
public void setUpTestPlugin() throws Exception {
givenThat(any(anyUrl()).willReturn(aResponse().withStatus(HttpStatus.SC_NO_CONTENT)));
+ FileBasedConfig fileBasedConfig =
+ new FileBasedConfig(
+ sitePaths.etc_dir.resolve(Configuration.PLUGIN_CONFIG_FILE).toFile(), FS.DETECTED);
+ fileBasedConfig.setString("peerInfo", "static", "url", URL);
+ fileBasedConfig.setInt("http", null, "retryInterval", 100);
+ fileBasedConfig.save();
beforeAction();
super.setUpTestPlugin();
}
@Test
@UseLocalDisk
- @GlobalPluginConfig(pluginName = "high-availability", name = "peerInfo.static.url", value = URL)
- @GlobalPluginConfig(pluginName = "high-availability", name = "http.retryInterval", value = "100")
public void testIndexForwarding() throws Exception {
String expectedRequest = getExpectedRequest();
CountDownLatch expectedRequestLatch = new CountDownLatch(1);
