Add project custom enforcement policy
Adds the ability of project or ref fine-grained policy
enforcement. This is typically needed whenever some of the projects
and refs requires different level of "consistency" across sites.
For example, the draft comments in All-Users are changed very often
across nodes and thus very likely to be misaligned. The loss of a draft
comment isn't nice, but is not so severe as the loss of a committed
code change.
Bug: Issue 10711
Change-Id: Ib477439b9c90af6e5c1719854e778c4b0aa6855e
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/Configuration.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/Configuration.java
index 3756097..72fec47 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/Configuration.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/Configuration.java
@@ -22,12 +22,16 @@
import com.google.common.base.CaseFormat;
import com.google.common.base.Strings;
import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.MultimapBuilder;
import com.google.gerrit.server.config.SitePaths;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.google.inject.spi.Message;
import com.googlesource.gerrit.plugins.multisite.forwarder.events.EventFamily;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefEnforcement.EnforcePolicy;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
@@ -517,6 +521,8 @@
public static final String KEY_MIGRATE = "migrate";
public final String TRANSACTION_LOCK_TIMEOUT_KEY = "transactionLockTimeoutMs";
+ public static final String SUBSECTION_ENFORCEMENT_RULES = "enforcementRules";
+
private final String connectionString;
private final String root;
private final int sessionTimeoutMs;
@@ -529,6 +535,8 @@
private final int casMaxRetries;
private final boolean enabled;
+ private final Multimap<EnforcePolicy, String> enforcementRules;
+
private final Long transactionLockTimeOut;
private CuratorFramework build;
@@ -600,7 +608,14 @@
checkArgument(StringUtils.isNotEmpty(connectionString), "zookeeper.%s contains no servers");
- enabled = Configuration.getBoolean(cfg, SECTION, SUBSECTION, ENABLE_KEY, true);
+ enabled = Configuration.getBoolean(cfg, SECTION, null, ENABLE_KEY, true);
+
+ enforcementRules = MultimapBuilder.hashKeys().arrayListValues().build();
+ for (EnforcePolicy policy : EnforcePolicy.values()) {
+ enforcementRules.putAll(
+ policy,
+ Configuration.getList(cfg, SECTION, SUBSECTION_ENFORCEMENT_RULES, policy.name()));
+ }
}
public CuratorFramework buildCurator() {
@@ -632,6 +647,15 @@
public boolean isEnabled() {
return enabled;
}
+
+ public Multimap<EnforcePolicy, String> getEnforcementRules() {
+ return enforcementRules;
+ }
+ }
+
+ static List<String> getList(
+ Supplier<Config> cfg, String section, String subsection, String name) {
+ return ImmutableList.copyOf(cfg.get().getStringList(section, subsection, name));
}
static boolean getBoolean(
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationModule.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationModule.java
index 8a652a9..7d8df66 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationModule.java
@@ -18,6 +18,7 @@
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.inject.Scopes;
import com.googlesource.gerrit.plugins.multisite.Configuration;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.CustomSharedRefEnforcementByProject;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.DefaultSharedRefEnforcement;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefEnforcement;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper.ZkValidationModule;
@@ -43,8 +44,14 @@
if (!disableGitRepositoryValidation) {
bind(GitRepositoryManager.class).to(MultiSiteGitRepositoryManager.class);
}
+ if (cfg.getZookeeperConfig().getEnforcementRules().isEmpty()) {
+ bind(SharedRefEnforcement.class).to(DefaultSharedRefEnforcement.class).in(Scopes.SINGLETON);
+ } else {
+ bind(SharedRefEnforcement.class)
+ .to(CustomSharedRefEnforcementByProject.class)
+ .in(Scopes.SINGLETON);
+ }
- bind(SharedRefEnforcement.class).to(DefaultSharedRefEnforcement.class).in(Scopes.SINGLETON);
install(new ZkValidationModule(cfg));
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/CustomSharedRefEnforcementByProject.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/CustomSharedRefEnforcementByProject.java
new file mode 100644
index 0000000..7a806a8
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/CustomSharedRefEnforcementByProject.java
@@ -0,0 +1,103 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb;
+
+import static com.google.common.base.Suppliers.memoize;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Splitter;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.multisite.Configuration;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+public class CustomSharedRefEnforcementByProject implements SharedRefEnforcement {
+ private static final String ALL = ".*";
+
+ private final Supplier<Map<String, Map<String, EnforcePolicy>>> predefEnforcements;
+
+ @Inject
+ public CustomSharedRefEnforcementByProject(Configuration config) {
+ this.predefEnforcements = memoize(() -> parseDryRunEnforcementsToMap(config));
+ }
+
+ private static Map<String, Map<String, EnforcePolicy>> parseDryRunEnforcementsToMap(
+ Configuration config) {
+ Map<String, Map<String, EnforcePolicy>> enforcementMap = new HashMap<>();
+
+ for (Map.Entry<EnforcePolicy, String> enforcementEntry :
+ config.getZookeeperConfig().getEnforcementRules().entries()) {
+ parseEnforcementEntry(enforcementMap, enforcementEntry);
+ }
+
+ return enforcementMap;
+ }
+
+ private static void parseEnforcementEntry(
+ Map<String, Map<String, EnforcePolicy>> enforcementMap,
+ Map.Entry<EnforcePolicy, String> enforcementEntry) {
+ Iterator<String> projectAndRef = Splitter.on(':').split(enforcementEntry.getValue()).iterator();
+ EnforcePolicy enforcementPolicy = enforcementEntry.getKey();
+
+ if (projectAndRef.hasNext()) {
+ String projectName = emptyToAll(projectAndRef.next());
+ String refName = emptyToAll(projectAndRef.hasNext() ? projectAndRef.next() : ALL);
+
+ Map<String, EnforcePolicy> existingOrDefaultRef =
+ enforcementMap.getOrDefault(projectName, new HashMap<>());
+
+ existingOrDefaultRef.put(refName, enforcementPolicy);
+
+ enforcementMap.put(projectName, existingOrDefaultRef);
+ }
+ }
+
+ private static String emptyToAll(String value) {
+ return value.trim().isEmpty() ? ALL : value;
+ }
+
+ @Override
+ public EnforcePolicy getPolicy(String projectName, String refName) {
+ if (isRefToBeIgnoredBySharedRefDb(refName)) {
+ return EnforcePolicy.IGNORED;
+ }
+
+ return getRefEnforcePolicy(projectName, refName);
+ }
+
+ private EnforcePolicy getRefEnforcePolicy(String projectName, String refName) {
+ Map<String, EnforcePolicy> orDefault =
+ predefEnforcements
+ .get()
+ .getOrDefault(
+ projectName, predefEnforcements.get().getOrDefault(ALL, ImmutableMap.of()));
+
+ return MoreObjects.firstNonNull(
+ orDefault.getOrDefault(refName, orDefault.get(ALL)), EnforcePolicy.REQUIRED);
+ }
+
+ @Override
+ public EnforcePolicy getPolicy(String projectName) {
+ Map<String, EnforcePolicy> policiesForProject =
+ predefEnforcements
+ .get()
+ .getOrDefault(
+ projectName, predefEnforcements.get().getOrDefault(ALL, ImmutableMap.of()));
+ return policiesForProject.getOrDefault(ALL, EnforcePolicy.REQUIRED);
+ }
+}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index e3abe59..be32c27 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -187,6 +187,34 @@
: Enable the use of a shared ref-database
Defaults: true
+```ref-database.enforcementRules.<policy>```
+: Level of consistency enforcement across sites on a project:refs basis.
+ Supports multiple values for enforcing the policy on multiple projects or refs.
+ If the project or ref is omitted, apply the policy to all projects or all refs.
+
+ The <policy> can be one of the following values:
+
+ 1. REQUIRED - Throw an exception if a git ref-update is processed again
+ a local ref not yet in sync with the shared ref-database.
+ The user transaction is cancelled. The Gerrit GUI (or the Git client)
+ receives an HTTP 500 - Internal Server Error.
+
+ 2. DESIRED - Validate the git ref-update against the shared ref-database.
+ Any misaligned is logged in errors_log file but the user operation is allowed
+ to continue successfully.
+
+ 3. IGNORED - Ignore any validation against the shared ref-database.
+
+ *Example:*
+ ```
+ [ref-database "enforcementRules"]
+ DESIRED = AProject:/refs/heads/feature
+ ```
+
+ Relax the alignment with the shared ref-database for AProject on refs/heads/feature.
+
+ Defaults: No rules = All projects are REQUIRED to be consistent on all refs.
+
```ref-database.zookeeper.connectString```
: Connection string to Zookeeper
@@ -230,14 +258,14 @@
operations on Zookeeper
Defaults: 1000
-
+
```ref-database.zookeeper.casRetryPolicyMaxSleepTimeMs```
: Configuration for the maximum sleep timeout in milliseconds of the
BoundedExponentialBackoffRetry policy used for the Compare and Swap
operations on Zookeeper
Defaults: 3000
-
+
```ref-database.zookeeper.casRetryPolicyMaxRetries```
: Configuration for the maximum number of retries of the
BoundedExponentialBackoffRetry policy used for the Compare and Swap
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/CustomSharedRefEnforcementByProjectTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/CustomSharedRefEnforcementByProjectTest.java
new file mode 100644
index 0000000..e4d7861
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/dfsrefdb/zookeeper/CustomSharedRefEnforcementByProjectTest.java
@@ -0,0 +1,159 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefDatabase.newRef;
+
+import com.googlesource.gerrit.plugins.multisite.Configuration;
+import com.googlesource.gerrit.plugins.multisite.Configuration.ZookeeperConfig;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.CustomSharedRefEnforcementByProject;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefEnforcement;
+import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefEnforcement.EnforcePolicy;
+import java.util.Arrays;
+import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.lib.Ref;
+import org.junit.Before;
+import org.junit.Test;
+
+public class CustomSharedRefEnforcementByProjectTest implements RefFixture {
+
+ SharedRefEnforcement refEnforcement;
+
+ @Before
+ public void setUp() {
+ Config multiSiteConfig = new Config();
+ multiSiteConfig.setStringList(
+ ZookeeperConfig.SECTION,
+ ZookeeperConfig.SUBSECTION_ENFORCEMENT_RULES,
+ EnforcePolicy.DESIRED.name(),
+ Arrays.asList(
+ "ProjectOne",
+ "ProjectTwo:refs/heads/master/test",
+ "ProjectTwo:refs/heads/master/test2"));
+ multiSiteConfig.setString(
+ ZookeeperConfig.SECTION,
+ ZookeeperConfig.SUBSECTION_ENFORCEMENT_RULES,
+ EnforcePolicy.IGNORED.name(),
+ ":refs/heads/master/test");
+
+ refEnforcement = newCustomRefEnforcement(multiSiteConfig);
+ }
+
+ @Test
+ public void projectOneShouldReturnDesiredForAllRefs() {
+ Ref aRef = newRef("refs/heads/master/2", AN_OBJECT_ID_1);
+ assertThat(refEnforcement.getPolicy("ProjectOne", aRef.getName()))
+ .isEqualTo(EnforcePolicy.DESIRED);
+ }
+
+ @Test
+ public void projectOneEnforcementShouldAlwaysPrevail() {
+ Ref aRef = newRef("refs/heads/master/test", AN_OBJECT_ID_1);
+ assertThat(refEnforcement.getPolicy("ProjectOne", aRef.getName()))
+ .isEqualTo(EnforcePolicy.DESIRED);
+ }
+
+ @Test
+ public void aNonListedProjectShouldIgnoreRefForMasterTest() {
+ Ref aRef = newRef("refs/heads/master/test", AN_OBJECT_ID_1);
+ assertThat(refEnforcement.getPolicy("NonListedProject", aRef.getName()))
+ .isEqualTo(EnforcePolicy.IGNORED);
+ }
+
+ @Test
+ public void projectTwoSpecificRefShouldReturnDesiredPolicy() {
+ Ref refOne = newRef("refs/heads/master/test", AN_OBJECT_ID_1);
+ Ref refTwo = newRef("refs/heads/master/test2", AN_OBJECT_ID_1);
+
+ assertThat(refEnforcement.getPolicy("ProjectTwo", refOne.getName()))
+ .isEqualTo(EnforcePolicy.DESIRED);
+ assertThat(refEnforcement.getPolicy("ProjectTwo", refTwo.getName()))
+ .isEqualTo(EnforcePolicy.DESIRED);
+ }
+
+ @Test
+ public void aNonListedProjectShouldReturnRequired() {
+ Ref refOne = newRef("refs/heads/master/newChange", AN_OBJECT_ID_1);
+ assertThat(refEnforcement.getPolicy("NonListedProject", refOne.getName()))
+ .isEqualTo(EnforcePolicy.REQUIRED);
+ }
+
+ @Test
+ public void aNonListedRefInProjectShouldReturnRequired() {
+ Ref refOne = newRef("refs/heads/master/test3", AN_OBJECT_ID_1);
+ assertThat(refEnforcement.getPolicy("ProjectTwo", refOne.getName()))
+ .isEqualTo(EnforcePolicy.REQUIRED);
+ }
+
+ @Test
+ public void aNonListedProjectAndRefShouldReturnRequired() {
+ Ref refOne = newRef("refs/heads/master/test3", AN_OBJECT_ID_1);
+ assertThat(refEnforcement.getPolicy("NonListedProject", refOne.getName()))
+ .isEqualTo(EnforcePolicy.REQUIRED);
+ }
+
+ @Test
+ public void getProjectPolicyForProjectOneShouldRetrunDesired() {
+ assertThat(refEnforcement.getPolicy("ProjectOne")).isEqualTo(EnforcePolicy.DESIRED);
+ }
+
+ @Test
+ public void getProjectPolicyForProjectTwoShouldReturnRequired() {
+ assertThat(refEnforcement.getPolicy("ProjectTwo")).isEqualTo(EnforcePolicy.REQUIRED);
+ }
+
+ @Test
+ public void getProjectPolicyForNonListedProjectShouldReturnRequired() {
+ assertThat(refEnforcement.getPolicy("NonListedProject")).isEqualTo(EnforcePolicy.REQUIRED);
+ }
+
+ @Test
+ public void getProjectPolicyForNonListedProjectWhenSingleProject() {
+ SharedRefEnforcement customEnforcement =
+ newCustomRefEnforcementWithValue(EnforcePolicy.DESIRED, ":refs/heads/master");
+
+ assertThat(customEnforcement.getPolicy("NonListedProject")).isEqualTo(EnforcePolicy.REQUIRED);
+ }
+
+ @Test
+ public void getANonListedProjectWhenOnlyOneProjectIsListedShouldReturnRequired() {
+ SharedRefEnforcement customEnforcement =
+ newCustomRefEnforcementWithValue(EnforcePolicy.DESIRED, "AProject:");
+ assertThat(customEnforcement.getPolicy("NonListedProject", "refs/heads/master"))
+ .isEqualTo(EnforcePolicy.REQUIRED);
+ }
+
+ private SharedRefEnforcement newCustomRefEnforcementWithValue(
+ EnforcePolicy policy, String... projectAndRefs) {
+ Config multiSiteConfig = new Config();
+ multiSiteConfig.setStringList(
+ ZookeeperConfig.SECTION,
+ ZookeeperConfig.SUBSECTION_ENFORCEMENT_RULES,
+ policy.name(),
+ Arrays.asList(projectAndRefs));
+ return newCustomRefEnforcement(multiSiteConfig);
+ }
+
+ private SharedRefEnforcement newCustomRefEnforcement(Config multiSiteConfig) {
+ return new CustomSharedRefEnforcementByProject(
+ new Configuration(multiSiteConfig, new Config()));
+ }
+
+ @Override
+ public String testBranch() {
+ return "fooBranch";
+ }
+}
