Merge branch 'stable-3.10' into stable-3.11 * stable-3.10: Add a remote config to exclude replicating desired refs Auto-format source code using gjf Change-Id: I0ba471d5649b5852c57e40244353210dbc330ecc
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java index 32c439b..0bf9026 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
@@ -46,9 +46,10 @@ new AtomicReference<>(newSecureCredentialsFactory(site, secureStore, config)); if (config.useLegacyCredentials()) { logger.atWarning().log( - "Using legacy credentials in clear text in secure.config. Please encrypt your credentials using " - + "'java -jar gerrit.war passwd' for each remote, remove the gerrit.useLegacyCredentials in replication.config " - + "and then reload the replication plugin."); + "Using legacy credentials in clear text in secure.config. Please encrypt your credentials" + + " using 'java -jar gerrit.war passwd' for each remote, remove the" + + " gerrit.useLegacyCredentials in replication.config and then reload the replication" + + " plugin."); } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java index 00f8baf..8cbb2a2 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java
@@ -51,8 +51,10 @@ } public boolean create() { - return destinations.getURIs(Optional.of(config.getName()), project, FilterType.PROJECT_CREATION) - .values().stream() + return destinations + .getURIs(Optional.of(config.getName()), project, FilterType.PROJECT_CREATION) + .values() + .stream() .map(u -> createProject(u, project, head)) .reduce(true, (a, b) -> a && b); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java index 3d7ae36..d535934 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
@@ -85,6 +85,7 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.function.Function; import java.util.function.Supplier; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Ref; @@ -890,6 +891,10 @@ return config.replicateNoteDbMetaRefs(); } + ImmutableList<Pattern> excludedRefsPattern() { + return config.excludedRefsPattern(); + } + private static boolean matches(URIish uri, String urlMatch) { if (urlMatch == null || urlMatch.equals("") || urlMatch.equals("*")) { return true;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java index 4d72ff0..41ab4c4 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java
@@ -20,8 +20,12 @@ import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; import com.google.gerrit.server.config.ConfigUtil; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.transport.RemoteConfig; @@ -51,6 +55,7 @@ private final int maxRetries; private final int slowLatencyThreshold; private final Supplier<Integer> pushBatchSize; + private final ImmutableList<Pattern> excludedRefsPattern; protected DestinationConfiguration(RemoteConfig remoteConfig, Config cfg) { this.remoteConfig = remoteConfig; @@ -116,6 +121,7 @@ } return 0; }); + excludedRefsPattern = getExcludedRefsPattern(cfg, name); } @Override @@ -215,4 +221,21 @@ public int getPushBatchSize() { return pushBatchSize.get(); } + + @Override + public ImmutableList<Pattern> excludedRefsPattern() { + return excludedRefsPattern; + } + + private ImmutableList<Pattern> getExcludedRefsPattern(Config cfg, String name) { + List<Pattern> patterns = new ArrayList<>(); + for (String regex : cfg.getStringList("remote", name, "excludedRefsPattern")) { + try { + patterns.add(Pattern.compile(regex)); + } catch (PatternSyntaxException e) { + repLog.atWarning().log("Invalid excludedRefsPattern '%s' is ignored", regex); + } + } + return ImmutableList.copyOf(patterns); + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/FanoutConfigResource.java b/src/main/java/com/googlesource/gerrit/plugins/replication/FanoutConfigResource.java index c634cce..b8392c7 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/FanoutConfigResource.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/FanoutConfigResource.java
@@ -112,7 +112,8 @@ Set<String> remoteNames = config.getSubsections("remote"); if (remoteNames.size() > 0) { logger.atSevere().log( - "When replication directory is present replication.config file cannot contain remote configuration. Ignoring: %s", + "When replication directory is present replication.config file cannot contain remote" + + " configuration. Ignoring: %s", String.join(",", remoteNames)); for (String name : remoteNames) { @@ -207,7 +208,8 @@ return hasher.hash().toString(); } catch (IOException e) { logger.atSevere().withCause(e).log( - "Cannot list remote configuration files from %s. Returning replication.config file version", + "Cannot list remote configuration files from %s. Returning replication.config file" + + " version", remoteConfigsDirPath); return parentVersion; }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java index c9b9994..9c4ad4d 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
@@ -751,7 +751,8 @@ private boolean canPushRef(String ref, boolean noPerms) { return !(noPerms && RefNames.REFS_CONFIG.equals(ref)) && !ref.startsWith(RefNames.REFS_CACHE_AUTOMERGE) - && !(!pool.replicateNoteDbMetaRefs() && RefNames.isNoteDbMetaRef(ref)); + && !(!pool.replicateNoteDbMetaRefs() && RefNames.isNoteDbMetaRef(ref)) + && pool.excludedRefsPattern().stream().noneMatch(p -> p.matcher(ref).matches()); } private Map<String, Ref> listRemote(Transport tn)
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteConfiguration.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteConfiguration.java index 726bcf5..307017b 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteConfiguration.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteConfiguration.java
@@ -14,6 +14,7 @@ package com.googlesource.gerrit.plugins.replication; import com.google.common.collect.ImmutableList; +import java.util.regex.Pattern; import org.eclipse.jgit.transport.RemoteConfig; /** Remote configuration for a replication endpoint */ @@ -25,6 +26,7 @@ * @return the delay value in seconds */ int getDelay(); + /** * Time to wait before rescheduling a remote replication operation, which might have failed the * first time round. Setting to 0 effectively disables the delay. @@ -32,6 +34,7 @@ * @return the delay value in seconds */ int getRescheduleDelay(); + /** * Time to wait before retrying a failed remote replication operation, Setting to 0 effectively * disables the delay. @@ -39,12 +42,14 @@ * @return the delay value in seconds */ int getRetryDelay(); + /** * List of the remote endpoint addresses used for replication. * * @return list of remote URL strings */ ImmutableList<String> getUrls(); + /** * List of alternative remote endpoint addresses, used for admin operations, such as repository * creation @@ -52,36 +57,42 @@ * @return list of remote URL strings */ ImmutableList<String> getAdminUrls(); + /** * List of repositories that should be replicated * * @return list of project strings */ ImmutableList<String> getProjects(); + /** * List of groups that should be used to access the repositories. * * @return list of group strings */ ImmutableList<String> getAuthGroupNames(); + /** * Influence how the name of the remote repository should be computed. * * @return a string representing a remote style name */ String getRemoteNameStyle(); + /** * If true, permissions-only projects and the refs/meta/config branch will also be replicated * * @return a string representing a remote style name */ boolean replicatePermissions(); + /** * the JGIT remote configuration representing the replication for this endpoint * * @return The remote config {@link RemoteConfig} */ RemoteConfig getRemoteConfig(); + /** * Number of times to retry a replication operation * @@ -132,4 +143,11 @@ } return ret; } + + /** + * List of patterns that will be used to exclude refs from being replicated + * + * @return list of successfully compiled patterns + */ + ImmutableList<Pattern> excludedRefsPattern(); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java index 40f5278..b60cc57 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
@@ -405,7 +405,8 @@ if (isMultiPrimary() && e instanceof NoSuchFileException) { logger.atFine().log( message - + " (expected after recovery from another node's startup with multi-primaries and distributor enabled)", + + " (expected after recovery from another node's startup with multi-primaries and" + + " distributor enabled)", taskKey); } else { logger.atSevere().withCause(e).log(message, taskKey);
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index bb4be9a..6cd0db1 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -604,6 +604,11 @@ By default, true. +remote.NAME.excludedRefsPattern +: Refs that match the pattern provided using this config will not be replicated. + This option is useful when admins want to skip replicating certain refs, for + example refs created by plugins. Multiple excludedRefsPattern keys can be + supplied, to specify multiple patterns to match against. Directory `replication` --------------------
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java index aa29846..3b07f49 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java
@@ -53,6 +53,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; import org.eclipse.jgit.errors.NotSupportedException; import org.eclipse.jgit.errors.RepositoryNotFoundException; import org.eclipse.jgit.errors.TransportException; @@ -75,6 +76,7 @@ import org.eclipse.jgit.util.FS; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -294,6 +296,40 @@ } @Test + public void skipPushingExcludedRefs() throws InterruptedException, IOException { + when(destinationMock.excludedRefsPattern()) + .thenReturn( + ImmutableList.of(Pattern.compile("refs/foo/.*"), Pattern.compile("refs/bar/.*"))); + PushOne pushOne = Mockito.spy(createPushOne(null)); + + Ref ref1 = + new ObjectIdRef.Unpeeled( + NEW, + "refs/heads/master", + ObjectId.fromString("0000000000000000000000000000000000000002")); + Ref ref2 = + new ObjectIdRef.Unpeeled( + NEW, "refs/foo/test", ObjectId.fromString("0000000000000000000000000000000000000003")); + Ref ref3 = + new ObjectIdRef.Unpeeled( + NEW, "refs/bar/test", ObjectId.fromString("0000000000000000000000000000000000000004")); + + localRefs.add(ref1); + localRefs.add(ref2); + localRefs.add(ref3); + + pushOne.addRefBatch(ImmutableSet.of(ref1.getName(), ref2.getName(), ref3.getName())); + pushOne.run(); + + isCallFinished.await(10, TimeUnit.SECONDS); + + ArgumentCaptor<Ref> refCaptor = ArgumentCaptor.forClass(Ref.class); + verify(transportMock, atLeastOnce()).push(any(), any()); + verify(pushOne, times(1)).push(any(), any(), refCaptor.capture()); + assertThat(refCaptor.getValue().getName()).isEqualTo("refs/heads/master"); + } + + @Test public void shouldNotAttemptDuplicateRemoteRefUpdate() throws InterruptedException, IOException { PushOne pushOne = Mockito.spy(createPushOne(null)); @@ -459,6 +495,7 @@ private void setupDestinationMock() { destinationMock = mock(Destination.class); when(destinationMock.requestRunway(any())).thenReturn(RunwayStatus.allowed()); + when(destinationMock.excludedRefsPattern()).thenReturn(ImmutableList.of()); } private void setupPermissionBackedMock() {
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/PushReplicationTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/PushReplicationTest.java index 122465b..736ad1a 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/PushReplicationTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/PushReplicationTest.java
@@ -45,7 +45,8 @@ assertThat(escape("name\nwith-LF")).isEqualTo("name%0Awith-LF"); assertThat( escape( - "key=a-value=1, --option1 \"OPTION_VALUE_1\" --option-2 <option_VALUE-2> --option-without-value")) + "key=a-value=1, --option1 \"OPTION_VALUE_1\" --option-2 <option_VALUE-2>" + + " --option-without-value")) .isEqualTo( "key=a-value=1,%20--option1%20%22OPTION_VALUE_1%22%20--option-2%20%3Coption_VALUE-2%3E%20--option-without-value"); } @@ -56,7 +57,8 @@ String url = urlBase + "/${name}.git"; URIish template = new URIish(url); String name = - "key=a-value=1, --option1 \"OPTION_VALUE_1\" --option-2 <option_VALUE-2> --option-without-value"; + "key=a-value=1, --option1 \"OPTION_VALUE_1\" --option-2 <option_VALUE-2>" + + " --option-without-value"; String expectedAsciiName = "key=a-value=1,%20--option1%20\"OPTION_VALUE_1\"%20--option-2%20<option_VALUE-2>%20--option-without-value"; String expectedEscapedName =
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorageTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorageTest.java index 6e02573..6d334fa 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorageTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorageTest.java
@@ -170,7 +170,8 @@ String url = urlBase + "/${name}.git"; URIish template = new URIish(url); String strangeValidName = - "project/with/a/strange/name key=a-value=1, --option1 \"OPTION_VALUE_1\" --option-2 <option_VALUE-2> --option-without-value"; + "project/with/a/strange/name key=a-value=1, --option1 \"OPTION_VALUE_1\" --option-2" + + " <option_VALUE-2> --option-without-value"; Project.NameKey project = Project.nameKey(strangeValidName); URIish expanded = Destination.getURI(template, project, "slash", false); ReplicateRefUpdate update =