Merge branch 'stable-3.4' into stable-3.5
* stable-3.4:
Make sure project is indexed after creation
Allow creation of new projects
Introduce concept of permanent failure
Document need for Access Database capability
Do not try to delete inexistent refs
Make variable used by constructor final
Stream events listener should respect excluded refs param
Stream events listener should respect `remote.NAME.projects` param
Add ref deletion functionality to the StreamEventsListener
Add guard clauses to improve readability
Additionally removed superfluous mocking in `DeleteRefCommandTest`
as imposed by ErrorProne.
Change-Id: If3daf65f9e10a1583cd163bcf7f1fa0515227411
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java
index 551b36c..43db907 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java
@@ -35,6 +35,7 @@
import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics;
import com.googlesource.gerrit.plugins.replication.pull.fetch.Fetch;
import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.PermanentTransportException;
import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState;
import java.io.IOException;
import java.util.Collection;
@@ -337,8 +338,10 @@
"Cannot replicate [{}] {}; Remote repository error: {}", taskIdHex, projectName, msg);
} catch (NotSupportedException e) {
stateLog.error("Cannot replicate from " + uri, e, getStatesAsArray());
+ } catch (PermanentTransportException e) {
+ repLog.error(
+ String.format("Terminal failure. Cannot replicate [%s] from %s", taskIdHex, uri), e);
} catch (TransportException e) {
- Throwable cause = e.getCause();
if (e instanceof LockFailureException) {
lockRetryCount++;
// The LockFailureException message contains both URI and reason
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
index 3170eb5..3d9f5c3 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
@@ -623,6 +623,10 @@
return configSettingsAllowReplication(project);
}
+ public boolean wouldCreateProject(Project.NameKey project) {
+ return configSettingsAllowReplication(project);
+ }
+
private boolean configSettingsAllowReplication(Project.NameKey project) {
// by default fetch all projects
List<String> projects = config.getProjects();
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java
index d4b7819..f897012 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java
@@ -40,6 +40,7 @@
import java.io.IOException;
import java.util.Optional;
import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.RefUpdate;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.URIish;
@@ -82,6 +83,12 @@
throw new ResourceNotFoundException(String.format("Project %s was not found", name));
}
+ Optional<Ref> ref = getRef(name, refName);
+ if (!ref.isPresent()) {
+ logger.atFine().log("Ref %s was not found in project %s", refName, name);
+ return;
+ }
+
Source source =
sourcesCollection
.getByRemoteName(sourceLabel)
@@ -94,7 +101,7 @@
try {
Context.setLocalEvent(true);
- deleteRef(name, refName);
+ deleteRef(name, ref.get());
eventDispatcher
.get()
@@ -138,17 +145,24 @@
}
}
- private RefUpdateState deleteRef(Project.NameKey name, String refName) throws IOException {
+ private Optional<Ref> getRef(Project.NameKey repo, String refName) throws IOException {
+ try (Repository repository = gitManager.openRepository(repo)) {
+ Ref ref = repository.exactRef(refName);
+ return Optional.ofNullable(ref);
+ }
+ }
+ private RefUpdateState deleteRef(Project.NameKey name, Ref ref) throws IOException {
try (Repository repository = gitManager.openRepository(name)) {
+
RefUpdate.Result result;
- RefUpdate u = repository.updateRef(refName);
- u.setExpectedOldObjectId(repository.exactRef(refName).getObjectId());
+ RefUpdate u = repository.updateRef(ref.getName());
+ u.setExpectedOldObjectId(ref.getObjectId());
u.setNewObjectId(ObjectId.zeroId());
u.setForceUpdate(true);
result = u.delete();
- return new RefUpdateState(refName, result);
+ return new RefUpdateState(ref.getName(), result);
}
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java
index 2214fb3..8711379 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java
@@ -22,6 +22,7 @@
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.restapi.AuthException;
import com.google.gerrit.extensions.restapi.Url;
+import com.google.gerrit.index.project.ProjectIndexer;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.permissions.GlobalPermission;
import com.google.gerrit.server.permissions.PermissionBackend;
@@ -49,15 +50,18 @@
private final GerritConfigOps gerritConfigOps;
private final Provider<CurrentUser> userProvider;
private final PermissionBackend permissionBackend;
+ private final ProjectIndexer projectIndexer;
@Inject
ProjectInitializationAction(
GerritConfigOps gerritConfigOps,
Provider<CurrentUser> userProvider,
- PermissionBackend permissionBackend) {
+ PermissionBackend permissionBackend,
+ ProjectIndexer projectIndexer) {
this.gerritConfigOps = gerritConfigOps;
this.userProvider = userProvider;
this.permissionBackend = permissionBackend;
+ this.projectIndexer = projectIndexer;
}
@Override
@@ -106,6 +110,10 @@
}
LocalFS localFS = new LocalFS(maybeUri.get());
Project.NameKey projectNameKey = Project.NameKey.parse(projectName);
- return localFS.createProject(projectNameKey, RefNames.HEAD);
+ if (localFS.createProject(projectNameKey, RefNames.HEAD)) {
+ projectIndexer.index(projectNameKey);
+ return true;
+ }
+ return false;
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
index c1ffa44..2ea8a33 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
@@ -22,45 +22,62 @@
import com.google.gerrit.entities.Project.NameKey;
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.restapi.AuthException;
+import com.google.gerrit.extensions.restapi.RestApiException;
import com.google.gerrit.server.config.GerritInstanceId;
import com.google.gerrit.server.events.Event;
import com.google.gerrit.server.events.EventListener;
import com.google.gerrit.server.events.ProjectCreatedEvent;
+import com.google.gerrit.server.events.ProjectEvent;
import com.google.gerrit.server.events.RefUpdatedEvent;
import com.google.gerrit.server.git.WorkQueue;
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.googlesource.gerrit.plugins.replication.pull.FetchOne;
+import com.googlesource.gerrit.plugins.replication.pull.Source;
+import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection;
+import com.googlesource.gerrit.plugins.replication.pull.api.DeleteRefCommand;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchJob;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchJob.Factory;
import com.googlesource.gerrit.plugins.replication.pull.api.ProjectInitializationAction;
import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics;
+import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter;
+import java.io.IOException;
+import java.util.Optional;
import org.eclipse.jgit.lib.ObjectId;
public class StreamEventListener implements EventListener {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+ private static final String ZERO_ID_NAME = ObjectId.zeroId().name();
- private String instanceId;
- private WorkQueue workQueue;
- private ProjectInitializationAction projectInitializationAction;
-
- private Factory fetchJobFactory;
+ private final DeleteRefCommand deleteCommand;
+ private final ExcludedRefsFilter refsFilter;
+ private final Factory fetchJobFactory;
+ private final ProjectInitializationAction projectInitializationAction;
private final Provider<PullReplicationApiRequestMetrics> metricsProvider;
+ private final SourcesCollection sources;
+ private final String instanceId;
+ private final WorkQueue workQueue;
@Inject
public StreamEventListener(
@Nullable @GerritInstanceId String instanceId,
+ DeleteRefCommand deleteCommand,
ProjectInitializationAction projectInitializationAction,
WorkQueue workQueue,
FetchJob.Factory fetchJobFactory,
- Provider<PullReplicationApiRequestMetrics> metricsProvider) {
+ Provider<PullReplicationApiRequestMetrics> metricsProvider,
+ SourcesCollection sources,
+ ExcludedRefsFilter excludedRefsFilter) {
this.instanceId = instanceId;
+ this.deleteCommand = deleteCommand;
this.projectInitializationAction = projectInitializationAction;
this.workQueue = workQueue;
this.fetchJobFactory = fetchJobFactory;
this.metricsProvider = metricsProvider;
+ this.sources = sources;
+ this.refsFilter = excludedRefsFilter;
requireNonNull(
Strings.emptyToNull(this.instanceId), "gerrit.instanceId cannot be null or empty");
@@ -79,40 +96,101 @@
}
public void fetchRefsForEvent(Event event) throws AuthException, PermissionBackendException {
- if (!instanceId.equals(event.instanceId)) {
- PullReplicationApiRequestMetrics metrics = metricsProvider.get();
- metrics.start(event);
- if (event instanceof RefUpdatedEvent) {
- RefUpdatedEvent refUpdatedEvent = (RefUpdatedEvent) event;
- if (!isProjectDelete(refUpdatedEvent)) {
- fetchRefsAsync(
- refUpdatedEvent.getRefName(),
- refUpdatedEvent.instanceId,
- refUpdatedEvent.getProjectNameKey(),
- metrics);
- }
+ if (instanceId.equals(event.instanceId) || !shouldReplicateProject(event)) {
+ return;
+ }
+
+ PullReplicationApiRequestMetrics metrics = metricsProvider.get();
+ metrics.start(event);
+ if (event instanceof RefUpdatedEvent) {
+ RefUpdatedEvent refUpdatedEvent = (RefUpdatedEvent) event;
+ if (!isRefToBeReplicated(refUpdatedEvent.getRefName())) {
+ logger.atFine().log(
+ "Skipping excluded ref '%s' for project '%s'",
+ refUpdatedEvent.getRefName(), refUpdatedEvent.getProjectNameKey());
+ return;
}
- if (event instanceof ProjectCreatedEvent) {
- ProjectCreatedEvent projectCreatedEvent = (ProjectCreatedEvent) event;
- try {
- projectInitializationAction.initProject(getProjectRepositoryName(projectCreatedEvent));
- fetchRefsAsync(
- FetchOne.ALL_REFS,
- projectCreatedEvent.instanceId,
- projectCreatedEvent.getProjectNameKey(),
- metrics);
- } catch (AuthException | PermissionBackendException e) {
- logger.atSevere().withCause(e).log(
- "Cannot initialise project:%s", projectCreatedEvent.projectName);
- throw e;
- }
+
+ if (isProjectDelete(refUpdatedEvent)) {
+ return;
+ }
+
+ if (isRefDelete(refUpdatedEvent)) {
+ deleteRef(refUpdatedEvent);
+ return;
+ }
+
+ fetchRefsAsync(
+ refUpdatedEvent.getRefName(),
+ refUpdatedEvent.instanceId,
+ refUpdatedEvent.getProjectNameKey(),
+ metrics);
+ } else if (event instanceof ProjectCreatedEvent) {
+ ProjectCreatedEvent projectCreatedEvent = (ProjectCreatedEvent) event;
+ try {
+ projectInitializationAction.initProject(getProjectRepositoryName(projectCreatedEvent));
+ fetchRefsAsync(
+ FetchOne.ALL_REFS,
+ projectCreatedEvent.instanceId,
+ projectCreatedEvent.getProjectNameKey(),
+ metrics);
+ } catch (AuthException | PermissionBackendException e) {
+ logger.atSevere().withCause(e).log(
+ "Cannot initialise project:%s", projectCreatedEvent.projectName);
+ throw e;
}
}
}
+ private void deleteRef(RefUpdatedEvent refUpdatedEvent) {
+ try {
+ deleteCommand.deleteRef(
+ refUpdatedEvent.getProjectNameKey(),
+ refUpdatedEvent.getRefName(),
+ refUpdatedEvent.instanceId);
+ } catch (IOException | RestApiException e) {
+ logger.atSevere().withCause(e).log(
+ "Cannot delete ref %s project:%s",
+ refUpdatedEvent.getRefName(), refUpdatedEvent.getProjectNameKey());
+ }
+ }
+
+ private boolean isRefToBeReplicated(String refName) {
+ return !refsFilter.match(refName);
+ }
+
+ private boolean shouldReplicateProject(Event event) {
+ if (!(event instanceof ProjectEvent)) {
+ return false;
+ }
+
+ Optional<Source> maybeSource =
+ sources.getAll().stream()
+ .filter(s -> s.getRemoteConfigName().equals(event.instanceId))
+ .findFirst();
+
+ if (!maybeSource.isPresent()) {
+ return false;
+ }
+
+ Source source = maybeSource.get();
+ if (event instanceof ProjectCreatedEvent) {
+ ProjectCreatedEvent projectCreatedEvent = (ProjectCreatedEvent) event;
+
+ return source.isCreateMissingRepositories()
+ && source.wouldCreateProject(projectCreatedEvent.getProjectNameKey());
+ }
+
+ ProjectEvent projectEvent = (ProjectEvent) event;
+ return source.wouldFetchProject(projectEvent.getProjectNameKey());
+ }
+
+ private boolean isRefDelete(RefUpdatedEvent event) {
+ return ZERO_ID_NAME.equals(event.refUpdate.get().newRev);
+ }
+
private boolean isProjectDelete(RefUpdatedEvent event) {
- return RefNames.isConfigRef(event.getRefName())
- && ObjectId.zeroId().equals(ObjectId.fromString(event.refUpdate.get().newRev));
+ return RefNames.isConfigRef(event.getRefName()) && isRefDelete(event);
}
protected void fetchRefsAsync(
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java
index 89972cf..74ad9fc 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java
@@ -21,6 +21,7 @@
import java.io.IOException;
import java.util.List;
import java.util.stream.Collectors;
+import org.eclipse.jgit.errors.TransportException;
import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.*;
@@ -51,6 +52,13 @@
private FetchResult fetchVia(Transport tn, List<RefSpec> fetchRefSpecs) throws IOException {
repLog.info("Fetch references {} from {}", fetchRefSpecs, uri);
- return tn.fetch(NullProgressMonitor.INSTANCE, fetchRefSpecs);
+ try {
+ return tn.fetch(NullProgressMonitor.INSTANCE, fetchRefSpecs);
+ } catch (TransportException e) {
+ if (PermanentTransportException.isPermanentFailure(e)) {
+ throw new PermanentTransportException("Terminal fetch failure", e);
+ }
+ throw e;
+ }
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/PermanentTransportException.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/PermanentTransportException.java
new file mode 100644
index 0000000..1d96a02
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/PermanentTransportException.java
@@ -0,0 +1,34 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.fetch;
+
+import com.jcraft.jsch.JSchException;
+import org.eclipse.jgit.errors.TransportException;
+import org.eclipse.jgit.internal.JGitText;
+
+public class PermanentTransportException extends TransportException {
+ private static final long serialVersionUID = 1L;
+
+ public PermanentTransportException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+
+ public static boolean isPermanentFailure(TransportException e) {
+ Throwable cause = e.getCause();
+ String message = e.getMessage();
+ return (cause instanceof JSchException && cause.getMessage().startsWith("UnknownHostKey:"))
+ || message.matches(JGitText.get().remoteDoesNotHaveSpec.replaceAll("\\{0\\}", ".+"));
+ }
+}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 51b10d8..b76585e 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -119,6 +119,18 @@
By default, fetches are retried indefinitely.
+ Note that only transient errors will be retried, whilst persistent errors will
+ cause a terminal failure, and the fetch will not be scheduled again. This is
+ only supported for JGit, not cGit. Currently, only the following failures are
+ considered permanent:
+
+ - UnknownHostKey: thrown by Jsch when establishing an SSH connection for an
+ unknown host.
+ - Jgit transport exception when the remote ref does not exist. The assumption
+ here is that the remote ref does not exist so it is not worth retrying. If the
+ exception arisen as a consequence of some ACLs (mis)configuration, then after
+ fixing the ACLs, an explicit replication must be manually triggered.
+
replication.instanceLabel
: Remote configuration name of the current server.
This label is passed as a part of the payload to notify other
@@ -380,6 +392,9 @@
By default, use replication.maxRetries.
+ Note that not all fetch failures are retriable. Please refer
+ to `replication.maxRetries` for more information on this.
+
remote.NAME.threads
: Number of worker threads to dedicate to fetching to the
repositories described by this remote. Each thread can fetch
@@ -558,6 +573,14 @@
remote.NAME.password
: Password to use for HTTP authentication on this remote.
+In both cases, the Global Capability `Access Database` [1] needs to be allowed
+in order to permit all `All-Users`' refs to be replicated. When _basic auth_ is
+used, the capability must be assigned to the `remote.NAME.username` used in
+configuration, whilst for _bearer token_, it needs to be assigned to
+the `Pull-replication Internal User` user.
+
+[1] https://gerrit-review.googlesource.com/Documentation/access-control.html#capability_accessDatabase
+
File `~/.ssh/config`
--------------------
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java
index 406cd8c..3f40848 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/CGitFetchIT.java
@@ -22,18 +22,12 @@
import static org.mockito.Mockito.when;
import com.google.common.collect.Lists;
-import com.google.common.flogger.FluentLogger;
-import com.google.gerrit.acceptance.LightweightPluginDaemonTest;
import com.google.gerrit.acceptance.PushOneCommit.Result;
import com.google.gerrit.acceptance.SkipProjectClone;
import com.google.gerrit.acceptance.TestPlugin;
import com.google.gerrit.acceptance.UseLocalDisk;
-import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
-import com.google.gerrit.entities.Project;
import com.google.gerrit.extensions.api.projects.BranchInput;
import com.google.gerrit.extensions.config.FactoryModule;
-import com.google.gerrit.server.config.SitePaths;
-import com.google.inject.Inject;
import com.google.inject.Scopes;
import com.google.inject.assistedinject.FactoryModuleBuilder;
import com.googlesource.gerrit.plugins.replication.AutoReloadSecureCredentialsFactoryDecorator;
@@ -46,12 +40,8 @@
import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchClientImplementation;
import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory;
import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState;
-import java.io.IOException;
import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.time.Duration;
import java.util.List;
-import java.util.function.Supplier;
import org.eclipse.jgit.errors.TransportException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.Ref;
@@ -68,27 +58,8 @@
@TestPlugin(
name = "pull-replication",
sysModule = "com.googlesource.gerrit.plugins.replication.pull.CGitFetchIT$TestModule")
-public class CGitFetchIT extends LightweightPluginDaemonTest {
+public class CGitFetchIT extends FetchITBase {
private static final String TEST_REPLICATION_SUFFIX = "suffix1";
- private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
- private static final int TEST_REPLICATION_DELAY = 60;
- private static final Duration TEST_TIMEOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 2);
-
- @Inject private SitePaths sitePaths;
- @Inject private ProjectOperations projectOperations;
- private FetchFactory fetchFactory;
- private Path gitPath;
- private Path testRepoPath;
-
- @Override
- public void setUpTestPlugin() throws Exception {
- gitPath = sitePaths.site_path.resolve("git");
- testRepoPath = gitPath.resolve(project + TEST_REPLICATION_SUFFIX + ".git");
-
- super.setUpTestPlugin();
- fetchFactory = plugin.getSysInjector().getInstance(FetchFactory.class);
- }
@Test
public void shouldFetchRef() throws Exception {
@@ -245,27 +216,6 @@
}
}
- private void waitUntil(Supplier<Boolean> waitCondition) throws InterruptedException {
- WaitUtil.waitUntil(waitCondition, TEST_TIMEOUT);
- }
-
- private Ref getRef(Repository repo, String branchName) throws IOException {
- return repo.getRefDatabase().exactRef(branchName);
- }
-
- private Ref checkedGetRef(Repository repo, String branchName) {
- try {
- return repo.getRefDatabase().exactRef(branchName);
- } catch (Exception e) {
- logger.atSevere().withCause(e).log("failed to get ref %s in repo %s", branchName, repo);
- return null;
- }
- }
-
- private Project.NameKey createTestProject(String name) throws Exception {
- return projectOperations.newProject().name(name).create();
- }
-
@SuppressWarnings("unused")
private static class TestModule extends FactoryModule {
@Override
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchITBase.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchITBase.java
new file mode 100644
index 0000000..2a11717
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchITBase.java
@@ -0,0 +1,73 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.acceptance.LightweightPluginDaemonTest;
+import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
+import com.google.gerrit.entities.Project;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.util.function.Supplier;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.Repository;
+
+public abstract class FetchITBase extends LightweightPluginDaemonTest {
+ private static final String TEST_REPLICATION_SUFFIX = "suffix1";
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+ private static final int TEST_REPLICATION_DELAY = 60;
+ private static final Duration TEST_TIMEOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 2);
+
+ @Inject private SitePaths sitePaths;
+ @Inject private ProjectOperations projectOperations;
+ FetchFactory fetchFactory;
+ private Path gitPath;
+ Path testRepoPath;
+
+ @Override
+ public void setUpTestPlugin() throws Exception {
+ gitPath = sitePaths.site_path.resolve("git");
+ testRepoPath = gitPath.resolve(project + TEST_REPLICATION_SUFFIX + ".git");
+
+ super.setUpTestPlugin();
+ fetchFactory = plugin.getSysInjector().getInstance(FetchFactory.class);
+ }
+
+ void waitUntil(Supplier<Boolean> waitCondition) throws InterruptedException {
+ WaitUtil.waitUntil(waitCondition, TEST_TIMEOUT);
+ }
+
+ Ref getRef(Repository repo, String branchName) throws IOException {
+ return repo.getRefDatabase().exactRef(branchName);
+ }
+
+ Ref checkedGetRef(Repository repo, String branchName) {
+ try {
+ return repo.getRefDatabase().exactRef(branchName);
+ } catch (Exception e) {
+ logger.atSevere().withCause(e).log("failed to get ref %s in repo %s", branchName, repo);
+ return null;
+ }
+ }
+
+ Project.NameKey createTestProject(String name) throws Exception {
+ return projectOperations.newProject().name(name).create();
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchIT.java
new file mode 100644
index 0000000..ec695a6
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/JGitFetchIT.java
@@ -0,0 +1,85 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import com.google.common.collect.Lists;
+import com.google.gerrit.acceptance.SkipProjectClone;
+import com.google.gerrit.acceptance.TestPlugin;
+import com.google.gerrit.acceptance.UseLocalDisk;
+import com.google.gerrit.extensions.config.FactoryModule;
+import com.google.inject.Scopes;
+import com.google.inject.assistedinject.FactoryModuleBuilder;
+import com.googlesource.gerrit.plugins.replication.AutoReloadSecureCredentialsFactoryDecorator;
+import com.googlesource.gerrit.plugins.replication.CredentialsFactory;
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig;
+import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.Fetch;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchClientImplementation;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.JGitFetch;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.PermanentTransportException;
+import java.net.URISyntaxException;
+import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.transport.RefSpec;
+import org.eclipse.jgit.transport.RemoteConfig;
+import org.eclipse.jgit.transport.URIish;
+import org.junit.Test;
+
+@SkipProjectClone
+@UseLocalDisk
+@TestPlugin(
+ name = "pull-replication",
+ sysModule = "com.googlesource.gerrit.plugins.replication.pull.JGitFetchIT$TestModule")
+public class JGitFetchIT extends FetchITBase {
+ private static final String TEST_REPLICATION_SUFFIX = "suffix1";
+
+ @Test(expected = PermanentTransportException.class)
+ public void shouldThrowPermanentTransportExceptionWhenRefDoesNotExists() throws Exception {
+
+ testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX));
+ String nonExistingRef = "refs/changes/02/20000/1:refs/changes/02/20000/1";
+ try (Repository repo = repoManager.openRepository(project)) {
+ Fetch objectUnderTest = fetchFactory.create(new URIish(testRepoPath.toString()), repo);
+ objectUnderTest.fetch(Lists.newArrayList(new RefSpec(nonExistingRef)));
+ }
+ }
+
+ @SuppressWarnings("unused")
+ private static class TestModule extends FactoryModule {
+ @Override
+ protected void configure() {
+ Config cf = new Config();
+ cf.setInt("remote", "test_config", "timeout", 0);
+ try {
+ RemoteConfig remoteConfig = new RemoteConfig(cf, "test_config");
+ SourceConfiguration sourceConfig = new SourceConfiguration(remoteConfig, cf);
+ bind(ReplicationConfig.class).to(ReplicationFileBasedConfig.class);
+ bind(CredentialsFactory.class)
+ .to(AutoReloadSecureCredentialsFactoryDecorator.class)
+ .in(Scopes.SINGLETON);
+
+ bind(SourceConfiguration.class).toInstance(sourceConfig);
+ install(
+ new FactoryModuleBuilder()
+ .implement(Fetch.class, JGitFetch.class)
+ .implement(Fetch.class, FetchClientImplementation.class, JGitFetch.class)
+ .build(FetchFactory.class));
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PermanentFailureExceptionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PermanentFailureExceptionTest.java
new file mode 100644
index 0000000..fb4fb04
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PermanentFailureExceptionTest.java
@@ -0,0 +1,42 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.googlesource.gerrit.plugins.replication.pull.fetch.PermanentTransportException;
+import com.jcraft.jsch.JSchException;
+import org.eclipse.jgit.errors.TransportException;
+import org.junit.Test;
+
+public class PermanentFailureExceptionTest {
+
+ @Test
+ public void shouldConsiderSchUnknownHostAsPermanent() {
+ assertThat(
+ PermanentTransportException.isPermanentFailure(
+ new TransportException(
+ "SSH error", new JSchException("UnknownHostKey: some.place"))))
+ .isTrue();
+ }
+
+ @Test
+ public void shouldConsiderNotExistingRefsAsPermanent() {
+ assertThat(
+ PermanentTransportException.isPermanentFailure(
+ new TransportException("Remote does not have refs/heads/foo available for fetch.")))
+ .isTrue();
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java
index 2adf72f..f1f9e44 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java
@@ -17,6 +17,7 @@
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -57,6 +58,8 @@
public class DeleteRefCommandTest {
private static final String TEST_SOURCE_LABEL = "test-source-label";
private static final String TEST_REF_NAME = "refs/changes/01/1/1";
+
+ private static final String NON_EXISTING_REF_NAME = "refs/changes/01/11101/1";
private static final NameKey TEST_PROJECT_NAME = Project.nameKey("test-project");
private static URIish TEST_REMOTE_URI;
@@ -88,9 +91,6 @@
when(sourceCollection.getByRemoteName(TEST_SOURCE_LABEL)).thenReturn(Optional.of(source));
TEST_REMOTE_URI = new URIish("git://some.remote.uri");
when(source.getURI(TEST_PROJECT_NAME)).thenReturn(TEST_REMOTE_URI);
- when(permissionBackend.currentUser()).thenReturn(currentUser);
- when(currentUser.project(any())).thenReturn(forProject);
- when(forProject.ref(any())).thenReturn(forRef);
when(gitManager.openRepository(any())).thenReturn(repository);
when(repository.updateRef(any())).thenReturn(refUpdate);
when(repository.getRefDatabase()).thenReturn(refDb);
@@ -119,4 +119,13 @@
assertThat(fetchEvent.getProjectNameKey()).isEqualTo(TEST_PROJECT_NAME);
assertThat(fetchEvent.getRefName()).isEqualTo(TEST_REF_NAME);
}
+
+ @Test
+ public void shouldHandleNonExistingRef() throws Exception {
+ when(refDb.exactRef(anyString())).thenReturn(null);
+
+ objectUnderTest.deleteRef(TEST_PROJECT_NAME, NON_EXISTING_REF_NAME, TEST_SOURCE_LABEL);
+
+ verify(eventDispatcher, never()).postEvent(any());
+ }
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java
index c673011..92314a4 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java
@@ -21,9 +21,11 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import com.google.common.collect.Lists;
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.restapi.AuthException;
+import com.google.gerrit.extensions.restapi.RestApiException;
import com.google.gerrit.server.data.RefUpdateAttribute;
import com.google.gerrit.server.events.Event;
import com.google.gerrit.server.events.ProjectCreatedEvent;
@@ -31,10 +33,15 @@
import com.google.gerrit.server.git.WorkQueue;
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.googlesource.gerrit.plugins.replication.pull.FetchOne;
+import com.googlesource.gerrit.plugins.replication.pull.Source;
+import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection;
+import com.googlesource.gerrit.plugins.replication.pull.api.DeleteRefCommand;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.Input;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchJob;
import com.googlesource.gerrit.plugins.replication.pull.api.ProjectInitializationAction;
import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics;
+import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter;
+import java.io.IOException;
import java.util.concurrent.ScheduledExecutorService;
import org.eclipse.jgit.lib.ObjectId;
import org.junit.Before;
@@ -51,6 +58,7 @@
private static final String TEST_REF_NAME = "refs/changes/01/1/1";
private static final String TEST_PROJECT = "test-project";
private static final String INSTANCE_ID = "node_instance_id";
+ private static final String NEW_REV = "0000000000000000000000000000000000000001";
private static final String REMOTE_INSTANCE_ID = "remote_node_instance_id";
@Mock private ProjectInitializationAction projectInitializationAction;
@@ -58,8 +66,12 @@
@Mock private ScheduledExecutorService executor;
@Mock private FetchJob fetchJob;
@Mock private FetchJob.Factory fetchJobFactory;
+ @Mock private DeleteRefCommand deleteRefCommand;
@Captor ArgumentCaptor<Input> inputCaptor;
@Mock private PullReplicationApiRequestMetrics metrics;
+ @Mock private SourcesCollection sources;
+ @Mock private Source source;
+ @Mock private ExcludedRefsFilter refsFilter;
private StreamEventListener objectUnderTest;
@@ -68,9 +80,22 @@
when(workQueue.getDefaultQueue()).thenReturn(executor);
when(fetchJobFactory.create(eq(Project.nameKey(TEST_PROJECT)), any(), any()))
.thenReturn(fetchJob);
+ when(sources.getAll()).thenReturn(Lists.newArrayList(source));
+ when(source.wouldFetchProject(any())).thenReturn(true);
+ when(source.wouldCreateProject(any())).thenReturn(true);
+ when(source.isCreateMissingRepositories()).thenReturn(true);
+ when(source.getRemoteConfigName()).thenReturn(REMOTE_INSTANCE_ID);
+ when(refsFilter.match(any())).thenReturn(false);
objectUnderTest =
new StreamEventListener(
- INSTANCE_ID, projectInitializationAction, workQueue, fetchJobFactory, () -> metrics);
+ INSTANCE_ID,
+ deleteRefCommand,
+ projectInitializationAction,
+ workQueue,
+ fetchJobFactory,
+ () -> metrics,
+ sources,
+ refsFilter);
}
@Test
@@ -80,6 +105,7 @@
objectUnderTest.onEvent(event);
verify(executor, never()).submit(any(Runnable.class));
+ verify(sources, never()).getAll();
}
@Test
@@ -99,11 +125,49 @@
}
@Test
+ public void shouldSkipEventWhenNotOnAllowedProjectsList() {
+ when(source.wouldFetchProject(any())).thenReturn(false);
+
+ RefUpdatedEvent event = new RefUpdatedEvent();
+ RefUpdateAttribute refUpdate = new RefUpdateAttribute();
+ refUpdate.refName = TEST_REF_NAME;
+ refUpdate.project = TEST_PROJECT;
+ refUpdate.oldRev = ObjectId.zeroId().getName();
+ refUpdate.newRev = NEW_REV;
+
+ event.instanceId = REMOTE_INSTANCE_ID;
+ event.refUpdate = () -> refUpdate;
+
+ objectUnderTest.onEvent(event);
+
+ verify(executor, never()).submit(any(Runnable.class));
+ }
+
+ @Test
+ public void shouldDeleteRefForRefDeleteEvent() throws IOException, RestApiException {
+ RefUpdatedEvent event = new RefUpdatedEvent();
+ RefUpdateAttribute refUpdate = new RefUpdateAttribute();
+ refUpdate.refName = TEST_REF_NAME;
+ refUpdate.newRev = ObjectId.zeroId().getName();
+ refUpdate.project = TEST_PROJECT;
+
+ event.instanceId = REMOTE_INSTANCE_ID;
+ event.refUpdate = () -> refUpdate;
+
+ objectUnderTest.onEvent(event);
+
+ verify(deleteRefCommand)
+ .deleteRef(Project.nameKey(TEST_PROJECT), refUpdate.refName, REMOTE_INSTANCE_ID);
+ }
+
+ @Test
public void shouldScheduleFetchJobForRefUpdateEvent() {
RefUpdatedEvent event = new RefUpdatedEvent();
RefUpdateAttribute refUpdate = new RefUpdateAttribute();
refUpdate.refName = TEST_REF_NAME;
refUpdate.project = TEST_PROJECT;
+ refUpdate.oldRev = ObjectId.zeroId().getName();
+ refUpdate.newRev = NEW_REV;
event.instanceId = REMOTE_INSTANCE_ID;
event.refUpdate = () -> refUpdate;
@@ -120,6 +184,24 @@
}
@Test
+ public void shouldSkipRefUpdateEventForExcludedRef() {
+ when(refsFilter.match(any())).thenReturn(true);
+ RefUpdatedEvent event = new RefUpdatedEvent();
+ RefUpdateAttribute refUpdate = new RefUpdateAttribute();
+ refUpdate.refName = TEST_REF_NAME;
+ refUpdate.project = TEST_PROJECT;
+ refUpdate.oldRev = ObjectId.zeroId().getName();
+ refUpdate.newRev = NEW_REV;
+
+ event.instanceId = REMOTE_INSTANCE_ID;
+ event.refUpdate = () -> refUpdate;
+
+ objectUnderTest.onEvent(event);
+
+ verify(executor, never()).submit(any(Runnable.class));
+ }
+
+ @Test
public void shouldCreateProjectForProjectCreatedEvent()
throws AuthException, PermissionBackendException {
ProjectCreatedEvent event = new ProjectCreatedEvent();
@@ -132,6 +214,34 @@
}
@Test
+ public void shouldNotCreateProjectWhenCreateMissingRepositoriesNotSet()
+ throws AuthException, PermissionBackendException {
+ when(source.isCreateMissingRepositories()).thenReturn(false);
+
+ ProjectCreatedEvent event = new ProjectCreatedEvent();
+ event.instanceId = REMOTE_INSTANCE_ID;
+ event.projectName = TEST_PROJECT;
+
+ objectUnderTest.onEvent(event);
+
+ verify(projectInitializationAction, never()).initProject(any());
+ }
+
+ @Test
+ public void shouldNotCreateProjectWhenReplicationNotAllowed()
+ throws AuthException, PermissionBackendException {
+ when(source.isCreateMissingRepositories()).thenReturn(false);
+
+ ProjectCreatedEvent event = new ProjectCreatedEvent();
+ event.instanceId = REMOTE_INSTANCE_ID;
+ event.projectName = TEST_PROJECT;
+
+ objectUnderTest.onEvent(event);
+
+ verify(projectInitializationAction, never()).initProject(any());
+ }
+
+ @Test
public void shouldScheduleAllRefsFetchForProjectCreatedEvent() {
ProjectCreatedEvent event = new ProjectCreatedEvent();
event.instanceId = REMOTE_INSTANCE_ID;
