Merge branch 'stable-3.4' into stable-3.5 * stable-3.4: Split integrations tests into separate Bazel targets. Extract base test class for regular and async acceptance tests Allow additional refs fallback to apply-objects Stop creating redundant replication tasks from stream-events Add event creation time to the apply object payload Don't read git submodule commits during replication Expose implicit dependency with the multi-site plugin Ignore remote ref-updated stream events for replication Increase test time to 900 seconds Change-Id: I7b98c49d875e4f2917af21abec562a21582f6b28
diff --git a/BUILD b/BUILD index aad6d61..7646282 100644 --- a/BUILD +++ b/BUILD
@@ -25,7 +25,6 @@ name = "pull_replication_tests", srcs = glob([ "src/test/java/**/*Test.java", - "src/test/java/**/*IT.java", ]), tags = ["pull-replication"], visibility = ["//visibility:public"], @@ -37,6 +36,18 @@ ], ) +[junit_tests( + name = f[:f.index(".")].replace("/", "_"), + srcs = [f], + tags = ["pull-replication"], + visibility = ["//visibility:public"], + deps = PLUGIN_TEST_DEPS + PLUGIN_DEPS + [ + ":pull-replication__plugin", + ":pull_replication_util", + "//plugins/replication", + ], +) for f in glob(["src/test/java/**/*IT.java"])] + java_library( name = "pull_replication_util", testonly = True,
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectCacheModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectCacheModule.java new file mode 100644 index 0000000..944326e --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectCacheModule.java
@@ -0,0 +1,29 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull; + +import com.google.gerrit.server.cache.CacheModule; +import java.time.Duration; + +public class ApplyObjectCacheModule extends CacheModule { + public static final String APPLY_OBJECTS_CACHE = "apply_objects"; + public static final Duration APPLY_OBJECTS_CACHE_MAX_AGE = Duration.ofMinutes(1); + + @Override + protected void configure() { + cache(APPLY_OBJECTS_CACHE, ApplyObjectsCacheKey.class, Long.class) + .expireAfterWrite(APPLY_OBJECTS_CACHE_MAX_AGE); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectsCacheKey.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectsCacheKey.java new file mode 100644 index 0000000..9e83b4d --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectsCacheKey.java
@@ -0,0 +1,31 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull; + +import com.google.auto.value.AutoValue; + +@AutoValue +public abstract class ApplyObjectsCacheKey { + + public static ApplyObjectsCacheKey create(String objectId, String refName, String project) { + return new AutoValue_ApplyObjectsCacheKey(objectId, refName, project); + } + + public abstract String objectId(); + + public abstract String refName(); + + public abstract String project(); +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java index ed3aecb..d1c5ca1 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java
@@ -80,6 +80,7 @@ bind(RevisionReader.class).in(Scopes.SINGLETON); bind(ApplyObject.class); install(new FactoryModuleBuilder().build(FetchJob.Factory.class)); + install(new ApplyObjectCacheModule()); install(new PullReplicationApiModule()); install(new FetchRefReplicatedEventModule());
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java index b90969b..fc77503 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
@@ -25,6 +25,7 @@ import com.google.gerrit.extensions.events.ProjectDeletedListener; import com.google.gerrit.extensions.registration.DynamicItem; import com.google.gerrit.metrics.Timer1.Context; +import com.google.gerrit.server.config.GerritInstanceId; import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.server.events.EventListener; import com.google.gerrit.server.events.RefUpdatedEvent; @@ -37,6 +38,7 @@ import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException; import com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient; import com.googlesource.gerrit.plugins.replication.pull.client.HttpResult; +import com.googlesource.gerrit.plugins.replication.pull.filter.ApplyObjectsRefsFilter; import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter; import java.io.IOException; import java.net.URISyntaxException; @@ -90,6 +92,8 @@ private Provider<RevisionReader> revReaderProvider; private final ApplyObjectMetrics applyObjectMetrics; private final FetchReplicationMetrics fetchMetrics; + private final String instanceId; + private ApplyObjectsRefsFilter applyObjectsRefsFilter; @Inject ReplicationQueue( @@ -101,7 +105,9 @@ ExcludedRefsFilter refsFilter, Provider<RevisionReader> revReaderProvider, ApplyObjectMetrics applyObjectMetrics, - FetchReplicationMetrics fetchMetrics) { + FetchReplicationMetrics fetchMetrics, + @GerritInstanceId String instanceId, + ApplyObjectsRefsFilter applyObjectsRefsFilter) { workQueue = wq; dispatcher = dis; sources = rd; @@ -112,6 +118,8 @@ this.revReaderProvider = revReaderProvider; this.applyObjectMetrics = applyObjectMetrics; this.fetchMetrics = fetchMetrics; + this.instanceId = instanceId; + this.applyObjectsRefsFilter = applyObjectsRefsFilter; } @Override @@ -151,7 +159,7 @@ @Override public void onEvent(com.google.gerrit.server.events.Event e) { - if (e.type.equals(REF_UDPATED_EVENT_TYPE)) { + if (e.type.equals(REF_UDPATED_EVENT_TYPE) && instanceId.equals(e.instanceId)) { RefUpdatedEvent event = (RefUpdatedEvent) e; if (isRefToBeReplicated(event.getRefName())) { @@ -166,6 +174,7 @@ event.refUpdate.get().project, ObjectId.fromString(event.refUpdate.get().newRev), event.getRefName(), + event.eventCreatedOn, ZEROS_OBJECTID.equals(event.refUpdate.get().newRev)); } } @@ -195,16 +204,22 @@ return !refsFilter.match(refName); } - private void fire(String projectName, ObjectId objectId, String refName, boolean isDelete) { + private void fire( + String projectName, + ObjectId objectId, + String refName, + long eventCreatedOn, + boolean isDelete) { ReplicationState state = new ReplicationState(new GitUpdateProcessing(dispatcher.get())); - fire(Project.nameKey(projectName), objectId, refName, isDelete, state); + fire(Project.nameKey(projectName), objectId, refName, eventCreatedOn, isDelete, state); state.markAllFetchTasksScheduled(); } private void fire( - Project.NameKey project, + NameKey project, ObjectId objectId, String refName, + long eventCreatedOn, boolean isDelete, ReplicationState state) { if (!running) { @@ -212,7 +227,7 @@ "Replication plugin did not finish startup before event, event replication is postponed", state); beforeStartupEventsQueue.add( - ReferenceUpdatedEvent.create(project.get(), refName, objectId, isDelete)); + ReferenceUpdatedEvent.create(project.get(), refName, objectId, eventCreatedOn, isDelete)); return; } ForkJoinPool fetchCallsPool = null; @@ -220,7 +235,7 @@ fetchCallsPool = new ForkJoinPool(sources.get().getAll().size()); final Consumer<Source> callFunction = - callFunction(project, objectId, refName, isDelete, state); + callFunction(project, objectId, refName, eventCreatedOn, isDelete, state); fetchCallsPool .submit(() -> sources.get().getAll().parallelStream().forEach(callFunction)) .get(fetchCallsTimeout, TimeUnit.MILLISECONDS); @@ -242,9 +257,11 @@ NameKey project, ObjectId objectId, String refName, + long eventCreatedOn, boolean isDelete, ReplicationState state) { - CallFunction call = getCallFunction(project, objectId, refName, isDelete, state); + CallFunction call = + getCallFunction(project, objectId, refName, eventCreatedOn, isDelete, state); return (source) -> { boolean callSuccessful; @@ -269,10 +286,12 @@ NameKey project, ObjectId objectId, String refName, + long eventCreatedOn, boolean isDelete, ReplicationState state) { if (isDelete) { - return ((source) -> callSendObject(source, project, refName, isDelete, null, state)); + return ((source) -> + callSendObject(source, project, refName, eventCreatedOn, isDelete, null, state)); } try { @@ -287,7 +306,13 @@ if (revisionData.isPresent()) { return ((source) -> callSendObject( - source, project, refName, isDelete, Arrays.asList(revisionData.get()), state)); + source, + project, + refName, + eventCreatedOn, + isDelete, + Arrays.asList(revisionData.get()), + state)); } } catch (InvalidObjectIdException | IOException e) { stateLog.error( @@ -303,8 +328,9 @@ private boolean callSendObject( Source source, - Project.NameKey project, + NameKey project, String refName, + long eventCreatedOn, boolean isDelete, List<RevisionData> revision, ReplicationState state) @@ -324,8 +350,9 @@ Context<String> apiTimer = applyObjectMetrics.startEnd2End(source.getRemoteConfigName()); HttpResult result = isDelete - ? fetchClient.callSendObject(project, refName, isDelete, null, uri) - : fetchClient.callSendObjects(project, refName, revision, uri); + ? fetchClient.callSendObject( + project, refName, eventCreatedOn, isDelete, null, uri) + : fetchClient.callSendObjects(project, refName, eventCreatedOn, revision, uri); boolean resultSuccessful = result.isSuccessful(); repLog.info( "Pull replication REST API apply object to {} COMPLETED for {}:{} - {}, HTTP Result:" @@ -347,7 +374,8 @@ if (!resultSuccessful) { if (result.isParentObjectMissing()) { - if (RefNames.isNoteDbMetaRef(refName) && revision.size() == 1) { + if ((RefNames.isNoteDbMetaRef(refName) || applyObjectsRefsFilter.match(refName)) + && revision.size() == 1) { List<RevisionData> allRevisions = fetchWholeMetaHistory(project, refName, revision.get(0)); repLog.info( @@ -356,7 +384,8 @@ project, refName, allRevisions); - return callSendObject(source, project, refName, isDelete, allRevisions, state); + return callSendObject( + source, project, refName, eventCreatedOn, isDelete, allRevisions, state); } throw new MissingParentObjectException( @@ -497,7 +526,12 @@ String eventKey = String.format("%s:%s", event.projectName(), event.refName()); if (!eventsReplayed.contains(eventKey)) { repLog.info("Firing pending task {}", event); - fire(event.projectName(), event.objectId(), event.refName(), event.isDelete()); + fire( + event.projectName(), + event.objectId(), + event.refName(), + event.eventCreatedOn(), + event.isDelete()); eventsReplayed.add(eventKey); } } @@ -518,9 +552,13 @@ abstract static class ReferenceUpdatedEvent { static ReferenceUpdatedEvent create( - String projectName, String refName, ObjectId objectId, boolean isDelete) { + String projectName, + String refName, + ObjectId objectId, + long eventCreatedOn, + boolean isDelete) { return new AutoValue_ReplicationQueue_ReferenceUpdatedEvent( - projectName, refName, objectId, isDelete); + projectName, refName, objectId, eventCreatedOn, isDelete); } public abstract String projectName(); @@ -529,6 +567,8 @@ public abstract ObjectId objectId(); + public abstract long eventCreatedOn(); + public abstract boolean isDelete(); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java index cd6a0ea..3f03ed6 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java
@@ -38,6 +38,7 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.RepositoryNotFoundException; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.Ref; @@ -214,6 +215,24 @@ return blobs; } + /** + * Reads and evaluates the git objects in this revision. The following are filtered out: + * <li>DELETE changes + * <li>git submodule commits, because the git commit hash is not present in this repo. + * + * <p>The method keeps track of the total size of all objects it has processed, and verifies + * it is below the acceptable threshold. + * + * @param projectName - the name of the project, used to check total object size threshold + * @param refName - the ref name, used to check total object size threshold + * @param git - this git repo, used to load the objects + * @param totalRefSize - tracks the total size of objects processed + * @param diffEntries - a list of the diff entries for this revision + * @return a List of `RevisionObjectData`, an object that includes the git object SHA, the git + * object change type and the object contents. + * @throws MissingObjectException - if the object can't be found + * @throws IOException - if processing failed for another reason + */ private List<RevisionObjectData> readBlobs( Project.NameKey projectName, String refName, @@ -223,7 +242,7 @@ throws MissingObjectException, IOException { List<RevisionObjectData> blobs = Lists.newLinkedList(); for (DiffEntry diffEntry : diffEntries) { - if (!ChangeType.DELETE.equals(diffEntry.getChangeType())) { + if (!(ChangeType.DELETE.equals(diffEntry.getChangeType()) || gitSubmoduleCommit(diffEntry))) { ObjectId diffObjectId = diffEntry.getNewId().toObjectId(); ObjectLoader objectLoader = git.open(diffObjectId); totalRefSize += objectLoader.getSize(); @@ -237,6 +256,10 @@ return blobs; } + private boolean gitSubmoduleCommit(DiffEntry diffEntry) { + return diffEntry.getNewMode().equals(FileMode.GITLINK); + } + private RevTree getParentTree(Repository git, RevCommit commit) throws MissingObjectException, IOException { RevCommit parent = commit.getParent(0);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java index 04cc9e8..b3379f2 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java
@@ -97,7 +97,11 @@ } applyObjectCommand.applyObject( - resource.getNameKey(), input.getRefName(), input.getRevisionData(), input.getLabel()); + resource.getNameKey(), + input.getRefName(), + input.getRevisionData(), + input.getLabel(), + input.getEventCreatedOn()); return Response.created(input); } catch (MissingParentObjectException e) { repLog.error(
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java index c268ba1..968a03c 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java
@@ -14,9 +14,11 @@ package com.googlesource.gerrit.plugins.replication.pull.api; +import static com.googlesource.gerrit.plugins.replication.pull.ApplyObjectCacheModule.APPLY_OBJECTS_CACHE; import static com.googlesource.gerrit.plugins.replication.pull.PullReplicationLogger.repLog; import static java.util.concurrent.TimeUnit.NANOSECONDS; +import com.google.common.cache.Cache; import com.google.common.collect.ImmutableSet; import com.google.common.flogger.FluentLogger; import com.google.gerrit.entities.Project; @@ -25,21 +27,20 @@ import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.server.permissions.PermissionBackendException; import com.google.inject.Inject; -import com.googlesource.gerrit.plugins.replication.pull.ApplyObjectMetrics; -import com.googlesource.gerrit.plugins.replication.pull.Context; -import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent; -import com.googlesource.gerrit.plugins.replication.pull.PullReplicationStateLogger; -import com.googlesource.gerrit.plugins.replication.pull.ReplicationState; +import com.google.inject.name.Named; +import com.googlesource.gerrit.plugins.replication.pull.*; import com.googlesource.gerrit.plugins.replication.pull.ReplicationState.RefFetchResult; import com.googlesource.gerrit.plugins.replication.pull.Source; import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData; import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException; import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException; import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject; import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState; import java.io.IOException; import java.util.Arrays; +import java.util.List; import java.util.Set; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.transport.RefSpec; @@ -47,7 +48,7 @@ public class ApplyObjectCommand { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - + private final Cache<ApplyObjectsCacheKey, Long> refUpdatesSucceededCache; private static final Set<RefUpdate.Result> SUCCESSFUL_RESULTS = ImmutableSet.of( RefUpdate.Result.NEW, @@ -67,22 +68,32 @@ ApplyObject applyObject, ApplyObjectMetrics metrics, DynamicItem<EventDispatcher> eventDispatcher, - SourcesCollection sourcesCollection) { + SourcesCollection sourcesCollection, + @Named(APPLY_OBJECTS_CACHE) Cache<ApplyObjectsCacheKey, Long> refUpdatesSucceededCache) { this.fetchStateLog = fetchStateLog; this.applyObject = applyObject; this.metrics = metrics; this.eventDispatcher = eventDispatcher; this.sourcesCollection = sourcesCollection; + this.refUpdatesSucceededCache = refUpdatesSucceededCache; } public void applyObject( - Project.NameKey name, String refName, RevisionData revisionsData, String sourceLabel) + Project.NameKey name, + String refName, + RevisionData revisionsData, + String sourceLabel, + long eventCreatedOn) throws IOException, RefUpdateException, MissingParentObjectException { - applyObjects(name, refName, new RevisionData[] {revisionsData}, sourceLabel); + applyObjects(name, refName, new RevisionData[] {revisionsData}, sourceLabel, eventCreatedOn); } public void applyObjects( - Project.NameKey name, String refName, RevisionData[] revisionsData, String sourceLabel) + Project.NameKey name, + String refName, + RevisionData[] revisionsData, + String sourceLabel, + long eventCreatedOn) throws IOException, RefUpdateException, MissingParentObjectException { repLog.info( @@ -94,7 +105,26 @@ Timer1.Context<String> context = metrics.start(sourceLabel); RefUpdateState refUpdateState = applyObject.apply(name, new RefSpec(refName), revisionsData); + Boolean isRefUpdateSuccessful = isSuccessful(refUpdateState.getResult()); + if (isRefUpdateSuccessful) { + for (RevisionData revisionData : revisionsData) { + RevisionObjectData commitObj = revisionData.getCommitObject(); + List<RevisionObjectData> blobs = revisionData.getBlobs(); + + if (commitObj != null) { + refUpdatesSucceededCache.put( + ApplyObjectsCacheKey.create( + revisionData.getCommitObject().getSha1(), refName, name.get()), + eventCreatedOn); + } else if (blobs != null) { + for (RevisionObjectData blob : blobs) { + refUpdatesSucceededCache.put( + ApplyObjectsCacheKey.create(blob.getSha1(), refName, name.get()), eventCreatedOn); + } + } + } + } long elapsed = NANOSECONDS.toMillis(context.stop()); try { @@ -122,7 +152,7 @@ Context.unsetLocalEvent(); } - if (!isSuccessful(refUpdateState.getResult())) { + if (!isRefUpdateSuccessful) { String message = String.format( "RefUpdate failed with result %s for: sourceLcabel=%s, project=%s, refName=%s",
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectsAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectsAction.java index a1e1f5b..dd155d4 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectsAction.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectsAction.java
@@ -97,7 +97,11 @@ } command.applyObjects( - resource.getNameKey(), input.getRefName(), input.getRevisionsData(), input.getLabel()); + resource.getNameKey(), + input.getRefName(), + input.getRevisionsData(), + input.getLabel(), + input.getEventCreatedOn()); return Response.created(input); } catch (MissingParentObjectException e) { repLog.error(
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationEndpoints.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationEndpoints.java new file mode 100644 index 0000000..fc97945 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationEndpoints.java
@@ -0,0 +1,39 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api; + +import static com.google.gerrit.common.UsedAt.Project.PLUGIN_MULTI_SITE; + +import com.google.gerrit.common.UsedAt; + +/** + * Temporary solution for stable branches for allowing the multi-site plugin to understand his + * caller identity. + * + * <p>TODO: To be removed from v3.9/master, where this problem does not exist anymore because of a + * clearer definition of responsibilities between the multi-site and the pull-replication plugins. + */ +public interface PullReplicationEndpoints { + + @UsedAt(PLUGIN_MULTI_SITE) + public static final String APPLY_OBJECT_API_ENDPOINT = "apply-object"; + + @UsedAt(PLUGIN_MULTI_SITE) + public static final String APPLY_OBJECTS_API_ENDPOINT = "apply-objects"; + + public static final String FETCH_ENDPOINT = "fetch"; + public static final String INIT_PROJECT_ENDPOINT = "init-project"; + public static final String DELETE_PROJECT_ENDPOINT = "delete-project"; +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java index 20122a7..4d932d4 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java
@@ -56,7 +56,6 @@ import java.io.EOFException; import java.io.IOException; import java.io.PrintWriter; -import java.nio.file.InvalidPathException; import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; @@ -69,7 +68,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -public class PullReplicationFilter extends AllRequestFilter { +public class PullReplicationFilter extends AllRequestFilter implements PullReplicationEndpoints { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private static final Pattern projectNameInGerritUrl = Pattern.compile(".*/projects/([^/]+)/.*"); @@ -159,7 +158,8 @@ RestApiServlet.replyError( httpRequest, httpResponse, SC_BAD_REQUEST, "Project name not present in the url", e); } catch (Exception e) { - if (e instanceof InvalidPathException || e.getCause() instanceof InvalidPathException) { + if (e instanceof IllegalArgumentException + || e.getCause() instanceof IllegalArgumentException) { RestApiServlet.replyError( httpRequest, httpResponse, SC_BAD_REQUEST, "Invalid repository path in request", e); } else { @@ -299,19 +299,25 @@ } private boolean isApplyObjectAction(HttpServletRequest httpRequest) { - return httpRequest.getRequestURI().endsWith(String.format("/%s~apply-object", pluginName)); + return httpRequest + .getRequestURI() + .endsWith(String.format("/%s~" + APPLY_OBJECT_API_ENDPOINT, pluginName)); } private boolean isApplyObjectsAction(HttpServletRequest httpRequest) { - return httpRequest.getRequestURI().endsWith(String.format("/%s~apply-objects", pluginName)); + return httpRequest + .getRequestURI() + .endsWith(String.format("/%s~" + APPLY_OBJECTS_API_ENDPOINT, pluginName)); } private boolean isFetchAction(HttpServletRequest httpRequest) { - return httpRequest.getRequestURI().endsWith(String.format("/%s~fetch", pluginName)); + return httpRequest.getRequestURI().endsWith(String.format("/%s~" + FETCH_ENDPOINT, pluginName)); } private boolean isInitProjectAction(HttpServletRequest httpRequest) { - return httpRequest.getRequestURI().contains(String.format("/%s/init-project/", pluginName)); + return httpRequest + .getRequestURI() + .contains(String.format("/%s/" + INIT_PROJECT_ENDPOINT + "/", pluginName)); } private boolean isUpdateHEADAction(HttpServletRequest httpRequest) { @@ -320,7 +326,9 @@ } private boolean isDeleteProjectAction(HttpServletRequest httpRequest) { - return httpRequest.getRequestURI().endsWith(String.format("/%s~delete-project", pluginName)) + return httpRequest + .getRequestURI() + .endsWith(String.format("/%s~" + DELETE_PROJECT_ENDPOINT, pluginName)) && "DELETE".equals(httpRequest.getMethod()); } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java index c18e11d..6883f2c 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java
@@ -23,11 +23,14 @@ private String refName; + private long eventCreatedOn; private RevisionData revisionData; - public RevisionInput(String label, String refName, RevisionData revisionData) { + public RevisionInput( + String label, String refName, long eventCreatedOn, RevisionData revisionData) { this.label = label; this.refName = refName; + this.eventCreatedOn = eventCreatedOn; this.revisionData = revisionData; } @@ -43,6 +46,10 @@ return revisionData; } + public long getEventCreatedOn() { + return eventCreatedOn; + } + public void validate() { validate(refName, revisionData); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionsInput.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionsInput.java index 2361f6b..d4626f4 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionsInput.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionsInput.java
@@ -21,11 +21,14 @@ private String refName; + private long eventCreatedOn; private RevisionData[] revisionsData; - public RevisionsInput(String label, String refName, RevisionData[] revisionsData) { + public RevisionsInput( + String label, String refName, long eventCreatedOn, RevisionData[] revisionsData) { this.label = label; this.refName = refName; + this.eventCreatedOn = eventCreatedOn; this.revisionsData = revisionsData; } @@ -37,6 +40,10 @@ return refName; } + public long getEventCreatedOn() { + return eventCreatedOn; + } + public RevisionData[] getRevisionsData() { return revisionsData; }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java index 6000eb9..1991260 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java
@@ -17,6 +17,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import com.google.gerrit.entities.Project; +import com.google.gerrit.entities.Project.NameKey; import com.googlesource.gerrit.plugins.replication.pull.Source; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; import java.io.IOException; @@ -46,14 +47,19 @@ HttpResult updateHead(Project.NameKey project, String newHead, URIish apiUri) throws IOException; HttpResult callSendObject( - Project.NameKey project, + NameKey project, String refName, + long eventCreatedOn, boolean isDelete, RevisionData revisionData, URIish targetUri) throws ClientProtocolException, IOException; HttpResult callSendObjects( - Project.NameKey project, String refName, List<RevisionData> revisionData, URIish targetUri) + NameKey project, + String refName, + long eventCreatedOn, + List<RevisionData> revisionData, + URIish targetUri) throws ClientProtocolException, IOException; }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java index cbc2cf7..b606ba8 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
@@ -171,8 +171,9 @@ */ @Override public HttpResult callSendObject( - Project.NameKey project, + NameKey project, String refName, + long eventCreatedOn, boolean isDelete, @Nullable RevisionData revisionData, URIish targetUri) @@ -184,7 +185,7 @@ } else { requireNull(revisionData, "DELETE ref-updates cannot be associated with a RevisionData"); } - RevisionInput input = new RevisionInput(instanceId, refName, revisionData); + RevisionInput input = new RevisionInput(instanceId, refName, eventCreatedOn, revisionData); String url = formatUrl(targetUri.toString(), project, "apply-object"); @@ -196,14 +197,20 @@ @Override public HttpResult callSendObjects( - NameKey project, String refName, List<RevisionData> revisionData, URIish targetUri) + NameKey project, + String refName, + long eventCreatedOn, + List<RevisionData> revisionData, + URIish targetUri) throws ClientProtocolException, IOException { if (revisionData.size() == 1) { - return callSendObject(project, refName, false, revisionData.get(0), targetUri); + return callSendObject( + project, refName, eventCreatedOn, false, revisionData.get(0), targetUri); } RevisionData[] inputData = new RevisionData[revisionData.size()]; - RevisionsInput input = new RevisionsInput(instanceId, refName, revisionData.toArray(inputData)); + RevisionsInput input = + new RevisionsInput(instanceId, refName, eventCreatedOn, revisionData.toArray(inputData)); String url = formatUrl(targetUri.toString(), project, "apply-objects"); HttpPost post = new HttpPost(url);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java index 2ea8a33..b14d4a1 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
@@ -14,9 +14,11 @@ package com.googlesource.gerrit.plugins.replication.pull.event; +import static com.googlesource.gerrit.plugins.replication.pull.ApplyObjectCacheModule.APPLY_OBJECTS_CACHE; import static java.util.Objects.requireNonNull; import com.google.common.base.Strings; +import com.google.common.cache.Cache; import com.google.common.flogger.FluentLogger; import com.google.gerrit.common.Nullable; import com.google.gerrit.entities.Project.NameKey; @@ -24,6 +26,7 @@ import com.google.gerrit.extensions.restapi.AuthException; import com.google.gerrit.extensions.restapi.RestApiException; import com.google.gerrit.server.config.GerritInstanceId; +import com.google.gerrit.server.data.RefUpdateAttribute; import com.google.gerrit.server.events.Event; import com.google.gerrit.server.events.EventListener; import com.google.gerrit.server.events.ProjectCreatedEvent; @@ -33,6 +36,8 @@ import com.google.gerrit.server.permissions.PermissionBackendException; import com.google.inject.Inject; import com.google.inject.Provider; +import com.google.inject.name.Named; +import com.googlesource.gerrit.plugins.replication.pull.ApplyObjectsCacheKey; import com.googlesource.gerrit.plugins.replication.pull.FetchOne; import com.googlesource.gerrit.plugins.replication.pull.Source; import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection; @@ -59,6 +64,7 @@ private final SourcesCollection sources; private final String instanceId; private final WorkQueue workQueue; + private final Cache<ApplyObjectsCacheKey, Long> refUpdatesSucceededCache; @Inject public StreamEventListener( @@ -69,7 +75,8 @@ FetchJob.Factory fetchJobFactory, Provider<PullReplicationApiRequestMetrics> metricsProvider, SourcesCollection sources, - ExcludedRefsFilter excludedRefsFilter) { + ExcludedRefsFilter excludedRefsFilter, + @Named(APPLY_OBJECTS_CACHE) Cache<ApplyObjectsCacheKey, Long> refUpdatesSucceededCache) { this.instanceId = instanceId; this.deleteCommand = deleteCommand; this.projectInitializationAction = projectInitializationAction; @@ -78,6 +85,7 @@ this.metricsProvider = metricsProvider; this.sources = sources; this.refsFilter = excludedRefsFilter; + this.refUpdatesSucceededCache = refUpdatesSucceededCache; requireNonNull( Strings.emptyToNull(this.instanceId), "gerrit.instanceId cannot be null or empty"); @@ -120,6 +128,17 @@ return; } + if (isApplyObjectsCacheHit(refUpdatedEvent)) { + logger.atFine().log( + "Skipping refupdate '%s' '%s'=>'%s' (eventCreatedOn=%d) for project '%s' because has been already replicated via apply-object", + refUpdatedEvent.getRefName(), + refUpdatedEvent.refUpdate.get().oldRev, + refUpdatedEvent.refUpdate.get().newRev, + refUpdatedEvent.eventCreatedOn, + refUpdatedEvent.getProjectNameKey()); + return; + } + fetchRefsAsync( refUpdatedEvent.getRefName(), refUpdatedEvent.instanceId, @@ -207,4 +226,15 @@ private String getProjectRepositoryName(ProjectCreatedEvent projectCreatedEvent) { return String.format("%s.git", projectCreatedEvent.projectName); } + + private boolean isApplyObjectsCacheHit(RefUpdatedEvent refUpdateEvent) { + RefUpdateAttribute refUpdateAttribute = refUpdateEvent.refUpdate.get(); + Long refUpdateSuccededTimestamp = + refUpdatesSucceededCache.getIfPresent( + ApplyObjectsCacheKey.create( + refUpdateAttribute.newRev, refUpdateAttribute.refName, refUpdateAttribute.project)); + + return refUpdateSuccededTimestamp != null + && refUpdateEvent.eventCreatedOn <= refUpdateSuccededTimestamp; + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/ApplyObjectsRefsFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/ApplyObjectsRefsFilter.java new file mode 100644 index 0000000..ff5899b --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/ApplyObjectsRefsFilter.java
@@ -0,0 +1,37 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.filter; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.googlesource.gerrit.plugins.replication.ReplicationConfig; +import java.util.List; +import org.eclipse.jgit.lib.Config; + +@Singleton +public class ApplyObjectsRefsFilter extends RefsFilter { + @Inject + public ApplyObjectsRefsFilter(ReplicationConfig replicationConfig) { + super(replicationConfig); + } + + @Override + protected List<String> getRefNamePatterns(Config cfg) { + String[] applyObjectsRefs = + cfg.getStringList("replication", null, "fallbackToApplyObjectsRefs"); + return ImmutableList.copyOf(applyObjectsRefs); + } +}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index eab5fe6..b85aeb6 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -49,6 +49,37 @@ To manually trigger replication at runtime, see SSH command [start](cmd-start.md). +File `gerrit.config` +------------------------- + +cache.@PLUGIN@-apply_objects.maxAge +: Maximum age to keep history of the latest successful apply-object refs. + Values should use common unit suffixes to express their setting: + + s, sec, second, seconds + + m, min, minute, minutes + + h, hr, hour, hours + + d, day, days + + w, week, weeks (1 week is treated as 7 days) + + mon, month, months (1 month is treated as 30 days) + + y, year, years (1 year is treated as 365 days) + + If a unit suffix is not specified, seconds is assumed. If 0 is supplied, the maximum age + is infinite and items are never purged except when the cache is full. + + Default is 60s. + +cache.@PLUGIN@-apply_objects.memoryLimit +: The maximum number of apply-object refs retained in memory. + + Default is 1024. + File `@PLUGIN@.config` ------------------------- @@ -261,6 +292,33 @@ By default, set to '*' (all refs are replicated synchronously). +replication.fallbackToApplyObjectsRefs +: Specify for which refs will fallback to apply-objects REST-API that is capable of applying +an entire chain of commits on the ref chain so that the Git fetch can be avoided altogether. +It can be provided more than once, and supports three formats: regular expressions, +wildcard matching, and single ref matching. All three formats matches are case-sensitive. + + Values starting with a caret `^` are treated as regular + expressions. For the regular expressions details please follow + official [java documentation](https://docs.oracle.com/javase/tutorial/essential/regex/). + + Please note that regular expressions could also be used + with inverse match. + + Values that are not regular expressions and end in `*` are + treated as wildcard matches. Wildcards match refs whose + name agrees from the beginning until the trailing `*`. So + `foo/b*` would match the refs `foo/b`, `foo/bar`, and + `foo/baz`, but neither `foobar`, nor `bar/foo/baz`. + + Values that are neither regular expressions nor wildcards are + treated as single ref matches. So `foo/bar` matches only + the ref `foo/bar`, but no other refs. + + By default, set to 'refs/changes/**/meta'. + + Note that the refs/changes/**/meta always fallback to apply-objects REST-API + replication.maxApiPayloadSize : Maximum size in bytes of the ref to be sent as a REST Api call payload. For refs larger than threshold git fetch operation
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FakeGitReferenceUpdatedEvent.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FakeGitReferenceUpdatedEvent.java index 47c00c7..1472be2 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FakeGitReferenceUpdatedEvent.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FakeGitReferenceUpdatedEvent.java
@@ -21,12 +21,17 @@ public class FakeGitReferenceUpdatedEvent extends RefUpdatedEvent { FakeGitReferenceUpdatedEvent( - Project.NameKey project, String ref, String oldObjectId, String newObjectId) { + Project.NameKey project, + String ref, + String oldObjectId, + String newObjectId, + String instanceId) { RefUpdateAttribute upd = new RefUpdateAttribute(); upd.newRev = newObjectId; upd.oldRev = oldObjectId; upd.project = project.get(); upd.refName = ref; this.refUpdate = Suppliers.ofInstance(upd); + this.instanceId = instanceId; } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java index 58be58e..be017d0 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java
@@ -28,7 +28,7 @@ name = "pull-replication", sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule", httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule") -public class PullReplicationAsyncIT extends PullReplicationIT { +public class PullReplicationAsyncIT extends PullReplicationITAbstract { @Inject private SitePaths sitePaths; @Override
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigIT.java index 1aaf434..b9ea0c8 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigIT.java
@@ -105,7 +105,11 @@ ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); FakeGitReferenceUpdatedEvent event = new FakeGitReferenceUpdatedEvent( - project, sourceRef, ObjectId.zeroId().getName(), sourceCommit.getId().getName()); + project, + sourceRef, + ObjectId.zeroId().getName(), + sourceCommit.getId().getName(), + TEST_REPLICATION_REMOTE); pullReplicationQueue.onEvent(event); try (Repository repo = repoManager.openRepository(project)) { @@ -137,7 +141,11 @@ ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); FakeGitReferenceUpdatedEvent event = new FakeGitReferenceUpdatedEvent( - project, sourceRef, ObjectId.zeroId().getName(), sourceCommit.getId().getName()); + project, + sourceRef, + ObjectId.zeroId().getName(), + sourceCommit.getId().getName(), + TEST_REPLICATION_REMOTE); pullReplicationQueue.onEvent(event); try (Repository repo = repoManager.openRepository(project)) { @@ -166,7 +174,11 @@ plugin.getSysInjector().getInstance(ReplicationQueue.class); FakeGitReferenceUpdatedEvent event = new FakeGitReferenceUpdatedEvent( - project, newBranch, ObjectId.zeroId().getName(), branchRevision); + project, + newBranch, + ObjectId.zeroId().getName(), + branchRevision, + TEST_REPLICATION_REMOTE); pullReplicationQueue.onEvent(event); try (Repository repo = repoManager.openRepository(project);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java index 180094d..6ee3c43 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java
@@ -1,4 +1,4 @@ -// Copyright (C) 2020 The Android Open Source Project +// Copyright (C) 2022 The Android Open Source Project // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,41 +14,9 @@ package com.googlesource.gerrit.plugins.replication.pull; -import static com.google.common.truth.Truth.assertThat; -import static com.google.gerrit.acceptance.GitUtil.fetch; -import static com.google.gerrit.acceptance.GitUtil.pushOne; -import static com.google.gerrit.acceptance.testsuite.project.TestProjectUpdate.allow; -import static com.google.gerrit.server.group.SystemGroupBackend.REGISTERED_USERS; - -import com.google.gerrit.acceptance.PushOneCommit.Result; import com.google.gerrit.acceptance.SkipProjectClone; import com.google.gerrit.acceptance.TestPlugin; import com.google.gerrit.acceptance.UseLocalDisk; -import com.google.gerrit.acceptance.config.GerritConfig; -import com.google.gerrit.entities.Permission; -import com.google.gerrit.entities.Project.NameKey; -import com.google.gerrit.entities.RefNames; -import com.google.gerrit.extensions.api.changes.NotifyHandling; -import com.google.gerrit.extensions.api.projects.BranchInput; -import com.google.gerrit.extensions.events.HeadUpdatedListener; -import com.google.gerrit.extensions.events.ProjectDeletedListener; -import com.google.gerrit.extensions.restapi.RestApiException; -import com.googlesource.gerrit.plugins.replication.AutoReloadConfigDecorator; -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository; -import org.eclipse.jgit.junit.TestRepository; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.transport.PushResult; -import org.eclipse.jgit.transport.RemoteRefUpdate; -import org.eclipse.jgit.transport.RemoteRefUpdate.Status; -import org.junit.Test; @SkipProjectClone @UseLocalDisk @@ -56,300 +24,4 @@ name = "pull-replication", sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule", httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule") -public class PullReplicationIT extends PullReplicationSetupBase { - - @Override - protected void setReplicationSource( - String remoteName, List<String> replicaSuffixes, Optional<String> project) - throws IOException { - List<String> fetchUrls = - buildReplicaURLs(replicaSuffixes, s -> gitPath.resolve("${name}" + s + ".git").toString()); - config.setStringList("remote", remoteName, "url", fetchUrls); - config.setString("remote", remoteName, "apiUrl", adminRestSession.url()); - config.setString("remote", remoteName, "fetch", "+refs/*:refs/*"); - config.setInt("remote", remoteName, "timeout", 600); - config.setInt("remote", remoteName, "replicationDelay", TEST_REPLICATION_DELAY); - project.ifPresent(prj -> config.setString("remote", remoteName, "projects", prj)); - config.setBoolean("gerrit", null, "autoReload", true); - config.save(); - } - - @Override - public void setUpTestPlugin() throws Exception { - setUpTestPlugin(false); - } - - @Test - @GerritConfig(name = "gerrit.instanceId", value = TEST_REPLICATION_REMOTE) - public void shouldReplicateNewChangeRef() throws Exception { - testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX)); - - Result pushResult = createChange(); - RevCommit sourceCommit = pushResult.getCommit(); - String sourceRef = pushResult.getPatchSet().refName(); - - ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( - project, sourceRef, ObjectId.zeroId().getName(), sourceCommit.getId().getName()); - pullReplicationQueue.onEvent(event); - - try (Repository repo = repoManager.openRepository(project)) { - waitUntil(() -> checkedGetRef(repo, sourceRef) != null); - - Ref targetBranchRef = getRef(repo, sourceRef); - assertThat(targetBranchRef).isNotNull(); - assertThat(targetBranchRef.getObjectId()).isEqualTo(sourceCommit.getId()); - } - } - - @Test - @GerritConfig(name = "gerrit.instanceId", value = TEST_REPLICATION_REMOTE) - public void shouldReplicateNewBranch() throws Exception { - String testProjectName = project + TEST_REPLICATION_SUFFIX; - createTestProject(testProjectName); - - String newBranch = "refs/heads/mybranch"; - String master = "refs/heads/master"; - BranchInput input = new BranchInput(); - input.revision = master; - gApi.projects().name(testProjectName).branch(newBranch).create(input); - String branchRevision = gApi.projects().name(testProjectName).branch(newBranch).get().revision; - - ReplicationQueue pullReplicationQueue = - plugin.getSysInjector().getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( - project, newBranch, ObjectId.zeroId().getName(), branchRevision); - pullReplicationQueue.onEvent(event); - - try (Repository repo = repoManager.openRepository(project); - Repository sourceRepo = repoManager.openRepository(project)) { - waitUntil(() -> checkedGetRef(repo, newBranch) != null); - - Ref targetBranchRef = getRef(repo, newBranch); - assertThat(targetBranchRef).isNotNull(); - assertThat(targetBranchRef.getObjectId().getName()).isEqualTo(branchRevision); - } - } - - @Test - @UseLocalDisk - @GerritConfig(name = "gerrit.instanceId", value = TEST_REPLICATION_REMOTE) - public void shouldReplicateForceUpdatedBranch() throws Exception { - boolean forcedPush = true; - String testProjectName = project + TEST_REPLICATION_SUFFIX; - NameKey testProjectNameKey = createTestProject(testProjectName); - - String newBranch = "refs/heads/mybranch"; - String master = "refs/heads/master"; - BranchInput input = new BranchInput(); - input.revision = master; - gApi.projects().name(testProjectName).branch(newBranch).create(input); - - projectOperations - .project(testProjectNameKey) - .forUpdate() - .add(allow(Permission.PUSH).ref(newBranch).group(REGISTERED_USERS).force(true)) - .update(); - - String branchRevision = gApi.projects().name(testProjectName).branch(newBranch).get().revision; - - ReplicationQueue pullReplicationQueue = - plugin.getSysInjector().getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( - project, newBranch, ObjectId.zeroId().getName(), branchRevision); - pullReplicationQueue.onEvent(event); - - try (Repository repo = repoManager.openRepository(project)) { - waitUntil(() -> checkedGetRef(repo, newBranch) != null); - - Ref targetBranchRef = getRef(repo, newBranch); - assertThat(targetBranchRef).isNotNull(); - assertThat(targetBranchRef.getObjectId().getName()).isEqualTo(branchRevision); - } - - TestRepository<InMemoryRepository> testProject = cloneProject(testProjectNameKey); - fetch(testProject, RefNames.REFS_HEADS + "*:" + RefNames.REFS_HEADS + "*"); - RevCommit amendedCommit = testProject.amendRef(newBranch).message("Amended commit").create(); - PushResult pushResult = - pushOne(testProject, newBranch, newBranch, false, forcedPush, Collections.emptyList()); - Collection<RemoteRefUpdate> pushedRefs = pushResult.getRemoteUpdates(); - assertThat(pushedRefs).hasSize(1); - assertThat(pushedRefs.iterator().next().getStatus()).isEqualTo(Status.OK); - - FakeGitReferenceUpdatedEvent forcedPushEvent = - new FakeGitReferenceUpdatedEvent( - project, newBranch, branchRevision, amendedCommit.getId().getName()); - pullReplicationQueue.onEvent(forcedPushEvent); - - try (Repository repo = repoManager.openRepository(project); - Repository sourceRepo = repoManager.openRepository(project)) { - waitUntil( - () -> - checkedGetRef(repo, newBranch) != null - && checkedGetRef(repo, newBranch) - .getObjectId() - .getName() - .equals(amendedCommit.getId().getName())); - } - } - - @Test - @GerritConfig(name = "gerrit.instanceId", value = TEST_REPLICATION_REMOTE) - public void shouldReplicateNewChangeRefCGitClient() throws Exception { - AutoReloadConfigDecorator autoReloadConfigDecorator = - getInstance(AutoReloadConfigDecorator.class); - - config.setBoolean("replication", null, "useCGitClient", true); - config.save(); - - autoReloadConfigDecorator.reload(); - - testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX)); - - Result pushResult = createChange(); - RevCommit sourceCommit = pushResult.getCommit(); - String sourceRef = pushResult.getPatchSet().refName(); - - ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( - project, sourceRef, ObjectId.zeroId().getName(), sourceCommit.getId().getName()); - pullReplicationQueue.onEvent(event); - - try (Repository repo = repoManager.openRepository(project)) { - waitUntil(() -> checkedGetRef(repo, sourceRef) != null); - - Ref targetBranchRef = getRef(repo, sourceRef); - assertThat(targetBranchRef).isNotNull(); - assertThat(targetBranchRef.getObjectId()).isEqualTo(sourceCommit.getId()); - } - } - - @Test - @GerritConfig(name = "gerrit.instanceId", value = TEST_REPLICATION_REMOTE) - public void shouldReplicateNewBranchCGitClient() throws Exception { - AutoReloadConfigDecorator autoReloadConfigDecorator = - getInstance(AutoReloadConfigDecorator.class); - - config.setBoolean("replication", null, "useCGitClient", true); - config.save(); - - autoReloadConfigDecorator.reload(); - - String testProjectName = project + TEST_REPLICATION_SUFFIX; - createTestProject(testProjectName); - - String newBranch = "refs/heads/mybranch"; - String master = "refs/heads/master"; - BranchInput input = new BranchInput(); - input.revision = master; - gApi.projects().name(testProjectName).branch(newBranch).create(input); - String branchRevision = gApi.projects().name(testProjectName).branch(newBranch).get().revision; - - ReplicationQueue pullReplicationQueue = - plugin.getSysInjector().getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( - project, newBranch, ObjectId.zeroId().getName(), branchRevision); - pullReplicationQueue.onEvent(event); - - try (Repository repo = repoManager.openRepository(project); - Repository sourceRepo = repoManager.openRepository(project)) { - waitUntil(() -> checkedGetRef(repo, newBranch) != null); - - Ref targetBranchRef = getRef(repo, newBranch); - assertThat(targetBranchRef).isNotNull(); - assertThat(targetBranchRef.getObjectId().getName()).isEqualTo(branchRevision); - } - } - - @Test - @GerritConfig(name = "gerrit.instanceId", value = TEST_REPLICATION_REMOTE) - public void shouldReplicateProjectDeletion() throws Exception { - String projectToDelete = project.get(); - setReplicationSource(TEST_REPLICATION_REMOTE, "", Optional.of(projectToDelete)); - config.save(); - AutoReloadConfigDecorator autoReloadConfigDecorator = - getInstance(AutoReloadConfigDecorator.class); - autoReloadConfigDecorator.reload(); - - ProjectDeletedListener.Event event = - new ProjectDeletedListener.Event() { - @Override - public String getProjectName() { - return projectToDelete; - } - - @Override - public NotifyHandling getNotify() { - return NotifyHandling.NONE; - } - }; - for (ProjectDeletedListener l : deletedListeners) { - l.onProjectDeleted(event); - } - - waitUntil(() -> !repoManager.list().contains(project)); - } - - @Test - @GerritConfig(name = "gerrit.instanceId", value = TEST_REPLICATION_REMOTE) - public void shouldReplicateHeadUpdate() throws Exception { - String testProjectName = project.get(); - setReplicationSource(TEST_REPLICATION_REMOTE, "", Optional.of(testProjectName)); - config.save(); - AutoReloadConfigDecorator autoReloadConfigDecorator = - getInstance(AutoReloadConfigDecorator.class); - autoReloadConfigDecorator.reload(); - - String newBranch = "refs/heads/mybranch"; - String master = "refs/heads/master"; - BranchInput input = new BranchInput(); - input.revision = master; - gApi.projects().name(testProjectName).branch(newBranch).create(input); - String branchRevision = gApi.projects().name(testProjectName).branch(newBranch).get().revision; - - ReplicationQueue pullReplicationQueue = - plugin.getSysInjector().getInstance(ReplicationQueue.class); - - HeadUpdatedListener.Event event = new FakeHeadUpdateEvent(master, newBranch, testProjectName); - pullReplicationQueue.onHeadUpdated(event); - - waitUntil( - () -> { - try { - return gApi.projects().name(testProjectName).head().equals(newBranch); - } catch (RestApiException e) { - return false; - } - }); - } - - @Test - @GerritConfig(name = "gerrit.instanceId", value = TEST_REPLICATION_REMOTE) - @GerritConfig(name = "container.replica", value = "true") - public void shouldReplicateNewChangeRefToReplica() throws Exception { - testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX)); - - Result pushResult = createChange(); - RevCommit sourceCommit = pushResult.getCommit(); - String sourceRef = pushResult.getPatchSet().refName(); - - ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( - project, sourceRef, ObjectId.zeroId().getName(), sourceCommit.getId().getName()); - pullReplicationQueue.onEvent(event); - - try (Repository repo = repoManager.openRepository(project)) { - waitUntil(() -> checkedGetRef(repo, sourceRef) != null); - - Ref targetBranchRef = getRef(repo, sourceRef); - assertThat(targetBranchRef).isNotNull(); - assertThat(targetBranchRef.getObjectId()).isEqualTo(sourceCommit.getId()); - } - } -} +public class PullReplicationIT extends PullReplicationITAbstract {}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationITAbstract.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationITAbstract.java new file mode 100644 index 0000000..803a756 --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationITAbstract.java
@@ -0,0 +1,376 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.gerrit.acceptance.GitUtil.fetch; +import static com.google.gerrit.acceptance.GitUtil.pushOne; +import static com.google.gerrit.acceptance.testsuite.project.TestProjectUpdate.allow; +import static com.google.gerrit.server.group.SystemGroupBackend.REGISTERED_USERS; + +import com.google.gerrit.acceptance.PushOneCommit.Result; +import com.google.gerrit.acceptance.UseLocalDisk; +import com.google.gerrit.acceptance.config.GerritConfig; +import com.google.gerrit.entities.Permission; +import com.google.gerrit.entities.Project.NameKey; +import com.google.gerrit.entities.RefNames; +import com.google.gerrit.extensions.api.changes.NotifyHandling; +import com.google.gerrit.extensions.api.projects.BranchInput; +import com.google.gerrit.extensions.events.HeadUpdatedListener; +import com.google.gerrit.extensions.events.ProjectDeletedListener; +import com.google.gerrit.extensions.restapi.RestApiException; +import com.googlesource.gerrit.plugins.replication.AutoReloadConfigDecorator; +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.transport.PushResult; +import org.eclipse.jgit.transport.RemoteRefUpdate; +import org.eclipse.jgit.transport.RemoteRefUpdate.Status; +import org.junit.Test; + +/** Base class to run regular and async acceptance tests */ +public abstract class PullReplicationITAbstract extends PullReplicationSetupBase { + + @Override + protected void setReplicationSource( + String remoteName, List<String> replicaSuffixes, Optional<String> project) + throws IOException { + List<String> fetchUrls = + buildReplicaURLs(replicaSuffixes, s -> gitPath.resolve("${name}" + s + ".git").toString()); + config.setStringList("remote", remoteName, "url", fetchUrls); + config.setString("remote", remoteName, "apiUrl", adminRestSession.url()); + config.setString("remote", remoteName, "fetch", "+refs/*:refs/*"); + config.setInt("remote", remoteName, "timeout", 600); + config.setInt("remote", remoteName, "replicationDelay", TEST_REPLICATION_DELAY); + project.ifPresent(prj -> config.setString("remote", remoteName, "projects", prj)); + config.setBoolean("gerrit", null, "autoReload", true); + config.save(); + } + + @Override + public void setUpTestPlugin() throws Exception { + setUpTestPlugin(false); + } + + @Test + @GerritConfig(name = "gerrit.instanceId", value = TEST_REPLICATION_REMOTE) + public void shouldReplicateNewChangeRef() throws Exception { + testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX)); + + Result pushResult = createChange(); + RevCommit sourceCommit = pushResult.getCommit(); + String sourceRef = pushResult.getPatchSet().refName(); + + ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); + FakeGitReferenceUpdatedEvent event = + new FakeGitReferenceUpdatedEvent( + project, + sourceRef, + ObjectId.zeroId().getName(), + sourceCommit.getId().getName(), + TEST_REPLICATION_REMOTE); + pullReplicationQueue.onEvent(event); + + try (Repository repo = repoManager.openRepository(project)) { + waitUntil(() -> checkedGetRef(repo, sourceRef) != null); + + Ref targetBranchRef = getRef(repo, sourceRef); + assertThat(targetBranchRef).isNotNull(); + assertThat(targetBranchRef.getObjectId()).isEqualTo(sourceCommit.getId()); + } + } + + @Test + @GerritConfig(name = "gerrit.instanceId", value = TEST_REPLICATION_REMOTE) + public void shouldReplicateNewBranch() throws Exception { + String testProjectName = project + TEST_REPLICATION_SUFFIX; + createTestProject(testProjectName); + + String newBranch = "refs/heads/mybranch"; + String master = "refs/heads/master"; + BranchInput input = new BranchInput(); + input.revision = master; + gApi.projects().name(testProjectName).branch(newBranch).create(input); + String branchRevision = gApi.projects().name(testProjectName).branch(newBranch).get().revision; + + ReplicationQueue pullReplicationQueue = + plugin.getSysInjector().getInstance(ReplicationQueue.class); + FakeGitReferenceUpdatedEvent event = + new FakeGitReferenceUpdatedEvent( + project, + newBranch, + ObjectId.zeroId().getName(), + branchRevision, + TEST_REPLICATION_REMOTE); + pullReplicationQueue.onEvent(event); + + try (Repository repo = repoManager.openRepository(project); + Repository sourceRepo = repoManager.openRepository(project)) { + waitUntil(() -> checkedGetRef(repo, newBranch) != null); + + Ref targetBranchRef = getRef(repo, newBranch); + assertThat(targetBranchRef).isNotNull(); + assertThat(targetBranchRef.getObjectId().getName()).isEqualTo(branchRevision); + } + } + + @Test + @UseLocalDisk + @GerritConfig(name = "gerrit.instanceId", value = TEST_REPLICATION_REMOTE) + public void shouldReplicateForceUpdatedBranch() throws Exception { + boolean forcedPush = true; + String testProjectName = project + TEST_REPLICATION_SUFFIX; + NameKey testProjectNameKey = createTestProject(testProjectName); + + String newBranch = "refs/heads/mybranch"; + String master = "refs/heads/master"; + BranchInput input = new BranchInput(); + input.revision = master; + gApi.projects().name(testProjectName).branch(newBranch).create(input); + + projectOperations + .project(testProjectNameKey) + .forUpdate() + .add(allow(Permission.PUSH).ref(newBranch).group(REGISTERED_USERS).force(true)) + .update(); + + String branchRevision = gApi.projects().name(testProjectName).branch(newBranch).get().revision; + + ReplicationQueue pullReplicationQueue = + plugin.getSysInjector().getInstance(ReplicationQueue.class); + FakeGitReferenceUpdatedEvent event = + new FakeGitReferenceUpdatedEvent( + project, + newBranch, + ObjectId.zeroId().getName(), + branchRevision, + TEST_REPLICATION_REMOTE); + pullReplicationQueue.onEvent(event); + + try (Repository repo = repoManager.openRepository(project)) { + waitUntil(() -> checkedGetRef(repo, newBranch) != null); + + Ref targetBranchRef = getRef(repo, newBranch); + assertThat(targetBranchRef).isNotNull(); + assertThat(targetBranchRef.getObjectId().getName()).isEqualTo(branchRevision); + } + + TestRepository<InMemoryRepository> testProject = cloneProject(testProjectNameKey); + fetch(testProject, RefNames.REFS_HEADS + "*:" + RefNames.REFS_HEADS + "*"); + RevCommit amendedCommit = testProject.amendRef(newBranch).message("Amended commit").create(); + PushResult pushResult = + pushOne(testProject, newBranch, newBranch, false, forcedPush, Collections.emptyList()); + Collection<RemoteRefUpdate> pushedRefs = pushResult.getRemoteUpdates(); + assertThat(pushedRefs).hasSize(1); + assertThat(pushedRefs.iterator().next().getStatus()).isEqualTo(Status.OK); + + FakeGitReferenceUpdatedEvent forcedPushEvent = + new FakeGitReferenceUpdatedEvent( + project, + newBranch, + branchRevision, + amendedCommit.getId().getName(), + TEST_REPLICATION_REMOTE); + pullReplicationQueue.onEvent(forcedPushEvent); + + try (Repository repo = repoManager.openRepository(project); + Repository sourceRepo = repoManager.openRepository(project)) { + waitUntil( + () -> + checkedGetRef(repo, newBranch) != null + && checkedGetRef(repo, newBranch) + .getObjectId() + .getName() + .equals(amendedCommit.getId().getName())); + } + } + + @Test + @GerritConfig(name = "gerrit.instanceId", value = TEST_REPLICATION_REMOTE) + public void shouldReplicateNewChangeRefCGitClient() throws Exception { + AutoReloadConfigDecorator autoReloadConfigDecorator = + getInstance(AutoReloadConfigDecorator.class); + + config.setBoolean("replication", null, "useCGitClient", true); + config.save(); + + autoReloadConfigDecorator.reload(); + + testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX)); + + Result pushResult = createChange(); + RevCommit sourceCommit = pushResult.getCommit(); + String sourceRef = pushResult.getPatchSet().refName(); + + ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); + FakeGitReferenceUpdatedEvent event = + new FakeGitReferenceUpdatedEvent( + project, + sourceRef, + ObjectId.zeroId().getName(), + sourceCommit.getId().getName(), + TEST_REPLICATION_REMOTE); + pullReplicationQueue.onEvent(event); + + try (Repository repo = repoManager.openRepository(project)) { + waitUntil(() -> checkedGetRef(repo, sourceRef) != null); + + Ref targetBranchRef = getRef(repo, sourceRef); + assertThat(targetBranchRef).isNotNull(); + assertThat(targetBranchRef.getObjectId()).isEqualTo(sourceCommit.getId()); + } + } + + @Test + @GerritConfig(name = "gerrit.instanceId", value = TEST_REPLICATION_REMOTE) + public void shouldReplicateNewBranchCGitClient() throws Exception { + AutoReloadConfigDecorator autoReloadConfigDecorator = + getInstance(AutoReloadConfigDecorator.class); + + config.setBoolean("replication", null, "useCGitClient", true); + config.save(); + + autoReloadConfigDecorator.reload(); + + String testProjectName = project + TEST_REPLICATION_SUFFIX; + createTestProject(testProjectName); + + String newBranch = "refs/heads/mybranch"; + String master = "refs/heads/master"; + BranchInput input = new BranchInput(); + input.revision = master; + gApi.projects().name(testProjectName).branch(newBranch).create(input); + String branchRevision = gApi.projects().name(testProjectName).branch(newBranch).get().revision; + + ReplicationQueue pullReplicationQueue = + plugin.getSysInjector().getInstance(ReplicationQueue.class); + FakeGitReferenceUpdatedEvent event = + new FakeGitReferenceUpdatedEvent( + project, + newBranch, + ObjectId.zeroId().getName(), + branchRevision, + TEST_REPLICATION_REMOTE); + pullReplicationQueue.onEvent(event); + + try (Repository repo = repoManager.openRepository(project); + Repository sourceRepo = repoManager.openRepository(project)) { + waitUntil(() -> checkedGetRef(repo, newBranch) != null); + + Ref targetBranchRef = getRef(repo, newBranch); + assertThat(targetBranchRef).isNotNull(); + assertThat(targetBranchRef.getObjectId().getName()).isEqualTo(branchRevision); + } + } + + @Test + @GerritConfig(name = "gerrit.instanceId", value = TEST_REPLICATION_REMOTE) + public void shouldReplicateProjectDeletion() throws Exception { + String projectToDelete = project.get(); + setReplicationSource(TEST_REPLICATION_REMOTE, "", Optional.of(projectToDelete)); + config.save(); + AutoReloadConfigDecorator autoReloadConfigDecorator = + getInstance(AutoReloadConfigDecorator.class); + autoReloadConfigDecorator.reload(); + + ProjectDeletedListener.Event event = + new ProjectDeletedListener.Event() { + @Override + public String getProjectName() { + return projectToDelete; + } + + @Override + public NotifyHandling getNotify() { + return NotifyHandling.NONE; + } + }; + for (ProjectDeletedListener l : deletedListeners) { + l.onProjectDeleted(event); + } + + waitUntil(() -> !repoManager.list().contains(project)); + } + + @Test + @GerritConfig(name = "gerrit.instanceId", value = TEST_REPLICATION_REMOTE) + public void shouldReplicateHeadUpdate() throws Exception { + String testProjectName = project.get(); + setReplicationSource(TEST_REPLICATION_REMOTE, "", Optional.of(testProjectName)); + config.save(); + AutoReloadConfigDecorator autoReloadConfigDecorator = + getInstance(AutoReloadConfigDecorator.class); + autoReloadConfigDecorator.reload(); + + String newBranch = "refs/heads/mybranch"; + String master = "refs/heads/master"; + BranchInput input = new BranchInput(); + input.revision = master; + gApi.projects().name(testProjectName).branch(newBranch).create(input); + String branchRevision = gApi.projects().name(testProjectName).branch(newBranch).get().revision; + + ReplicationQueue pullReplicationQueue = + plugin.getSysInjector().getInstance(ReplicationQueue.class); + + HeadUpdatedListener.Event event = new FakeHeadUpdateEvent(master, newBranch, testProjectName); + pullReplicationQueue.onHeadUpdated(event); + + waitUntil( + () -> { + try { + return gApi.projects().name(testProjectName).head().equals(newBranch); + } catch (RestApiException e) { + return false; + } + }); + } + + @Test + @GerritConfig(name = "gerrit.instanceId", value = TEST_REPLICATION_REMOTE) + @GerritConfig(name = "container.replica", value = "true") + public void shouldReplicateNewChangeRefToReplica() throws Exception { + testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX)); + + Result pushResult = createChange(); + RevCommit sourceCommit = pushResult.getCommit(); + String sourceRef = pushResult.getPatchSet().refName(); + + ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); + FakeGitReferenceUpdatedEvent event = + new FakeGitReferenceUpdatedEvent( + project, + sourceRef, + ObjectId.zeroId().getName(), + sourceCommit.getId().getName(), + TEST_REPLICATION_REMOTE); + pullReplicationQueue.onEvent(event); + + try (Repository repo = repoManager.openRepository(project)) { + waitUntil(() -> checkedGetRef(repo, sourceRef) != null); + + Ref targetBranchRef = getRef(repo, sourceRef); + assertThat(targetBranchRef).isNotNull(); + assertThat(targetBranchRef.getObjectId()).isEqualTo(sourceCommit.getId()); + } + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolIT.java index 40a9d03..e55e383 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolIT.java
@@ -82,7 +82,11 @@ ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); FakeGitReferenceUpdatedEvent event = new FakeGitReferenceUpdatedEvent( - project, sourceRef, ObjectId.zeroId().getName(), sourceCommit.getId().getName()); + project, + sourceRef, + ObjectId.zeroId().getName(), + sourceCommit.getId().getName(), + TEST_REPLICATION_REMOTE); pullReplicationQueue.onEvent(event); try (Repository repo = repoManager.openRepository(project)) {
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java index dc6e429..d65322f 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
@@ -17,6 +17,7 @@ import static com.google.common.truth.Truth.assertThat; import static java.nio.file.Files.createTempDirectory; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyInt; import static org.mockito.Mockito.anyString; @@ -51,6 +52,7 @@ import com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient; import com.googlesource.gerrit.plugins.replication.pull.client.FetchRestApiClient; import com.googlesource.gerrit.plugins.replication.pull.client.HttpResult; +import com.googlesource.gerrit.plugins.replication.pull.filter.ApplyObjectsRefsFilter; import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter; import java.io.IOException; import java.nio.file.Path; @@ -73,6 +75,9 @@ @RunWith(MockitoJUnitRunner.class) public class ReplicationQueueTest { private static int CONNECTION_TIMEOUT = 1000000; + private static final String LOCAL_INSTANCE_ID = "local instance id"; + private static final String FOREIGN_INSTANCE_ID = "any other instance id"; + private static final String TEST_REF_NAME = "refs/meta/heads/anyref"; @Mock private WorkQueue wq; @Mock private Source source; @@ -90,6 +95,7 @@ @Mock RevisionData revisionDataWithParents; List<ObjectId> revisionDataParentObjectIds; @Mock HttpResult httpResult; + @Mock ApplyObjectsRefsFilter applyObjectsRefsFilter; ApplyObjectMetrics applyObjectMetrics; FetchReplicationMetrics fetchMetrics; @@ -134,10 +140,12 @@ when(fetchClientFactory.create(any())).thenReturn(fetchRestApiClient); lenient() - .when(fetchRestApiClient.callSendObject(any(), anyString(), anyBoolean(), any(), any())) + .when( + fetchRestApiClient.callSendObject( + any(), anyString(), anyLong(), anyBoolean(), any(), any())) .thenReturn(httpResult); lenient() - .when(fetchRestApiClient.callSendObjects(any(), anyString(), any(), any())) + .when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any())) .thenReturn(httpResult); when(fetchRestApiClient.callFetch(any(), anyString(), any())).thenReturn(fetchHttpResult); when(fetchRestApiClient.initProject(any(), any())).thenReturn(successfulHttpResult); @@ -145,6 +153,7 @@ when(httpResult.isSuccessful()).thenReturn(true); when(fetchHttpResult.isSuccessful()).thenReturn(true); when(httpResult.isProjectMissing(any())).thenReturn(false); + when(applyObjectsRefsFilter.match(any())).thenReturn(false); applyObjectMetrics = new ApplyObjectMetrics("pull-replication", new DisabledMetricMaker()); fetchMetrics = new FetchReplicationMetrics("pull-replication", new DisabledMetricMaker()); @@ -159,7 +168,9 @@ refsFilter, () -> revReader, applyObjectMetrics, - fetchMetrics); + fetchMetrics, + LOCAL_INSTANCE_ID, + applyObjectsRefsFilter); } @Test @@ -168,7 +179,19 @@ objectUnderTest.start(); objectUnderTest.onEvent(event); - verify(fetchRestApiClient).callSendObjects(any(), anyString(), any(), any()); + verify(fetchRestApiClient).callSendObjects(any(), anyString(), anyLong(), any(), any()); + } + + @Test + public void shouldIgnoreEventWhenIsNotLocalInstanceId() + throws ClientProtocolException, IOException { + Event event = new TestEvent(); + event.instanceId = FOREIGN_INSTANCE_ID; + objectUnderTest.start(); + objectUnderTest.onEvent(event); + + verify(fetchRestApiClient, never()) + .callSendObjects(any(), anyString(), anyLong(), any(), any()); } @Test @@ -203,7 +226,7 @@ objectUnderTest.start(); objectUnderTest.onEvent(event); - verify(fetchRestApiClient).callSendObjects(any(), anyString(), any(), any()); + verify(fetchRestApiClient).callSendObjects(any(), anyString(), anyLong(), any(), any()); } @Test @@ -240,7 +263,7 @@ when(httpResult.isSuccessful()).thenReturn(false); when(httpResult.isParentObjectMissing()).thenReturn(true); - when(fetchRestApiClient.callSendObjects(any(), anyString(), any(), any())) + when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any())) .thenReturn(httpResult); objectUnderTest.onEvent(event); @@ -256,13 +279,41 @@ when(httpResult.isSuccessful()).thenReturn(false, true); when(httpResult.isParentObjectMissing()).thenReturn(true, false); - when(fetchRestApiClient.callSendObjects(any(), anyString(), any(), any())) + when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any())) .thenReturn(httpResult); objectUnderTest.onEvent(event); verify(fetchRestApiClient, times(2)) - .callSendObjects(any(), anyString(), revisionsDataCaptor.capture(), any()); + .callSendObjects(any(), anyString(), anyLong(), revisionsDataCaptor.capture(), any()); + List<List<RevisionData>> revisionsDataValues = revisionsDataCaptor.getAllValues(); + assertThat(revisionsDataValues).hasSize(2); + + List<RevisionData> firstRevisionsValues = revisionsDataValues.get(0); + assertThat(firstRevisionsValues).hasSize(1); + assertThat(firstRevisionsValues).contains(revisionData); + + List<RevisionData> secondRevisionsValues = revisionsDataValues.get(1); + assertThat(secondRevisionsValues).hasSize(1 + revisionDataParentObjectIds.size()); + } + + @Test + public void shouldFallbackToApplyAllParentObjectsWhenParentObjectIsMissingOnAllowedRefs() + throws ClientProtocolException, IOException { + String refName = "refs/tags/test-tag"; + Event event = new TestEvent(refName); + objectUnderTest.start(); + + when(httpResult.isSuccessful()).thenReturn(false, true); + when(httpResult.isParentObjectMissing()).thenReturn(true, false); + when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any())) + .thenReturn(httpResult); + when(applyObjectsRefsFilter.match(refName)).thenReturn(true); + + objectUnderTest.onEvent(event); + + verify(fetchRestApiClient, times(2)) + .callSendObjects(any(), anyString(), anyLong(), revisionsDataCaptor.capture(), any()); List<List<RevisionData>> revisionsDataValues = revisionsDataCaptor.getAllValues(); assertThat(revisionsDataValues).hasSize(2); @@ -293,7 +344,9 @@ refsFilter, () -> revReader, applyObjectMetrics, - fetchMetrics); + fetchMetrics, + LOCAL_INSTANCE_ID, + applyObjectsRefsFilter); Event event = new TestEvent("refs/multi-site/version"); objectUnderTest.onEvent(event); @@ -369,6 +422,11 @@ } private class TestEvent extends RefUpdatedEvent { + + public TestEvent() { + this(TEST_REF_NAME); + } + public TestEvent(String refName) { this( refName, @@ -383,6 +441,7 @@ upd.project = projectName; upd.refName = refName; this.refUpdate = Suppliers.ofInstance(upd); + this.instanceId = LOCAL_INSTANCE_ID; } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java index 1d8520f..fcd9b19 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java
@@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.truth.Truth8; import com.google.gerrit.acceptance.LightweightPluginDaemonTest; +import com.google.gerrit.acceptance.PushOneCommit; import com.google.gerrit.acceptance.PushOneCommit.Result; import com.google.gerrit.acceptance.TestPlugin; import com.google.gerrit.acceptance.UseLocalDisk; @@ -199,6 +200,39 @@ Truth8.assertThat(revisionDataOption).isEmpty(); } + @Test + public void shouldFilterOutGitSubmoduleCommitsWhenReadingTheBlobs() throws Exception { + String submodulePath = "submodule_path"; + final ObjectId GitSubmoduleCommit = + ObjectId.fromString("93e2901bc0b4719ef6081ee6353b49c9cdd97614"); + + PushOneCommit push = + pushFactory + .create(admin.newIdent(), testRepo) + .addGitSubmodule(submodulePath, GitSubmoduleCommit); + PushOneCommit.Result pushResult = push.to("refs/for/master"); + pushResult.assertOkStatus(); + Change.Id changeId = pushResult.getChange().getId(); + String refName = RefNames.patchSetRef(pushResult.getPatchSetId()); + + CommentInput comment = createCommentInput(1, 0, 1, 1, "Test comment"); + + ReviewInput reviewInput = new ReviewInput(); + reviewInput.comments = ImmutableMap.of(Patch.COMMIT_MSG, ImmutableList.of(comment)); + gApi.changes().id(changeId.get()).current().review(reviewInput); + + Optional<RevisionData> revisionDataOption = + refObjectId(refName).flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId, 0)); + + assertThat(revisionDataOption.isPresent()).isTrue(); + RevisionData revisionData = revisionDataOption.get(); + + assertThat(revisionData.getBlobs()).hasSize(1); + RevisionObjectData blobObject = revisionData.getBlobs().get(0); + assertThat(blobObject.getType()).isEqualTo(Constants.OBJ_BLOB); + assertThat(blobObject.getSha1()).isNotEqualTo(GitSubmoduleCommit.getName()); + } + private CommentInput createCommentInput( int startLine, int startCharacter, int endLine, int endCharacter, String message) { CommentInput comment = new CommentInput();
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionIT.java index 2ab5caf..453a6b0 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionIT.java
@@ -27,11 +27,12 @@ public class ApplyObjectActionIT extends ActionITBase { @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") public void shouldAcceptPayloadWithAsyncField() throws Exception { String payloadWithAsyncFieldTemplate = "{\"label\":\"" + TEST_REPLICATION_REMOTE - + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}, \"async\":true}"; + + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"sha1\":\"%s\",\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}, \"async\":true}"; String refName = createRef(); Optional<RevisionData> revisionDataOption = createRevisionData(refName); @@ -48,11 +49,12 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") public void shouldAcceptPayloadWithoutAsyncField() throws Exception { String payloadWithoutAsyncFieldTemplate = "{\"label\":\"" + TEST_REPLICATION_REMOTE - + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}}"; + + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"sha1\":\"%s\",\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}}"; String refName = createRef(); Optional<RevisionData> revisionDataOption = createRevisionData(refName); @@ -70,12 +72,13 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") public void shouldAcceptPayloadWhenNodeIsAReplica() throws Exception { String payloadWithoutAsyncFieldTemplate = "{\"label\":\"" + TEST_REPLICATION_REMOTE - + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}}"; + + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"sha1\":\"%s\",\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}}"; String refName = createRef(); Optional<RevisionData> revisionDataOption = createRevisionData(refName); @@ -93,12 +96,13 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") public void shouldAcceptPayloadWhenNodeIsAReplicaAndProjectNameContainsSlash() throws Exception { String payloadWithoutAsyncFieldTemplate = "{\"label\":\"" + TEST_REPLICATION_REMOTE - + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}}"; + + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"sha1\":\"%s\",\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}}"; NameKey projectName = Project.nameKey("test/repo"); String refName = createRef(projectName); Optional<RevisionData> revisionDataOption = createRevisionData(projectName, refName); @@ -119,12 +123,13 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") public void shouldReturnForbiddenWhenNodeIsAReplicaAndUSerIsAnonymous() throws Exception { String payloadWithoutAsyncFieldTemplate = "{\"label\":\"" + TEST_REPLICATION_REMOTE - + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}}"; + + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"sha1\":\"%s\",\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}}"; String refName = createRef(); Optional<RevisionData> revisionDataOption = createRevisionData(refName); @@ -140,9 +145,10 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") public void shouldReturnBadRequestCodeWhenMandatoryFieldLabelIsMissing() throws Exception { String payloadWithoutLabelFieldTemplate = - "{\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}, \"async\":true}"; + "{\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"sha1\":\"%s\",\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}, \"async\":true}"; String refName = createRef(); Optional<RevisionData> revisionDataOption = createRevisionData(refName); @@ -160,11 +166,12 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") public void shouldReturnBadRequestCodeWhenPayloadIsNotAProperJSON() throws Exception { String wrongPayloadTemplate = "{\"label\":\"" + TEST_REPLICATION_REMOTE - + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}, \"async\":true,}"; + + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"sha1\":\"%s\",\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}, \"async\":true,}"; String refName = createRef(); Optional<RevisionData> revisionDataOption = createRevisionData(refName); @@ -181,6 +188,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") @GerritConfig(name = "auth.bearerToken", value = "some-bearer-token") public void shouldAcceptPayloadWhenNodeIsAReplicaWithBearerToken() throws Exception { @@ -188,7 +196,7 @@ String payloadWithoutAsyncFieldTemplate = "{\"label\":\"" + TEST_REPLICATION_REMOTE - + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}}"; + + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"sha1\":\"%s\",\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}}"; String refName = createRef(); Optional<RevisionData> revisionDataOption = createRevisionData(refName); @@ -206,6 +214,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "false") @GerritConfig(name = "auth.bearerToken", value = "some-bearer-token") public void shouldAcceptPayloadWhenNodeIsAPrimaryWithBearerToken() throws Exception { @@ -213,7 +222,7 @@ String payloadWithoutAsyncFieldTemplate = "{\"label\":\"" + TEST_REPLICATION_REMOTE - + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}}"; + + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"sha1\":\"%s\",\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}}"; String refName = createRef(); Optional<RevisionData> revisionDataOption = createRevisionData(refName); @@ -236,6 +245,7 @@ String.format( wrongPayloadTemplate, refName, + revisionData.getCommitObject().getSha1(), encode(revisionData.getCommitObject().getContent()), encode(revisionData.getTreeObject().getContent())); return sendObjectPayload;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java index 814ba76..6ef5b2a 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java
@@ -17,6 +17,7 @@ import static com.google.common.truth.Truth.assertThat; import static org.apache.http.HttpStatus.SC_CREATED; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.when; @@ -48,6 +49,9 @@ @RunWith(MockitoJUnitRunner.class) public class ApplyObjectActionTest { + + private static final long DUMMY_EVENT_TIMESTAMP = 1684875939; + ApplyObjectAction applyObjectAction; String label = "instance-2-label"; String url = "file:///gerrit-host/instance-1/git/${name}.git"; @@ -55,7 +59,6 @@ String refMetaName = "refs/meta/version"; String location = "http://gerrit-host/a/config/server/tasks/08d173e9"; int taskId = 1234; - private String sampleCommitObjectId = "9f8d52853089a3cf00c02ff7bd0817bd4353a95a"; private String sampleTreeObjectId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; private String sampleBlobObjectId = "b5d7bcf1d1c5b0f0726d10a16c8315f06f900bfb"; @@ -95,7 +98,8 @@ @Test public void shouldReturnCreatedResponseCode() throws RestApiException { - RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData()); + RevisionInput inputParams = + new RevisionInput(label, refName, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); Response<?> response = applyObjectAction.apply(projectResource, inputParams); @@ -109,6 +113,7 @@ new RevisionInput( label, refMetaName, + DUMMY_EVENT_TIMESTAMP, createSampleRevisionDataBlob( new RevisionObjectData(sampleBlobObjectId, Constants.OBJ_BLOB, blobData))); @@ -120,7 +125,8 @@ @SuppressWarnings("cast") @Test public void shouldReturnSourceUrlAndrefNameAsAResponseBody() throws Exception { - RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData()); + RevisionInput inputParams = + new RevisionInput(label, refName, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); Response<?> response = applyObjectAction.apply(projectResource, inputParams); assertThat((RevisionInput) response.value()).isEqualTo(inputParams); @@ -128,28 +134,32 @@ @Test(expected = BadRequestException.class) public void shouldThrowBadRequestExceptionWhenMissingLabel() throws Exception { - RevisionInput inputParams = new RevisionInput(null, refName, createSampleRevisionData()); + RevisionInput inputParams = + new RevisionInput(null, refName, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); applyObjectAction.apply(projectResource, inputParams); } @Test(expected = BadRequestException.class) public void shouldThrowBadRequestExceptionWhenEmptyLabel() throws Exception { - RevisionInput inputParams = new RevisionInput("", refName, createSampleRevisionData()); + RevisionInput inputParams = + new RevisionInput("", refName, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); applyObjectAction.apply(projectResource, inputParams); } @Test(expected = BadRequestException.class) public void shouldThrowBadRequestExceptionWhenMissingRefName() throws Exception { - RevisionInput inputParams = new RevisionInput(label, null, createSampleRevisionData()); + RevisionInput inputParams = + new RevisionInput(label, null, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); applyObjectAction.apply(projectResource, inputParams); } @Test(expected = BadRequestException.class) public void shouldThrowBadRequestExceptionWhenEmptyRefName() throws Exception { - RevisionInput inputParams = new RevisionInput(label, "", createSampleRevisionData()); + RevisionInput inputParams = + new RevisionInput(label, "", DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); applyObjectAction.apply(projectResource, inputParams); } @@ -161,7 +171,8 @@ RevisionObjectData treeData = new RevisionObjectData(sampleTreeObjectId, Constants.OBJ_TREE, new byte[] {}); RevisionInput inputParams = - new RevisionInput(label, refName, createSampleRevisionData(commitData, treeData)); + new RevisionInput( + label, refName, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData(commitData, treeData)); applyObjectAction.apply(projectResource, inputParams); } @@ -172,7 +183,8 @@ new RevisionObjectData( sampleCommitObjectId, Constants.OBJ_COMMIT, sampleCommitContent.getBytes()); RevisionInput inputParams = - new RevisionInput(label, refName, createSampleRevisionData(commitData, null)); + new RevisionInput( + label, refName, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData(commitData, null)); applyObjectAction.apply(projectResource, inputParams); } @@ -180,7 +192,8 @@ @Test(expected = AuthException.class) public void shouldThrowAuthExceptionWhenCallFetchActionCapabilityNotAssigned() throws RestApiException { - RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData()); + RevisionInput inputParams = + new RevisionInput(label, refName, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); when(preConditions.canCallFetchApi()).thenReturn(false); @@ -190,13 +203,14 @@ @Test(expected = ResourceConflictException.class) public void shouldThrowResourceConflictExceptionWhenMissingParentObject() throws RestApiException, IOException, RefUpdateException, MissingParentObjectException { - RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData()); + RevisionInput inputParams = + new RevisionInput(label, refName, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); doThrow( new MissingParentObjectException( Project.nameKey("test_projects"), refName, ObjectId.zeroId())) .when(applyObjectCommand) - .applyObject(any(), anyString(), any(), anyString()); + .applyObject(any(), anyString(), any(), anyString(), anyLong()); applyObjectAction.apply(projectResource, inputParams); }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommandTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommandTest.java index 9b5ef46..7f5a67c 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommandTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommandTest.java
@@ -20,6 +20,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.collect.Lists; import com.google.gerrit.entities.Project; import com.google.gerrit.entities.Project.NameKey; @@ -29,6 +31,7 @@ import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.server.permissions.PermissionBackendException; import com.googlesource.gerrit.plugins.replication.pull.ApplyObjectMetrics; +import com.googlesource.gerrit.plugins.replication.pull.ApplyObjectsCacheKey; import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent; import com.googlesource.gerrit.plugins.replication.pull.PullReplicationStateLogger; import com.googlesource.gerrit.plugins.replication.pull.Source; @@ -61,10 +64,13 @@ private static final NameKey TEST_PROJECT_NAME = Project.nameKey("test-project"); private static final String TEST_REMOTE_NAME = "test-remote-name"; private static URIish TEST_REMOTE_URI; + private static final long TEST_EVENT_TIMESTAMP = 1L; private String sampleCommitObjectId = "9f8d52853089a3cf00c02ff7bd0817bd4353a95a"; private String sampleTreeObjectId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; private String sampleBlobObjectId = "b5d7bcf1d1c5b0f0726d10a16c8315f06f900bfb"; + private String sampleCommitObjectId2 = "9f8d52853089a3cf00c02ff7bd0817bd4353a95b"; + private String sampleTreeObjectId2 = "4b825dc642cb6eb9a060e54bf8d69288fbee4905"; @Mock private PullReplicationStateLogger fetchStateLog; @Mock private ApplyObject applyObject; @@ -75,11 +81,13 @@ @Mock private SourcesCollection sourceCollection; @Mock private Source source; @Captor ArgumentCaptor<Event> eventCaptor; + private Cache<ApplyObjectsCacheKey, Long> cache; private ApplyObjectCommand objectUnderTest; @Before public void setup() throws MissingParentObjectException, IOException, URISyntaxException { + cache = CacheBuilder.newBuilder().build(); RefUpdateState state = new RefUpdateState(TEST_REMOTE_NAME, RefUpdate.Result.NEW); TEST_REMOTE_URI = new URIish("git://some.remote.uri"); when(eventDispatcherDataItem.get()).thenReturn(eventDispatcher); @@ -91,15 +99,21 @@ objectUnderTest = new ApplyObjectCommand( - fetchStateLog, applyObject, metrics, eventDispatcherDataItem, sourceCollection); + fetchStateLog, applyObject, metrics, eventDispatcherDataItem, sourceCollection, cache); } @Test public void shouldSendEventWhenApplyObject() throws PermissionBackendException, IOException, RefUpdateException, MissingParentObjectException { + RevisionData sampleRevisionData = + createSampleRevisionData(sampleCommitObjectId, sampleTreeObjectId); objectUnderTest.applyObject( - TEST_PROJECT_NAME, TEST_REF_NAME, createSampleRevisionData(), TEST_SOURCE_LABEL); + TEST_PROJECT_NAME, + TEST_REF_NAME, + sampleRevisionData, + TEST_SOURCE_LABEL, + TEST_EVENT_TIMESTAMP); verify(eventDispatcher).postEvent(eventCaptor.capture()); Event sentEvent = eventCaptor.getValue(); @@ -110,11 +124,64 @@ assertThat(fetchEvent.targetUri).isEqualTo(TEST_REMOTE_URI.toASCIIString()); } - private RevisionData createSampleRevisionData() { + @Test + public void shouldInsertIntoApplyObjectsCacheWhenApplyObjectIsSuccessful() + throws IOException, RefUpdateException, MissingParentObjectException { + RevisionData sampleRevisionData = + createSampleRevisionData(sampleCommitObjectId, sampleTreeObjectId); + RevisionData sampleRevisionData2 = + createSampleRevisionData(sampleCommitObjectId2, sampleTreeObjectId2); + objectUnderTest.applyObjects( + TEST_PROJECT_NAME, + TEST_REF_NAME, + new RevisionData[] {sampleRevisionData, sampleRevisionData2}, + TEST_SOURCE_LABEL, + TEST_EVENT_TIMESTAMP); + + assertThat( + cache.getIfPresent( + ApplyObjectsCacheKey.create( + sampleRevisionData.getCommitObject().getSha1(), + TEST_REF_NAME, + TEST_PROJECT_NAME.get()))) + .isEqualTo(TEST_EVENT_TIMESTAMP); + assertThat( + cache.getIfPresent( + ApplyObjectsCacheKey.create( + sampleRevisionData2.getCommitObject().getSha1(), + TEST_REF_NAME, + TEST_PROJECT_NAME.get()))) + .isEqualTo(TEST_EVENT_TIMESTAMP); + } + + @Test(expected = RefUpdateException.class) + public void shouldNotInsertIntoApplyObjectsCacheWhenApplyObjectIsFailure() + throws IOException, RefUpdateException, MissingParentObjectException { + RevisionData sampleRevisionData = + createSampleRevisionData(sampleCommitObjectId, sampleTreeObjectId); + RefUpdateState failureState = new RefUpdateState(TEST_REMOTE_NAME, RefUpdate.Result.IO_FAILURE); + when(applyObject.apply(any(), any(), any())).thenReturn(failureState); + objectUnderTest.applyObject( + TEST_PROJECT_NAME, + TEST_REF_NAME, + sampleRevisionData, + TEST_SOURCE_LABEL, + TEST_EVENT_TIMESTAMP); + + assertThat( + cache.getIfPresent( + ApplyObjectsCacheKey.create( + sampleRevisionData.getCommitObject().getSha1(), + TEST_REF_NAME, + TEST_PROJECT_NAME.get()))) + .isNull(); + } + + private RevisionData createSampleRevisionData(String commitObjectId, String treeObjectId) { RevisionObjectData commitData = - new RevisionObjectData(sampleCommitObjectId, Constants.OBJ_COMMIT, new byte[] {}); + new RevisionObjectData(commitObjectId, Constants.OBJ_COMMIT, new byte[] {}); RevisionObjectData treeData = - new RevisionObjectData(sampleTreeObjectId, Constants.OBJ_TREE, new byte[] {}); + new RevisionObjectData(treeObjectId, Constants.OBJ_TREE, new byte[] {}); return new RevisionData(Collections.emptyList(), commitData, treeData, Lists.newArrayList()); } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionIT.java index cc01c2a..0a69c3d 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionIT.java
@@ -23,6 +23,7 @@ public class FetchActionIT extends ActionITBase { @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") public void shouldFetchRefWhenNodeIsAReplica() throws Exception { String refName = createRef(); @@ -41,6 +42,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") public void shouldFetchRefWhenNodeIsAReplicaAndProjectNameContainsSlash() throws Exception { NameKey projectName = Project.nameKey("test/repo"); @@ -63,6 +65,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") public void shouldReturnForbiddenWhenNodeIsAReplicaAndUSerIsAnonymous() throws Exception { String refName = createRef(); @@ -79,6 +82,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") @GerritConfig(name = "auth.bearerToken", value = "some-bearer-token") public void shouldFetchRefWhenNodeIsAReplicaWithBearerToken() throws Exception { @@ -99,6 +103,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "false") @GerritConfig(name = "auth.bearerToken", value = "some-bearer-token") public void shouldFetchRefWhenNodeIsAPrimaryWithBearerToken() throws Exception {
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionActionIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionActionIT.java index a46a721..10415e4 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionActionIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionActionIT.java
@@ -33,6 +33,7 @@ @Inject private ProjectOperations projectOperations; @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") public void shouldReturnUnauthorizedForUserWithoutPermissions() throws Exception { httpClientFactory .create(source) @@ -41,6 +42,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") public void shouldDeleteRepositoryWhenUserHasProjectDeletionCapabilities() throws Exception { String testProjectName = project.get(); url = getURLWithAuthenticationPrefix(testProjectName); @@ -64,6 +66,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") public void shouldReturnOKWhenProjectIsDeleted() throws Exception { String testProjectName = project.get(); url = getURLWithAuthenticationPrefix(testProjectName); @@ -76,6 +79,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @Ignore("Failing in RestApiServlet: to be enabled again once that is fixed in core") public void shouldReturnBadRequestWhenDeletingAnInvalidProjectName() throws Exception { url = getURLWithAuthenticationPrefix(INVALID_TEST_PROJECT_NAME); @@ -88,6 +92,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") public void shouldReturnForbiddenForUserWithoutPermissionsOnReplica() throws Exception { httpClientFactory @@ -96,6 +101,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") public void shouldReturnOKWhenProjectIsDeletedOnReplica() throws Exception { String testProjectName = project.get(); @@ -109,6 +115,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") public void shouldDeleteRepositoryWhenUserHasProjectDeletionCapabilitiesAndNodeIsAReplica() throws Exception { @@ -132,6 +139,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") public void shouldReturnBadRequestWhenDeletingAnInvalidProjectNameWhenNodeIsAReplica() throws Exception { @@ -145,6 +153,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") @GerritConfig(name = "auth.bearerToken", value = "some-bearer-token") public void shouldReturnOKWhenProjectIsDeletedOnReplicaWithBearerToken() throws Exception { @@ -159,6 +168,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "false") @GerritConfig(name = "auth.bearerToken", value = "some-bearer-token") public void shouldReturnOKWhenProjectIsDeletedOnPrimaryWithBearerToken() throws Exception {
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationActionIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationActionIT.java index 47037d6..c543969 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationActionIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationActionIT.java
@@ -31,9 +31,11 @@ import org.junit.Test; public class ProjectInitializationActionIT extends ActionITBase { + public static final String INVALID_TEST_PROJECT_NAME = "\0"; @Inject private ProjectOperations projectOperations; @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") public void shouldReturnUnauthorizedForUserWithoutPermissions() throws Exception { httpClientFactory .create(source) @@ -43,6 +45,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") public void shouldReturnBadRequestIfContentNotSet() throws Exception { httpClientFactory .create(source) @@ -52,6 +55,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") public void shouldCreateRepository() throws Exception { String newProjectName = "new/newProjectForPrimary"; url = getURLWithAuthenticationPrefix(newProjectName); @@ -71,6 +75,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") public void shouldCreateRepositoryWhenUserHasProjectCreationCapabilities() throws Exception { String newProjectName = "new/newProjectForUserWithCapabilities"; url = getURLWithAuthenticationPrefix(newProjectName); @@ -93,6 +98,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") public void shouldReturnForbiddenIfUserNotAuthorized() throws Exception { httpClientFactory .create(source) @@ -102,6 +108,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") public void shouldCreateRepositoryWhenNodeIsAReplica() throws Exception { String newProjectName = "new/newProjectForReplica"; @@ -114,6 +121,8 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") public void shouldReturnForbiddenIfUserNotAuthorizedAndNodeIsAReplica() throws Exception { httpClientFactory @@ -124,6 +133,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") public void shouldCreateRepositoryWhenUserHasProjectCreationCapabilitiesAndNodeIsAReplica() throws Exception { @@ -148,6 +158,20 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") + @GerritConfig(name = "container.replica", value = "true") + public void shouldReturnBadRequestIfProjectNameIsInvalidAndCannotBeCreatedWhenNodeIsAReplica() + throws Exception { + url = getURLWithAuthenticationPrefix(INVALID_TEST_PROJECT_NAME); + httpClientFactory + .create(source) + .execute( + withBasicAuthenticationAsAdmin(createPutRequestWithHeaders()), + assertHttpResponseCode(HttpServletResponse.SC_BAD_REQUEST)); + } + + @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") public void shouldReturnBadRequestIfContentNotSetWhenNodeIsAReplica() throws Exception { httpClientFactory @@ -158,6 +182,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") public void shouldReturnForbiddenForUserWithoutPermissionsWhenNodeIsAReplica() throws Exception { httpClientFactory @@ -168,6 +193,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") @GerritConfig(name = "auth.bearerToken", value = "some-bearer-token") public void shouldCreateRepositoryWhenNodeIsAReplicaWithBearerToken() throws Exception { @@ -181,6 +207,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "false") @GerritConfig(name = "auth.bearerToken", value = "some-bearer-token") public void shouldCreateRepositoryWhenNodeIsAPrimaryWithBearerToken() throws Exception {
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/UpdateHeadActionIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/UpdateHeadActionIT.java index 7c725b3..f6e631f 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/UpdateHeadActionIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/UpdateHeadActionIT.java
@@ -36,6 +36,7 @@ @Inject private ProjectOperations projectOperations; @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") public void shouldReturnUnauthorizedForUserWithoutPermissions() throws Exception { httpClientFactory .create(source) @@ -45,6 +46,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") public void shouldReturnBadRequestWhenInputIsEmpty() throws Exception { httpClientFactory .create(source) @@ -54,6 +56,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") public void shouldReturnOKWhenHeadIsUpdated() throws Exception { String testProjectName = project.get(); String newBranch = "refs/heads/mybranch"; @@ -71,6 +74,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") public void shouldReturnBadRequestWhenInputIsEmptyInReplica() throws Exception { httpClientFactory @@ -81,6 +85,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") public void shouldReturnOKWhenHeadIsUpdatedInReplica() throws Exception { String testProjectName = project.get(); @@ -99,6 +104,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") public void shouldReturnForbiddenWhenMissingPermissions() throws Exception { httpClientFactory .create(source) @@ -108,6 +114,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") public void shouldReturnOKWhenRegisteredUserHasPermissions() throws Exception { String testProjectName = project.get(); String newBranch = "refs/heads/mybranch"; @@ -132,6 +139,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") public void shouldReturnForbiddenWhenMissingPermissionsInReplica() throws Exception { httpClientFactory @@ -142,6 +150,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "true") @GerritConfig(name = "auth.bearerToken", value = "some-bearer-token") @Ignore("Waiting for resolving: Issue 16332: Not able to update the HEAD from internal user") @@ -164,6 +173,7 @@ } @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") @GerritConfig(name = "container.replica", value = "false") @GerritConfig(name = "auth.bearerToken", value = "some-bearer-token") @Ignore("Waiting for resolving: Issue 16332: Not able to update the HEAD from internal user")
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java index a2389d7..cdb238e 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java
@@ -74,6 +74,7 @@ String pluginName = "pull-replication"; String instanceId = "Replication"; String refName = RefNames.REFS_HEADS + "master"; + long eventCreatedOn = 1684875939; String expectedPayload = "{\"label\":\"Replication\", \"ref_name\": \"" + refName + "\", \"async\":false}"; @@ -87,7 +88,9 @@ String blobObjectId = "bb383f5249c68a4cc8c82bdd1228b4a8883ff6e8"; String expectedSendObjectPayload = - "{\"label\":\"Replication\",\"ref_name\":\"refs/heads/master\",\"revision_data\":{\"commit_object\":{\"sha1\":\"" + "{\"label\":\"Replication\",\"ref_name\":\"refs/heads/master\",\"event_created_on\":" + + eventCreatedOn + + ",\"revision_data\":{\"commit_object\":{\"sha1\":\"" + commitObjectId + "\",\"type\":1,\"content\":\"dHJlZSA3NzgxNGQyMTZhNmNhYjJkZGI5ZjI4NzdmYmJkMGZlYmRjMGZhNjA4CnBhcmVudCA5ODNmZjFhM2NmNzQ3MjVhNTNhNWRlYzhkMGMwNjEyMjEyOGY1YThkCmF1dGhvciBHZXJyaXQgVXNlciAxMDAwMDAwIDwxMDAwMDAwQDY5ZWMzOGYwLTM1MGUtNGQ5Yy05NmQ0LWJjOTU2ZjJmYWFhYz4gMTYxMDU3ODY0OCArMDEwMApjb21taXR0ZXIgR2Vycml0IENvZGUgUmV2aWV3IDxyb290QG1hY3plY2gtWFBTLTE1PiAxNjEwNTc4NjQ4ICswMTAwCgpVcGRhdGUgcGF0Y2ggc2V0IDEKClBhdGNoIFNldCAxOgoKKDEgY29tbWVudCkKClBhdGNoLXNldDogMQo\\u003d\"},\"tree_object\":{\"sha1\":\"" + treeObjectId @@ -265,6 +268,7 @@ objectUnderTest.callSendObject( Project.nameKey("test_repo"), refName, + eventCreatedOn, IS_REF_UPDATE, createSampleRevisionData(), new URIish(api)); @@ -287,6 +291,7 @@ objectUnderTest.callSendObject( Project.nameKey("test_repo"), refName, + eventCreatedOn, IS_REF_UPDATE, createSampleRevisionData(), new URIish(api));
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java index 92314a4..8dc0359 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java
@@ -21,6 +21,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.collect.Lists; import com.google.gerrit.entities.Project; import com.google.gerrit.entities.RefNames; @@ -32,6 +34,7 @@ import com.google.gerrit.server.events.RefUpdatedEvent; import com.google.gerrit.server.git.WorkQueue; import com.google.gerrit.server.permissions.PermissionBackendException; +import com.googlesource.gerrit.plugins.replication.pull.ApplyObjectsCacheKey; import com.googlesource.gerrit.plugins.replication.pull.FetchOne; import com.googlesource.gerrit.plugins.replication.pull.Source; import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection; @@ -60,6 +63,7 @@ private static final String INSTANCE_ID = "node_instance_id"; private static final String NEW_REV = "0000000000000000000000000000000000000001"; private static final String REMOTE_INSTANCE_ID = "remote_node_instance_id"; + private static final long TEST_EVENT_TIMESTAMP = 1684879097024L; @Mock private ProjectInitializationAction projectInitializationAction; @Mock private WorkQueue workQueue; @@ -72,11 +76,13 @@ @Mock private SourcesCollection sources; @Mock private Source source; @Mock private ExcludedRefsFilter refsFilter; + private Cache<ApplyObjectsCacheKey, Long> cache; private StreamEventListener objectUnderTest; @Before public void setup() { + cache = CacheBuilder.newBuilder().build(); when(workQueue.getDefaultQueue()).thenReturn(executor); when(fetchJobFactory.create(eq(Project.nameKey(TEST_PROJECT)), any(), any())) .thenReturn(fetchJob); @@ -95,7 +101,8 @@ fetchJobFactory, () -> metrics, sources, - refsFilter); + refsFilter, + cache); } @Test @@ -257,4 +264,58 @@ verify(executor).submit(any(FetchJob.class)); } + + @Test + public void shouldSkipEventWhenFoundInApplyObjectsCacheWithTheSameTimestamp() { + sendRefUpdateEventWithTimestamp(TEST_EVENT_TIMESTAMP, TEST_EVENT_TIMESTAMP); + verify(executor, never()).submit(any(Runnable.class)); + } + + @Test + public void shouldSkipEventWhenFoundInApplyObjectsCacheWithOlderTimestamp() { + sendRefUpdateEventWithTimestamp(TEST_EVENT_TIMESTAMP - 1, TEST_EVENT_TIMESTAMP); + verify(executor, never()).submit(any(Runnable.class)); + } + + @Test + public void shouldProcessEventWhenFoundInApplyObjectsCacheWithNewerTimestamp() { + sendRefUpdateEventWithTimestamp(TEST_EVENT_TIMESTAMP + 1, TEST_EVENT_TIMESTAMP); + verify(executor).submit(any(Runnable.class)); + } + + private void sendRefUpdateEventWithTimestamp(long eventTimestamp, long cachedTimestamp) { + RefUpdatedEvent event = new RefUpdatedEvent(); + RefUpdateAttribute refUpdate = new RefUpdateAttribute(); + refUpdate.refName = TEST_REF_NAME; + refUpdate.project = TEST_PROJECT; + refUpdate.oldRev = ObjectId.zeroId().getName(); + refUpdate.newRev = NEW_REV; + event.eventCreatedOn = eventTimestamp; + + cache.put( + ApplyObjectsCacheKey.create(refUpdate.newRev, refUpdate.refName, refUpdate.project), + cachedTimestamp); + + event.instanceId = REMOTE_INSTANCE_ID; + event.refUpdate = () -> refUpdate; + + objectUnderTest.onEvent(event); + } + + @Test + public void shouldScheduleAllRefsFetchWhenNotFoundInApplyObjectsCache() { + RefUpdatedEvent event = new RefUpdatedEvent(); + RefUpdateAttribute refUpdate = new RefUpdateAttribute(); + refUpdate.refName = TEST_REF_NAME; + refUpdate.project = TEST_PROJECT; + refUpdate.oldRev = ObjectId.zeroId().getName(); + refUpdate.newRev = NEW_REV; + + event.instanceId = REMOTE_INSTANCE_ID; + event.refUpdate = () -> refUpdate; + + objectUnderTest.onEvent(event); + + verify(executor).submit(any(FetchJob.class)); + } }