Merge branch 'stable-3.8' into stable-3.9 * stable-3.8: Fix ordering of parameters for apply objects log Change-Id: I014a22727a8152d43e05a7a546f638d41d6be950
diff --git a/BUILD b/BUILD index dbbc7e9..53b55dc 100644 --- a/BUILD +++ b/BUILD
@@ -18,6 +18,7 @@ deps = [ ":events-broker-neverlink", "//lib/commons:io", + "//plugins/delete-project", "//plugins/replication", "@commons-lang3//jar", ], @@ -34,6 +35,7 @@ deps = PLUGIN_TEST_DEPS + PLUGIN_DEPS + [ ":pull-replication__plugin", ":pull_replication_util", + "//plugins/delete-project", "//plugins/replication", "//plugins/events-broker", ], @@ -63,6 +65,7 @@ ), deps = PLUGIN_TEST_DEPS + PLUGIN_DEPS + [ ":pull-replication__plugin", + "//plugins/delete-project", "//plugins/replication", ], )
diff --git a/example-setup/broker/Dockerfile b/example-setup/broker/Dockerfile index d615273..9ed7944 100644 --- a/example-setup/broker/Dockerfile +++ b/example-setup/broker/Dockerfile
@@ -1,4 +1,4 @@ -FROM gerritcodereview/gerrit:3.8.1-almalinux9 +FROM gerritcodereview/gerrit:3.9.0-rc4-almalinux9 USER root
diff --git a/example-setup/broker/configs/replication.config.template b/example-setup/broker/configs/replication.config.template index e77edeb..92047f8 100644 --- a/example-setup/broker/configs/replication.config.template +++ b/example-setup/broker/configs/replication.config.template
@@ -14,7 +14,7 @@ url = http://$REMOTE_URL:8080/#{name}#.git apiUrl = http://$REMOTE_URL:8080/ fetch = +refs/*:refs/* - mirror = true + mirror = $MIRROR timeout = 60 # In seconds connectionTimeout = 120000 # In mseconds rescheduleDelay = 15
diff --git a/example-setup/broker/docker-compose.yaml b/example-setup/broker/docker-compose.yaml index 705aea6..5d2e68a 100644 --- a/example-setup/broker/docker-compose.yaml +++ b/example-setup/broker/docker-compose.yaml
@@ -11,6 +11,7 @@ - BROKER_HOST=broker - BROKER_PORT=9092 - REPLICATE_ON_STARTUP=false + - MIRROR=true ports: - "8080:8080" - "29418:29418" @@ -28,6 +29,7 @@ - BROKER_HOST=broker - BROKER_PORT=9092 - REPLICATE_ON_STARTUP=true + - MIRROR=true ports: - "8081:8080" - "29419:29418"
diff --git a/example-setup/http/Dockerfile b/example-setup/http/Dockerfile index 6f1d1f9..32a475e 100644 --- a/example-setup/http/Dockerfile +++ b/example-setup/http/Dockerfile
@@ -1,4 +1,4 @@ -FROM gerritcodereview/gerrit:3.8.1-almalinux9 +FROM gerritcodereview/gerrit:3.9.0-rc4-almalinux9 USER root
diff --git a/example-setup/http/configs/replication.config.template b/example-setup/http/configs/replication.config.template index 6768146..53809ca 100644 --- a/example-setup/http/configs/replication.config.template +++ b/example-setup/http/configs/replication.config.template
@@ -13,7 +13,7 @@ url = http://$REMOTE_URL:8080/#{name}#.git apiUrl = http://$REMOTE_URL:8080 fetch = +refs/*:refs/* - mirror = true + mirror = $MIRROR timeout = 60 # In seconds connectionTimeout = 120000 # In mseconds rescheduleDelay = 15
diff --git a/example-setup/http/docker-compose.yaml b/example-setup/http/docker-compose.yaml index ccb6b86..62fed3c 100644 --- a/example-setup/http/docker-compose.yaml +++ b/example-setup/http/docker-compose.yaml
@@ -9,6 +9,7 @@ - REMOTE_URL=gerrit2 - DEBUG_PORT=5005 - REPLICATE_ON_STARTUP=false + - MIRROR=true ports: - "8080:8080" - "29418:29418" @@ -22,6 +23,7 @@ - REMOTE_URL=gerrit1 - DEBUG_PORT=5006 - REPLICATE_ON_STARTUP=true + - MIRROR=true ports: - "8081:8080" - "29419:29418"
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchReplicationMetrics.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchReplicationMetrics.java index 22bb073..13be628 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchReplicationMetrics.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchReplicationMetrics.java
@@ -87,16 +87,6 @@ } /** - * Start the end-to-end replication latency timer from a source. - * - * @param name the source name. - * @return the timer context. - */ - public Timer1.Context<String> startEnd2End(String name) { - return end2EndExecutionTime.start(name); - } - - /** * Record the end-to-end replication latency timer from a source. * * @param name the source name.
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java index 005d383..cbdfe1b 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java
@@ -53,7 +53,6 @@ import com.googlesource.gerrit.plugins.replication.pull.client.HttpClient; import com.googlesource.gerrit.plugins.replication.pull.client.SourceHttpClient; import com.googlesource.gerrit.plugins.replication.pull.event.EventsBrokerConsumerModule; -import com.googlesource.gerrit.plugins.replication.pull.event.FetchRefReplicatedEventModule; import com.googlesource.gerrit.plugins.replication.pull.event.StreamEventModule; import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject; import java.io.File; @@ -94,8 +93,6 @@ install(new FactoryModuleBuilder().build(FetchJob.Factory.class)); install(new ApplyObjectCacheModule()); - install(new FetchRefReplicatedEventModule()); - install( new FactoryModuleBuilder() .implement(HttpClient.class, SourceHttpClient.class)
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java index ef95596..8fb6b0d 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
@@ -26,6 +26,9 @@ import com.google.gerrit.extensions.registration.DynamicItem; import com.google.gerrit.metrics.Timer1.Context; import com.google.gerrit.server.config.GerritInstanceId; +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.data.RefUpdateAttribute; +import com.google.gerrit.server.events.BatchRefUpdateEvent; import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.server.events.EventListener; import com.google.gerrit.server.events.RefUpdatedEvent; @@ -35,15 +38,17 @@ import com.google.inject.Singleton; import com.googlesource.gerrit.plugins.replication.ObservableQueue; import com.googlesource.gerrit.plugins.replication.pull.FetchResultProcessing.GitUpdateProcessing; +import com.googlesource.gerrit.plugins.replication.pull.api.data.BatchApplyObjectData; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException; import com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient; import com.googlesource.gerrit.plugins.replication.pull.client.HttpResult; +import com.googlesource.gerrit.plugins.replication.pull.client.HttpResultUtils; import com.googlesource.gerrit.plugins.replication.pull.filter.ApplyObjectsRefsFilter; import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter; import java.io.IOException; +import java.io.UncheckedIOException; import java.net.URISyntaxException; -import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -55,12 +60,12 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.function.Consumer; -import org.apache.http.client.ClientProtocolException; +import java.util.stream.Collectors; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.IncorrectObjectTypeException; -import org.eclipse.jgit.errors.InvalidObjectIdException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.RepositoryNotFoundException; +import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.transport.URIish; import org.slf4j.Logger; @@ -78,6 +83,7 @@ static final Logger repLog = LoggerFactory.getLogger(PULL_REPLICATION_LOG_NAME); private static final Integer DEFAULT_FETCH_CALLS_TIMEOUT = 0; + private static final String BATCH_REF_UPDATED_EVENT_TYPE = BatchRefUpdateEvent.TYPE; private static final String REF_UDPATED_EVENT_TYPE = new RefUpdatedEvent().type; private static final String ZEROS_OBJECTID = ObjectId.zeroId().getName(); private final ReplicationStateListener stateLog; @@ -88,15 +94,15 @@ private final Provider<SourcesCollection> sources; // For Guice circular dependency private volatile boolean running; private volatile boolean replaying; - private final Queue<ReferenceUpdatedEvent> beforeStartupEventsQueue; + private final Queue<ReferenceBatchUpdatedEvent> beforeStartupEventsQueue; private FetchApiClient.Factory fetchClientFactory; private Integer fetchCallsTimeout; private ExcludedRefsFilter refsFilter; private Provider<RevisionReader> revReaderProvider; private final ApplyObjectMetrics applyObjectMetrics; - private final FetchReplicationMetrics fetchMetrics; private final ReplicationQueueMetrics queueMetrics; private final String instanceId; + private final boolean useBatchUpdateEvents; private ApplyObjectsRefsFilter applyObjectsRefsFilter; @Inject @@ -109,9 +115,9 @@ ExcludedRefsFilter refsFilter, Provider<RevisionReader> revReaderProvider, ApplyObjectMetrics applyObjectMetrics, - FetchReplicationMetrics fetchMetrics, ReplicationQueueMetrics queueMetrics, @GerritInstanceId String instanceId, + @GerritServerConfig Config gerritConfig, ApplyObjectsRefsFilter applyObjectsRefsFilter, ShutdownState shutdownState) { workQueue = wq; @@ -124,9 +130,10 @@ this.refsFilter = refsFilter; this.revReaderProvider = revReaderProvider; this.applyObjectMetrics = applyObjectMetrics; - this.fetchMetrics = fetchMetrics; this.queueMetrics = queueMetrics; this.instanceId = instanceId; + this.useBatchUpdateEvents = + gerritConfig.getBoolean("event", "stream-events", "enableBatchRefUpdatedEvents", false); this.applyObjectsRefsFilter = applyObjectsRefsFilter; } @@ -170,18 +177,65 @@ @Override public void onEvent(com.google.gerrit.server.events.Event e) { - if (e.type.equals(REF_UDPATED_EVENT_TYPE) && instanceId.equals(e.instanceId)) { + if (!instanceId.equals(e.instanceId)) { + return; + } + + if (useBatchUpdateEvents) { + if (e.type.equals(BATCH_REF_UPDATED_EVENT_TYPE)) { + BatchRefUpdateEvent event = (BatchRefUpdateEvent) e; + repLog.info( + "Batch ref event received on project {} for refs: {}", + event.getProjectNameKey().get(), + String.join(",", event.getRefNames())); + + long eventCreatedOn = e.eventCreatedOn; + List<ReferenceUpdatedEvent> refs = + event.refUpdates.get().stream() + .filter(u -> isRefToBeReplicated(u.refName)) + .map( + u -> { + repLog.info( + "Ref event received: {} on project {}:{} - {} => {}", + refUpdateType(u.oldRev, u.newRev), + event.getProjectNameKey().get(), + u.refName, + u.oldRev, + u.newRev); + return ReferenceUpdatedEvent.from(u, eventCreatedOn); + }) + .sorted(ReplicationQueue::sortByMetaRefAsLast) + .collect(Collectors.toList()); + + if (!refs.isEmpty()) { + ReferenceBatchUpdatedEvent referenceBatchUpdatedEvent = + ReferenceBatchUpdatedEvent.create( + event.getProjectNameKey().get(), refs, eventCreatedOn); + fire(referenceBatchUpdatedEvent); + } + } + return; + } + + if (e.type.equals(REF_UDPATED_EVENT_TYPE)) { RefUpdatedEvent event = (RefUpdatedEvent) e; if (isRefToBeReplicated(event.getRefName())) { + RefUpdateAttribute refUpdateAttribute = event.refUpdate.get(); repLog.info( "Ref event received: {} on project {}:{} - {} => {}", - refUpdateType(event), - event.refUpdate.get().project, - event.getRefName(), - event.refUpdate.get().oldRev, - event.refUpdate.get().newRev); - fire(ReferenceUpdatedEvent.from(event)); + refUpdateType(refUpdateAttribute.oldRev, refUpdateAttribute.newRev), + event.getProjectNameKey().get(), + refUpdateAttribute.refName, + refUpdateAttribute.oldRev, + refUpdateAttribute.newRev); + + ReferenceBatchUpdatedEvent referenceBatchUpdatedEvent = + ReferenceBatchUpdatedEvent.create( + event.getProjectNameKey().get(), + List.of(ReferenceUpdatedEvent.from(refUpdateAttribute, e.eventCreatedOn)), + e.eventCreatedOn); + fire(referenceBatchUpdatedEvent); } } } @@ -196,10 +250,16 @@ source.getApis().forEach(apiUrl -> source.scheduleDeleteProject(apiUrl, project))); } - private static String refUpdateType(RefUpdatedEvent event) { - if (ZEROS_OBJECTID.equals(event.refUpdate.get().oldRev)) { + private static int sortByMetaRefAsLast(ReferenceUpdatedEvent a, ReferenceUpdatedEvent b) { + repLog.debug("sortByMetaRefAsLast({} <=> {})", a.refName(), b.refName()); + return Boolean.compare( + RefNames.isNoteDbMetaRef(a.refName()), RefNames.isNoteDbMetaRef(b.refName())); + } + + private static String refUpdateType(String oldRev, String newRev) { + if (ZEROS_OBJECTID.equals(oldRev)) { return "CREATE"; - } else if (ZEROS_OBJECTID.equals(event.refUpdate.get().newRev)) { + } else if (ZEROS_OBJECTID.equals(newRev)) { return "DELETE"; } else { return "UPDATE"; @@ -210,13 +270,13 @@ return !refsFilter.match(refName); } - private void fire(ReferenceUpdatedEvent event) { + private void fire(ReferenceBatchUpdatedEvent event) { ReplicationState state = new ReplicationState(new GitUpdateProcessing(dispatcher.get())); fire(event, state); state.markAllFetchTasksScheduled(); } - private void fire(ReferenceUpdatedEvent event, ReplicationState state) { + private void fire(ReferenceBatchUpdatedEvent event, ReplicationState state) { if (!running) { stateLog.warn( String.format( @@ -241,12 +301,7 @@ final Consumer<Source> callFunction = callFunction( - Project.nameKey(event.projectName()), - event.objectId(), - event.refName(), - event.eventCreatedOn(), - event.isDelete(), - state); + Project.nameKey(event.projectName()), event.refs(), event.eventCreatedOn(), state); fetchCallsPool .submit(() -> allSources.parallelStream().forEach(callFunction)) .get(fetchCallsTimeout, TimeUnit.MILLISECONDS); @@ -266,13 +321,10 @@ private Consumer<Source> callFunction( NameKey project, - ObjectId objectId, - String refName, + List<ReferenceUpdatedEvent> refs, long eventCreatedOn, - boolean isDelete, ReplicationState state) { - CallFunction call = - getCallFunction(project, objectId, refName, eventCreatedOn, isDelete, state); + CallFunction call = getCallFunction(project, refs, eventCreatedOn, state); return (source) -> { boolean callSuccessful; @@ -281,161 +333,250 @@ } catch (Exception e) { repLog.warn( String.format( - "Failed to apply object %s on project %s:%s, falling back to git fetch", - objectId.name(), project, refName), + "Failed to batch apply object %s on project %s, falling back to git fetch", + refs.stream() + .map(event -> String.format("%s:%s", event.refName(), event.objectId())) + .collect(Collectors.joining(",")), + project), e); callSuccessful = false; } if (!callSuccessful) { - callFetch(source, project, refName, state); + if (source.enableBatchedRefs()) { + callBatchFetch(source, project, refs, state); + } else { + callFetch(source, project, refs, state); + } } }; } private CallFunction getCallFunction( NameKey project, - ObjectId objectId, - String refName, + List<ReferenceUpdatedEvent> refs, long eventCreatedOn, - boolean isDelete, ReplicationState state) { - if (isDelete) { - return ((source) -> - callSendObject(source, project, refName, eventCreatedOn, isDelete, null, state)); - } try { - Optional<RevisionData> revisionData = - revReaderProvider.get().read(project, objectId, refName, 0); - repLog.info( - "RevisionData is {} for {}:{}", - revisionData.map(RevisionData::toString).orElse("ABSENT"), - project, - refName); + List<BatchApplyObjectData> refsBatch = + refs.stream() + .map(ref -> toBatchApplyObject(project, ref, state)) + .collect(Collectors.toList()); - if (revisionData.isPresent()) { - return ((source) -> - callSendObject( - source, - project, - refName, - eventCreatedOn, - isDelete, - Arrays.asList(revisionData.get()), - state)); + if (!containsLargeRef(refsBatch)) { + return ((source) -> callBatchSendObject(source, project, refsBatch, eventCreatedOn, state)); } - } catch (InvalidObjectIdException | IOException e) { + } catch (UncheckedIOException e) { + stateLog.error("Falling back to calling fetch", e, state); + } + return ((source) -> callBatchFetch(source, project, refs, state)); + } + + private BatchApplyObjectData toBatchApplyObject( + NameKey project, ReferenceUpdatedEvent event, ReplicationState state) { + if (event.isDelete()) { + Optional<RevisionData> noRevisionData = Optional.empty(); + return BatchApplyObjectData.create(event.refName(), noRevisionData, event.isDelete()); + } + try { + Optional<RevisionData> maybeRevisionData = + revReaderProvider.get().read(project, event.objectId(), event.refName(), 0); + return BatchApplyObjectData.create(event.refName(), maybeRevisionData, event.isDelete()); + } catch (IOException e) { stateLog.error( String.format( "Exception during reading ref: %s, project:%s, message: %s", - refName, project.get(), e.getMessage()), + event.refName(), project.get(), e.getMessage()), e, state); + throw new UncheckedIOException(e); } - - return (source) -> callFetch(source, project, refName, state); } - private boolean callSendObject( - Source source, + private boolean containsLargeRef(List<BatchApplyObjectData> batchApplyObjectData) { + return batchApplyObjectData.stream().anyMatch(e -> e.revisionData().isEmpty() && !e.isDelete()); + } + + private Optional<HttpResult> callSendObject( + FetchApiClient fetchClient, + String remoteName, + URIish uri, NameKey project, String refName, long eventCreatedOn, boolean isDelete, - List<RevisionData> revision, + List<RevisionData> revision) + throws IOException { + String revisionDataStr = + Optional.ofNullable(revision).orElse(ImmutableList.of()).stream() + .map(RevisionData::toString) + .collect(Collectors.joining(",")); + repLog.info( + "Pull replication REST API apply object to {} for {}:{} - {}", + uri, + project, + refName, + revisionDataStr); + Context<String> apiTimer = applyObjectMetrics.startEnd2End(remoteName); + HttpResult result = + isDelete + ? fetchClient.callSendObject(project, refName, eventCreatedOn, isDelete, null, uri) + : fetchClient.callSendObjects(project, refName, eventCreatedOn, revision, uri); + repLog.info( + "Pull replication REST API apply object to {} COMPLETED for {}:{} - {}, HTTP Result:" + + " {} - time:{} ms", + uri, + project, + refName, + revisionDataStr, + result, + apiTimer.stop() / 1000000.0); + + return Optional.of(result); + } + + private boolean callBatchSendObject( + Source source, + NameKey project, + List<BatchApplyObjectData> refsBatch, + long eventCreatedOn, ReplicationState state) throws MissingParentObjectException { - boolean resultIsSuccessful = true; - if (source.wouldFetchProject(project) && source.wouldFetchRef(refName)) { - for (String apiUrl : source.getApis()) { - try { - URIish uri = new URIish(apiUrl); - FetchApiClient fetchClient = fetchClientFactory.create(source); + boolean batchResultSuccessful = true; + + List<BatchApplyObjectData> filteredRefsBatch = + refsBatch.stream() + .filter(r -> source.wouldFetchProject(project) && source.wouldFetchRef(r.refName())) + .collect(Collectors.toList()); + + String batchApplyObjectStr = + filteredRefsBatch.stream() + .map(BatchApplyObjectData::toString) + .collect(Collectors.joining(",")); + FetchApiClient fetchClient = fetchClientFactory.create(source); + String remoteName = source.getRemoteConfigName(); + + for (String apiUrl : source.getApis()) { + try { + boolean resultSuccessful = true; + Optional<HttpResult> result = Optional.empty(); + URIish uri = new URIish(apiUrl); + if (source.enableBatchedRefs()) { repLog.info( - "Pull replication REST API apply object to {} for {}:{} - {}", + "Pull replication REST API batch apply object to {} for {}:[{}]", apiUrl, project, - refName, - revision); - Context<String> apiTimer = applyObjectMetrics.startEnd2End(source.getRemoteConfigName()); - HttpResult result = - isDelete - ? fetchClient.callSendObject( - project, refName, eventCreatedOn, isDelete, null, uri) - : fetchClient.callSendObjects(project, refName, eventCreatedOn, revision, uri); - boolean resultSuccessful = result.isSuccessful(); + batchApplyObjectStr); + Context<String> apiTimer = applyObjectMetrics.startEnd2End(remoteName); + result = + Optional.of( + fetchClient.callBatchSendObject(project, filteredRefsBatch, eventCreatedOn, uri)); + resultSuccessful = HttpResultUtils.isSuccessful(result); repLog.info( - "Pull replication REST API apply object to {} COMPLETED for {}:{} - {}, HTTP Result:" + "Pull replication REST API batch apply object to {} COMPLETED for {}:[{}], HTTP Result:" + " {} - time:{} ms", apiUrl, project, - refName, - revision, - result, + batchApplyObjectStr, + HttpResultUtils.status(result), apiTimer.stop() / 1000000.0); - - if (!resultSuccessful - && result.isProjectMissing(project) - && source.isCreateMissingRepositories()) { - result = initProject(project, uri, fetchClient, result); - repLog.info("Missing project {} created, HTTP Result:{}", project, result); - } - - if (!resultSuccessful) { - if (result.isParentObjectMissing()) { - - if ((RefNames.isNoteDbMetaRef(refName) || applyObjectsRefsFilter.match(refName)) - && revision.size() == 1) { - List<RevisionData> allRevisions = - fetchWholeMetaHistory(project, refName, revision.get(0)); - repLog.info( - "Pull replication REST API apply object to {} for {}:{} - {}", - apiUrl, + } else { + repLog.info( + "REST API batch apply object not enabled for source {}, using REST API apply object to {} for {}:[{}]", + remoteName, + apiUrl, + project, + batchApplyObjectStr); + for (BatchApplyObjectData batchApplyObject : filteredRefsBatch) { + result = + callSendObject( + fetchClient, + remoteName, + uri, project, - refName, - allRevisions); - return callSendObject( - source, project, refName, eventCreatedOn, isDelete, allRevisions, state); - } + batchApplyObject.refName(), + eventCreatedOn, + batchApplyObject.isDelete(), + batchApplyObject.revisionData().map(ImmutableList::of).orElse(null)); + resultSuccessful = HttpResultUtils.isSuccessful(result); + if (!resultSuccessful) { + break; + } + } + } + + if (!resultSuccessful + && HttpResultUtils.isProjectMissing(result, project) + && source.isCreateMissingRepositories()) { + result = initProject(project, uri, fetchClient, result); + repLog.info( + "Missing project {} created, HTTP Result:{}", + project, + HttpResultUtils.status(result)); + } + + if (!resultSuccessful && HttpResultUtils.isParentObjectMissing(result)) { + resultSuccessful = true; + for (BatchApplyObjectData batchApplyObject : filteredRefsBatch) { + String refName = batchApplyObject.refName(); + if ((RefNames.isNoteDbMetaRef(refName) || applyObjectsRefsFilter.match(refName)) + && batchApplyObject.revisionData().isPresent()) { + + Optional<RevisionData> maybeRevisionData = batchApplyObject.revisionData(); + List<RevisionData> allRevisions = + fetchWholeMetaHistory(project, refName, maybeRevisionData.get()); + + Optional<HttpResult> sendObjectResult = + callSendObject( + fetchClient, + remoteName, + uri, + project, + refName, + eventCreatedOn, + batchApplyObject.isDelete(), + allRevisions); + resultSuccessful = HttpResultUtils.isSuccessful(sendObjectResult); + if (!resultSuccessful) { + break; + } + } else { throw new MissingParentObjectException( project, refName, source.getRemoteConfigName()); } } - - resultIsSuccessful &= resultSuccessful; - } catch (URISyntaxException e) { - repLog.warn( - "Pull replication REST API apply object to {} *FAILED* for {}:{} - {}", - apiUrl, - project, - refName, - revision, - e); - stateLog.error(String.format("Cannot parse pull replication api url:%s", apiUrl), state); - resultIsSuccessful = false; - } catch (IOException e) { - repLog.warn( - "Pull replication REST API apply object to {} *FAILED* for {}:{} - {}", - apiUrl, - project, - refName, - revision, - e); - stateLog.error( - String.format( - "Exception during the pull replication fetch rest api call. Endpoint url:%s," - + " message:%s", - apiUrl, e.getMessage()), - e, - state); - resultIsSuccessful = false; } + + batchResultSuccessful &= resultSuccessful; + } catch (URISyntaxException e) { + repLog.warn( + "Pull replication REST API batch apply object to {} *FAILED* for {}:[{}]", + apiUrl, + project, + batchApplyObjectStr, + e); + stateLog.error(String.format("Cannot parse pull replication api url:%s", apiUrl), state); + batchResultSuccessful = false; + } catch (IOException | IllegalArgumentException e) { + repLog.warn( + "Pull replication REST API batch apply object to {} *FAILED* for {}:[{}]", + apiUrl, + project, + batchApplyObjectStr, + e); + stateLog.error( + String.format( + "Exception during the pull replication fetch rest api call. Endpoint url:%s," + + " message:%s", + apiUrl, e.getMessage()), + e, + state); + batchResultSuccessful = false; } } - - return resultIsSuccessful; + return batchResultSuccessful; } private List<RevisionData> fetchWholeMetaHistory( @@ -460,53 +601,129 @@ return revisionDataBuilder.build(); } - private boolean callFetch( - Source source, Project.NameKey project, String refName, ReplicationState state) { - boolean resultIsSuccessful = true; - if (source.wouldFetchProject(project) && source.wouldFetchRef(refName)) { - for (String apiUrl : source.getApis()) { - try { - URIish uri = new URIish(apiUrl); - FetchApiClient fetchClient = fetchClientFactory.create(source); - repLog.info("Pull replication REST API fetch to {} for {}:{}", apiUrl, project, refName); - Context<String> timer = fetchMetrics.startEnd2End(source.getRemoteConfigName()); - HttpResult result = fetchClient.callFetch(project, refName, uri); - long elapsedMs = TimeUnit.NANOSECONDS.toMillis(timer.stop()); - boolean resultSuccessful = result.isSuccessful(); - repLog.info( - "Pull replication REST API fetch to {} COMPLETED for {}:{}, HTTP Result:" - + " {} - time:{} ms", - apiUrl, - project, - refName, - result, - elapsedMs); - if (!resultSuccessful - && result.isProjectMissing(project) - && source.isCreateMissingRepositories()) { - result = initProject(project, uri, fetchClient, result); - } - if (!resultSuccessful) { - stateLog.warn( - String.format( - "Pull replication rest api fetch call failed. Endpoint url: %s, reason:%s", - apiUrl, result.getMessage().orElse("unknown")), - state); - } + private boolean callBatchFetch( + Source source, + Project.NameKey project, + List<ReferenceUpdatedEvent> refs, + ReplicationState state) { - resultIsSuccessful &= result.isSuccessful(); - } catch (URISyntaxException e) { - stateLog.error(String.format("Cannot parse pull replication api url:%s", apiUrl), state); - resultIsSuccessful = false; - } catch (Exception e) { - stateLog.error( + boolean resultIsSuccessful = true; + + List<String> filteredRefs = + refs.stream() + .map(ReferenceUpdatedEvent::refName) + .filter(refName -> source.wouldFetchProject(project) && source.wouldFetchRef(refName)) + .collect(Collectors.toList()); + + String refsStr = String.join(",", filteredRefs); + FetchApiClient fetchClient = fetchClientFactory.create(source); + + for (String apiUrl : source.getApis()) { + try { + URIish uri = new URIish(apiUrl); + Optional<HttpResult> result = Optional.empty(); + repLog.info( + "Pull replication REST API batch fetch to {} for {}:[{}]", apiUrl, project, refsStr); + long startTime = System.currentTimeMillis(); + result = Optional.of(fetchClient.callBatchFetch(project, filteredRefs, uri)); + long endTime = System.currentTimeMillis(); + boolean resultSuccessful = HttpResultUtils.isSuccessful(result); + repLog.info( + "Pull replication REST API batch fetch to {} COMPLETED for {}:[{}], HTTP Result:" + + " {} - time:{} ms", + apiUrl, + project, + refsStr, + HttpResultUtils.status(result), + endTime - startTime); + if (!resultSuccessful + && HttpResultUtils.isProjectMissing(result, project) + && source.isCreateMissingRepositories()) { + result = initProject(project, uri, fetchClient, result); + resultSuccessful = HttpResultUtils.isSuccessful(result); + } + if (!resultSuccessful) { + stateLog.warn( String.format( - "Exception during the pull replication fetch rest api call. Endpoint url:%s," - + " message:%s", - apiUrl, e.getMessage()), - e, + "Pull replication REST API batch fetch call failed. Endpoint url: %s, reason:%s", + apiUrl, HttpResultUtils.errorMsg(result)), state); - resultIsSuccessful = false; + } + resultIsSuccessful &= resultSuccessful; + } catch (URISyntaxException e) { + stateLog.error( + String.format("Cannot parse pull replication batch api url:%s", apiUrl), state); + resultIsSuccessful = false; + } catch (Exception e) { + stateLog.error( + String.format( + "Exception during the pull replication batch fetch rest api call. Endpoint url:%s," + + " message:%s", + apiUrl, e.getMessage()), + e, + state); + resultIsSuccessful = false; + } + } + + return resultIsSuccessful; + } + + private boolean callFetch( + Source source, + Project.NameKey project, + List<ReferenceUpdatedEvent> refs, + ReplicationState state) { + boolean resultIsSuccessful = true; + for (ReferenceUpdatedEvent refEvent : refs) { + String refName = refEvent.refName(); + if (source.wouldFetchProject(project) && source.wouldFetchRef(refName)) { + for (String apiUrl : source.getApis()) { + try { + URIish uri = new URIish(apiUrl); + FetchApiClient fetchClient = fetchClientFactory.create(source); + repLog.info( + "Pull replication REST API fetch to {} for {}:{}", apiUrl, project, refName); + long startTime = System.currentTimeMillis(); + Optional<HttpResult> result = Optional.of(fetchClient.callFetch(project, refName, uri)); + long endTime = System.currentTimeMillis(); + boolean resultSuccessful = HttpResultUtils.isSuccessful(result); + repLog.info( + "Pull replication REST API fetch to {} COMPLETED for {}:{}, HTTP Result:" + + " {} - time: {} ms", + apiUrl, + project, + refName, + HttpResultUtils.status(result), + endTime - startTime); + if (!resultSuccessful + && HttpResultUtils.isProjectMissing(result, project) + && source.isCreateMissingRepositories()) { + result = initProject(project, uri, fetchClient, result); + } + if (!resultSuccessful) { + stateLog.warn( + String.format( + "Pull replication rest api fetch call failed. Endpoint url: %s, reason:%s", + apiUrl, HttpResultUtils.errorMsg(result)), + state); + } + + resultIsSuccessful &= HttpResultUtils.isSuccessful(result); + } catch (URISyntaxException e) { + stateLog.error( + String.format("Cannot parse pull replication api url:%s", apiUrl), state); + resultIsSuccessful = false; + } catch (Exception e) { + stateLog.error( + String.format( + "Exception during the pull replication fetch rest api call. Endpoint url:%s," + + " message:%s", + apiUrl, e.getMessage()), + e, + state); + resultIsSuccessful = false; + } } } } @@ -518,9 +735,9 @@ return maxRetries == 0 || attempt < maxRetries; } - private HttpResult initProject( - Project.NameKey project, URIish uri, FetchApiClient fetchClient, HttpResult result) - throws IOException, ClientProtocolException { + private Optional<HttpResult> initProject( + Project.NameKey project, URIish uri, FetchApiClient fetchClient, Optional<HttpResult> result) + throws IOException { RevisionData refsMetaConfigRevisionData = revReaderProvider .get() @@ -536,7 +753,7 @@ HttpResult initProjectResult = fetchClient.initProject(project, uri, System.currentTimeMillis(), refsMetaConfigDataList); if (initProjectResult.isSuccessful()) { - result = fetchClient.callFetch(project, FetchOne.ALL_REFS, uri); + result = Optional.of(fetchClient.callFetch(project, FetchOne.ALL_REFS, uri)); } else { String errorMessage = initProjectResult.getMessage().map(e -> " - Error: " + e).orElse(""); repLog.error("Cannot create project " + project + errorMessage); @@ -546,9 +763,15 @@ private void fireBeforeStartupEvents() { Set<String> eventsReplayed = new HashSet<>(); - ReferenceUpdatedEvent event; + ReferenceBatchUpdatedEvent event; while ((event = beforeStartupEventsQueue.poll()) != null) { - String eventKey = String.format("%s:%s", event.projectName(), event.refName()); + String eventKey = + String.format( + "%s:%s", + event.projectName(), + event.refs().stream() + .map(ReferenceUpdatedEvent::refName) + .collect(Collectors.joining())); if (!eventsReplayed.contains(eventKey)) { repLog.info("Firing pending task {}", event); fire(event); @@ -573,6 +796,22 @@ } @AutoValue + abstract static class ReferenceBatchUpdatedEvent { + + static ReferenceBatchUpdatedEvent create( + String projectName, List<ReferenceUpdatedEvent> refs, long eventCreatedOn) { + return new AutoValue_ReplicationQueue_ReferenceBatchUpdatedEvent( + projectName, refs, eventCreatedOn); + } + + public abstract String projectName(); + + public abstract List<ReferenceUpdatedEvent> refs(); + + public abstract long eventCreatedOn(); + } + + @AutoValue abstract static class ReferenceUpdatedEvent { static ReferenceUpdatedEvent create( @@ -585,13 +824,13 @@ projectName, refName, objectId, eventCreatedOn, isDelete); } - static ReferenceUpdatedEvent from(RefUpdatedEvent event) { + static ReferenceUpdatedEvent from(RefUpdateAttribute refUpdateAttribute, long eventCreatedOn) { return ReferenceUpdatedEvent.create( - event.refUpdate.get().project, - event.getRefName(), - ObjectId.fromString(event.refUpdate.get().newRev), - event.eventCreatedOn, - ZEROS_OBJECTID.equals(event.refUpdate.get().newRev)); + refUpdateAttribute.project, + refUpdateAttribute.refName, + ObjectId.fromString(refUpdateAttribute.newRev), + eventCreatedOn, + ZEROS_OBJECTID.equals(refUpdateAttribute.newRev)); } public abstract String projectName();
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java index 012a046..9984a03 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
@@ -906,6 +906,14 @@ return config.replicateProjectDeletions(); } + public boolean isMirror() { + return config.getRemoteConfig().isMirror(); + } + + public boolean enableBatchedRefs() { + return config.enableBatchedRefs(); + } + void scheduleUpdateHead(String apiUrl, Project.NameKey project, String newHead) { try { URIish apiURI = new URIish(apiUrl);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java index 0a22a5a..1921e7b 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java
@@ -16,6 +16,7 @@ import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; +import com.google.common.flogger.FluentLogger; import com.google.gerrit.server.config.ConfigUtil; import com.googlesource.gerrit.plugins.replication.RemoteConfiguration; import java.util.concurrent.TimeUnit; @@ -23,6 +24,7 @@ import org.eclipse.jgit.transport.RemoteConfig; public class SourceConfiguration implements RemoteConfiguration { + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); static final int DEFAULT_REPLICATION_DELAY = 4; static final int DEFAULT_RESCHEDULE_DELAY = 3; static final int DEFAULT_SLOW_LATENCY_THRESHOLD_SECS = 900; @@ -56,6 +58,7 @@ private int slowLatencyThreshold; private boolean useCGitClient; private int refsBatchSize; + private boolean enableBatchedRefs; public SourceConfiguration(RemoteConfig remoteConfig, Config cfg) { this.remoteConfig = remoteConfig; @@ -110,6 +113,16 @@ "shutDownDrainTimeout", DEFAULT_DRAIN_SHUTDOWN_TIMEOUT_SECS, TimeUnit.SECONDS); + + enableBatchedRefs = cfg.getBoolean("remote", name, "enableBatchedRefs", false); + if (!enableBatchedRefs) { + logger.atWarning().log( + "You haven't enabled batched refs in the %s node, as such you are not " + + "leveraging the performance improvements introduced by the batch-apply-object API. Consider " + + "upgrading the plugin to the latest version and consult the plugin's documentation for more " + + "details on the `enableBatchedRefs` configuration.", + name); + } } @Override @@ -237,4 +250,8 @@ public int getShutDownDrainTimeout() { return shutDownDrainTimeout; } + + public boolean enableBatchedRefs() { + return enableBatchedRefs; + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java index 968a03c..75de6c8 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java
@@ -23,12 +23,18 @@ import com.google.common.flogger.FluentLogger; import com.google.gerrit.entities.Project; import com.google.gerrit.extensions.registration.DynamicItem; +import com.google.gerrit.extensions.restapi.ResourceNotFoundException; import com.google.gerrit.metrics.Timer1; import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.server.permissions.PermissionBackendException; import com.google.inject.Inject; import com.google.inject.name.Named; -import com.googlesource.gerrit.plugins.replication.pull.*; +import com.googlesource.gerrit.plugins.replication.pull.ApplyObjectMetrics; +import com.googlesource.gerrit.plugins.replication.pull.ApplyObjectsCacheKey; +import com.googlesource.gerrit.plugins.replication.pull.Context; +import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent; +import com.googlesource.gerrit.plugins.replication.pull.PullReplicationStateLogger; +import com.googlesource.gerrit.plugins.replication.pull.ReplicationState; import com.googlesource.gerrit.plugins.replication.pull.ReplicationState.RefFetchResult; import com.googlesource.gerrit.plugins.replication.pull.Source; import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection; @@ -84,7 +90,8 @@ RevisionData revisionsData, String sourceLabel, long eventCreatedOn) - throws IOException, RefUpdateException, MissingParentObjectException { + throws IOException, RefUpdateException, MissingParentObjectException, + ResourceNotFoundException { applyObjects(name, refName, new RevisionData[] {revisionsData}, sourceLabel, eventCreatedOn); } @@ -94,7 +101,8 @@ RevisionData[] revisionsData, String sourceLabel, long eventCreatedOn) - throws IOException, RefUpdateException, MissingParentObjectException { + throws IOException, RefUpdateException, MissingParentObjectException, + ResourceNotFoundException { repLog.info( "Apply object from {} for {}:{} - {}",
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchApplyObjectAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchApplyObjectAction.java new file mode 100644 index 0000000..717b70e --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchApplyObjectAction.java
@@ -0,0 +1,57 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api; + +import static com.googlesource.gerrit.plugins.replication.pull.PullReplicationLogger.repLog; + +import com.google.gerrit.extensions.restapi.Response; +import com.google.gerrit.extensions.restapi.RestApiException; +import com.google.gerrit.extensions.restapi.RestModifyView; +import com.google.gerrit.server.project.ProjectResource; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +@Singleton +class BatchApplyObjectAction implements RestModifyView<ProjectResource, List<RevisionInput>> { + + private final ApplyObjectAction applyObjectAction; + + @Inject + BatchApplyObjectAction(ApplyObjectAction applyObjectAction) { + this.applyObjectAction = applyObjectAction; + } + + @Override + public Response<?> apply(ProjectResource resource, List<RevisionInput> inputs) + throws RestApiException { + + repLog.info( + "Batch Apply object API from {} for refs {}", + resource.getNameKey(), + inputs.stream().map(RevisionInput::getRefName).collect(Collectors.joining(","))); + + List<Response<?>> allResponses = new ArrayList<>(); + for (RevisionInput input : inputs) { + Response<?> individualResponse = applyObjectAction.apply(resource, input); + allResponses.add(individualResponse); + } + + return Response.ok(allResponses); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchAction.java new file mode 100644 index 0000000..c6ad47d --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchAction.java
@@ -0,0 +1,47 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api; + +import com.google.gerrit.extensions.restapi.Response; +import com.google.gerrit.extensions.restapi.RestApiException; +import com.google.gerrit.extensions.restapi.RestModifyView; +import com.google.gerrit.server.project.ProjectResource; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.Input; +import java.util.ArrayList; +import java.util.List; + +@Singleton +public class BatchFetchAction implements RestModifyView<ProjectResource, List<Input>> { + private final FetchAction fetchAction; + + @Inject + public BatchFetchAction(FetchAction fetchAction) { + this.fetchAction = fetchAction; + } + + @Override + public Response<?> apply(ProjectResource resource, List<Input> inputs) throws RestApiException { + + List<Response<?>> allResponses = new ArrayList<>(); + for (Input input : inputs) { + Response<?> res = fetchAction.apply(resource, input); + allResponses.add(res); + } + + return Response.ok(allResponses); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BearerAuthenticationFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BearerAuthenticationFilter.java index 68dac6a..64a0bd8 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BearerAuthenticationFilter.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BearerAuthenticationFilter.java
@@ -125,7 +125,9 @@ return (requestURI.contains(pluginName) && (requestURI.endsWith(String.format("/%s~apply-object", pluginName)) || requestURI.endsWith(String.format("/%s~apply-objects", pluginName)) + || requestURI.endsWith(String.format("/%s~batch-apply-object", pluginName)) || requestURI.endsWith(String.format("/%s~fetch", pluginName)) + || requestURI.endsWith(String.format("/%s~batch-fetch", pluginName)) || requestURI.endsWith(String.format("/%s~delete-project", pluginName)) || requestURI.contains(String.format("/%s/init-project/", pluginName)))) || (requestURI.matches(String.format(".*/projects/[^/]+/%s~HEAD", pluginName))
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java index e49c8b6..3150fe1 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java
@@ -23,7 +23,6 @@ import com.google.gerrit.extensions.restapi.RestApiException; import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.server.git.GitRepositoryManager; -import com.google.gerrit.server.permissions.PermissionBackend; import com.google.gerrit.server.permissions.PermissionBackendException; import com.google.gerrit.server.project.ProjectCache; import com.google.gerrit.server.project.ProjectState; @@ -51,7 +50,6 @@ private final DynamicItem<EventDispatcher> eventDispatcher; private final ProjectCache projectCache; private final SourcesCollection sourcesCollection; - private final PermissionBackend permissionBackend; private final GitRepositoryManager gitManager; @Inject @@ -59,19 +57,29 @@ PullReplicationStateLogger fetchStateLog, ProjectCache projectCache, SourcesCollection sourcesCollection, - PermissionBackend permissionBackend, DynamicItem<EventDispatcher> eventDispatcher, LocalGitRepositoryManagerProvider gitManagerProvider) { this.fetchStateLog = fetchStateLog; this.projectCache = projectCache; this.eventDispatcher = eventDispatcher; this.sourcesCollection = sourcesCollection; - this.permissionBackend = permissionBackend; this.gitManager = gitManagerProvider.get(); } public void deleteRef(Project.NameKey name, String refName, String sourceLabel) throws IOException, RestApiException { + Source source = + sourcesCollection + .getByRemoteName(sourceLabel) + .orElseThrow( + () -> + new IllegalStateException( + String.format("Could not find URI for %s remote", sourceLabel))); + if (!source.isMirror()) { + repLog.info( + "Ignoring ref {} deletion from project {}, as mirror option is false", refName, name); + return; + } try { repLog.info("Delete ref from {} for project {}, ref name {}", sourceLabel, name, refName); Optional<ProjectState> projectState = projectCache.get(name); @@ -85,13 +93,6 @@ return; } - Source source = - sourcesCollection - .getByRemoteName(sourceLabel) - .orElseThrow( - () -> - new IllegalStateException( - String.format("Could not find URI for %s remote", sourceLabel))); URIish sourceUri = source.getURI(name); try {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionAction.java index adb333c..6a21104 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionAction.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionAction.java
@@ -14,6 +14,8 @@ package com.googlesource.gerrit.plugins.replication.pull.api; +import static com.googlesource.gerrit.plugins.replication.pull.PullReplicationLogger.repLog; + import com.google.gerrit.extensions.api.access.PluginPermission; import com.google.gerrit.extensions.restapi.AuthException; import com.google.gerrit.extensions.restapi.BadRequestException; @@ -28,9 +30,12 @@ import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.Singleton; -import com.googlesource.gerrit.plugins.replication.LocalFS; +import com.googlesource.gerrit.plugins.deleteproject.cache.CacheDeleteHandler; +import com.googlesource.gerrit.plugins.deleteproject.fs.RepositoryDelete; import com.googlesource.gerrit.plugins.replication.pull.GerritConfigOps; +import java.io.IOException; import java.util.Optional; +import org.eclipse.jgit.errors.RepositoryNotFoundException; import org.eclipse.jgit.transport.URIish; @Singleton @@ -44,15 +49,21 @@ private final Provider<CurrentUser> userProvider; private final GerritConfigOps gerritConfigOps; private final PermissionBackend permissionBackend; + private final RepositoryDelete repositoryDelete; + private final CacheDeleteHandler cacheDeleteHandler; @Inject ProjectDeletionAction( GerritConfigOps gerritConfigOps, PermissionBackend permissionBackend, - Provider<CurrentUser> userProvider) { + Provider<CurrentUser> userProvider, + RepositoryDelete repositoryDelete, + CacheDeleteHandler cacheDeleteHandler) { this.gerritConfigOps = gerritConfigOps; this.permissionBackend = permissionBackend; this.userProvider = userProvider; + this.repositoryDelete = repositoryDelete; + this.cacheDeleteHandler = cacheDeleteHandler; } @Override @@ -69,11 +80,24 @@ gerritConfigOps.getGitRepositoryURI(String.format("%s.git", projectResource.getName())); if (maybeRepoURI.isPresent()) { - if (new LocalFS(maybeRepoURI.get()).deleteProject(projectResource.getNameKey())) { + try { + // reuse repo deletion logic from delete-project plugin, as it can successfully delete + // the git directories hosted on nfs. + repositoryDelete.execute(projectResource.getNameKey()); + // delete the project from the local project cache, otherwise future ops + // will fail as the replica will think that the project still exists locally. + cacheDeleteHandler.delete(projectResource.getProjectState().getProject()); + repLog.info( + "Deleted local repository {} and removed it from the local project cache", + projectResource.getName()); return Response.ok(); + } catch (RepositoryNotFoundException e) { + throw new ResourceNotFoundException( + String.format("Repository %s not found", projectResource.getName()), e); + } catch (IOException e) { + throw new UnprocessableEntityException( + String.format("Could not delete project %s", projectResource.getName())); } - throw new UnprocessableEntityException( - String.format("Could not delete project %s", projectResource.getName())); } throw new ResourceNotFoundException( String.format("Could not compute URI for repo: %s", projectResource.getName()));
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java index e7289ac..9afae71 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java
@@ -20,6 +20,7 @@ import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST; import static javax.servlet.http.HttpServletResponse.SC_CONFLICT; import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN; +import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; import com.google.common.base.Strings; import com.google.common.flogger.FluentLogger; @@ -28,6 +29,7 @@ import com.google.gerrit.entities.RefNames; import com.google.gerrit.extensions.restapi.AuthException; import com.google.gerrit.extensions.restapi.BadRequestException; +import com.google.gerrit.extensions.restapi.ResourceNotFoundException; import com.google.gerrit.extensions.restapi.Url; import com.google.gerrit.index.project.ProjectIndexer; import com.google.gerrit.server.CurrentUser; @@ -129,13 +131,16 @@ logExceptionAndUpdateResponse(httpServletResponse, e, SC_BAD_REQUEST, gitRepositoryName); } catch (RefUpdateException | MissingParentObjectException e) { logExceptionAndUpdateResponse(httpServletResponse, e, SC_CONFLICT, gitRepositoryName); + } catch (ResourceNotFoundException e) { + logExceptionAndUpdateResponse( + httpServletResponse, e, SC_INTERNAL_SERVER_ERROR, gitRepositoryName); } catch (AuthException | PermissionBackendException e) { logExceptionAndUpdateResponse(httpServletResponse, e, SC_FORBIDDEN, gitRepositoryName); } } public boolean initProject(String gitRepositoryName) - throws AuthException, PermissionBackendException { + throws AuthException, PermissionBackendException, IOException { if (initProject(gitRepositoryName, true)) { repLog.info("Init project API from {}", gitRepositoryName); return true; @@ -146,7 +151,7 @@ private boolean initProjectWithConfiguration( HttpServletRequest httpServletRequest, String gitRepositoryName) throws AuthException, PermissionBackendException, IOException, BadRequestException, - MissingParentObjectException, RefUpdateException { + MissingParentObjectException, RefUpdateException, ResourceNotFoundException { RevisionsInput input = PayloadSerDes.parseRevisionsInput(httpServletRequest); validateInput(input); @@ -172,7 +177,7 @@ } private boolean initProject(String gitRepositoryName, boolean needsProjectReindexing) - throws AuthException, PermissionBackendException { + throws AuthException, PermissionBackendException, IOException { // When triggered internally(for example by consuming stream events) user is not provided // and internal user is returned. Project creation should be always allowed for internal user. if (!userProvider.get().isInternalUser()) { @@ -187,7 +192,7 @@ Project.NameKey projectNameKey = Project.NameKey.parse(gitRepositoryName); if (localFS.createProject(projectNameKey, RefNames.HEAD)) { if (needsProjectReindexing) { - projectIndexer.index(projectNameKey); + projectCache.onCreateProject(projectNameKey); } return true; }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationEndpoints.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationEndpoints.java index fc97945..9fdf5d9 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationEndpoints.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationEndpoints.java
@@ -33,7 +33,10 @@ @UsedAt(PLUGIN_MULTI_SITE) public static final String APPLY_OBJECTS_API_ENDPOINT = "apply-objects"; + public static final String BATCH_APPLY_OBJECT_API_ENDPOINT = "batch-apply-object"; + public static final String FETCH_ENDPOINT = "fetch"; + public static final String BATCH_FETCH_ENDPOINT = "batch-fetch"; public static final String INIT_PROJECT_ENDPOINT = "init-project"; public static final String DELETE_PROJECT_ENDPOINT = "delete-project"; }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java index f501e90..86fba22 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java
@@ -18,8 +18,10 @@ import static com.googlesource.gerrit.plugins.replication.pull.api.HttpServletOps.checkAcceptHeader; import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST; import static javax.servlet.http.HttpServletResponse.SC_CONFLICT; +import static javax.servlet.http.HttpServletResponse.SC_CREATED; import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN; -import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; +import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND; +import static javax.servlet.http.HttpServletResponse.SC_OK; import static javax.servlet.http.HttpServletResponse.SC_UNAUTHORIZED; import com.google.common.flogger.FluentLogger; @@ -36,22 +38,31 @@ import com.google.gerrit.extensions.restapi.UnprocessableEntityException; import com.google.gerrit.httpd.AllRequestFilter; import com.google.gerrit.httpd.restapi.RestApiServlet; +import com.google.gerrit.json.OutputFormat; import com.google.gerrit.server.AnonymousUser; import com.google.gerrit.server.CurrentUser; import com.google.gerrit.server.permissions.PermissionBackendException; import com.google.gerrit.server.project.ProjectCache; import com.google.gerrit.server.project.ProjectResource; import com.google.gerrit.server.project.ProjectState; +import com.google.gson.Gson; import com.google.gson.JsonParseException; +import com.google.gson.stream.JsonReader; import com.google.gson.stream.MalformedJsonException; import com.google.inject.Inject; import com.google.inject.Provider; +import com.google.inject.TypeLiteral; import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.Input; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionsInput; import com.googlesource.gerrit.plugins.replication.pull.api.exception.UnauthorizedAuthException; import com.googlesource.gerrit.plugins.replication.pull.api.util.PayloadSerDes; +import java.io.BufferedReader; +import java.io.EOFException; import java.io.IOException; +import java.io.PrintWriter; +import java.lang.reflect.Type; +import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; @@ -68,24 +79,27 @@ private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private static final Pattern projectNameInGerritUrl = Pattern.compile(".*/projects/([^/]+)/.*"); - private static final Pattern projectNameInitProjectUrl = - Pattern.compile(".*/init-project/([^/]+.git)"); private FetchAction fetchAction; + private BatchFetchAction batchFetchAction; private ApplyObjectAction applyObjectAction; private ApplyObjectsAction applyObjectsAction; + private BatchApplyObjectAction batchApplyObjectAction; private ProjectInitializationAction projectInitializationAction; private UpdateHeadAction updateHEADAction; private ProjectDeletionAction projectDeletionAction; private ProjectCache projectCache; + private Gson gson; private String pluginName; private final Provider<CurrentUser> currentUserProvider; @Inject public PullReplicationFilter( FetchAction fetchAction, + BatchFetchAction batchFetchAction, ApplyObjectAction applyObjectAction, ApplyObjectsAction applyObjectsAction, + BatchApplyObjectAction batchApplyObjectAction, ProjectInitializationAction projectInitializationAction, UpdateHeadAction updateHEADAction, ProjectDeletionAction projectDeletionAction, @@ -93,13 +107,16 @@ @PluginName String pluginName, Provider<CurrentUser> currentUserProvider) { this.fetchAction = fetchAction; + this.batchFetchAction = batchFetchAction; this.applyObjectAction = applyObjectAction; this.applyObjectsAction = applyObjectsAction; + this.batchApplyObjectAction = batchApplyObjectAction; this.projectInitializationAction = projectInitializationAction; this.updateHEADAction = updateHEADAction; this.projectDeletionAction = projectDeletionAction; this.projectCache = projectCache; this.pluginName = pluginName; + this.gson = OutputFormat.JSON.newGsonBuilder().create(); this.currentUserProvider = currentUserProvider; } @@ -117,12 +134,18 @@ if (isFetchAction(httpRequest)) { failIfcurrentUserIsAnonymous(); PayloadSerDes.writeResponse(httpResponse, doFetch(httpRequest)); + } else if (isBatchFetchAction(httpRequest)) { + failIfcurrentUserIsAnonymous(); + PayloadSerDes.writeResponse(httpResponse, doBatchFetch(httpRequest)); } else if (isApplyObjectAction(httpRequest)) { failIfcurrentUserIsAnonymous(); PayloadSerDes.writeResponse(httpResponse, doApplyObject(httpRequest)); } else if (isApplyObjectsAction(httpRequest)) { failIfcurrentUserIsAnonymous(); PayloadSerDes.writeResponse(httpResponse, doApplyObjects(httpRequest)); + } else if (isBatchApplyObjectsAction(httpRequest)) { + failIfcurrentUserIsAnonymous(); + PayloadSerDes.writeResponse(httpResponse, doBatchApplyObject(httpRequest)); } else if (isInitProjectAction(httpRequest)) { failIfcurrentUserIsAnonymous(); if (!checkAcceptHeader(httpRequest, httpResponse)) { @@ -159,7 +182,7 @@ httpRequest, httpResponse, SC_CONFLICT, e.getMessage(), e.caching(), e); } catch (ResourceNotFoundException e) { RestApiServlet.replyError( - httpRequest, httpResponse, SC_INTERNAL_SERVER_ERROR, e.getMessage(), e.caching(), e); + httpRequest, httpResponse, SC_NOT_FOUND, e.getMessage(), e.caching(), e); } catch (NoSuchElementException e) { RestApiServlet.replyError( httpRequest, httpResponse, SC_BAD_REQUEST, "Project name not present in the url", e); @@ -205,6 +228,17 @@ } @SuppressWarnings("unchecked") + private Response<Map<String, Object>> doBatchApplyObject(HttpServletRequest httpRequest) + throws RestApiException, IOException { + TypeLiteral<List<RevisionInput>> collectionType = new TypeLiteral<>() {}; + List<RevisionInput> inputs = readJson(httpRequest, collectionType.getType()); + IdString id = getProjectName(httpRequest).get(); + + return (Response<Map<String, Object>>) + batchApplyObjectAction.apply(parseProjectResource(id), inputs); + } + + @SuppressWarnings("unchecked") private Response<String> doUpdateHEAD(HttpServletRequest httpRequest) throws Exception { HeadInput input = PayloadSerDes.parseHeadInput(httpRequest); IdString id = getProjectName(httpRequest).get(); @@ -237,15 +271,58 @@ return new ProjectResource(project.get(), currentUserProvider.get()); } - /** - * Return project name from request URI. Request URI format: - * /a/projects/<project_name>/pull-replication~apply-object - * - * @param req - * @return project name - */ - private Optional<IdString> getInitProjectName(HttpServletRequest req) { - return extractProjectName(req, projectNameInitProjectUrl); + @SuppressWarnings("unchecked") + private Response<Map<String, Object>> doBatchFetch(HttpServletRequest httpRequest) + throws IOException, RestApiException { + TypeLiteral<List<Input>> collectionType = new TypeLiteral<>() {}; + List<Input> inputs = readJson(httpRequest, collectionType.getType()); + IdString id = getProjectName(httpRequest).get(); + + return (Response<Map<String, Object>>) batchFetchAction.apply(parseProjectResource(id), inputs); + } + + private <T> void writeResponse(HttpServletResponse httpResponse, Response<T> response) + throws IOException { + String responseJson = gson.toJson(response); + if (response.statusCode() == SC_OK || response.statusCode() == SC_CREATED) { + + httpResponse.setContentType("application/json"); + httpResponse.setStatus(response.statusCode()); + PrintWriter writer = httpResponse.getWriter(); + writer.print(new String(RestApiServlet.JSON_MAGIC)); + writer.print(responseJson); + } else { + httpResponse.sendError(response.statusCode(), responseJson); + } + } + + private <T> T readJson(HttpServletRequest httpRequest, Type typeToken) + throws IOException, BadRequestException { + + try (BufferedReader br = httpRequest.getReader(); + JsonReader json = new JsonReader(br)) { + try { + json.setLenient(true); + + try { + json.peek(); + } catch (EOFException e) { + throw new BadRequestException("Expected JSON object", e); + } + + return gson.fromJson(json, typeToken); + } finally { + try { + // Reader.close won't consume the rest of the input. Explicitly consume the request + // body. + br.skip(Long.MAX_VALUE); + } catch (Exception e) { + // ignore, e.g. trying to consume the rest of the input may fail if the request was + // cancelled + logger.atFine().withCause(e).log("Exception during the parsing of the request json"); + } + } + } } private Optional<IdString> getProjectName(HttpServletRequest req) { @@ -275,10 +352,22 @@ .endsWith(String.format("/%s~" + APPLY_OBJECTS_API_ENDPOINT, pluginName)); } + private boolean isBatchApplyObjectsAction(HttpServletRequest httpRequest) { + return httpRequest + .getRequestURI() + .endsWith(String.format("/%s~" + BATCH_APPLY_OBJECT_API_ENDPOINT, pluginName)); + } + private boolean isFetchAction(HttpServletRequest httpRequest) { return httpRequest.getRequestURI().endsWith(String.format("/%s~" + FETCH_ENDPOINT, pluginName)); } + private boolean isBatchFetchAction(HttpServletRequest httpRequest) { + return httpRequest + .getRequestURI() + .endsWith(String.format("/%s~" + BATCH_FETCH_ENDPOINT, pluginName)); + } + private boolean isInitProjectAction(HttpServletRequest httpRequest) { return httpRequest .getRequestURI()
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/BatchApplyObjectData.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/BatchApplyObjectData.java new file mode 100644 index 0000000..7a613c0 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/BatchApplyObjectData.java
@@ -0,0 +1,45 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api.data; + +import com.google.auto.value.AutoValue; +import java.util.Optional; + +@AutoValue +public abstract class BatchApplyObjectData { + + public static BatchApplyObjectData create( + String refName, Optional<RevisionData> revisionData, boolean isDelete) + throws IllegalArgumentException { + if (isDelete && revisionData.isPresent()) { + throw new IllegalArgumentException( + "DELETE ref-updates cannot be associated with a RevisionData"); + } + return new AutoValue_BatchApplyObjectData(refName, revisionData, isDelete); + } + + public abstract String refName(); + + public abstract Optional<RevisionData> revisionData(); + + public abstract boolean isDelete(); + + @Override + public String toString() { + return String.format( + "%s:%s isDelete=%s", + refName(), revisionData().map(RevisionData::toString).orElse("ABSENT"), isDelete()); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java index f7ed4cb..27e13a7 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java
@@ -19,10 +19,10 @@ import com.google.gerrit.entities.Project; import com.google.gerrit.entities.Project.NameKey; import com.googlesource.gerrit.plugins.replication.pull.Source; +import com.googlesource.gerrit.plugins.replication.pull.api.data.BatchApplyObjectData; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; import java.io.IOException; import java.util.List; -import org.apache.http.client.ClientProtocolException; import org.eclipse.jgit.transport.URIish; public interface FetchApiClient { @@ -33,13 +33,23 @@ HttpResult callFetch( Project.NameKey project, String refName, URIish targetUri, long startTimeNanos) - throws ClientProtocolException, IOException; + throws IOException; default HttpResult callFetch(Project.NameKey project, String refName, URIish targetUri) - throws ClientProtocolException, IOException { + throws IOException { return callFetch(project, refName, targetUri, MILLISECONDS.toNanos(System.currentTimeMillis())); } + HttpResult callBatchFetch( + Project.NameKey project, List<String> refsInBatch, URIish targetUri, long startTimeNanos) + throws IOException; + + default HttpResult callBatchFetch( + Project.NameKey project, List<String> refsInBatch, URIish targetUri) throws IOException { + return callBatchFetch( + project, refsInBatch, targetUri, MILLISECONDS.toNanos(System.currentTimeMillis())); + } + /** * Replicates the creation of a project, including the configuration stored in refs/meta/config. * @@ -68,7 +78,14 @@ boolean isDelete, RevisionData revisionData, URIish targetUri) - throws ClientProtocolException, IOException; + throws IOException; + + HttpResult callBatchSendObject( + NameKey project, + List<BatchApplyObjectData> batchApplyObjects, + long eventCreatedOn, + URIish targetUri) + throws IOException; HttpResult callSendObjects( NameKey project, @@ -76,5 +93,5 @@ long eventCreatedOn, List<RevisionData> revisionData, URIish targetUri) - throws ClientProtocolException, IOException; + throws IOException; }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java index 7607e4b..081661f 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
@@ -14,8 +14,10 @@ package com.googlesource.gerrit.plugins.replication.pull.client; +import static com.google.common.net.HttpHeaders.CONTENT_TYPE; import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES; import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import com.google.common.base.Strings; import com.google.common.flogger.FluentLogger; @@ -36,6 +38,7 @@ import com.googlesource.gerrit.plugins.replication.pull.BearerTokenProvider; import com.googlesource.gerrit.plugins.replication.pull.Source; import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics; +import com.googlesource.gerrit.plugins.replication.pull.api.data.BatchApplyObjectData; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionsInput; @@ -43,13 +46,14 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import org.apache.http.HttpHeaders; import org.apache.http.HttpResponse; import org.apache.http.ParseException; import org.apache.http.auth.AuthenticationException; import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.ClientProtocolException; import org.apache.http.client.ResponseHandler; import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpPost; @@ -123,13 +127,81 @@ "{\"label\":\"%s\", \"ref_name\": \"%s\", \"async\":%s}", instanceId, refName, callAsync), StandardCharsets.UTF_8)); - post.addHeader(new BasicHeader("Content-Type", "application/json")); + post.addHeader(new BasicHeader(CONTENT_TYPE, "application/json")); post.addHeader( PullReplicationApiRequestMetrics.HTTP_HEADER_X_START_TIME_NANOS, Long.toString(startTimeNanos)); return executeRequest(post, bearerTokenProvider.get(), targetUri); } + private Map<Boolean, List<String>> partitionRefsToAsyncAndSync(List<String> refsInBatch) { + return refsInBatch.stream() + .collect( + Collectors.partitioningBy( + refName -> !syncRefsFilter.match(refName), + Collectors.mapping( + refName -> + String.format( + "{\"label\":\"%s\", \"ref_name\": \"%s\", \"async\":%s}", + instanceId, refName, !syncRefsFilter.match(refName)), + Collectors.toList()))); + } + + @Override + public HttpResult callBatchFetch( + NameKey project, List<String> refsInBatch, URIish targetUri, long startTimeNanos) + throws IOException { + Map<Boolean, List<String>> refsPartitionedInAsyncAndSync = + partitionRefsToAsyncAndSync(refsInBatch); + boolean async = true; + List<String> asyncRefs = refsPartitionedInAsyncAndSync.get(async); + List<String> syncRefs = refsPartitionedInAsyncAndSync.get(!async); + + if (asyncRefs.isEmpty() && syncRefs.isEmpty()) { + throw new IllegalArgumentException( + "At least one ref should be provided during a batch-fetch operation"); + } + + String url = formatUrl(targetUri.toString(), project, "batch-fetch"); + + if (asyncRefs.isEmpty()) { + HttpPost syncPost = + createPostRequest(url, "[" + String.join(",", syncRefs) + "]", startTimeNanos); + return executeRequest(syncPost, bearerTokenProvider.get(), targetUri); + } + if (syncRefs.isEmpty()) { + HttpPost asyncPost = + createPostRequest(url, "[" + String.join(",", asyncRefs) + "]", startTimeNanos); + return executeRequest(asyncPost, bearerTokenProvider.get(), targetUri); + } + + // first execute for async refs, then for sync + HttpPost asyncPost = + createPostRequest(url, "[" + String.join(",", asyncRefs) + "]", startTimeNanos); + HttpResult asyncResult = executeRequest(asyncPost, bearerTokenProvider.get(), targetUri); + + if (asyncResult.isSuccessful()) { + HttpPost syncPost = + createPostRequest( + url, + "[" + String.join(",", syncRefs) + "]", + MILLISECONDS.toNanos(System.currentTimeMillis())); + return executeRequest(syncPost, bearerTokenProvider.get(), targetUri); + } + + return asyncResult; + } + + private HttpPost createPostRequest(String url, String msgBody, long startTimeNanos) { + HttpPost post = new HttpPost(url); + post.setEntity(new StringEntity(msgBody, StandardCharsets.UTF_8)); + post.addHeader(new BasicHeader(CONTENT_TYPE, "application/json")); + post.addHeader( + PullReplicationApiRequestMetrics.HTTP_HEADER_X_START_TIME_NANOS, + Long.toString(startTimeNanos)); + return post; + } + /* (non-Javadoc) * @see com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient#initProject(com.google.gerrit.entities.Project.NameKey, org.eclipse.jgit.transport.URIish, long, java.util.List<com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData>) */ @@ -153,7 +225,7 @@ HttpPut put = new HttpPut(url); put.setEntity(new StringEntity(GSON.toJson(input))); put.addHeader(new BasicHeader("Accept", MediaType.ANY_TEXT_TYPE.toString())); - put.addHeader(new BasicHeader("Content-Type", MediaType.JSON_UTF_8.toString())); + put.addHeader(new BasicHeader(CONTENT_TYPE, MediaType.JSON_UTF_8.toString())); return executeRequest(put, bearerTokenProvider.get(), uri); } @@ -178,7 +250,7 @@ HttpPut req = new HttpPut(url); req.setEntity( new StringEntity(String.format("{\"ref\": \"%s\"}", newHead), StandardCharsets.UTF_8)); - req.addHeader(new BasicHeader("Content-Type", MediaType.JSON_UTF_8.toString())); + req.addHeader(new BasicHeader(CONTENT_TYPE, MediaType.JSON_UTF_8.toString())); return executeRequest(req, bearerTokenProvider.get(), apiUri); } @@ -193,7 +265,7 @@ boolean isDelete, @Nullable RevisionData revisionData, URIish targetUri) - throws ClientProtocolException, IOException { + throws IOException { if (!isDelete) { requireNonNull( @@ -207,7 +279,33 @@ HttpPost post = new HttpPost(url); post.setEntity(new StringEntity(GSON.toJson(input))); - post.addHeader(new BasicHeader("Content-Type", MediaType.JSON_UTF_8.toString())); + post.addHeader(new BasicHeader(CONTENT_TYPE, MediaType.JSON_UTF_8.toString())); + return executeRequest(post, bearerTokenProvider.get(), targetUri); + } + + @Override + public HttpResult callBatchSendObject( + NameKey project, + List<BatchApplyObjectData> batchedRefs, + long eventCreatedOn, + URIish targetUri) + throws IOException { + List<RevisionInput> inputs = + batchedRefs.stream() + .map( + batchApplyObject -> + new RevisionInput( + instanceId, + batchApplyObject.refName(), + eventCreatedOn, + batchApplyObject.revisionData().orElse(null))) + .collect(Collectors.toList()); + + String url = formatUrl(targetUri.toString(), project, "batch-apply-object"); + + HttpPost post = new HttpPost(url); + post.setEntity(new StringEntity(GSON.toJson(inputs))); + post.addHeader(new BasicHeader(CONTENT_TYPE, MediaType.JSON_UTF_8.toString())); return executeRequest(post, bearerTokenProvider.get(), targetUri); } @@ -218,7 +316,7 @@ long eventCreatedOn, List<RevisionData> revisionData, URIish targetUri) - throws ClientProtocolException, IOException { + throws IOException { if (revisionData.size() == 1) { return callSendObject( project, refName, eventCreatedOn, false, revisionData.get(0), targetUri); @@ -231,7 +329,7 @@ String url = formatUrl(targetUri.toString(), project, "apply-objects"); HttpPost post = new HttpPost(url); post.setEntity(new StringEntity(GSON.toJson(input))); - post.addHeader(new BasicHeader("Content-Type", MediaType.JSON_UTF_8.toString())); + post.addHeader(new BasicHeader(CONTENT_TYPE, MediaType.JSON_UTF_8.toString())); return executeRequest(post, bearerTokenProvider.get(), targetUri); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultUtils.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultUtils.java new file mode 100644 index 0000000..257e7e4 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultUtils.java
@@ -0,0 +1,43 @@ +// +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.client; + +import com.google.gerrit.entities.Project; +import java.util.Optional; + +public class HttpResultUtils { + + public static String status(Optional<HttpResult> maybeResult) { + return maybeResult.map(HttpResult::toString).orElse("unknown"); + } + + public static boolean isSuccessful(Optional<HttpResult> maybeResult) { + return maybeResult.map(HttpResult::isSuccessful).orElse(false); + } + + public static boolean isProjectMissing( + Optional<HttpResult> maybeResult, Project.NameKey project) { + return maybeResult.map(r -> r.isProjectMissing(project)).orElse(false); + } + + public static boolean isParentObjectMissing(Optional<HttpResult> maybeResult) { + return maybeResult.map(HttpResult::isParentObjectMissing).orElse(false); + } + + public static String errorMsg(Optional<HttpResult> maybeResult) { + return maybeResult.flatMap(HttpResult::getMessage).orElse("unknown"); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/EventsBrokerMessageConsumer.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/EventsBrokerMessageConsumer.java index a21a48c..a9b4945 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/EventsBrokerMessageConsumer.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/EventsBrokerMessageConsumer.java
@@ -19,7 +19,6 @@ import com.gerritforge.gerrit.eventbroker.BrokerApi; import com.gerritforge.gerrit.eventbroker.ExtendedBrokerApi; -import com.google.common.flogger.FluentLogger; import com.google.gerrit.common.Nullable; import com.google.gerrit.extensions.events.LifecycleListener; import com.google.gerrit.extensions.registration.DynamicItem; @@ -29,11 +28,10 @@ import com.google.inject.Inject; import com.google.inject.name.Named; import com.googlesource.gerrit.plugins.replication.pull.ShutdownState; +import java.io.IOException; import java.util.function.Consumer; public class EventsBrokerMessageConsumer implements Consumer<Event>, LifecycleListener { - - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private final DynamicItem<BrokerApi> eventsBrokerDi; private final StreamEventListener eventListener; private final ShutdownState shutdownState; @@ -60,7 +58,7 @@ try { eventListener.fetchRefsForEvent(event); if (shutdownState.isShuttingDown()) stop(); - } catch (AuthException | PermissionBackendException e) { + } catch (AuthException | PermissionBackendException | IOException e) { throw new EventRejectedException(event, e); } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandler.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandler.java deleted file mode 100644 index a618e16..0000000 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandler.java +++ /dev/null
@@ -1,67 +0,0 @@ -// Copyright (C) 2021 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.googlesource.gerrit.plugins.replication.pull.event; - -import com.google.common.flogger.FluentLogger; -import com.google.gerrit.entities.Change; -import com.google.gerrit.entities.Project; -import com.google.gerrit.entities.RefNames; -import com.google.gerrit.server.events.Event; -import com.google.gerrit.server.events.EventListener; -import com.google.gerrit.server.index.change.ChangeIndexer; -import com.google.inject.Inject; -import com.googlesource.gerrit.plugins.replication.pull.Context; -import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent; -import com.googlesource.gerrit.plugins.replication.pull.ReplicationState; - -public class FetchRefReplicatedEventHandler implements EventListener { - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private ChangeIndexer changeIndexer; - - @Inject - FetchRefReplicatedEventHandler(ChangeIndexer changeIndexer) { - this.changeIndexer = changeIndexer; - } - - @Override - public void onEvent(Event event) { - if (event instanceof FetchRefReplicatedEvent && isLocalEvent()) { - FetchRefReplicatedEvent fetchRefReplicatedEvent = (FetchRefReplicatedEvent) event; - if (!RefNames.isNoteDbMetaRef(fetchRefReplicatedEvent.getRefName()) - || !fetchRefReplicatedEvent - .getStatus() - .equals(ReplicationState.RefFetchResult.SUCCEEDED.toString())) { - return; - } - - Project.NameKey projectNameKey = fetchRefReplicatedEvent.getProjectNameKey(); - logger.atFine().log( - "Indexing ref '%s' for project %s", - fetchRefReplicatedEvent.getRefName(), projectNameKey.get()); - Change.Id changeId = Change.Id.fromRef(fetchRefReplicatedEvent.getRefName()); - if (changeId != null) { - changeIndexer.index(projectNameKey, changeId); - } else { - logger.atWarning().log( - "Couldn't get changeId from refName. Skipping indexing of change %s for project %s", - fetchRefReplicatedEvent.getRefName(), projectNameKey.get()); - } - } - } - - private boolean isLocalEvent() { - return Context.isLocalEvent(); - } -}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventModule.java deleted file mode 100644 index 675563a..0000000 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventModule.java +++ /dev/null
@@ -1,27 +0,0 @@ -// Copyright (C) 2021 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.googlesource.gerrit.plugins.replication.pull.event; - -import com.google.gerrit.extensions.registration.DynamicSet; -import com.google.gerrit.lifecycle.LifecycleModule; -import com.google.gerrit.server.events.EventListener; - -public class FetchRefReplicatedEventModule extends LifecycleModule { - - @Override - protected void configure() { - DynamicSet.bind(binder(), EventListener.class).to(FetchRefReplicatedEventHandler.class); - } -}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java index b14d4a1..4c621ac 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
@@ -95,7 +95,7 @@ public void onEvent(Event event) { try { fetchRefsForEvent(event); - } catch (AuthException | PermissionBackendException e) { + } catch (AuthException | PermissionBackendException | IOException e) { logger.atSevere().withCause(e).log( "This is the event handler of Gerrit's event-bus. It isn't" + "supposed to throw any exception, otherwise the other handlers " @@ -103,7 +103,8 @@ } } - public void fetchRefsForEvent(Event event) throws AuthException, PermissionBackendException { + public void fetchRefsForEvent(Event event) + throws AuthException, PermissionBackendException, IOException { if (instanceId.equals(event.instanceId) || !shouldReplicateProject(event)) { return; } @@ -153,7 +154,7 @@ projectCreatedEvent.instanceId, projectCreatedEvent.getProjectNameKey(), metrics); - } catch (AuthException | PermissionBackendException e) { + } catch (AuthException | PermissionBackendException | IOException e) { logger.atSevere().withCause(e).log( "Cannot initialise project:%s", projectCreatedEvent.projectName); throw e;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java index 05033b4..dc90c7f 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java
@@ -15,6 +15,8 @@ package com.googlesource.gerrit.plugins.replication.pull.fetch; import com.google.gerrit.entities.Project; +import com.google.gerrit.extensions.restapi.IdString; +import com.google.gerrit.extensions.restapi.ResourceNotFoundException; import com.google.gerrit.server.git.GitRepositoryManager; import com.google.inject.Inject; import com.googlesource.gerrit.plugins.replication.pull.LocalGitRepositoryManagerProvider; @@ -22,6 +24,7 @@ import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData; import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException; import java.io.IOException; +import org.eclipse.jgit.errors.RepositoryNotFoundException; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.RefUpdate; @@ -43,7 +46,7 @@ } public RefUpdateState apply(Project.NameKey name, RefSpec refSpec, RevisionData[] revisionsData) - throws MissingParentObjectException, IOException { + throws MissingParentObjectException, IOException, ResourceNotFoundException { try (Repository git = gitManager.openRepository(name)) { ObjectId refHead = null; @@ -90,6 +93,8 @@ RefUpdate.Result result = ru.update(); return new RefUpdateState(refSpec.getSource(), result); } + } catch (RepositoryNotFoundException e) { + throw new ResourceNotFoundException(IdString.fromDecoded(name.get()), e); } } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java index 1845230..039c395 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java
@@ -25,7 +25,10 @@ import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.transport.*; +import org.eclipse.jgit.transport.FetchResult; +import org.eclipse.jgit.transport.RefSpec; +import org.eclipse.jgit.transport.Transport; +import org.eclipse.jgit.transport.URIish; public class JGitFetch implements Fetch { URIish uri;
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index 61d62b4..99d8b78 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -591,6 +591,21 @@ By default, replicates without matching, i.e. replicates everything from all remotes. +remote.NAME.enableBatchedRefs +: Choose whether the batch-apply-object endpoint is enabled. + If you set this to `true`, then there will be a single call + to the batch-apply-object endpoint with all the refs from + the batch ref update included. The default behaviour means + one call to the apply object(s) endpoint per ref. + + *NOTE*: the default value is only needed for backwards + compatibility to allow migrating transparently to the + latest pull-replication plugin version. Once the migration is + over, this value should be set to `true` to leverage the + performance improvements introduced by the `batch-apply-object` API. + + By default, false. + Directory `replication` -------------------- The optional directory `$site_path/etc/replication` contains Git-style
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FakeGitReferenceUpdatedEvent.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FakeGitReferenceUpdatedEvent.java deleted file mode 100644 index 1472be2..0000000 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FakeGitReferenceUpdatedEvent.java +++ /dev/null
@@ -1,37 +0,0 @@ -// Copyright (C) 2020 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.googlesource.gerrit.plugins.replication.pull; - -import com.google.common.base.Suppliers; -import com.google.gerrit.entities.Project; -import com.google.gerrit.server.data.RefUpdateAttribute; -import com.google.gerrit.server.events.RefUpdatedEvent; - -public class FakeGitReferenceUpdatedEvent extends RefUpdatedEvent { - FakeGitReferenceUpdatedEvent( - Project.NameKey project, - String ref, - String oldObjectId, - String newObjectId, - String instanceId) { - RefUpdateAttribute upd = new RefUpdateAttribute(); - upd.newRev = newObjectId; - upd.oldRev = oldObjectId; - upd.project = project.get(); - upd.refName = ref; - this.refUpdate = Suppliers.ofInstance(upd); - this.instanceId = instanceId; - } -}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchOneTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchOneTest.java index dd9f98e..3445604 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchOneTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchOneTest.java
@@ -16,7 +16,13 @@ import static com.google.common.truth.Truth.assertThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.anyList; +import static org.mockito.Mockito.argThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import com.google.gerrit.acceptance.TestMetricMaker; import com.google.gerrit.entities.Project; @@ -29,7 +35,11 @@ import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory; import com.googlesource.gerrit.plugins.replication.pull.fetch.InexistentRefTransportException; import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState; -import java.util.*; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.Callable; import java.util.stream.Collectors; import java.util.stream.IntStream;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationBatchRefUpdatedAsyncIT.java similarity index 90% rename from src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java rename to src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationBatchRefUpdatedAsyncIT.java index 0ebd0ae..a72a97e 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationBatchRefUpdatedAsyncIT.java
@@ -29,10 +29,15 @@ sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationITAbstract$PullReplicationTestModule", httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule") -public class PullReplicationAsyncIT extends PullReplicationITAbstract { +public class PullReplicationBatchRefUpdatedAsyncIT extends PullReplicationITAbstract { @Inject private SitePaths sitePaths; @Override + protected boolean useBatchRefUpdateEvent() { + return true; + } + + @Override public void setUpTestPlugin() throws Exception { FileBasedConfig config = new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationBatchRefUpdatedIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationBatchRefUpdatedIT.java new file mode 100644 index 0000000..2e727d1 --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationBatchRefUpdatedIT.java
@@ -0,0 +1,33 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull; + +import com.google.gerrit.acceptance.SkipProjectClone; +import com.google.gerrit.acceptance.TestPlugin; +import com.google.gerrit.acceptance.UseLocalDisk; + +@SkipProjectClone +@UseLocalDisk +@TestPlugin( + name = "pull-replication", + sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule", + httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule") +public class PullReplicationBatchRefUpdatedIT extends PullReplicationITBase { + + @Override + protected boolean useBatchRefUpdateEvent() { + return true; + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigBase.java similarity index 72% rename from src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigIT.java rename to src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigBase.java index 0721dd2..4349cde 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigBase.java
@@ -17,26 +17,16 @@ import static com.google.common.truth.Truth.assertThat; import static java.util.stream.Collectors.toList; -import com.google.common.flogger.FluentLogger; -import com.google.gerrit.acceptance.LightweightPluginDaemonTest; import com.google.gerrit.acceptance.PushOneCommit.Result; -import com.google.gerrit.acceptance.SkipProjectClone; -import com.google.gerrit.acceptance.TestPlugin; -import com.google.gerrit.acceptance.UseLocalDisk; import com.google.gerrit.acceptance.config.GerritConfig; -import com.google.gerrit.acceptance.testsuite.project.ProjectOperations; -import com.google.gerrit.entities.Project; import com.google.gerrit.extensions.api.projects.BranchInput; -import com.google.gerrit.server.config.SitePaths; -import com.google.inject.Inject; +import com.google.gerrit.server.events.ProjectEvent; import com.googlesource.gerrit.plugins.replication.AutoReloadConfigDecorator; +import java.io.IOException; import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; import java.util.Arrays; import java.util.List; import java.util.Optional; -import java.util.function.Supplier; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; @@ -45,33 +35,14 @@ import org.eclipse.jgit.util.FS; import org.junit.Test; -@SkipProjectClone -@UseLocalDisk -@TestPlugin( - name = "pull-replication", - sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule", - httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule") -public class PullReplicationFanoutConfigIT extends LightweightPluginDaemonTest { - private static final Optional<String> ALL_PROJECTS = Optional.empty(); - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); +abstract class PullReplicationFanoutConfigBase extends PullReplicationSetupBase { private static final int TEST_REPLICATION_DELAY = 60; - private static final Duration TEST_TIMEOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 2); - private static final String TEST_REPLICATION_SUFFIX = "suffix1"; - private static final String TEST_REPLICATION_REMOTE = "remote1"; - @Inject private SitePaths sitePaths; - @Inject private ProjectOperations projectOperations; - private Path gitPath; - private FileBasedConfig config; private FileBasedConfig remoteConfig; - private FileBasedConfig secureConfig; @Override public void setUpTestPlugin() throws Exception { gitPath = sitePaths.site_path.resolve("git"); - - config = - new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED); remoteConfig = new FileBasedConfig( sitePaths @@ -80,17 +51,8 @@ .toFile(), FS.DETECTED); - setReplicationSource( - TEST_REPLICATION_REMOTE); // Simulates a full replication.config initialization - setRemoteConfig(TEST_REPLICATION_SUFFIX, ALL_PROJECTS); - - secureConfig = - new FileBasedConfig(sitePaths.etc_dir.resolve("secure.config").toFile(), FS.DETECTED); - setReplicationCredentials(TEST_REPLICATION_REMOTE, admin.username(), admin.httpPassword()); - secureConfig.save(); - - super.setUpTestPlugin(); + super.setUpTestPlugin(false); } @Test @@ -103,8 +65,8 @@ String sourceRef = pushResult.getPatchSet().refName(); ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( + ProjectEvent event = + generateUpdateEvent( project, sourceRef, ObjectId.zeroId().getName(), @@ -139,8 +101,8 @@ RevCommit sourceCommit = pushResult.getCommit(); final String sourceRef = pushResult.getPatchSet().refName(); ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( + ProjectEvent event = + generateUpdateEvent( project, sourceRef, ObjectId.zeroId().getName(), @@ -172,8 +134,8 @@ ReplicationQueue pullReplicationQueue = plugin.getSysInjector().getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( + ProjectEvent event = + generateUpdateEvent( project, newBranch, ObjectId.zeroId().getName(), @@ -250,37 +212,31 @@ waitUntil(() -> sources.getAll().size() == 1); } - private Ref getRef(Repository repo, String branchName) throws Exception { - return repo.getRefDatabase().exactRef(branchName); + @Override + protected void setReplicationSource( + String remoteName, List<String> replicaSuffixes, Optional<String> project) + throws IOException { + setReplicationSource(remoteName); } - private Ref checkedGetRef(Repository repo, String branchName) { - try { - return repo.getRefDatabase().exactRef(branchName); - } catch (Exception e) { - logger.atSevere().withCause(e).log("failed to get ref %s in repo %s", branchName, repo); - return null; - } - } - - private void setReplicationSource(String remoteName) throws Exception { + private void setReplicationSource(String remoteName) throws IOException { config.setBoolean("gerrit", null, "autoReload", true); config.save(); } - private void setRemoteConfig(String replicaSuffix, Optional<String> project) throws Exception { + private void setRemoteConfig(String replicaSuffix, Optional<String> project) throws IOException { setRemoteConfig(remoteConfig, replicaSuffix, project); } private void setRemoteConfig( FileBasedConfig remoteConfig, String replicaSuffix, Optional<String> project) - throws Exception { + throws IOException { setRemoteConfig(remoteConfig, Arrays.asList(replicaSuffix), project); } private void setRemoteConfig( FileBasedConfig remoteConfig, List<String> replicaSuffixes, Optional<String> project) - throws Exception { + throws IOException { List<String> replicaUrls = replicaSuffixes.stream() .map(suffix -> gitPath.resolve("${name}" + suffix + ".git").toString()) @@ -293,23 +249,4 @@ project.ifPresent(prj -> remoteConfig.setString("remote", null, "projects", prj)); remoteConfig.save(); } - - private void setReplicationCredentials(String remoteName, String username, String password) - throws Exception { - secureConfig.setString("remote", remoteName, "username", username); - secureConfig.setString("remote", remoteName, "password", password); - secureConfig.save(); - } - - private void waitUntil(Supplier<Boolean> waitCondition) throws Exception { - WaitUtil.waitUntil(waitCondition, TEST_TIMEOUT); - } - - private <T> T getInstance(Class<T> classObj) { - return plugin.getSysInjector().getInstance(classObj); - } - - private Project.NameKey createTestProject(String name) throws Exception { - return projectOperations.newProject().name(name).create(); - } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigBatchRefUpdateEventIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigBatchRefUpdateEventIT.java new file mode 100644 index 0000000..7babf34 --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigBatchRefUpdateEventIT.java
@@ -0,0 +1,33 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.googlesource.gerrit.plugins.replication.pull; + +import com.google.gerrit.acceptance.SkipProjectClone; +import com.google.gerrit.acceptance.TestPlugin; +import com.google.gerrit.acceptance.UseLocalDisk; + +@SkipProjectClone +@UseLocalDisk +@TestPlugin( + name = "pull-replication", + sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule", + httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule") +public class PullReplicationFanoutConfigBatchRefUpdateEventIT + extends PullReplicationFanoutConfigBase { + + @Override + protected boolean useBatchRefUpdateEvent() { + return true; + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigRefUpdatedEventIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigRefUpdatedEventIT.java new file mode 100644 index 0000000..16b0b02 --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigRefUpdatedEventIT.java
@@ -0,0 +1,32 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.googlesource.gerrit.plugins.replication.pull; + +import com.google.gerrit.acceptance.SkipProjectClone; +import com.google.gerrit.acceptance.TestPlugin; +import com.google.gerrit.acceptance.UseLocalDisk; + +@SkipProjectClone +@UseLocalDisk +@TestPlugin( + name = "pull-replication", + sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule", + httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule") +public class PullReplicationFanoutConfigRefUpdatedEventIT extends PullReplicationFanoutConfigBase { + + @Override + protected boolean useBatchRefUpdateEvent() { + return false; + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationITAbstract.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationITAbstract.java index 8160304..43f6ef2 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationITAbstract.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationITAbstract.java
@@ -37,6 +37,7 @@ import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.events.Event; import com.google.gerrit.server.events.EventListener; +import com.google.gerrit.server.events.ProjectEvent; import com.google.inject.Inject; import com.google.inject.Singleton; import com.googlesource.gerrit.plugins.replication.AutoReloadConfigDecorator; @@ -165,8 +166,8 @@ String sourceRef = pushResult.getPatchSet().refName(); ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( + ProjectEvent event = + generateUpdateEvent( project, sourceRef, ObjectId.zeroId().getName(), @@ -213,8 +214,8 @@ ReplicationQueue pullReplicationQueue = plugin.getSysInjector().getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( + ProjectEvent event = + generateUpdateEvent( project, newBranch, ObjectId.zeroId().getName(), @@ -245,8 +246,8 @@ ReplicationQueue pullReplicationQueue = plugin.getSysInjector().getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( + ProjectEvent event = + generateUpdateEvent( project, newBranch, ObjectId.zeroId().getName(), @@ -292,8 +293,8 @@ ReplicationQueue pullReplicationQueue = plugin.getSysInjector().getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( + ProjectEvent event = + generateUpdateEvent( project, newBranch, ObjectId.zeroId().getName(), @@ -322,8 +323,8 @@ assertThat(pushedRefs).hasSize(1); assertThat(pushedRefs.iterator().next().getStatus()).isEqualTo(Status.OK); - FakeGitReferenceUpdatedEvent forcedPushEvent = - new FakeGitReferenceUpdatedEvent( + ProjectEvent forcedPushEvent = + generateUpdateEvent( project, newBranch, branchRevision, @@ -367,8 +368,8 @@ String sourceRef = pushResult.getPatchSet().refName(); ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( + ProjectEvent event = + generateUpdateEvent( project, sourceRef, ObjectId.zeroId().getName(), @@ -414,8 +415,8 @@ ReplicationQueue pullReplicationQueue = plugin.getSysInjector().getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( + ProjectEvent event = + generateUpdateEvent( project, newBranch, ObjectId.zeroId().getName(), @@ -524,8 +525,8 @@ String sourceRef = pushResult.getPatchSet().refName(); ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( + ProjectEvent event = + generateUpdateEvent( project, sourceRef, ObjectId.zeroId().getName(),
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationITBase.java similarity index 93% rename from src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java rename to src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationITBase.java index 5717a8b..0754870 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationITBase.java
@@ -21,8 +21,6 @@ import static com.google.gerrit.server.group.SystemGroupBackend.REGISTERED_USERS; import com.google.gerrit.acceptance.PushOneCommit.Result; -import com.google.gerrit.acceptance.SkipProjectClone; -import com.google.gerrit.acceptance.TestPlugin; import com.google.gerrit.acceptance.UseLocalDisk; import com.google.gerrit.acceptance.config.GerritConfig; import com.google.gerrit.entities.Permission; @@ -34,6 +32,7 @@ import com.google.gerrit.extensions.events.HeadUpdatedListener; import com.google.gerrit.extensions.events.ProjectDeletedListener; import com.google.gerrit.extensions.restapi.RestApiException; +import com.google.gerrit.server.events.ProjectEvent; import com.googlesource.gerrit.plugins.replication.AutoReloadConfigDecorator; import com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient; import java.io.IOException; @@ -54,14 +53,7 @@ import org.junit.Ignore; import org.junit.Test; -@SkipProjectClone -@UseLocalDisk -@TestPlugin( - name = "pull-replication", - sysModule = - "com.googlesource.gerrit.plugins.replication.pull.PullReplicationITAbstract$PullReplicationTestModule", - httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule") -public class PullReplicationIT extends PullReplicationSetupBase { +abstract class PullReplicationITBase extends PullReplicationSetupBase { @Override protected void setReplicationSource( @@ -94,8 +86,8 @@ String sourceRef = pushResult.getPatchSet().refName(); ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( + ProjectEvent event = + generateUpdateEvent( project, sourceRef, ObjectId.zeroId().getName(), @@ -127,8 +119,8 @@ ReplicationQueue pullReplicationQueue = plugin.getSysInjector().getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( + ProjectEvent event = + generateUpdateEvent( project, newBranch, ObjectId.zeroId().getName(), @@ -170,8 +162,8 @@ ReplicationQueue pullReplicationQueue = plugin.getSysInjector().getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( + ProjectEvent event = + generateUpdateEvent( project, newBranch, ObjectId.zeroId().getName(), @@ -196,8 +188,8 @@ assertThat(pushedRefs).hasSize(1); assertThat(pushedRefs.iterator().next().getStatus()).isEqualTo(Status.OK); - FakeGitReferenceUpdatedEvent forcedPushEvent = - new FakeGitReferenceUpdatedEvent( + ProjectEvent forcedPushEvent = + generateUpdateEvent( project, newBranch, branchRevision, @@ -235,8 +227,8 @@ String sourceRef = pushResult.getPatchSet().refName(); ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( + ProjectEvent event = + generateUpdateEvent( project, sourceRef, ObjectId.zeroId().getName(), @@ -276,8 +268,8 @@ ReplicationQueue pullReplicationQueue = plugin.getSysInjector().getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( + ProjectEvent event = + generateUpdateEvent( project, newBranch, ObjectId.zeroId().getName(), @@ -390,8 +382,8 @@ String sourceRef = pushResult.getPatchSet().refName(); ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( + ProjectEvent event = + generateUpdateEvent( project, sourceRef, ObjectId.zeroId().getName(),
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationRefUpdatedAsyncIT.java similarity index 90% copy from src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java copy to src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationRefUpdatedAsyncIT.java index 0ebd0ae..fba8783 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationRefUpdatedAsyncIT.java
@@ -29,10 +29,15 @@ sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationITAbstract$PullReplicationTestModule", httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule") -public class PullReplicationAsyncIT extends PullReplicationITAbstract { +public class PullReplicationRefUpdatedAsyncIT extends PullReplicationITAbstract { @Inject private SitePaths sitePaths; @Override + protected boolean useBatchRefUpdateEvent() { + return false; + } + + @Override public void setUpTestPlugin() throws Exception { FileBasedConfig config = new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationRefUpdatedIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationRefUpdatedIT.java new file mode 100644 index 0000000..6e7c369 --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationRefUpdatedIT.java
@@ -0,0 +1,33 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull; + +import com.google.gerrit.acceptance.SkipProjectClone; +import com.google.gerrit.acceptance.TestPlugin; +import com.google.gerrit.acceptance.UseLocalDisk; + +@SkipProjectClone +@UseLocalDisk +@TestPlugin( + name = "pull-replication", + sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule", + httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule") +public class PullReplicationRefUpdatedIT extends PullReplicationITBase { + + @Override + protected boolean useBatchRefUpdateEvent() { + return false; + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationSetupBase.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationSetupBase.java index fea576a..9cb2dc9 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationSetupBase.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationSetupBase.java
@@ -16,13 +16,20 @@ import static java.util.stream.Collectors.toList; +import com.google.common.base.Suppliers; import com.google.common.flogger.FluentLogger; import com.google.gerrit.acceptance.LightweightPluginDaemonTest; import com.google.gerrit.acceptance.testsuite.project.ProjectOperations; +import com.google.gerrit.entities.Project; import com.google.gerrit.entities.Project.NameKey; import com.google.gerrit.extensions.events.ProjectDeletedListener; import com.google.gerrit.extensions.registration.DynamicSet; import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.data.AccountAttribute; +import com.google.gerrit.server.data.RefUpdateAttribute; +import com.google.gerrit.server.events.BatchRefUpdateEvent; +import com.google.gerrit.server.events.ProjectEvent; +import com.google.gerrit.server.events.RefUpdatedEvent; import com.google.inject.Inject; import java.io.File; import java.io.IOException; @@ -53,6 +60,22 @@ protected FileBasedConfig config; protected FileBasedConfig secureConfig; + protected abstract boolean useBatchRefUpdateEvent(); + + protected ProjectEvent generateUpdateEvent( + Project.NameKey project, + String ref, + String oldObjectId, + String newObjectId, + String instanceId) { + + if (useBatchRefUpdateEvent()) { + return generateBatchRefUpdateEvent(project, ref, oldObjectId, newObjectId, instanceId); + } + + return generateRefUpdateEvent(project, ref, oldObjectId, newObjectId, instanceId); + } + protected void setUpTestPlugin(boolean loadExisting) throws Exception { gitPath = sitePaths.site_path.resolve("git"); @@ -72,6 +95,9 @@ setReplicationCredentials(TEST_REPLICATION_REMOTE, admin.username(), admin.httpPassword()); secureConfig.save(); + cfg.setBoolean( + "event", "stream-events", "enableBatchRefUpdatedEvents", useBatchRefUpdateEvent()); + super.setUpTestPlugin(); } @@ -119,4 +145,38 @@ List<String> replicaSuffixes, Function<String, String> toURL) { return replicaSuffixes.stream().map(suffix -> toURL.apply(suffix)).collect(toList()); } + + private BatchRefUpdateEvent generateBatchRefUpdateEvent( + Project.NameKey project, + String ref, + String oldObjectId, + String newObjectId, + String instanceId) { + RefUpdateAttribute upd = new RefUpdateAttribute(); + upd.newRev = newObjectId; + upd.oldRev = oldObjectId; + upd.project = project.get(); + upd.refName = ref; + BatchRefUpdateEvent event = + new BatchRefUpdateEvent( + project, + Suppliers.ofInstance(List.of(upd)), + Suppliers.ofInstance(new AccountAttribute(admin.id().get()))); + event.instanceId = instanceId; + return event; + } + + private ProjectEvent generateRefUpdateEvent( + NameKey project, String ref, String oldObjectId, String newObjectId, String instanceId) { + RefUpdateAttribute upd = new RefUpdateAttribute(); + upd.newRev = newObjectId; + upd.oldRev = oldObjectId; + upd.project = project.get(); + upd.refName = ref; + RefUpdatedEvent event = new RefUpdatedEvent(); + event.refUpdate = Suppliers.ofInstance(upd); + event.submitter = Suppliers.ofInstance(new AccountAttribute(admin.id().get())); + event.instanceId = instanceId; + return event; + } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolBase.java similarity index 94% rename from src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolIT.java rename to src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolBase.java index e55e383..2e95ef1 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolBase.java
@@ -21,6 +21,7 @@ import com.google.gerrit.acceptance.TestPlugin; import com.google.gerrit.acceptance.UseLocalDisk; import com.google.gerrit.acceptance.config.GerritConfig; +import com.google.gerrit.server.events.ProjectEvent; import java.io.IOException; import java.util.List; import java.util.Optional; @@ -36,7 +37,8 @@ name = "pull-replication", sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule", httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule") -public class PullReplicationWithGitHttpTransportProtocolIT extends PullReplicationSetupBase { +public abstract class PullReplicationWithGitHttpTransportProtocolBase + extends PullReplicationSetupBase { @Override protected void setReplicationSource( @@ -80,8 +82,8 @@ String sourceRef = pushResult.getPatchSet().refName(); ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); - FakeGitReferenceUpdatedEvent event = - new FakeGitReferenceUpdatedEvent( + ProjectEvent event = + generateUpdateEvent( project, sourceRef, ObjectId.zeroId().getName(),
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolBatchRefUpdatedIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolBatchRefUpdatedIT.java new file mode 100644 index 0000000..8c8ca37 --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolBatchRefUpdatedIT.java
@@ -0,0 +1,34 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull; + +import com.google.gerrit.acceptance.SkipProjectClone; +import com.google.gerrit.acceptance.TestPlugin; +import com.google.gerrit.acceptance.UseLocalDisk; + +@SkipProjectClone +@UseLocalDisk +@TestPlugin( + name = "pull-replication", + sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule", + httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule") +public class PullReplicationWithGitHttpTransportProtocolBatchRefUpdatedIT + extends PullReplicationWithGitHttpTransportProtocolBase { + + @Override + protected boolean useBatchRefUpdateEvent() { + return true; + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolRefUpdatedIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolRefUpdatedIT.java new file mode 100644 index 0000000..5f3c7b6 --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolRefUpdatedIT.java
@@ -0,0 +1,34 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull; + +import com.google.gerrit.acceptance.SkipProjectClone; +import com.google.gerrit.acceptance.TestPlugin; +import com.google.gerrit.acceptance.UseLocalDisk; + +@SkipProjectClone +@UseLocalDisk +@TestPlugin( + name = "pull-replication", + sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule", + httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule") +public class PullReplicationWithGitHttpTransportProtocolRefUpdatedIT + extends PullReplicationWithGitHttpTransportProtocolBase { + + @Override + protected boolean useBatchRefUpdateEvent() { + return false; + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java index a07aa55..90c9b9c 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
@@ -26,7 +26,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import com.google.common.base.Suppliers; @@ -35,12 +35,13 @@ import com.google.gerrit.entities.Project; import com.google.gerrit.entities.RefNames; import com.google.gerrit.extensions.api.changes.NotifyHandling; -import com.google.gerrit.extensions.common.AccountInfo; import com.google.gerrit.extensions.events.ProjectDeletedListener; import com.google.gerrit.extensions.registration.DynamicItem; import com.google.gerrit.metrics.DisabledMetricMaker; import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.data.AccountAttribute; import com.google.gerrit.server.data.RefUpdateAttribute; +import com.google.gerrit.server.events.BatchRefUpdateEvent; import com.google.gerrit.server.events.Event; import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.server.events.RefUpdatedEvent; @@ -48,6 +49,7 @@ import com.google.inject.Provider; import com.googlesource.gerrit.plugins.replication.ReplicationConfig; import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig; +import com.googlesource.gerrit.plugins.replication.pull.api.data.BatchApplyObjectData; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; import com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient; import com.googlesource.gerrit.plugins.replication.pull.client.FetchRestApiClient; @@ -55,12 +57,16 @@ import com.googlesource.gerrit.plugins.replication.pull.filter.ApplyObjectsRefsFilter; import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter; import java.io.IOException; +import java.net.URISyntaxException; import java.nio.file.Path; import java.util.Arrays; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; +import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.transport.URIish; import org.eclipse.jgit.util.FS; import org.junit.Before; import org.junit.Test; @@ -77,6 +83,10 @@ private static final String FOREIGN_INSTANCE_ID = "any other instance id"; private static final String TEST_REF_NAME = "refs/meta/heads/anyref"; + private static final Project.NameKey PROJECT = Project.nameKey("defaultProject"); + private static final String NEW_OBJECT_ID = + ObjectId.fromString("3c1ddc050d7906adb0e29bc3bc46af8749b2f63b").getName(); + @Mock private WorkQueue wq; @Mock private Source source; @Mock private SourcesCollection sourceCollection; @@ -85,23 +95,27 @@ @Mock ReplicationStateListeners sl; @Mock FetchRestApiClient fetchRestApiClient; @Mock FetchApiClient.Factory fetchClientFactory; - @Mock AccountInfo accountInfo; + @Mock AccountAttribute accountAttribute; @Mock RevisionReader revReader; @Mock RevisionData revisionData; @Mock HttpResult successfulHttpResult; @Mock HttpResult fetchHttpResult; + @Mock HttpResult batchFetchHttpResult; @Mock RevisionData revisionDataWithParents; List<ObjectId> revisionDataParentObjectIds; @Mock HttpResult httpResult; + @Mock HttpResult batchHttpResult; @Mock ApplyObjectsRefsFilter applyObjectsRefsFilter; + + @Mock Config config; ApplyObjectMetrics applyObjectMetrics; - FetchReplicationMetrics fetchMetrics; ReplicationQueueMetrics queueMetrics; ShutdownState shutdownState; @Captor ArgumentCaptor<String> stringCaptor; @Captor ArgumentCaptor<Project.NameKey> projectNameKeyCaptor; @Captor ArgumentCaptor<List<RevisionData>> revisionsDataCaptor; + @Captor ArgumentCaptor<List<BatchApplyObjectData>> batchRefsCaptor; private ExcludedRefsFilter refsFilter; private ReplicationQueue objectUnderTest; @@ -120,8 +134,11 @@ when(source.wouldFetchRef(anyString())).thenReturn(true); ImmutableList<String> apis = ImmutableList.of("http://localhost:18080"); when(source.getApis()).thenReturn(apis); + when(source.enableBatchedRefs()).thenReturn(true); when(sourceCollection.getAll()).thenReturn(Lists.newArrayList(source)); when(rd.get()).thenReturn(sourceCollection); + when(config.getBoolean("event", "stream-events", "enableBatchRefUpdatedEvents", false)) + .thenReturn(true); lenient() .when(revReader.read(any(), any(), anyString(), eq(0))) .thenReturn(Optional.of(revisionData)); @@ -147,17 +164,23 @@ lenient() .when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any())) .thenReturn(httpResult); + lenient() + .when(fetchRestApiClient.callBatchSendObject(any(), any(), anyLong(), any())) + .thenReturn(batchHttpResult); when(fetchRestApiClient.callFetch(any(), anyString(), any())).thenReturn(fetchHttpResult); + when(fetchRestApiClient.callBatchFetch(any(), any(), any())).thenReturn(batchFetchHttpResult); when(fetchRestApiClient.initProject(any(), any(), anyLong(), any())) .thenReturn(successfulHttpResult); when(successfulHttpResult.isSuccessful()).thenReturn(true); when(httpResult.isSuccessful()).thenReturn(true); + when(batchHttpResult.isSuccessful()).thenReturn(true); when(fetchHttpResult.isSuccessful()).thenReturn(true); + when(batchFetchHttpResult.isSuccessful()).thenReturn(true); when(httpResult.isProjectMissing(any())).thenReturn(false); + when(batchHttpResult.isProjectMissing(any())).thenReturn(false); when(applyObjectsRefsFilter.match(any())).thenReturn(false); applyObjectMetrics = new ApplyObjectMetrics("pull-replication", new DisabledMetricMaker()); - fetchMetrics = new FetchReplicationMetrics("pull-replication", new DisabledMetricMaker()); queueMetrics = new ReplicationQueueMetrics("pull-replication", new DisabledMetricMaker()); shutdownState = new ShutdownState(); @@ -171,38 +194,65 @@ refsFilter, () -> revReader, applyObjectMetrics, - fetchMetrics, queueMetrics, LOCAL_INSTANCE_ID, + config, applyObjectsRefsFilter, shutdownState); } @Test - public void shouldCallSendObjectWhenMetaRef() throws Exception { + public void shouldCallBatchSendObjectWhenMetaRef() throws Exception { + Event event = generateBatchRefUpdateEvent("refs/changes/01/1/meta"); + objectUnderTest.start(); + objectUnderTest.onEvent(event); + + verify(fetchRestApiClient).callBatchSendObject(any(), any(), anyLong(), any()); + } + + @Test + public void shouldCallSendObjectWhenMetaRefAndRefUpdateEvent() throws Exception { + when(config.getBoolean("event", "stream-events", "enableBatchRefUpdatedEvents", false)) + .thenReturn(false); + + objectUnderTest = + new ReplicationQueue( + wq, + rd, + dis, + sl, + fetchClientFactory, + refsFilter, + () -> revReader, + applyObjectMetrics, + queueMetrics, + LOCAL_INSTANCE_ID, + config, + applyObjectsRefsFilter, + shutdownState); + Event event = new TestEvent("refs/changes/01/1/meta"); objectUnderTest.start(); objectUnderTest.onEvent(event); - verify(fetchRestApiClient).callSendObjects(any(), anyString(), anyLong(), any(), any()); + verify(fetchRestApiClient).callBatchSendObject(any(), any(), anyLong(), any()); } @Test public void shouldIgnoreEventWhenIsNotLocalInstanceId() throws Exception { - Event event = new TestEvent(); + Event event = generateBatchRefUpdateEvent(TEST_REF_NAME); event.instanceId = FOREIGN_INSTANCE_ID; objectUnderTest.start(); objectUnderTest.onEvent(event); - verify(fetchRestApiClient, never()) - .callSendObjects(any(), anyString(), anyLong(), any(), any()); + verify(fetchRestApiClient, never()).callBatchSendObject(any(), any(), anyLong(), any()); } @Test public void shouldCallInitProjectWhenProjectIsMissing() throws Exception { - Event event = new TestEvent("refs/changes/01/1/meta"); - when(httpResult.isSuccessful()).thenReturn(false); - when(httpResult.isProjectMissing(any())).thenReturn(true); + Event event = generateBatchRefUpdateEvent("refs/changes/01/1/meta"); + when(batchHttpResult.isSuccessful()).thenReturn(false); + when(batchHttpResult.isProjectMissing(any())).thenReturn(true); when(source.isCreateMissingRepositories()).thenReturn(true); objectUnderTest.start(); @@ -212,11 +262,24 @@ } @Test + public void shouldCallSendObjectReorderingRefsHavingMetaAtTheEnd() throws Exception { + Event event = generateBatchRefUpdateEvent("refs/changes/01/1/meta", "refs/changes/01/1/1"); + objectUnderTest.start(); + objectUnderTest.onEvent(event); + verifySendObjectOrdering("refs/changes/01/1/1", "refs/changes/01/1/meta"); + } + + @Test + public void shouldCallSendObjectKeepingMetaAtTheEnd() throws Exception { + Event event = generateBatchRefUpdateEvent("refs/changes/01/1/1", "refs/changes/01/1/meta"); + objectUnderTest.start(); + objectUnderTest.onEvent(event); + verifySendObjectOrdering("refs/changes/01/1/1", "refs/changes/01/1/meta"); + } + + @Test public void shouldNotCallInitProjectWhenReplicateNewRepositoriesNotSet() throws Exception { Event event = new TestEvent("refs/changes/01/1/meta"); - when(httpResult.isSuccessful()).thenReturn(false); - when(httpResult.isProjectMissing(any())).thenReturn(true); - when(source.isCreateMissingRepositories()).thenReturn(false); objectUnderTest.start(); objectUnderTest.onEvent(event); @@ -227,10 +290,12 @@ @Test public void shouldNotCallInitProjectWhenProjectWithoutConfiguration() throws Exception { Event event = new TestEvent("refs/changes/01/1/meta"); - when(httpResult.isSuccessful()).thenReturn(false); - when(httpResult.isProjectMissing(any())).thenReturn(true); - when(source.isCreateMissingRepositories()).thenReturn(true); - when(revReader.read(any(), any(), eq(RefNames.REFS_CONFIG), anyInt())) + lenient().when(httpResult.isSuccessful()).thenReturn(false); + + lenient().when(httpResult.isProjectMissing(any())).thenReturn(true); + lenient().when(source.isCreateMissingRepositories()).thenReturn(true); + lenient() + .when(revReader.read(any(), any(), eq(RefNames.REFS_CONFIG), anyInt())) .thenReturn(Optional.empty()); objectUnderTest.start(); @@ -240,105 +305,199 @@ } @Test - public void shouldCallSendObjectWhenPatchSetRef() throws Exception { - Event event = new TestEvent("refs/changes/01/1/1"); + public void shouldCallBatchSendObjectWhenPatchSetRefAndRefUpdateEvent() throws Exception { + when(config.getBoolean("event", "stream-events", "enableBatchRefUpdatedEvents", false)) + .thenReturn(false); + + objectUnderTest = + new ReplicationQueue( + wq, + rd, + dis, + sl, + fetchClientFactory, + refsFilter, + () -> revReader, + applyObjectMetrics, + queueMetrics, + LOCAL_INSTANCE_ID, + config, + applyObjectsRefsFilter, + shutdownState); + } + + @Test + public void shouldCallBatchSendObjectWhenPatchSetRef() throws Exception { + Event event = generateBatchRefUpdateEvent("refs/changes/01/1/1"); objectUnderTest.start(); objectUnderTest.onEvent(event); - verify(fetchRestApiClient).callSendObjects(any(), anyString(), anyLong(), any(), any()); + verify(fetchRestApiClient).callBatchSendObject(any(), any(), anyLong(), any()); } @Test public void shouldFallbackToCallFetchWhenIOException() throws Exception { - Event event = new TestEvent("refs/changes/01/1/meta"); + Event event = generateBatchRefUpdateEvent("refs/changes/01/1/meta"); + objectUnderTest.start(); when(revReader.read(any(), any(), anyString(), anyInt())).thenThrow(IOException.class); objectUnderTest.onEvent(event); - verify(fetchRestApiClient).callFetch(any(), anyString(), any()); + verify(fetchRestApiClient).callBatchFetch(any(), any(), any()); } @Test - public void shouldFallbackToCallFetchWhenLargeRef() throws Exception { - Event event = new TestEvent("refs/changes/01/1/1"); + public void shouldFallbackToCallBatchFetchWhenLargeRef() throws Exception { + Event event = generateBatchRefUpdateEvent("refs/changes/01/1/1"); + objectUnderTest.start(); when(revReader.read(any(), any(), anyString(), anyInt())).thenReturn(Optional.empty()); objectUnderTest.onEvent(event); - verify(fetchRestApiClient).callFetch(any(), anyString(), any()); + verify(fetchRestApiClient).callBatchFetch(any(), any(), any()); } @Test - public void shouldFallbackToCallFetchWhenParentObjectIsMissing() throws Exception { - Event event = new TestEvent("refs/changes/01/1/1"); + public void + shouldFallbackToCallBatchFetchWhenParentObjectIsMissingAndRefDoesntMatchApplyObjectsRefsFilter() + throws Exception { + Event event = generateBatchRefUpdateEvent("refs/changes/01/1/1"); objectUnderTest.start(); - when(httpResult.isSuccessful()).thenReturn(false); - when(httpResult.isParentObjectMissing()).thenReturn(true); - when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any())) - .thenReturn(httpResult); + when(batchHttpResult.isSuccessful()).thenReturn(false); + when(batchHttpResult.isParentObjectMissing()).thenReturn(true); objectUnderTest.onEvent(event); - verify(fetchRestApiClient).callFetch(any(), anyString(), any()); + verify(fetchRestApiClient).callBatchFetch(any(), any(), any()); + } + + @Test + public void + shouldFallbackToApplyObjectsForEachRefWhenParentObjectIsMissingAndRefMatchesApplyObjectsRefFilter() + throws Exception { + Event event = generateBatchRefUpdateEvent("refs/changes/01/1/1", "refs/changes/02/1/1"); + objectUnderTest.start(); + + when(batchHttpResult.isSuccessful()).thenReturn(false); + when(batchHttpResult.isParentObjectMissing()).thenReturn(true); + when(applyObjectsRefsFilter.match(any())).thenReturn(true); + + objectUnderTest.onEvent(event); + + verify(fetchRestApiClient, times(2)) + .callSendObjects(any(), anyString(), anyLong(), any(), any()); + verify(fetchRestApiClient, never()).callFetch(any(), anyString(), any()); + } + + @Test + public void shouldFallbackToCallBatchFetchWhenParentObjectNotMissingButApplyObjectFails() + throws Exception { + Event event = generateBatchRefUpdateEvent("refs/changes/01/1/1"); + objectUnderTest.start(); + + when(batchHttpResult.isSuccessful()).thenReturn(false); + when(batchHttpResult.isParentObjectMissing()).thenReturn(false); + lenient().when(httpResult.isSuccessful()).thenReturn(false); + lenient().when(httpResult.isParentObjectMissing()).thenReturn(false); + + objectUnderTest.onEvent(event); + + verify(fetchRestApiClient).callBatchFetch(any(), any(), any()); } @Test public void shouldFallbackToApplyAllParentObjectsWhenParentObjectIsMissingOnMetaRef() throws Exception { - Event event = new TestEvent("refs/changes/01/1/meta"); + Event event = generateBatchRefUpdateEvent("refs/changes/01/1/meta"); objectUnderTest.start(); - when(httpResult.isSuccessful()).thenReturn(false, true); - when(httpResult.isParentObjectMissing()).thenReturn(true, false); - when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any())) - .thenReturn(httpResult); + when(batchHttpResult.isSuccessful()).thenReturn(false); + when(batchHttpResult.isParentObjectMissing()).thenReturn(true); objectUnderTest.onEvent(event); - verify(fetchRestApiClient, times(2)) + verify(fetchRestApiClient, times(1)) .callSendObjects(any(), anyString(), anyLong(), revisionsDataCaptor.capture(), any()); List<List<RevisionData>> revisionsDataValues = revisionsDataCaptor.getAllValues(); - assertThat(revisionsDataValues).hasSize(2); + assertThat(revisionsDataValues).hasSize(1); List<RevisionData> firstRevisionsValues = revisionsDataValues.get(0); - assertThat(firstRevisionsValues).hasSize(1); + assertThat(firstRevisionsValues).hasSize(1 + revisionDataParentObjectIds.size()); assertThat(firstRevisionsValues).contains(revisionData); - - List<RevisionData> secondRevisionsValues = revisionsDataValues.get(1); - assertThat(secondRevisionsValues).hasSize(1 + revisionDataParentObjectIds.size()); } @Test public void shouldFallbackToApplyAllParentObjectsWhenParentObjectIsMissingOnAllowedRefs() throws Exception { String refName = "refs/tags/test-tag"; - Event event = new TestEvent(refName); + Event event = generateBatchRefUpdateEvent(refName); objectUnderTest.start(); - when(httpResult.isSuccessful()).thenReturn(false, true); - when(httpResult.isParentObjectMissing()).thenReturn(true, false); - when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any())) - .thenReturn(httpResult); + when(batchHttpResult.isSuccessful()).thenReturn(false); + when(batchHttpResult.isParentObjectMissing()).thenReturn(true); when(applyObjectsRefsFilter.match(refName)).thenReturn(true); objectUnderTest.onEvent(event); - verify(fetchRestApiClient, times(2)) + verify(fetchRestApiClient, times(1)) .callSendObjects(any(), anyString(), anyLong(), revisionsDataCaptor.capture(), any()); List<List<RevisionData>> revisionsDataValues = revisionsDataCaptor.getAllValues(); - assertThat(revisionsDataValues).hasSize(2); + assertThat(revisionsDataValues).hasSize(1); List<RevisionData> firstRevisionsValues = revisionsDataValues.get(0); - assertThat(firstRevisionsValues).hasSize(1); + assertThat(firstRevisionsValues).hasSize(1 + revisionDataParentObjectIds.size()); assertThat(firstRevisionsValues).contains(revisionData); + } - List<RevisionData> secondRevisionsValues = revisionsDataValues.get(1); - assertThat(secondRevisionsValues).hasSize(1 + revisionDataParentObjectIds.size()); + @Test + public void shouldCallSendObjectsIfBatchedRefsNotEnabledAtSource() throws Exception { + Event event = generateBatchRefUpdateEvent("refs/changes/01/1/1"); + when(source.enableBatchedRefs()).thenReturn(false); + objectUnderTest.start(); + objectUnderTest.onEvent(event); + + verify(fetchRestApiClient, never()).callBatchSendObject(any(), any(), anyLong(), any()); + verify(fetchRestApiClient).callSendObjects(any(), anyString(), anyLong(), any(), any()); + } + + @Test + public void shouldCallFetchIfBatchedRefsNotEnabledAtSource() throws Exception { + Event event = generateBatchRefUpdateEvent("refs/changes/01/1/1"); + when(source.enableBatchedRefs()).thenReturn(false); + when(httpResult.isSuccessful()).thenReturn(false); + when(httpResult.isParentObjectMissing()).thenReturn(false); + + objectUnderTest.start(); + objectUnderTest.onEvent(event); + + verify(fetchRestApiClient, never()).callBatchFetch(any(), any(), any()); + verify(fetchRestApiClient).callFetch(any(), anyString(), any()); + } + + @Test + public void shouldCallBatchFetchForAllTheRefsInTheBatchIfApplyObjectFails() + throws Exception, URISyntaxException { + Event event = generateBatchRefUpdateEvent("refs/changes/01/1/1", "refs/changes/02/1/1"); + when(batchHttpResult.isSuccessful()).thenReturn(false); + when(batchHttpResult.isParentObjectMissing()).thenReturn(true); + when(applyObjectsRefsFilter.match(any())).thenReturn(true, true); + when(httpResult.isSuccessful()).thenReturn(true, false); + + objectUnderTest.start(); + objectUnderTest.onEvent(event); + + verify(fetchRestApiClient, times(2)) + .callSendObjects(any(), anyString(), anyLong(), any(), any()); + verify(fetchRestApiClient) + .callBatchFetch( + PROJECT, + List.of("refs/changes/01/1/1", "refs/changes/02/1/1"), + new URIish("http://localhost:18080")); } @Test @@ -360,15 +519,15 @@ refsFilter, () -> revReader, applyObjectMetrics, - fetchMetrics, queueMetrics, LOCAL_INSTANCE_ID, + config, applyObjectsRefsFilter, shutdownState); - Event event = new TestEvent("refs/multi-site/version"); + Event event = generateBatchRefUpdateEvent("refs/multi-site/version"); objectUnderTest.onEvent(event); - verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); + verifyNoInteractions(wq, rd, dis, sl, fetchClientFactory, accountAttribute); } @Test @@ -379,10 +538,10 @@ @Test public void shouldSkipEventWhenStarredChangesRef() { - Event event = new TestEvent("refs/starred-changes/41/2941/1000000"); + Event event = generateBatchRefUpdateEvent("refs/starred-changes/41/2941/1000000"); objectUnderTest.onEvent(event); - verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); + verifyNoInteractions(wq, rd, dis, sl, fetchClientFactory, accountAttribute); } @Test @@ -445,6 +604,36 @@ return createTempDirectory(prefix); } + private BatchRefUpdateEvent generateBatchRefUpdateEvent(String... refs) { + List<RefUpdateAttribute> refUpdates = + Arrays.stream(refs) + .map( + ref -> { + RefUpdateAttribute upd = new RefUpdateAttribute(); + upd.newRev = NEW_OBJECT_ID; + upd.oldRev = ObjectId.zeroId().getName(); + upd.project = PROJECT.get(); + upd.refName = ref; + return upd; + }) + .collect(Collectors.toList()); + + BatchRefUpdateEvent event = + new BatchRefUpdateEvent( + PROJECT, Suppliers.ofInstance(refUpdates), Suppliers.ofInstance(accountAttribute)); + event.instanceId = LOCAL_INSTANCE_ID; + return event; + } + + private void verifySendObjectOrdering(String firstRef, String secondRef) throws Exception { + verify(fetchRestApiClient) + .callBatchSendObject(any(), batchRefsCaptor.capture(), anyLong(), any()); + List<BatchApplyObjectData> batchRefs = batchRefsCaptor.getValue(); + + assertThat(batchRefs.get(0).refName()).isEqualTo(firstRef); + assertThat(batchRefs.get(1).refName()).isEqualTo(secondRef); + } + private class TestEvent extends RefUpdatedEvent { public TestEvent() {
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java index 96bd68e..8e7dc35 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java
@@ -33,7 +33,7 @@ import com.google.gerrit.extensions.api.changes.ReviewInput.CommentInput; import com.google.gerrit.extensions.client.Comment; import com.google.gerrit.extensions.config.FactoryModule; -import com.google.gerrit.server.notedb.Sequences; +import com.google.gerrit.server.Sequence; import com.google.inject.Scopes; import com.googlesource.gerrit.plugins.replication.ReplicationConfig; import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig; @@ -248,7 +248,7 @@ @Test public void shouldNotReadRefsSequences() throws Exception { createChange().assertOkStatus(); - String refName = RefNames.REFS_SEQUENCES + Sequences.NAME_CHANGES; + String refName = RefNames.REFS_SEQUENCES + Sequence.NAME_CHANGES; Optional<RevisionData> revisionDataOption = refObjectId(refName).flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId, 0));
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java index 245003a..a4fc41b 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java
@@ -27,6 +27,7 @@ import com.google.gerrit.extensions.restapi.AuthException; import com.google.gerrit.extensions.restapi.BadRequestException; import com.google.gerrit.extensions.restapi.ResourceConflictException; +import com.google.gerrit.extensions.restapi.ResourceNotFoundException; import com.google.gerrit.extensions.restapi.Response; import com.google.gerrit.server.project.ProjectResource; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; @@ -210,6 +211,18 @@ applyObjectAction.apply(projectResource, inputParams); } + @Test(expected = ResourceNotFoundException.class) + public void shouldRethrowResourceNotFoundException() throws Exception { + RevisionInput inputParams = + new RevisionInput(label, refName, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); + + doThrow(new ResourceNotFoundException("test_projects")) + .when(applyObjectCommand) + .applyObject(any(), anyString(), any(), anyString(), anyLong()); + + applyObjectAction.apply(projectResource, inputParams); + } + private RevisionData createSampleRevisionData() { RevisionObjectData commitData = new RevisionObjectData(
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchApplyObjectActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchApplyObjectActionTest.java new file mode 100644 index 0000000..d8dabf1 --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchApplyObjectActionTest.java
@@ -0,0 +1,214 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.http.HttpStatus.SC_OK; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.common.collect.Lists; +import com.google.gerrit.extensions.restapi.MergeConflictException; +import com.google.gerrit.extensions.restapi.Response; +import com.google.gerrit.extensions.restapi.RestApiException; +import com.google.gerrit.server.project.ProjectResource; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData; +import java.util.Collections; +import java.util.List; +import javax.servlet.http.HttpServletResponse; +import org.eclipse.jgit.lib.Constants; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +@RunWith(MockitoJUnitRunner.class) +public class BatchApplyObjectActionTest { + + private static final long DUMMY_EVENT_TIMESTAMP = 1684875939; + + private BatchApplyObjectAction batchApplyObjectAction; + private static final String LABEL = "instance-2-label"; + private static final String REF_NAME = "refs/heads/master"; + private static final String REF_META_NAME = "refs/meta/version"; + private static final String SAMPLE_COMMIT_OBJECT_ID = "9f8d52853089a3cf00c02ff7bd0817bd4353a95a"; + private static final String SAMPLE_TREE_OBJECT_ID = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; + + private static final String SAMPLE_COMMIT_CONTENT = + "tree " + + SAMPLE_TREE_OBJECT_ID + + "\n" + + "parent 20eb48d28be86dc88fb4bef747f08de0fbefe12d\n" + + "author Gerrit User 1000000 <1000000@69ec38f0-350e-4d9c-96d4-bc956f2faaac> 1610471611 +0100\n" + + "committer Gerrit Code Review <root@maczech-XPS-15> 1610471611 +0100\n" + + "\n" + + "Update patch set 1\n" + + "\n" + + "Change has been successfully merged by Administrator\n" + + "\n" + + "Patch-set: 1\n" + + "Status: merged\n" + + "Tag: autogenerated:gerrit:merged\n" + + "Reviewer: Gerrit User 1000000 <1000000@69ec38f0-350e-4d9c-96d4-bc956f2faaac>\n" + + "Label: SUBM=+1\n" + + "Submission-id: 1904-1610471611558-783c0a2f\n" + + "Submitted-with: OK\n" + + "Submitted-with: OK: Code-Review: Gerrit User 1000000 <1000000@69ec38f0-350e-4d9c-96d4-bc956f2faaac>"; + + @Mock private ApplyObjectAction applyObjectAction; + @Mock private ProjectResource projectResource; + + @Before + public void setup() { + batchApplyObjectAction = new BatchApplyObjectAction(applyObjectAction); + } + + @Test + public void shouldDelegateToApplyObjectActionForEveryRevision() throws RestApiException { + RevisionInput first = + new RevisionInput(LABEL, REF_NAME, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); + RevisionInput second = + new RevisionInput(LABEL, "foo", DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); + + batchApplyObjectAction.apply(projectResource, List.of(first, second)); + + verify(applyObjectAction).apply(projectResource, first); + verify(applyObjectAction).apply(projectResource, second); + } + + @Test + public void shouldReturnOkResponseCodeWhenAllRevisionsAreProcessedSuccessfully() + throws RestApiException { + RevisionInput first = + new RevisionInput(LABEL, REF_NAME, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); + RevisionInput second = + new RevisionInput(LABEL, "foo", DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); + + when(applyObjectAction.apply(projectResource, first)) + .thenAnswer((Answer<Response<?>>) invocation -> Response.created(first)); + when(applyObjectAction.apply(projectResource, second)) + .thenAnswer((Answer<Response<?>>) invocation -> Response.created(second)); + + Response<?> response = batchApplyObjectAction.apply(projectResource, List.of(first, second)); + + assertThat(response.statusCode()).isEqualTo(SC_OK); + } + + @Test + public void shouldReturnAListWithAllTheRevisionsInResponseBodyOnSuccess() + throws RestApiException { + RevisionInput first = + new RevisionInput(LABEL, REF_NAME, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); + RevisionInput second = + new RevisionInput(LABEL, "foo", DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); + Response<?> firstResponse = Response.created(first); + Response<?> secondResponse = Response.created(second); + + when(applyObjectAction.apply(projectResource, first)) + .thenAnswer((Answer<Response<?>>) invocation -> firstResponse); + when(applyObjectAction.apply(projectResource, second)) + .thenAnswer((Answer<Response<?>>) invocation -> secondResponse); + + Response<?> response = batchApplyObjectAction.apply(projectResource, List.of(first, second)); + + assertThat((List<Response<?>>) response.value()) + .isEqualTo(List.of(firstResponse, secondResponse)); + } + + @Test + public void shouldAcceptAMixOfCreatesAndDeletes() throws RestApiException { + RevisionInput delete = new RevisionInput(LABEL, REF_NAME, DUMMY_EVENT_TIMESTAMP, null); + RevisionInput create = + new RevisionInput(LABEL, "foo", DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); + Response<?> deleteResponse = Response.withStatusCode(HttpServletResponse.SC_NO_CONTENT, ""); + Response<?> createResponse = Response.created(create); + + when(applyObjectAction.apply(projectResource, delete)) + .thenAnswer((Answer<Response<?>>) invocation -> deleteResponse); + when(applyObjectAction.apply(projectResource, create)) + .thenAnswer((Answer<Response<?>>) invocation -> createResponse); + + Response<?> response = batchApplyObjectAction.apply(projectResource, List.of(delete, create)); + + assertThat((List<Response<?>>) response.value()) + .isEqualTo(List.of(deleteResponse, createResponse)); + } + + @Test + public void shouldReturnOneOkCodeEvenIfInputContainsBothCreatesAndDeletes() + throws RestApiException { + RevisionInput create = + new RevisionInput(LABEL, REF_NAME, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); + RevisionInput delete = new RevisionInput(LABEL, REF_META_NAME, DUMMY_EVENT_TIMESTAMP + 1, null); + + List<RevisionInput> inputs = List.of(create, delete); + + Response<?> response = batchApplyObjectAction.apply(projectResource, inputs); + + assertThat(response.statusCode()).isEqualTo(SC_OK); + } + + @Test(expected = RestApiException.class) + public void shouldThrowARestApiExceptionIfProcessingFailsForAnyOfTheRevisions() + throws RestApiException { + RevisionInput good = + new RevisionInput(LABEL, REF_NAME, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); + RevisionInput bad = + new RevisionInput(LABEL, "bad", DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); + + when(applyObjectAction.apply(projectResource, good)) + .thenAnswer((Answer<Response<?>>) invocation -> Response.created(good)); + when(applyObjectAction.apply(projectResource, bad)) + .thenThrow(new MergeConflictException("BOOM")); + + batchApplyObjectAction.apply(projectResource, List.of(good, bad)); + } + + @Test + public void shouldStopProcessingWhenAFailureOccurs() throws RestApiException { + RevisionInput good = + new RevisionInput(LABEL, REF_NAME, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); + RevisionInput bad = + new RevisionInput(LABEL, "bad", DUMMY_EVENT_TIMESTAMP, createSampleRevisionData()); + + when(applyObjectAction.apply(projectResource, bad)) + .thenThrow(new MergeConflictException("BOOM")); + + try { + batchApplyObjectAction.apply(projectResource, List.of(bad, good)); + } catch (MergeConflictException e) { + verify(applyObjectAction, never()).apply(projectResource, good); + } + } + + private RevisionData createSampleRevisionData() { + RevisionObjectData commitData = + new RevisionObjectData( + SAMPLE_COMMIT_OBJECT_ID, Constants.OBJ_COMMIT, SAMPLE_COMMIT_CONTENT.getBytes()); + RevisionObjectData treeData = + new RevisionObjectData(SAMPLE_TREE_OBJECT_ID, Constants.OBJ_TREE, new byte[] {}); + return createSampleRevisionData(commitData, treeData); + } + + private RevisionData createSampleRevisionData( + RevisionObjectData commitData, RevisionObjectData treeData) { + return new RevisionData(Collections.emptyList(), commitData, treeData, Lists.newArrayList()); + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchActionTest.java new file mode 100644 index 0000000..738815a --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchActionTest.java
@@ -0,0 +1,133 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.googlesource.gerrit.plugins.replication.pull.api; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.http.HttpStatus.SC_OK; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.gerrit.extensions.restapi.MergeConflictException; +import com.google.gerrit.extensions.restapi.Response; +import com.google.gerrit.extensions.restapi.RestApiException; +import com.google.gerrit.server.project.ProjectResource; +import java.util.List; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +@RunWith(MockitoJUnitRunner.class) +public class BatchFetchActionTest { + + BatchFetchAction batchFetchAction; + String label = "instance-2-label"; + String master = "refs/heads/master"; + String test = "refs/heads/test"; + + @Mock ProjectResource projectResource; + @Mock FetchAction fetchAction; + + @Before + public void setup() { + batchFetchAction = new BatchFetchAction(fetchAction); + } + + @Test + public void shouldDelegateToFetchActionForEveryFetchInput() throws RestApiException { + FetchAction.Input first = createInput(master); + FetchAction.Input second = createInput(test); + + batchFetchAction.apply(projectResource, List.of(first, second)); + + verify(fetchAction).apply(projectResource, first); + verify(fetchAction).apply(projectResource, second); + } + + @Test + public void shouldReturnOkResponseCodeWhenAllInputsAreProcessedSuccessfully() + throws RestApiException { + FetchAction.Input first = createInput(master); + FetchAction.Input second = createInput(test); + + when(fetchAction.apply(any(), any())) + .thenAnswer((Answer<Response<?>>) invocation -> Response.accepted("some-url")); + Response<?> response = batchFetchAction.apply(projectResource, List.of(first, second)); + + assertThat(response.statusCode()).isEqualTo(SC_OK); + } + + @Test + public void shouldReturnAListWithAllResponsesOnSuccess() throws RestApiException { + FetchAction.Input first = createInput(master); + FetchAction.Input second = createInput(test); + String masterUrl = "master-url"; + String testUrl = "test-url"; + Response.Accepted firstResponse = Response.accepted(masterUrl); + Response.Accepted secondResponse = Response.accepted(testUrl); + + when(fetchAction.apply(projectResource, first)) + .thenAnswer((Answer<Response<?>>) invocation -> firstResponse); + when(fetchAction.apply(projectResource, second)) + .thenAnswer((Answer<Response<?>>) invocation -> secondResponse); + Response<?> response = batchFetchAction.apply(projectResource, List.of(first, second)); + + assertThat((List<Response<?>>) response.value()) + .isEqualTo(List.of(firstResponse, secondResponse)); + } + + @Test + public void shouldReturnAMixOfSyncAndAsyncResponses() throws RestApiException { + FetchAction.Input async = createInput(master); + FetchAction.Input sync = createInput(test); + String masterUrl = "master-url"; + Response.Accepted asyncResponse = Response.accepted(masterUrl); + Response<?> syncResponse = Response.created(sync); + + when(fetchAction.apply(projectResource, async)) + .thenAnswer((Answer<Response<?>>) invocation -> asyncResponse); + when(fetchAction.apply(projectResource, sync)) + .thenAnswer((Answer<Response<?>>) invocation -> syncResponse); + Response<?> response = batchFetchAction.apply(projectResource, List.of(async, sync)); + + assertThat((List<Response<?>>) response.value()) + .isEqualTo(List.of(asyncResponse, syncResponse)); + } + + @Test(expected = RestApiException.class) + public void shouldThrowRestApiExceptionWhenProcessingFailsForAnInput() throws RestApiException { + FetchAction.Input first = createInput(master); + FetchAction.Input second = createInput(test); + String masterUrl = "master-url"; + + when(fetchAction.apply(projectResource, first)) + .thenAnswer((Answer<Response<?>>) invocation -> Response.accepted(masterUrl)); + when(fetchAction.apply(projectResource, second)).thenThrow(new MergeConflictException("BOOM")); + + batchFetchAction.apply(projectResource, List.of(first, second)); + } + + private FetchAction.Input createInput(String refName) { + FetchAction.Input input = new FetchAction.Input(); + input.label = label; + input.refName = refName; + return input; + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BearerAuthenticationFilterTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BearerAuthenticationFilterTest.java index 95b86cc..ea5149b 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BearerAuthenticationFilterTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BearerAuthenticationFilterTest.java
@@ -85,6 +85,11 @@ } @Test + public void shouldAuthenticateWhenBatchFetch() throws Exception { + authenticateAndFilter("any-prefix/pull-replication~batch-fetch", NO_QUERY_PARAMETERS); + } + + @Test public void shouldAuthenticateWhenApplyObject() throws Exception { authenticateAndFilter("any-prefix/pull-replication~apply-object", NO_QUERY_PARAMETERS); } @@ -95,6 +100,11 @@ } @Test + public void shouldAuthenticateWhenBatchApplyObject() throws Exception { + authenticateAndFilter("any-prefix/pull-replication~batch-apply-object", NO_QUERY_PARAMETERS); + } + + @Test public void shouldAuthenticateWhenDeleteProject() throws Exception { authenticateAndFilter("any-prefix/pull-replication~delete-project", NO_QUERY_PARAMETERS); }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java index fc1b02c..40912ac 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java
@@ -19,6 +19,7 @@ import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import com.google.gerrit.entities.Project; @@ -27,7 +28,6 @@ import com.google.gerrit.server.events.Event; import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.server.git.LocalDiskRepositoryManager; -import com.google.gerrit.server.permissions.PermissionBackend; import com.google.gerrit.server.permissions.PermissionBackend.ForProject; import com.google.gerrit.server.permissions.PermissionBackend.ForRef; import com.google.gerrit.server.permissions.PermissionBackend.WithUser; @@ -41,7 +41,6 @@ import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject; import java.util.Optional; import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.RefUpdate.Result; import org.eclipse.jgit.lib.Repository; @@ -71,7 +70,6 @@ @Mock private ProjectState projectState; @Mock private SourcesCollection sourceCollection; @Mock private Source source; - @Mock private PermissionBackend permissionBackend; @Mock private WithUser currentUser; @Mock private ForProject forProject; @Mock private ForRef forRef; @@ -79,7 +77,6 @@ @Mock private RefUpdate refUpdate; @Mock private Repository repository; @Mock private Ref currentRef; - @Mock private RefDatabase refDb; @Captor ArgumentCaptor<Event> eventCaptor; private DeleteRefCommand objectUnderTest; @@ -93,8 +90,7 @@ when(source.getURI(TEST_PROJECT_NAME)).thenReturn(TEST_REMOTE_URI); when(gitManager.openRepository(any())).thenReturn(repository); when(repository.updateRef(any())).thenReturn(refUpdate); - when(repository.getRefDatabase()).thenReturn(refDb); - when(refDb.exactRef(anyString())).thenReturn(currentRef); + when(repository.exactRef(anyString())).thenReturn(currentRef); when(refUpdate.delete()).thenReturn(Result.FORCED); objectUnderTest = @@ -102,13 +98,14 @@ fetchStateLog, projectCache, sourceCollection, - permissionBackend, eventDispatcherDataItem, new LocalGitRepositoryManagerProvider(gitManager)); } @Test public void shouldSendEventWhenDeletingRef() throws Exception { + when(source.isMirror()).thenReturn(true); + objectUnderTest.deleteRef(TEST_PROJECT_NAME, TEST_REF_NAME, TEST_SOURCE_LABEL); verify(eventDispatcher).postEvent(eventCaptor.capture()); @@ -120,9 +117,18 @@ } @Test - public void shouldHandleNonExistingRef() throws Exception { - when(refDb.exactRef(anyString())).thenReturn(null); + public void shouldNotSendNotSendEventWhenMirroringIsDisabled() throws Exception { + when(source.isMirror()).thenReturn(false); + objectUnderTest.deleteRef(TEST_PROJECT_NAME, TEST_REF_NAME, TEST_SOURCE_LABEL); + + verifyNoInteractions(eventDispatcher); + } + + @Test + public void shouldHandleNonExistingRef() throws Exception { + when(source.isMirror()).thenReturn(true); + when(repository.exactRef(anyString())).thenReturn(null); objectUnderTest.deleteRef(TEST_PROJECT_NAME, NON_EXISTING_REF_NAME, TEST_SOURCE_LABEL); verify(eventDispatcher, never()).postEvent(any());
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionActionIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionActionIT.java index 7c7846c..3bd9d35 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionActionIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionActionIT.java
@@ -14,6 +14,7 @@ package com.googlesource.gerrit.plugins.replication.pull.api; +import static com.google.common.truth.Truth.assertThat; import static com.google.gerrit.acceptance.testsuite.project.TestProjectUpdate.allowCapability; import com.google.gerrit.acceptance.config.GerritConfig; @@ -181,6 +182,50 @@ assertHttpResponseCode(HttpServletResponse.SC_OK)); } + @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") + public void shouldRemoveFromTheProjectCacheWhenProjectIsSuccessfullyDeleted() throws Exception { + String testProjectName = project.get(); + url = getURLWithAuthenticationPrefix(testProjectName); + assertThat(projectCache.get(project).isPresent()).isTrue(); + + httpClientFactory + .create(source) + .execute( + withBasicAuthenticationAsAdmin(createDeleteRequest()), + assertHttpResponseCode(HttpServletResponse.SC_OK)); + assertThat(projectCache.get(project).isPresent()).isFalse(); + } + + @Test + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") + @GerritConfig(name = "container.replica", value = "true") + public void shouldRemoveFromtheReplicaProjectCacheWhenProjectIsSuccessfullyDeletedFromTheReplica() + throws Exception { + String testProjectName = project.get(); + url = getURLWithAuthenticationPrefix(testProjectName); + assertThat(projectCache.get(project).isPresent()).isTrue(); + + httpClientFactory + .create(source) + .execute( + withBasicAuthenticationAsAdmin(createDeleteRequest()), + assertHttpResponseCode(HttpServletResponse.SC_OK)); + assertThat(projectCache.get(project).isPresent()).isFalse(); + } + + @Test + @GerritConfig(name = "container.replica", value = "true") + @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId") + public void shouldNotRemoveFromTheReplicaCacheIfAProjectCannotBeDeleted() throws Exception { + assertThat(projectCache.get(project).isPresent()).isTrue(); + httpClientFactory + .create(source) + .execute( + createDeleteRequest(), assertHttpResponseCode(HttpServletResponse.SC_UNAUTHORIZED)); + assertThat(projectCache.get(project).isPresent()).isTrue(); + } + @Override protected String getURLWithAuthenticationPrefix(String projectName) { return String.format(
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilterTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilterTest.java index 9492d49..34885b2 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilterTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilterTest.java
@@ -8,11 +8,14 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.mockito.internal.verification.VerificationModeFactory.atLeastOnce; -import static org.mockito.internal.verification.VerificationModeFactory.times; import com.google.common.net.MediaType; import com.google.gerrit.entities.Project; -import com.google.gerrit.extensions.restapi.*; +import com.google.gerrit.extensions.restapi.AuthException; +import com.google.gerrit.extensions.restapi.ResourceConflictException; +import com.google.gerrit.extensions.restapi.ResourceNotFoundException; +import com.google.gerrit.extensions.restapi.Response; +import com.google.gerrit.extensions.restapi.UnprocessableEntityException; import com.google.gerrit.server.AnonymousUser; import com.google.gerrit.server.CurrentUser; import com.google.gerrit.server.IdentifiedUser; @@ -20,7 +23,11 @@ import com.google.gerrit.server.project.ProjectResource; import com.google.gerrit.server.project.ProjectState; import com.google.inject.util.Providers; -import java.io.*; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PrintWriter; import java.nio.charset.StandardCharsets; import java.util.Optional; import javax.servlet.FilterChain; @@ -39,8 +46,10 @@ @Mock HttpServletResponse response; @Mock FilterChain filterChain; @Mock private FetchAction fetchAction; + @Mock private BatchFetchAction batchFetchAction; @Mock private ApplyObjectAction applyObjectAction; @Mock private ApplyObjectsAction applyObjectsAction; + @Mock private BatchApplyObjectAction batchApplyObjectAction; @Mock private ProjectInitializationAction projectInitializationAction; @Mock private UpdateHeadAction updateHEADAction; @Mock private ProjectDeletionAction projectDeletionAction; @@ -55,12 +64,17 @@ private final String PROJECT_NAME_GIT = "some-project.git"; private final String FETCH_URI = String.format("any-prefix/projects/%s/%s~fetch", PROJECT_NAME, PLUGIN_NAME); + private final String BATCH_FETCH_URI = + String.format("any-prefix/projects/%s/%s~batch-fetch", PROJECT_NAME, PLUGIN_NAME); private final String APPLY_OBJECT_URI = String.format("any-prefix/projects/%s/%s~apply-object", PROJECT_NAME, PLUGIN_NAME); private final String APPLY_OBJECTS_URI = String.format("any-prefix/projects/%s/%s~apply-objects", PROJECT_NAME, PLUGIN_NAME); private final String HEAD_URI = String.format("any-prefix/projects/%s/%s~HEAD", PROJECT_NAME, PLUGIN_NAME); + + private final String BATCH_APPLY_OBJECT_URI = + String.format("any-prefix/projects/%s/%s~batch-apply-object", PROJECT_NAME, PLUGIN_NAME); private final String DELETE_PROJECT_URI = String.format("any-prefix/projects/%s/%s~delete-project", PROJECT_NAME, PLUGIN_NAME); private final String INIT_PROJECT_URI = @@ -75,8 +89,10 @@ private PullReplicationFilter createPullReplicationFilter(CurrentUser currentUser) { return new PullReplicationFilter( fetchAction, + batchFetchAction, applyObjectAction, applyObjectsAction, + batchApplyObjectAction, projectInitializationAction, updateHEADAction, projectDeletionAction, @@ -124,6 +140,31 @@ } @Test + public void shouldFilterBatchFetchAction() throws Exception { + byte[] payloadBatchFetch = + ("[{" + + "\"label\":\"Replication\", " + + "\"ref_name\": \"refs/heads/master\", " + + "\"async\":false" + + "}," + + "{" + + "\"label\":\"Replication\", " + + "\"ref_name\": \"refs/heads/test\", " + + "\"async\":false" + + "}]") + .getBytes(StandardCharsets.UTF_8); + + defineBehaviours(payloadBatchFetch, BATCH_FETCH_URI); + when(batchFetchAction.apply(any(), any())).thenReturn(OK_RESPONSE); + + PullReplicationFilter pullReplicationFilter = createPullReplicationFilter(); + pullReplicationFilter.doFilter(request, response, filterChain); + + verifyBehaviours(); + verify(batchFetchAction).apply(any(ProjectResource.class), any()); + } + + @Test public void shouldFilterApplyObjectAction() throws Exception { byte[] payloadApplyObject = @@ -206,7 +247,6 @@ final PullReplicationFilter pullReplicationFilter = createPullReplicationFilter(); pullReplicationFilter.doFilter(request, response, filterChain); - verify(request, times(7)).getRequestURI(); verify(projectCache).get(Project.nameKey(PROJECT_NAME)); verify(projectDeletionAction).apply(any(ProjectResource.class), any()); verify(response).getWriter(); @@ -249,7 +289,7 @@ } @Test - public void shouldBe500WhenResourceNotFound() throws Exception { + public void shouldBe404WhenResourceNotFound() throws Exception { when(request.getRequestURI()).thenReturn(DELETE_PROJECT_URI); when(request.getMethod()).thenReturn("DELETE"); when(projectCache.get(Project.nameKey(PROJECT_NAME))).thenReturn(Optional.of(projectState)); @@ -260,7 +300,7 @@ final PullReplicationFilter pullReplicationFilter = createPullReplicationFilter(); pullReplicationFilter.doFilter(request, response, filterChain); - verify(response).setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + verify(response).setStatus(HttpServletResponse.SC_NOT_FOUND); } @Test @@ -346,4 +386,33 @@ verify(response).setStatus(HttpServletResponse.SC_BAD_REQUEST); } + + @Test + public void shouldFilterBatchApplyObjectAction() throws Exception { + + byte[] payloadApplyObject = + ("[{\"label\":\"Replication\",\"ref_name\":\"refs/heads/foo\"," + + "\"revision_data\":{" + + "\"commit_object\":{\"type\":1,\"content\":\"some-content\"}," + + "\"tree_object\":{\"type\":2,\"content\":\"some-content\"}," + + "\"blobs\":[]}" + + "}," + + "{\"label\":\"Replication\",\"ref_name\":\"refs/heads/bar\"," + + "\"revision_data\":{" + + "\"commit_object\":{\"type\":1,\"content\":\"some-content\"}," + + "\"tree_object\":{\"type\":2,\"content\":\"some-content\"}," + + "\"blobs\":[]}" + + "}]") + .getBytes(StandardCharsets.UTF_8); + + defineBehaviours(payloadApplyObject, BATCH_APPLY_OBJECT_URI); + + when(batchApplyObjectAction.apply(any(), any())).thenReturn(OK_RESPONSE); + + PullReplicationFilter pullReplicationFilter = createPullReplicationFilter(); + pullReplicationFilter.doFilter(request, response, filterChain); + + verifyBehaviours(); + verify(batchApplyObjectAction).apply(any(ProjectResource.class), any()); + } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/data/BatchApplyObjectDataTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/data/BatchApplyObjectDataTest.java new file mode 100644 index 0000000..bf74b56 --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/data/BatchApplyObjectDataTest.java
@@ -0,0 +1,32 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api.data; + +import java.util.Optional; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class BatchApplyObjectDataTest { + + @Mock private RevisionData revisionData; + + @Test(expected = IllegalArgumentException.class) + public void shouldFailIfRevisionDataIsPresentForADelete() { + BatchApplyObjectData.create("foo", Optional.of(revisionData), true); + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java index 65ccdfb..b061de0 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java
@@ -31,12 +31,16 @@ import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig; import com.googlesource.gerrit.plugins.replication.pull.BearerTokenProvider; import com.googlesource.gerrit.plugins.replication.pull.Source; +import com.googlesource.gerrit.plugins.replication.pull.api.data.BatchApplyObjectData; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData; import com.googlesource.gerrit.plugins.replication.pull.filter.SyncRefsFilter; import java.io.InputStreamReader; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; +import java.util.Optional; import org.apache.http.Header; import org.apache.http.HttpHeaders; import org.apache.http.client.methods.HttpDelete; @@ -176,6 +180,25 @@ } @Test + public void shouldCallBatchFetchEndpoint() throws Exception { + + objectUnderTest.callBatchFetch( + Project.nameKey("test_repo"), + List.of(refName, RefNames.REFS_HEADS + "test"), + new URIish(api)); + + verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any()); + + HttpPost httpPost = httpPostCaptor.getValue(); + assertThat(httpPost.getURI().getHost()).isEqualTo("gerrit-host"); + assertThat(httpPost.getURI().getPath()) + .isEqualTo( + String.format( + "%s/projects/test_repo/pull-replication~batch-fetch", urlAuthenticationPrefix())); + assertAuthentication(httpPost); + } + + @Test public void shouldByDefaultCallSyncFetchForAllRefs() throws Exception { objectUnderTest.callFetch(Project.nameKey("test_repo"), refName, new URIish(api)); @@ -212,6 +235,41 @@ } @Test + public void shouldCallAsyncBatchFetchForAllRefs() throws Exception { + + when(config.getStringList("replication", null, "syncRefs")) + .thenReturn(new String[] {"NO_SYNC_REFS"}); + syncRefsFilter = new SyncRefsFilter(replicationConfig); + objectUnderTest = + new FetchRestApiClient( + credentials, + httpClientFactory, + replicationConfig, + syncRefsFilter, + pluginName, + instanceId, + bearerTokenProvider, + source); + + String testRef = RefNames.REFS_HEADS + "test"; + List<String> refs = List.of(refName, testRef); + objectUnderTest.callBatchFetch(Project.nameKey("test_repo"), refs, new URIish(api)); + + verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any()); + + HttpPost httpPost = httpPostCaptor.getValue(); + String expectedPayload = + "[{\"label\":\"Replication\", \"ref_name\": \"" + + refName + + "\", \"async\":true}," + + "{\"label\":\"Replication\", \"ref_name\": \"" + + refs.get(1) + + "\", \"async\":true}" + + "]"; + assertThat(readPayload(httpPost)).isEqualTo(expectedPayload); + } + + @Test public void shouldCallSyncFetchOnlyForMetaRef() throws Exception { String metaRefName = "refs/changes/01/101/meta"; String expectedMetaRefPayload = @@ -243,6 +301,33 @@ } @Test + public void shouldCallSyncBatchFetchOnlyForMetaRef() throws Exception { + String metaRefName = "refs/changes/01/101/meta"; + String expectedMetaRefPayload = + "[{\"label\":\"Replication\", \"ref_name\": \"" + metaRefName + "\", \"async\":false}]"; + + when(config.getStringList("replication", null, "syncRefs")) + .thenReturn(new String[] {"^refs\\/changes\\/.*\\/meta"}); + syncRefsFilter = new SyncRefsFilter(replicationConfig); + objectUnderTest = + new FetchRestApiClient( + credentials, + httpClientFactory, + replicationConfig, + syncRefsFilter, + pluginName, + instanceId, + bearerTokenProvider, + source); + + objectUnderTest.callBatchFetch( + Project.nameKey("test_repo"), List.of(metaRefName), new URIish(api)); + verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any()); + HttpPost httpPost = httpPostCaptor.getValue(); + assertThat(readPayload(httpPost)).isEqualTo(expectedMetaRefPayload); + } + + @Test public void shouldCallFetchEndpointWithPayload() throws Exception { objectUnderTest.callFetch(Project.nameKey("test_repo"), refName, new URIish(api)); @@ -254,6 +339,92 @@ } @Test + public void shouldCallBatchFetchEndpointWithPayload() throws Exception { + + String testRef = RefNames.REFS_HEADS + "test"; + List<String> refs = List.of(refName, testRef); + objectUnderTest.callBatchFetch(Project.nameKey("test_repo"), refs, new URIish(api)); + + verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any()); + + HttpPost httpPost = httpPostCaptor.getValue(); + String expectedPayload = + "[{\"label\":\"Replication\", \"ref_name\": \"" + + refName + + "\", \"async\":false}," + + "{\"label\":\"Replication\", \"ref_name\": \"" + + refs.get(1) + + "\", \"async\":false}" + + "]"; + assertThat(readPayload(httpPost)).isEqualTo(expectedPayload); + } + + @Test + public void shouldExecuteOneFetchCallForAsyncAndOneForSyncRefsDuringBatchFetch() + throws Exception { + + when(config.getStringList("replication", null, "syncRefs")) + .thenReturn(new String[] {"^refs\\/heads\\/test"}); + syncRefsFilter = new SyncRefsFilter(replicationConfig); + String testRef = RefNames.REFS_HEADS + "test"; + List<String> refs = List.of(refName, testRef); + objectUnderTest = + new FetchRestApiClient( + credentials, + httpClientFactory, + replicationConfig, + syncRefsFilter, + pluginName, + instanceId, + bearerTokenProvider, + source); + objectUnderTest.callBatchFetch(Project.nameKey("test_repo"), refs, new URIish(api)); + + verify(httpClient, times(2)).execute(httpPostCaptor.capture(), any()); + + List<HttpPost> httpPosts = httpPostCaptor.getAllValues(); + String expectedSyncPayload = + "[{\"label\":\"Replication\", \"ref_name\": \"" + + refs.get(1) + + "\", \"async\":false}" + + "]"; + String expectedAsyncPayload = + "[{\"label\":\"Replication\", \"ref_name\": \"" + refName + "\", \"async\":true}]"; + + assertThat(readPayload(httpPosts.get(0))).isEqualTo(expectedAsyncPayload); + assertThat(readPayload(httpPosts.get(1))).isEqualTo(expectedSyncPayload); + } + + @Test + public void shouldNotExecuteSyncFetchCallWhenAsyncCallFailsDuringBatchFetch() throws Exception { + when(config.getStringList("replication", null, "syncRefs")) + .thenReturn(new String[] {"^refs\\/heads\\/test"}); + when(httpClient.execute(any(), any())).thenReturn(new HttpResult(500, Optional.of("BOOM"))); + syncRefsFilter = new SyncRefsFilter(replicationConfig); + String testRef = RefNames.REFS_HEADS + "test"; + List<String> refs = List.of(refName, testRef); + objectUnderTest = + new FetchRestApiClient( + credentials, + httpClientFactory, + replicationConfig, + syncRefsFilter, + pluginName, + instanceId, + bearerTokenProvider, + source); + objectUnderTest.callBatchFetch(Project.nameKey("test_repo"), refs, new URIish(api)); + + verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any()); + + HttpPost httpPost = httpPostCaptor.getValue(); + String expectedAsyncPayload = + "[{\"label\":\"Replication\", \"ref_name\": \"" + refName + "\", \"async\":true}]"; + + assertThat(readPayload(httpPost)).isEqualTo(expectedAsyncPayload); + } + + @Test public void shouldSetContentTypeHeader() throws Exception { objectUnderTest.callFetch(Project.nameKey("test_repo"), refName, new URIish(api)); @@ -266,6 +437,18 @@ } @Test + public void shouldSetContentTypeHeaderInBatchFetch() throws Exception { + + objectUnderTest.callBatchFetch(Project.nameKey("test_repo"), List.of(refName), new URIish(api)); + + verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any()); + + HttpPost httpPost = httpPostCaptor.getValue(); + assertThat(httpPost.getLastHeader("Content-Type").getValue()) + .isEqualTo(expectedHeader.getValue()); + } + + @Test public void shouldCallSendObjectEndpoint() throws Exception { objectUnderTest.callSendObject( @@ -449,19 +632,139 @@ assertAuthentication(httpPut); } + @Test + public void shouldCallBatchSendObjectEndpoint() throws Exception { + + List<BatchApplyObjectData> batchApplyObjects = new ArrayList<>(); + batchApplyObjects.add( + BatchApplyObjectData.create(refName, Optional.of(createSampleRevisionData("a")), false)); + + objectUnderTest.callBatchSendObject( + Project.nameKey("test_repo"), batchApplyObjects, eventCreatedOn, new URIish(api)); + + verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any()); + + HttpPost httpPost = httpPostCaptor.getValue(); + assertThat(httpPost.getURI().getHost()).isEqualTo("gerrit-host"); + assertThat(httpPost.getURI().getPath()) + .isEqualTo( + String.format( + "%s/projects/test_repo/pull-replication~batch-apply-object", + urlAuthenticationPrefix())); + assertAuthentication(httpPost); + } + + @Test + public void shouldCallBatchApplyObjectEndpointWithAListOfRefsInPayload() throws Exception { + List<BatchApplyObjectData> batchApplyObjects = new ArrayList<>(); + RevisionData revisionA = createSampleRevisionData("a"); + RevisionData revisionB = createSampleRevisionData("b"); + String refNameB = "refs/heads/b"; + batchApplyObjects.add(BatchApplyObjectData.create(refName, Optional.of(revisionA), false)); + batchApplyObjects.add(BatchApplyObjectData.create(refNameB, Optional.of(revisionB), false)); + + objectUnderTest.callBatchSendObject( + Project.nameKey("test_repo"), batchApplyObjects, eventCreatedOn, new URIish(api)); + + verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any()); + + HttpPost httpPost = httpPostCaptor.getValue(); + + String expectedSendObjectsPayload = + "[{\"label\":\"Replication\",\"ref_name\":\"" + + refName + + "\",\"event_created_on\":" + + eventCreatedOn + + ",\"revision_data\":{\"commit_object\":{\"sha1\":\"" + + revisionA.getCommitObject().getSha1() + + "\",\"type\":1,\"content\":\"Y29tbWl0YWNvbnRlbnQ\\u003d\"},\"tree_object\":{\"sha1\":\"" + + revisionA.getTreeObject().getSha1() + + "\",\"type\":2,\"content\":\"dHJlZWFjb250ZW50\"},\"blobs\":[{\"sha1\":\"" + + revisionA.getBlobs().get(0).getSha1() + + "\",\"type\":3,\"content\":\"YmxvYmFjb250ZW50\"}]}}," + + "{\"label\":\"Replication\",\"ref_name\":\"" + + refNameB + + "\",\"event_created_on\":" + + eventCreatedOn + + ",\"revision_data\":{\"commit_object\":{\"sha1\":\"" + + revisionB.getCommitObject().getSha1() + + "\",\"type\":1,\"content\":\"Y29tbWl0YmNvbnRlbnQ\\u003d\"},\"tree_object\":{\"sha1\":\"" + + revisionB.getTreeObject().getSha1() + + "\",\"type\":2,\"content\":\"dHJlZWJjb250ZW50\"},\"blobs\":[{\"sha1\":\"" + + revisionB.getBlobs().get(0).getSha1() + + "\",\"type\":3,\"content\":\"YmxvYmJjb250ZW50\"}]}}]"; + assertThat(readPayload(httpPost)).isEqualTo(expectedSendObjectsPayload); + } + + @Test + public void shouldCallBatchApplyObjectEndpointWithNoRevisionDataForDeletes() throws Exception { + List<BatchApplyObjectData> batchApplyObjects = new ArrayList<>(); + batchApplyObjects.add(BatchApplyObjectData.create(refName, Optional.empty(), true)); + + objectUnderTest.callBatchSendObject( + Project.nameKey("test_repo"), batchApplyObjects, eventCreatedOn, new URIish(api)); + + verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any()); + + HttpPost httpPost = httpPostCaptor.getValue(); + + String expectedSendObjectsPayload = + "[{\"label\":\"Replication\",\"ref_name\":\"" + + refName + + "\",\"event_created_on\":" + + eventCreatedOn + + "}]"; + assertThat(readPayload(httpPost)).isEqualTo(expectedSendObjectsPayload); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldThrowExceptionIfDeleteFlagIsSetButRevisionDataIsPresentForBatchSendEndpoint() + throws Exception { + List<BatchApplyObjectData> batchApplyObjects = new ArrayList<>(); + batchApplyObjects.add( + BatchApplyObjectData.create(refName, Optional.of(createSampleRevisionData()), true)); + + objectUnderTest.callBatchSendObject( + Project.nameKey("test_repo"), batchApplyObjects, eventCreatedOn, new URIish(api)); + } + public String readPayload(HttpPost entity) throws Exception { ByteBuffer buf = IO.readWholeStream(entity.getEntity().getContent(), 1024); return RawParseUtils.decode(buf.array(), buf.arrayOffset(), buf.limit()).trim(); } - private RevisionData createSampleRevisionData() { + private RevisionData createSampleRevisionData(String prefix) { + String commitPrefix = "commit" + prefix; + String treePrefix = "tree" + prefix; + String blobPrefix = "blob" + prefix; + return createSampleRevisionData( + commitPrefix, + commitPrefix + "content", + treePrefix, + treePrefix + "content", + blobPrefix, + blobPrefix + "content"); + } + + private RevisionData createSampleRevisionData( + String commitObjectId, + String commitContent, + String treeObjectId, + String treeContent, + String blobObjectId, + String blobContent) { RevisionObjectData commitData = - new RevisionObjectData(commitObjectId, Constants.OBJ_COMMIT, commitObject.getBytes()); + new RevisionObjectData(commitObjectId, Constants.OBJ_COMMIT, commitContent.getBytes()); RevisionObjectData treeData = - new RevisionObjectData(treeObjectId, Constants.OBJ_TREE, treeObject.getBytes()); + new RevisionObjectData(treeObjectId, Constants.OBJ_TREE, treeContent.getBytes()); RevisionObjectData blobData = - new RevisionObjectData(blobObjectId, Constants.OBJ_BLOB, blobObject.getBytes()); + new RevisionObjectData(blobObjectId, Constants.OBJ_BLOB, blobContent.getBytes()); return new RevisionData( Collections.emptyList(), commitData, treeData, Lists.newArrayList(blobData)); } + + private RevisionData createSampleRevisionData() { + return createSampleRevisionData( + commitObjectId, commitObject, treeObjectId, treeObject, blobObjectId, blobObject); + } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientWithBasicAuthenticationTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientWithBasicAuthenticationTest.java index b4d4e7a..165ed80 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientWithBasicAuthenticationTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientWithBasicAuthenticationTest.java
@@ -16,7 +16,10 @@ import static com.google.common.truth.Truth.assertThat; import static javax.servlet.http.HttpServletResponse.SC_CREATED; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import com.googlesource.gerrit.plugins.replication.pull.filter.SyncRefsFilter; import java.util.Optional; @@ -36,7 +39,7 @@ @Before public void setup() throws Exception { when(bearerTokenProvider.get()).thenReturn(Optional.empty()); - when(credentialProvider.supports(any())) + when(credentialProvider.supports(any(CredentialItem[].class))) .thenAnswer( new Answer<Boolean>() { @@ -51,7 +54,7 @@ } }); - when(credentialProvider.get(any(), any(CredentialItem.class))).thenReturn(true); + when(credentialProvider.get(any(), any(CredentialItem[].class))).thenReturn(true); when(credentials.create(anyString())).thenReturn(credentialProvider); when(replicationConfig.getConfig()).thenReturn(config); when(config.getStringList("replication", null, "syncRefs")).thenReturn(new String[0]);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientWithBearerTokenTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientWithBearerTokenTest.java index fe50b5c..9c662a9 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientWithBearerTokenTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientWithBearerTokenTest.java
@@ -16,7 +16,9 @@ import static com.google.common.truth.Truth.assertThat; import static javax.servlet.http.HttpServletResponse.SC_CREATED; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import com.googlesource.gerrit.plugins.replication.pull.filter.SyncRefsFilter; import java.util.Optional;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultTest.java index d82f3a5..3f73729 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultTest.java
@@ -35,6 +35,7 @@ {HttpServletResponse.SC_ACCEPTED, true}, {HttpServletResponse.SC_NO_CONTENT, true}, {HttpServletResponse.SC_BAD_REQUEST, false}, + {HttpServletResponse.SC_NOT_FOUND, false}, {HttpServletResponse.SC_CONFLICT, false} }); }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandlerTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandlerTest.java deleted file mode 100644 index 81a4fc0..0000000 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandlerTest.java +++ /dev/null
@@ -1,136 +0,0 @@ -// Copyright (C) 2021 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.googlesource.gerrit.plugins.replication.pull.event; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import com.google.gerrit.entities.Change; -import com.google.gerrit.entities.Project; -import com.google.gerrit.server.index.change.ChangeIndexer; -import com.googlesource.gerrit.plugins.replication.pull.Context; -import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent; -import com.googlesource.gerrit.plugins.replication.pull.ReplicationState; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.transport.URIish; -import org.junit.Before; -import org.junit.Test; - -public class FetchRefReplicatedEventHandlerTest { - private ChangeIndexer changeIndexerMock; - private FetchRefReplicatedEventHandler fetchRefReplicatedEventHandler; - private static URIish sourceUri; - - @Before - public void setUp() throws Exception { - changeIndexerMock = mock(ChangeIndexer.class); - fetchRefReplicatedEventHandler = new FetchRefReplicatedEventHandler(changeIndexerMock); - sourceUri = new URIish("git://aSourceNode/testProject.git"); - } - - @Test - public void onEventShouldIndexExistingChange() { - Project.NameKey projectNameKey = Project.nameKey("testProject"); - String ref = "refs/changes/41/41/meta"; - Change.Id changeId = Change.Id.fromRef(ref); - try { - Context.setLocalEvent(true); - fetchRefReplicatedEventHandler.onEvent( - new FetchRefReplicatedEvent( - projectNameKey.get(), - ref, - sourceUri, - ReplicationState.RefFetchResult.SUCCEEDED, - RefUpdate.Result.FAST_FORWARD)); - verify(changeIndexerMock, times(1)).index(eq(projectNameKey), eq(changeId)); - } finally { - Context.unsetLocalEvent(); - } - } - - @Test - public void onEventShouldNotIndexIfNotLocalEvent() { - Project.NameKey projectNameKey = Project.nameKey("testProject"); - String ref = "refs/changes/41/41/meta"; - Change.Id changeId = Change.Id.fromRef(ref); - fetchRefReplicatedEventHandler.onEvent( - new FetchRefReplicatedEvent( - projectNameKey.get(), - ref, - sourceUri, - ReplicationState.RefFetchResult.SUCCEEDED, - RefUpdate.Result.FAST_FORWARD)); - verify(changeIndexerMock, never()).index(eq(projectNameKey), eq(changeId)); - } - - @Test - public void onEventShouldIndexOnlyMetaRef() { - Project.NameKey projectNameKey = Project.nameKey("testProject"); - String ref = "refs/changes/41/41/1"; - Change.Id changeId = Change.Id.fromRef(ref); - fetchRefReplicatedEventHandler.onEvent( - new FetchRefReplicatedEvent( - projectNameKey.get(), - ref, - sourceUri, - ReplicationState.RefFetchResult.SUCCEEDED, - RefUpdate.Result.FAST_FORWARD)); - verify(changeIndexerMock, never()).index(eq(projectNameKey), eq(changeId)); - } - - @Test - public void onEventShouldNotIndexMissingChange() { - fetchRefReplicatedEventHandler.onEvent( - new FetchRefReplicatedEvent( - Project.nameKey("testProject").get(), - "invalidRef", - sourceUri, - ReplicationState.RefFetchResult.SUCCEEDED, - RefUpdate.Result.FAST_FORWARD)); - verify(changeIndexerMock, never()).index(any(), any()); - } - - @Test - public void onEventShouldNotIndexFailingChange() { - Project.NameKey projectNameKey = Project.nameKey("testProject"); - String ref = "refs/changes/41/41/meta"; - fetchRefReplicatedEventHandler.onEvent( - new FetchRefReplicatedEvent( - projectNameKey.get(), - ref, - sourceUri, - ReplicationState.RefFetchResult.FAILED, - RefUpdate.Result.FAST_FORWARD)); - verify(changeIndexerMock, never()).index(any(), any()); - } - - @Test - public void onEventShouldNotIndexNotAttemptedChange() { - Project.NameKey projectNameKey = Project.nameKey("testProject"); - String ref = "refs/changes/41/41/meta"; - fetchRefReplicatedEventHandler.onEvent( - new FetchRefReplicatedEvent( - projectNameKey.get(), - ref, - sourceUri, - ReplicationState.RefFetchResult.NOT_ATTEMPTED, - RefUpdate.Result.FAST_FORWARD)); - verify(changeIndexerMock, never()).index(any(), any()); - } -}