Merge branch 'stable-3.8' into stable-3.9
* stable-3.8:
Track replication started vs replication failed refs
Change-Id: I6c6dc8a5e86f5b00e780203cb9ece9e31665d8c6
diff --git a/BUILD b/BUILD
index dbbc7e9..53b55dc 100644
--- a/BUILD
+++ b/BUILD
@@ -18,6 +18,7 @@
deps = [
":events-broker-neverlink",
"//lib/commons:io",
+ "//plugins/delete-project",
"//plugins/replication",
"@commons-lang3//jar",
],
@@ -34,6 +35,7 @@
deps = PLUGIN_TEST_DEPS + PLUGIN_DEPS + [
":pull-replication__plugin",
":pull_replication_util",
+ "//plugins/delete-project",
"//plugins/replication",
"//plugins/events-broker",
],
@@ -63,6 +65,7 @@
),
deps = PLUGIN_TEST_DEPS + PLUGIN_DEPS + [
":pull-replication__plugin",
+ "//plugins/delete-project",
"//plugins/replication",
],
)
diff --git a/example-setup/broker/Dockerfile b/example-setup/broker/Dockerfile
index d615273..9ed7944 100644
--- a/example-setup/broker/Dockerfile
+++ b/example-setup/broker/Dockerfile
@@ -1,4 +1,4 @@
-FROM gerritcodereview/gerrit:3.8.1-almalinux9
+FROM gerritcodereview/gerrit:3.9.0-rc4-almalinux9
USER root
diff --git a/example-setup/broker/configs/replication.config.template b/example-setup/broker/configs/replication.config.template
index e77edeb..92047f8 100644
--- a/example-setup/broker/configs/replication.config.template
+++ b/example-setup/broker/configs/replication.config.template
@@ -14,7 +14,7 @@
url = http://$REMOTE_URL:8080/#{name}#.git
apiUrl = http://$REMOTE_URL:8080/
fetch = +refs/*:refs/*
- mirror = true
+ mirror = $MIRROR
timeout = 60 # In seconds
connectionTimeout = 120000 # In mseconds
rescheduleDelay = 15
diff --git a/example-setup/broker/docker-compose.yaml b/example-setup/broker/docker-compose.yaml
index 705aea6..5d2e68a 100644
--- a/example-setup/broker/docker-compose.yaml
+++ b/example-setup/broker/docker-compose.yaml
@@ -11,6 +11,7 @@
- BROKER_HOST=broker
- BROKER_PORT=9092
- REPLICATE_ON_STARTUP=false
+ - MIRROR=true
ports:
- "8080:8080"
- "29418:29418"
@@ -28,6 +29,7 @@
- BROKER_HOST=broker
- BROKER_PORT=9092
- REPLICATE_ON_STARTUP=true
+ - MIRROR=true
ports:
- "8081:8080"
- "29419:29418"
diff --git a/example-setup/http/Dockerfile b/example-setup/http/Dockerfile
index 6f1d1f9..32a475e 100644
--- a/example-setup/http/Dockerfile
+++ b/example-setup/http/Dockerfile
@@ -1,4 +1,4 @@
-FROM gerritcodereview/gerrit:3.8.1-almalinux9
+FROM gerritcodereview/gerrit:3.9.0-rc4-almalinux9
USER root
diff --git a/example-setup/http/configs/replication.config.template b/example-setup/http/configs/replication.config.template
index 6768146..53809ca 100644
--- a/example-setup/http/configs/replication.config.template
+++ b/example-setup/http/configs/replication.config.template
@@ -13,7 +13,7 @@
url = http://$REMOTE_URL:8080/#{name}#.git
apiUrl = http://$REMOTE_URL:8080
fetch = +refs/*:refs/*
- mirror = true
+ mirror = $MIRROR
timeout = 60 # In seconds
connectionTimeout = 120000 # In mseconds
rescheduleDelay = 15
diff --git a/example-setup/http/docker-compose.yaml b/example-setup/http/docker-compose.yaml
index ccb6b86..62fed3c 100644
--- a/example-setup/http/docker-compose.yaml
+++ b/example-setup/http/docker-compose.yaml
@@ -9,6 +9,7 @@
- REMOTE_URL=gerrit2
- DEBUG_PORT=5005
- REPLICATE_ON_STARTUP=false
+ - MIRROR=true
ports:
- "8080:8080"
- "29418:29418"
@@ -22,6 +23,7 @@
- REMOTE_URL=gerrit1
- DEBUG_PORT=5006
- REPLICATE_ON_STARTUP=true
+ - MIRROR=true
ports:
- "8081:8080"
- "29419:29418"
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchReplicationMetrics.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchReplicationMetrics.java
index 22bb073..13be628 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchReplicationMetrics.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchReplicationMetrics.java
@@ -87,16 +87,6 @@
}
/**
- * Start the end-to-end replication latency timer from a source.
- *
- * @param name the source name.
- * @return the timer context.
- */
- public Timer1.Context<String> startEnd2End(String name) {
- return end2EndExecutionTime.start(name);
- }
-
- /**
* Record the end-to-end replication latency timer from a source.
*
* @param name the source name.
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java
index 005d383..cbdfe1b 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java
@@ -53,7 +53,6 @@
import com.googlesource.gerrit.plugins.replication.pull.client.HttpClient;
import com.googlesource.gerrit.plugins.replication.pull.client.SourceHttpClient;
import com.googlesource.gerrit.plugins.replication.pull.event.EventsBrokerConsumerModule;
-import com.googlesource.gerrit.plugins.replication.pull.event.FetchRefReplicatedEventModule;
import com.googlesource.gerrit.plugins.replication.pull.event.StreamEventModule;
import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject;
import java.io.File;
@@ -94,8 +93,6 @@
install(new FactoryModuleBuilder().build(FetchJob.Factory.class));
install(new ApplyObjectCacheModule());
- install(new FetchRefReplicatedEventModule());
-
install(
new FactoryModuleBuilder()
.implement(HttpClient.class, SourceHttpClient.class)
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
index cdef514..2eb26e8 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
@@ -28,6 +28,9 @@
import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.metrics.Timer1.Context;
import com.google.gerrit.server.config.GerritInstanceId;
+import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.gerrit.server.data.RefUpdateAttribute;
+import com.google.gerrit.server.events.BatchRefUpdateEvent;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.events.EventListener;
import com.google.gerrit.server.events.RefUpdatedEvent;
@@ -37,16 +40,18 @@
import com.google.inject.Singleton;
import com.googlesource.gerrit.plugins.replication.ObservableQueue;
import com.googlesource.gerrit.plugins.replication.pull.FetchResultProcessing.GitUpdateProcessing;
+import com.googlesource.gerrit.plugins.replication.pull.api.data.BatchApplyObjectData;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData;
import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException;
import com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient;
import com.googlesource.gerrit.plugins.replication.pull.client.FetchRestApiClient;
import com.googlesource.gerrit.plugins.replication.pull.client.HttpResult;
+import com.googlesource.gerrit.plugins.replication.pull.client.HttpResultUtils;
import com.googlesource.gerrit.plugins.replication.pull.filter.ApplyObjectsRefsFilter;
import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter;
import java.io.IOException;
+import java.io.UncheckedIOException;
import java.net.URISyntaxException;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@@ -55,15 +60,14 @@
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ForkJoinPool;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.Consumer;
-import org.apache.http.client.ClientProtocolException;
+import java.util.stream.Collectors;
import org.eclipse.jgit.errors.CorruptObjectException;
import org.eclipse.jgit.errors.IncorrectObjectTypeException;
-import org.eclipse.jgit.errors.InvalidObjectIdException;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.errors.RepositoryNotFoundException;
+import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.transport.URIish;
import org.slf4j.Logger;
@@ -81,6 +85,7 @@
static final Logger repLog = LoggerFactory.getLogger(PULL_REPLICATION_LOG_NAME);
private static final Integer DEFAULT_FETCH_CALLS_TIMEOUT = 0;
+ private static final String BATCH_REF_UPDATED_EVENT_TYPE = BatchRefUpdateEvent.TYPE;
private static final String REF_UDPATED_EVENT_TYPE = new RefUpdatedEvent().type;
private static final String ZEROS_OBJECTID = ObjectId.zeroId().getName();
private final ReplicationStateListener stateLog;
@@ -91,15 +96,15 @@
private final Provider<SourcesCollection> sources; // For Guice circular dependency
private volatile boolean running;
private volatile boolean replaying;
- private final Queue<ReferenceUpdatedEvent> beforeStartupEventsQueue;
+ private final Queue<ReferenceBatchUpdatedEvent> beforeStartupEventsQueue;
private FetchApiClient.Factory fetchClientFactory;
private Integer fetchCallsTimeout;
private ExcludedRefsFilter refsFilter;
private Provider<RevisionReader> revReaderProvider;
private final ApplyObjectMetrics applyObjectMetrics;
- private final FetchReplicationMetrics fetchMetrics;
private final ReplicationQueueMetrics queueMetrics;
private final String instanceId;
+ private final boolean useBatchUpdateEvents;
private ApplyObjectsRefsFilter applyObjectsRefsFilter;
@Inject
@@ -112,9 +117,9 @@
ExcludedRefsFilter refsFilter,
Provider<RevisionReader> revReaderProvider,
ApplyObjectMetrics applyObjectMetrics,
- FetchReplicationMetrics fetchMetrics,
ReplicationQueueMetrics queueMetrics,
@GerritInstanceId String instanceId,
+ @GerritServerConfig Config gerritConfig,
ApplyObjectsRefsFilter applyObjectsRefsFilter,
ShutdownState shutdownState) {
workQueue = wq;
@@ -127,9 +132,10 @@
this.refsFilter = refsFilter;
this.revReaderProvider = revReaderProvider;
this.applyObjectMetrics = applyObjectMetrics;
- this.fetchMetrics = fetchMetrics;
this.queueMetrics = queueMetrics;
this.instanceId = instanceId;
+ this.useBatchUpdateEvents =
+ gerritConfig.getBoolean("event", "stream-events", "enableBatchRefUpdatedEvents", false);
this.applyObjectsRefsFilter = applyObjectsRefsFilter;
}
@@ -173,18 +179,65 @@
@Override
public void onEvent(com.google.gerrit.server.events.Event e) {
- if (e.type.equals(REF_UDPATED_EVENT_TYPE) && instanceId.equals(e.instanceId)) {
+ if (!instanceId.equals(e.instanceId)) {
+ return;
+ }
+
+ if (useBatchUpdateEvents) {
+ if (e.type.equals(BATCH_REF_UPDATED_EVENT_TYPE)) {
+ BatchRefUpdateEvent event = (BatchRefUpdateEvent) e;
+ repLog.info(
+ "Batch ref event received on project {} for refs: {}",
+ event.getProjectNameKey().get(),
+ String.join(",", event.getRefNames()));
+
+ long eventCreatedOn = e.eventCreatedOn;
+ List<ReferenceUpdatedEvent> refs =
+ event.refUpdates.get().stream()
+ .filter(u -> isRefToBeReplicated(u.refName))
+ .map(
+ u -> {
+ repLog.info(
+ "Ref event received: {} on project {}:{} - {} => {}",
+ refUpdateType(u.oldRev, u.newRev),
+ event.getProjectNameKey().get(),
+ u.refName,
+ u.oldRev,
+ u.newRev);
+ return ReferenceUpdatedEvent.from(u, eventCreatedOn);
+ })
+ .sorted(ReplicationQueue::sortByMetaRefAsLast)
+ .collect(Collectors.toList());
+
+ if (!refs.isEmpty()) {
+ ReferenceBatchUpdatedEvent referenceBatchUpdatedEvent =
+ ReferenceBatchUpdatedEvent.create(
+ event.getProjectNameKey().get(), refs, eventCreatedOn);
+ fire(referenceBatchUpdatedEvent);
+ }
+ }
+ return;
+ }
+
+ if (e.type.equals(REF_UDPATED_EVENT_TYPE)) {
RefUpdatedEvent event = (RefUpdatedEvent) e;
if (isRefToBeReplicated(event.getRefName())) {
+ RefUpdateAttribute refUpdateAttribute = event.refUpdate.get();
repLog.info(
"Ref event received: {} on project {}:{} - {} => {}",
- refUpdateType(event),
- event.refUpdate.get().project,
- event.getRefName(),
- event.refUpdate.get().oldRev,
- event.refUpdate.get().newRev);
- fire(ReferenceUpdatedEvent.from(event));
+ refUpdateType(refUpdateAttribute.oldRev, refUpdateAttribute.newRev),
+ event.getProjectNameKey().get(),
+ refUpdateAttribute.refName,
+ refUpdateAttribute.oldRev,
+ refUpdateAttribute.newRev);
+
+ ReferenceBatchUpdatedEvent referenceBatchUpdatedEvent =
+ ReferenceBatchUpdatedEvent.create(
+ event.getProjectNameKey().get(),
+ List.of(ReferenceUpdatedEvent.from(refUpdateAttribute, e.eventCreatedOn)),
+ e.eventCreatedOn);
+ fire(referenceBatchUpdatedEvent);
}
}
}
@@ -199,10 +252,16 @@
source.getApis().forEach(apiUrl -> source.scheduleDeleteProject(apiUrl, project)));
}
- private static String refUpdateType(RefUpdatedEvent event) {
- if (ZEROS_OBJECTID.equals(event.refUpdate.get().oldRev)) {
+ private static int sortByMetaRefAsLast(ReferenceUpdatedEvent a, ReferenceUpdatedEvent b) {
+ repLog.debug("sortByMetaRefAsLast({} <=> {})", a.refName(), b.refName());
+ return Boolean.compare(
+ RefNames.isNoteDbMetaRef(a.refName()), RefNames.isNoteDbMetaRef(b.refName()));
+ }
+
+ private static String refUpdateType(String oldRev, String newRev) {
+ if (ZEROS_OBJECTID.equals(oldRev)) {
return "CREATE";
- } else if (ZEROS_OBJECTID.equals(event.refUpdate.get().newRev)) {
+ } else if (ZEROS_OBJECTID.equals(newRev)) {
return "DELETE";
} else {
return "UPDATE";
@@ -213,13 +272,13 @@
return !refsFilter.match(refName);
}
- private void fire(ReferenceUpdatedEvent event) {
+ private void fire(ReferenceBatchUpdatedEvent event) {
ReplicationState state = new ReplicationState(new GitUpdateProcessing(dispatcher.get()));
fire(event, state);
state.markAllFetchTasksScheduled();
}
- private void fire(ReferenceUpdatedEvent event, ReplicationState state) {
+ private void fire(ReferenceBatchUpdatedEvent event, ReplicationState state) {
if (!running) {
stateLog.warn(
String.format(
@@ -244,15 +303,10 @@
final Consumer<Source> callFunction =
callFunction(
- Project.nameKey(event.projectName()),
- event.objectId(),
- event.refName(),
- event.eventCreatedOn(),
- event.isDelete(),
- state);
+ Project.nameKey(event.projectName()), event.refs(), event.eventCreatedOn(), state);
fetchCallsPool
.submit(() -> allSources.parallelStream().forEach(callFunction))
- .get(fetchCallsTimeout, TimeUnit.MILLISECONDS);
+ .get(fetchCallsTimeout, MILLISECONDS);
} catch (InterruptedException | ExecutionException | TimeoutException e) {
stateLog.error(
String.format(
@@ -269,13 +323,10 @@
private Consumer<Source> callFunction(
NameKey project,
- ObjectId objectId,
- String refName,
+ List<ReferenceUpdatedEvent> refs,
long eventCreatedOn,
- boolean isDelete,
ReplicationState state) {
- CallFunction call =
- getCallFunction(project, objectId, refName, eventCreatedOn, isDelete, state);
+ CallFunction call = getCallFunction(project, refs, eventCreatedOn, state);
return (source) -> {
boolean callSuccessful;
@@ -284,161 +335,250 @@
} catch (Exception e) {
repLog.warn(
String.format(
- "Failed to apply object %s on project %s:%s, falling back to git fetch",
- objectId.name(), project, refName),
+ "Failed to batch apply object %s on project %s, falling back to git fetch",
+ refs.stream()
+ .map(event -> String.format("%s:%s", event.refName(), event.objectId()))
+ .collect(Collectors.joining(",")),
+ project),
e);
callSuccessful = false;
}
if (!callSuccessful) {
- callFetch(source, project, refName, state, FetchRestApiClient.FORCE_ASYNC);
+ if (source.enableBatchedRefs()) {
+ callBatchFetch(source, project, refs, state);
+ } else {
+ callFetch(source, project, refs, state, FetchRestApiClient.FORCE_ASYNC);
+ }
}
};
}
private CallFunction getCallFunction(
NameKey project,
- ObjectId objectId,
- String refName,
+ List<ReferenceUpdatedEvent> refs,
long eventCreatedOn,
- boolean isDelete,
ReplicationState state) {
- if (isDelete) {
- return ((source) ->
- callSendObject(source, project, refName, eventCreatedOn, isDelete, null, state));
- }
try {
- Optional<RevisionData> revisionData =
- revReaderProvider.get().read(project, objectId, refName, 0);
- repLog.info(
- "RevisionData is {} for {}:{}",
- revisionData.map(RevisionData::toString).orElse("ABSENT"),
- project,
- refName);
+ List<BatchApplyObjectData> refsBatch =
+ refs.stream()
+ .map(ref -> toBatchApplyObject(project, ref, state))
+ .collect(Collectors.toList());
- if (revisionData.isPresent()) {
- return ((source) ->
- callSendObject(
- source,
- project,
- refName,
- eventCreatedOn,
- isDelete,
- Arrays.asList(revisionData.get()),
- state));
+ if (!containsLargeRef(refsBatch)) {
+ return ((source) -> callBatchSendObject(source, project, refsBatch, eventCreatedOn, state));
}
- } catch (InvalidObjectIdException | IOException e) {
+ } catch (UncheckedIOException e) {
+ stateLog.error("Falling back to calling fetch", e, state);
+ }
+ return ((source) -> callBatchFetch(source, project, refs, state));
+ }
+
+ private BatchApplyObjectData toBatchApplyObject(
+ NameKey project, ReferenceUpdatedEvent event, ReplicationState state) {
+ if (event.isDelete()) {
+ Optional<RevisionData> noRevisionData = Optional.empty();
+ return BatchApplyObjectData.create(event.refName(), noRevisionData, event.isDelete());
+ }
+ try {
+ Optional<RevisionData> maybeRevisionData =
+ revReaderProvider.get().read(project, event.objectId(), event.refName(), 0);
+ return BatchApplyObjectData.create(event.refName(), maybeRevisionData, event.isDelete());
+ } catch (IOException e) {
stateLog.error(
String.format(
"Exception during reading ref: %s, project:%s, message: %s",
- refName, project.get(), e.getMessage()),
+ event.refName(), project.get(), e.getMessage()),
e,
state);
+ throw new UncheckedIOException(e);
}
-
- return (source) -> callFetch(source, project, refName, state, false);
}
- private boolean callSendObject(
- Source source,
+ private boolean containsLargeRef(List<BatchApplyObjectData> batchApplyObjectData) {
+ return batchApplyObjectData.stream().anyMatch(e -> e.revisionData().isEmpty() && !e.isDelete());
+ }
+
+ private Optional<HttpResult> callSendObject(
+ FetchApiClient fetchClient,
+ String remoteName,
+ URIish uri,
NameKey project,
String refName,
long eventCreatedOn,
boolean isDelete,
- List<RevisionData> revision,
+ List<RevisionData> revision)
+ throws IOException {
+ String revisionDataStr =
+ Optional.ofNullable(revision).orElse(ImmutableList.of()).stream()
+ .map(RevisionData::toString)
+ .collect(Collectors.joining(","));
+ repLog.info(
+ "Pull replication REST API apply object to {} for {}:{} - {}",
+ uri,
+ project,
+ refName,
+ revisionDataStr);
+ Context<String> apiTimer = applyObjectMetrics.startEnd2End(remoteName);
+ HttpResult result =
+ isDelete
+ ? fetchClient.callSendObject(project, refName, eventCreatedOn, isDelete, null, uri)
+ : fetchClient.callSendObjects(project, refName, eventCreatedOn, revision, uri);
+ repLog.info(
+ "Pull replication REST API apply object to {} COMPLETED for {}:{} - {}, HTTP Result:"
+ + " {} - time:{} ms",
+ uri,
+ project,
+ refName,
+ revisionDataStr,
+ result,
+ apiTimer.stop() / 1000000.0);
+
+ return Optional.of(result);
+ }
+
+ private boolean callBatchSendObject(
+ Source source,
+ NameKey project,
+ List<BatchApplyObjectData> refsBatch,
+ long eventCreatedOn,
ReplicationState state)
throws MissingParentObjectException {
- boolean resultIsSuccessful = true;
- if (source.wouldFetchProject(project) && source.wouldFetchRef(refName)) {
- for (String apiUrl : source.getApis()) {
- try {
- URIish uri = new URIish(apiUrl);
- FetchApiClient fetchClient = fetchClientFactory.create(source);
+ boolean batchResultSuccessful = true;
+
+ List<BatchApplyObjectData> filteredRefsBatch =
+ refsBatch.stream()
+ .filter(r -> source.wouldFetchProject(project) && source.wouldFetchRef(r.refName()))
+ .collect(Collectors.toList());
+
+ String batchApplyObjectStr =
+ filteredRefsBatch.stream()
+ .map(BatchApplyObjectData::toString)
+ .collect(Collectors.joining(","));
+ FetchApiClient fetchClient = fetchClientFactory.create(source);
+ String remoteName = source.getRemoteConfigName();
+
+ for (String apiUrl : source.getApis()) {
+ try {
+ boolean resultSuccessful = true;
+ Optional<HttpResult> result = Optional.empty();
+ URIish uri = new URIish(apiUrl);
+ if (source.enableBatchedRefs()) {
repLog.info(
- "Pull replication REST API apply object to {} for {}:{} - {}",
+ "Pull replication REST API batch apply object to {} for {}:[{}]",
apiUrl,
project,
- refName,
- revision);
- Context<String> apiTimer = applyObjectMetrics.startEnd2End(source.getRemoteConfigName());
- HttpResult result =
- isDelete
- ? fetchClient.callSendObject(
- project, refName, eventCreatedOn, isDelete, null, uri)
- : fetchClient.callSendObjects(project, refName, eventCreatedOn, revision, uri);
- boolean resultSuccessful = result.isSuccessful();
+ batchApplyObjectStr);
+ Context<String> apiTimer = applyObjectMetrics.startEnd2End(remoteName);
+ result =
+ Optional.of(
+ fetchClient.callBatchSendObject(project, filteredRefsBatch, eventCreatedOn, uri));
+ resultSuccessful = HttpResultUtils.isSuccessful(result);
repLog.info(
- "Pull replication REST API apply object to {} COMPLETED for {}:{} - {}, HTTP Result:"
+ "Pull replication REST API batch apply object to {} COMPLETED for {}:[{}], HTTP Result:"
+ " {} - time:{} ms",
apiUrl,
project,
- refName,
- revision,
- result,
+ batchApplyObjectStr,
+ HttpResultUtils.status(result),
apiTimer.stop() / 1000000.0);
-
- if (!resultSuccessful
- && result.isProjectMissing(project)
- && source.isCreateMissingRepositories()) {
- result = initProject(project, uri, fetchClient, result);
- repLog.info("Missing project {} created, HTTP Result:{}", project, result);
- }
-
- if (!resultSuccessful) {
- if (result.isParentObjectMissing()) {
-
- if ((RefNames.isNoteDbMetaRef(refName) || applyObjectsRefsFilter.match(refName))
- && revision.size() == 1) {
- List<RevisionData> allRevisions =
- fetchWholeMetaHistory(project, refName, revision.get(0));
- repLog.info(
- "Pull replication REST API apply object to {} for {}:{} - {}",
- apiUrl,
+ } else {
+ repLog.info(
+ "REST API batch apply object not enabled for source {}, using REST API apply object to {} for {}:[{}]",
+ remoteName,
+ apiUrl,
+ project,
+ batchApplyObjectStr);
+ for (BatchApplyObjectData batchApplyObject : filteredRefsBatch) {
+ result =
+ callSendObject(
+ fetchClient,
+ remoteName,
+ uri,
project,
- refName,
- allRevisions);
- return callSendObject(
- source, project, refName, eventCreatedOn, isDelete, allRevisions, state);
- }
+ batchApplyObject.refName(),
+ eventCreatedOn,
+ batchApplyObject.isDelete(),
+ batchApplyObject.revisionData().map(ImmutableList::of).orElse(null));
+ resultSuccessful = HttpResultUtils.isSuccessful(result);
+ if (!resultSuccessful) {
+ break;
+ }
+ }
+ }
+
+ if (!resultSuccessful
+ && HttpResultUtils.isProjectMissing(result, project)
+ && source.isCreateMissingRepositories()) {
+ result = initProject(project, uri, fetchClient, result);
+ repLog.info(
+ "Missing project {} created, HTTP Result:{}",
+ project,
+ HttpResultUtils.status(result));
+ }
+
+ if (!resultSuccessful && HttpResultUtils.isParentObjectMissing(result)) {
+ resultSuccessful = true;
+ for (BatchApplyObjectData batchApplyObject : filteredRefsBatch) {
+ String refName = batchApplyObject.refName();
+ if ((RefNames.isNoteDbMetaRef(refName) || applyObjectsRefsFilter.match(refName))
+ && batchApplyObject.revisionData().isPresent()) {
+
+ Optional<RevisionData> maybeRevisionData = batchApplyObject.revisionData();
+ List<RevisionData> allRevisions =
+ fetchWholeMetaHistory(project, refName, maybeRevisionData.get());
+
+ Optional<HttpResult> sendObjectResult =
+ callSendObject(
+ fetchClient,
+ remoteName,
+ uri,
+ project,
+ refName,
+ eventCreatedOn,
+ batchApplyObject.isDelete(),
+ allRevisions);
+ resultSuccessful = HttpResultUtils.isSuccessful(sendObjectResult);
+ if (!resultSuccessful) {
+ break;
+ }
+ } else {
throw new MissingParentObjectException(
project, refName, source.getRemoteConfigName());
}
}
-
- resultIsSuccessful &= resultSuccessful;
- } catch (URISyntaxException e) {
- repLog.warn(
- "Pull replication REST API apply object to {} *FAILED* for {}:{} - {}",
- apiUrl,
- project,
- refName,
- revision,
- e);
- stateLog.error(String.format("Cannot parse pull replication api url:%s", apiUrl), state);
- resultIsSuccessful = false;
- } catch (IOException e) {
- repLog.warn(
- "Pull replication REST API apply object to {} *FAILED* for {}:{} - {}",
- apiUrl,
- project,
- refName,
- revision,
- e);
- stateLog.error(
- String.format(
- "Exception during the pull replication fetch rest api call. Endpoint url:%s,"
- + " message:%s",
- apiUrl, e.getMessage()),
- e,
- state);
- resultIsSuccessful = false;
}
+
+ batchResultSuccessful &= resultSuccessful;
+ } catch (URISyntaxException e) {
+ repLog.warn(
+ "Pull replication REST API batch apply object to {} *FAILED* for {}:[{}]",
+ apiUrl,
+ project,
+ batchApplyObjectStr,
+ e);
+ stateLog.error(String.format("Cannot parse pull replication api url:%s", apiUrl), state);
+ batchResultSuccessful = false;
+ } catch (IOException | IllegalArgumentException e) {
+ repLog.warn(
+ "Pull replication REST API batch apply object to {} *FAILED* for {}:[{}]",
+ apiUrl,
+ project,
+ batchApplyObjectStr,
+ e);
+ stateLog.error(
+ String.format(
+ "Exception during the pull replication fetch rest api call. Endpoint url:%s,"
+ + " message:%s",
+ apiUrl, e.getMessage()),
+ e,
+ state);
+ batchResultSuccessful = false;
}
}
-
- return resultIsSuccessful;
+ return batchResultSuccessful;
}
private List<RevisionData> fetchWholeMetaHistory(
@@ -463,63 +603,137 @@
return revisionDataBuilder.build();
}
+ private boolean callBatchFetch(
+ Source source,
+ Project.NameKey project,
+ List<ReferenceUpdatedEvent> refs,
+ ReplicationState state) {
+
+ boolean resultIsSuccessful = true;
+
+ List<String> filteredRefs =
+ refs.stream()
+ .map(ReferenceUpdatedEvent::refName)
+ .filter(refName -> source.wouldFetchProject(project) && source.wouldFetchRef(refName))
+ .collect(Collectors.toList());
+
+ String refsStr = String.join(",", filteredRefs);
+ FetchApiClient fetchClient = fetchClientFactory.create(source);
+
+ for (String apiUrl : source.getApis()) {
+ try {
+ URIish uri = new URIish(apiUrl);
+ Optional<HttpResult> result = Optional.empty();
+ repLog.info(
+ "Pull replication REST API batch fetch to {} for {}:[{}]", apiUrl, project, refsStr);
+ long startTime = System.currentTimeMillis();
+ result = Optional.of(fetchClient.callBatchFetch(project, filteredRefs, uri));
+ long endTime = System.currentTimeMillis();
+ boolean resultSuccessful = HttpResultUtils.isSuccessful(result);
+ repLog.info(
+ "Pull replication REST API batch fetch to {} COMPLETED for {}:[{}], HTTP Result:"
+ + " {} - time:{} ms",
+ apiUrl,
+ project,
+ refsStr,
+ HttpResultUtils.status(result),
+ endTime - startTime);
+ if (!resultSuccessful
+ && HttpResultUtils.isProjectMissing(result, project)
+ && source.isCreateMissingRepositories()) {
+ result = initProject(project, uri, fetchClient, result);
+ resultSuccessful = HttpResultUtils.isSuccessful(result);
+ }
+ if (!resultSuccessful) {
+ stateLog.warn(
+ String.format(
+ "Pull replication REST API batch fetch call failed. Endpoint url: %s, reason:%s",
+ apiUrl, HttpResultUtils.errorMsg(result)),
+ state);
+ }
+ resultIsSuccessful &= resultSuccessful;
+ } catch (URISyntaxException e) {
+ stateLog.error(
+ String.format("Cannot parse pull replication batch api url:%s", apiUrl), state);
+ resultIsSuccessful = false;
+ } catch (Exception e) {
+ stateLog.error(
+ String.format(
+ "Exception during the pull replication batch fetch rest api call. Endpoint url:%s,"
+ + " message:%s",
+ apiUrl, e.getMessage()),
+ e,
+ state);
+ resultIsSuccessful = false;
+ }
+ }
+
+ return resultIsSuccessful;
+ }
+
private boolean callFetch(
Source source,
Project.NameKey project,
- String refName,
+ List<ReferenceUpdatedEvent> refs,
ReplicationState state,
boolean forceAsyncCall) {
boolean resultIsSuccessful = true;
- if (source.wouldFetchProject(project) && source.wouldFetchRef(refName)) {
- for (String apiUrl : source.getApis()) {
- try {
- URIish uri = new URIish(apiUrl);
- FetchApiClient fetchClient = fetchClientFactory.create(source);
- repLog.info("Pull replication REST API fetch to {} for {}:{}", apiUrl, project, refName);
- Context<String> timer = fetchMetrics.startEnd2End(source.getRemoteConfigName());
- HttpResult result =
- fetchClient.callFetch(
- project,
- refName,
- uri,
- MILLISECONDS.toNanos(System.currentTimeMillis()),
- forceAsyncCall);
- long elapsedMs = TimeUnit.NANOSECONDS.toMillis(timer.stop());
- boolean resultSuccessful = result.isSuccessful();
- repLog.info(
- "Pull replication REST API fetch to {} COMPLETED for {}:{}, HTTP Result:"
- + " {} - time:{} ms",
- apiUrl,
- project,
- refName,
- result,
- elapsedMs);
- if (!resultSuccessful
- && result.isProjectMissing(project)
- && source.isCreateMissingRepositories()) {
- result = initProject(project, uri, fetchClient, result);
- }
- if (!resultSuccessful) {
- stateLog.warn(
- String.format(
- "Pull replication rest api fetch call failed. Endpoint url: %s, reason:%s",
- apiUrl, result.getMessage().orElse("unknown")),
- state);
- }
+ for (ReferenceUpdatedEvent refEvent : refs) {
+ String refName = refEvent.refName();
+ if (source.wouldFetchProject(project) && source.wouldFetchRef(refName)) {
+ for (String apiUrl : source.getApis()) {
+ try {
+ URIish uri = new URIish(apiUrl);
+ FetchApiClient fetchClient = fetchClientFactory.create(source);
+ repLog.info(
+ "Pull replication REST API fetch to {} for {}:{}", apiUrl, project, refName);
+ long startTime = System.currentTimeMillis();
+ Optional<HttpResult> result =
+ Optional.of(
+ fetchClient.callFetch(
+ project,
+ refName,
+ uri,
+ MILLISECONDS.toNanos(System.currentTimeMillis()),
+ forceAsyncCall));
+ long endTime = System.currentTimeMillis();
+ boolean resultSuccessful = HttpResultUtils.isSuccessful(result);
+ repLog.info(
+ "Pull replication REST API fetch to {} COMPLETED for {}:{}, HTTP Result:"
+ + " {} - time: {} ms",
+ apiUrl,
+ project,
+ refName,
+ HttpResultUtils.status(result),
+ endTime - startTime);
+ if (!resultSuccessful
+ && HttpResultUtils.isProjectMissing(result, project)
+ && source.isCreateMissingRepositories()) {
+ result = initProject(project, uri, fetchClient, result);
+ }
+ if (!resultSuccessful) {
+ stateLog.warn(
+ String.format(
+ "Pull replication rest api fetch call failed. Endpoint url: %s, reason:%s",
+ apiUrl, HttpResultUtils.errorMsg(result)),
+ state);
+ }
- resultIsSuccessful &= result.isSuccessful();
- } catch (URISyntaxException e) {
- stateLog.error(String.format("Cannot parse pull replication api url:%s", apiUrl), state);
- resultIsSuccessful = false;
- } catch (Exception e) {
- stateLog.error(
- String.format(
- "Exception during the pull replication fetch rest api call. Endpoint url:%s,"
- + " message:%s",
- apiUrl, e.getMessage()),
- e,
- state);
- resultIsSuccessful = false;
+ resultIsSuccessful &= HttpResultUtils.isSuccessful(result);
+ } catch (URISyntaxException e) {
+ stateLog.error(
+ String.format("Cannot parse pull replication api url:%s", apiUrl), state);
+ resultIsSuccessful = false;
+ } catch (Exception e) {
+ stateLog.error(
+ String.format(
+ "Exception during the pull replication fetch rest api call. Endpoint url:%s,"
+ + " message:%s",
+ apiUrl, e.getMessage()),
+ e,
+ state);
+ resultIsSuccessful = false;
+ }
}
}
}
@@ -531,9 +745,9 @@
return maxRetries == 0 || attempt < maxRetries;
}
- private HttpResult initProject(
- Project.NameKey project, URIish uri, FetchApiClient fetchClient, HttpResult result)
- throws IOException, ClientProtocolException {
+ private Optional<HttpResult> initProject(
+ Project.NameKey project, URIish uri, FetchApiClient fetchClient, Optional<HttpResult> result)
+ throws IOException {
RevisionData refsMetaConfigRevisionData =
revReaderProvider
.get()
@@ -549,7 +763,7 @@
HttpResult initProjectResult =
fetchClient.initProject(project, uri, System.currentTimeMillis(), refsMetaConfigDataList);
if (initProjectResult.isSuccessful()) {
- result = fetchClient.callFetch(project, FetchOne.ALL_REFS, uri);
+ result = Optional.of(fetchClient.callFetch(project, FetchOne.ALL_REFS, uri));
} else {
String errorMessage = initProjectResult.getMessage().map(e -> " - Error: " + e).orElse("");
repLog.error("Cannot create project " + project + errorMessage);
@@ -559,9 +773,15 @@
private void fireBeforeStartupEvents() {
Set<String> eventsReplayed = new HashSet<>();
- ReferenceUpdatedEvent event;
+ ReferenceBatchUpdatedEvent event;
while ((event = beforeStartupEventsQueue.poll()) != null) {
- String eventKey = String.format("%s:%s", event.projectName(), event.refName());
+ String eventKey =
+ String.format(
+ "%s:%s",
+ event.projectName(),
+ event.refs().stream()
+ .map(ReferenceUpdatedEvent::refName)
+ .collect(Collectors.joining()));
if (!eventsReplayed.contains(eventKey)) {
repLog.info("Firing pending task {}", event);
fire(event);
@@ -586,6 +806,22 @@
}
@AutoValue
+ abstract static class ReferenceBatchUpdatedEvent {
+
+ static ReferenceBatchUpdatedEvent create(
+ String projectName, List<ReferenceUpdatedEvent> refs, long eventCreatedOn) {
+ return new AutoValue_ReplicationQueue_ReferenceBatchUpdatedEvent(
+ projectName, refs, eventCreatedOn);
+ }
+
+ public abstract String projectName();
+
+ public abstract List<ReferenceUpdatedEvent> refs();
+
+ public abstract long eventCreatedOn();
+ }
+
+ @AutoValue
abstract static class ReferenceUpdatedEvent {
static ReferenceUpdatedEvent create(
@@ -598,13 +834,13 @@
projectName, refName, objectId, eventCreatedOn, isDelete);
}
- static ReferenceUpdatedEvent from(RefUpdatedEvent event) {
+ static ReferenceUpdatedEvent from(RefUpdateAttribute refUpdateAttribute, long eventCreatedOn) {
return ReferenceUpdatedEvent.create(
- event.refUpdate.get().project,
- event.getRefName(),
- ObjectId.fromString(event.refUpdate.get().newRev),
- event.eventCreatedOn,
- ZEROS_OBJECTID.equals(event.refUpdate.get().newRev));
+ refUpdateAttribute.project,
+ refUpdateAttribute.refName,
+ ObjectId.fromString(refUpdateAttribute.newRev),
+ eventCreatedOn,
+ ZEROS_OBJECTID.equals(refUpdateAttribute.newRev));
}
public abstract String projectName();
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
index 74a6be1..98e8d1d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
@@ -907,6 +907,14 @@
return config.replicateProjectDeletions();
}
+ public boolean isMirror() {
+ return config.getRemoteConfig().isMirror();
+ }
+
+ public boolean enableBatchedRefs() {
+ return config.enableBatchedRefs();
+ }
+
void scheduleUpdateHead(String apiUrl, Project.NameKey project, String newHead) {
try {
URIish apiURI = new URIish(apiUrl);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java
index 0a22a5a..1921e7b 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java
@@ -16,6 +16,7 @@
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableList;
+import com.google.common.flogger.FluentLogger;
import com.google.gerrit.server.config.ConfigUtil;
import com.googlesource.gerrit.plugins.replication.RemoteConfiguration;
import java.util.concurrent.TimeUnit;
@@ -23,6 +24,7 @@
import org.eclipse.jgit.transport.RemoteConfig;
public class SourceConfiguration implements RemoteConfiguration {
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
static final int DEFAULT_REPLICATION_DELAY = 4;
static final int DEFAULT_RESCHEDULE_DELAY = 3;
static final int DEFAULT_SLOW_LATENCY_THRESHOLD_SECS = 900;
@@ -56,6 +58,7 @@
private int slowLatencyThreshold;
private boolean useCGitClient;
private int refsBatchSize;
+ private boolean enableBatchedRefs;
public SourceConfiguration(RemoteConfig remoteConfig, Config cfg) {
this.remoteConfig = remoteConfig;
@@ -110,6 +113,16 @@
"shutDownDrainTimeout",
DEFAULT_DRAIN_SHUTDOWN_TIMEOUT_SECS,
TimeUnit.SECONDS);
+
+ enableBatchedRefs = cfg.getBoolean("remote", name, "enableBatchedRefs", false);
+ if (!enableBatchedRefs) {
+ logger.atWarning().log(
+ "You haven't enabled batched refs in the %s node, as such you are not "
+ + "leveraging the performance improvements introduced by the batch-apply-object API. Consider "
+ + "upgrading the plugin to the latest version and consult the plugin's documentation for more "
+ + "details on the `enableBatchedRefs` configuration.",
+ name);
+ }
}
@Override
@@ -237,4 +250,8 @@
public int getShutDownDrainTimeout() {
return shutDownDrainTimeout;
}
+
+ public boolean enableBatchedRefs() {
+ return enableBatchedRefs;
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java
index 5fef17d..157fac3 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java
@@ -23,12 +23,18 @@
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.entities.Project;
import com.google.gerrit.extensions.registration.DynamicItem;
+import com.google.gerrit.extensions.restapi.ResourceNotFoundException;
import com.google.gerrit.metrics.Timer1;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.inject.Inject;
import com.google.inject.name.Named;
-import com.googlesource.gerrit.plugins.replication.pull.*;
+import com.googlesource.gerrit.plugins.replication.pull.ApplyObjectMetrics;
+import com.googlesource.gerrit.plugins.replication.pull.ApplyObjectsCacheKey;
+import com.googlesource.gerrit.plugins.replication.pull.Context;
+import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent;
+import com.googlesource.gerrit.plugins.replication.pull.PullReplicationStateLogger;
+import com.googlesource.gerrit.plugins.replication.pull.ReplicationState;
import com.googlesource.gerrit.plugins.replication.pull.ReplicationState.RefFetchResult;
import com.googlesource.gerrit.plugins.replication.pull.Source;
import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection;
@@ -86,7 +92,7 @@
String sourceLabel,
long eventCreatedOn)
throws IOException, RefUpdateException, MissingParentObjectException,
- MissingLatestPatchSetException {
+ ResourceNotFoundException, MissingLatestPatchSetException {
applyObjects(name, refName, new RevisionData[] {revisionsData}, sourceLabel, eventCreatedOn);
}
@@ -97,7 +103,7 @@
String sourceLabel,
long eventCreatedOn)
throws IOException, RefUpdateException, MissingParentObjectException,
- MissingLatestPatchSetException {
+ ResourceNotFoundException, MissingLatestPatchSetException {
repLog.info(
"Apply object from {} for {}:{} - {}",
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchApplyObjectAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchApplyObjectAction.java
new file mode 100644
index 0000000..717b70e
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchApplyObjectAction.java
@@ -0,0 +1,57 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.api;
+
+import static com.googlesource.gerrit.plugins.replication.pull.PullReplicationLogger.repLog;
+
+import com.google.gerrit.extensions.restapi.Response;
+import com.google.gerrit.extensions.restapi.RestApiException;
+import com.google.gerrit.extensions.restapi.RestModifyView;
+import com.google.gerrit.server.project.ProjectResource;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+@Singleton
+class BatchApplyObjectAction implements RestModifyView<ProjectResource, List<RevisionInput>> {
+
+ private final ApplyObjectAction applyObjectAction;
+
+ @Inject
+ BatchApplyObjectAction(ApplyObjectAction applyObjectAction) {
+ this.applyObjectAction = applyObjectAction;
+ }
+
+ @Override
+ public Response<?> apply(ProjectResource resource, List<RevisionInput> inputs)
+ throws RestApiException {
+
+ repLog.info(
+ "Batch Apply object API from {} for refs {}",
+ resource.getNameKey(),
+ inputs.stream().map(RevisionInput::getRefName).collect(Collectors.joining(",")));
+
+ List<Response<?>> allResponses = new ArrayList<>();
+ for (RevisionInput input : inputs) {
+ Response<?> individualResponse = applyObjectAction.apply(resource, input);
+ allResponses.add(individualResponse);
+ }
+
+ return Response.ok(allResponses);
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchAction.java
new file mode 100644
index 0000000..c6ad47d
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchAction.java
@@ -0,0 +1,47 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.api;
+
+import com.google.gerrit.extensions.restapi.Response;
+import com.google.gerrit.extensions.restapi.RestApiException;
+import com.google.gerrit.extensions.restapi.RestModifyView;
+import com.google.gerrit.server.project.ProjectResource;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.Input;
+import java.util.ArrayList;
+import java.util.List;
+
+@Singleton
+public class BatchFetchAction implements RestModifyView<ProjectResource, List<Input>> {
+ private final FetchAction fetchAction;
+
+ @Inject
+ public BatchFetchAction(FetchAction fetchAction) {
+ this.fetchAction = fetchAction;
+ }
+
+ @Override
+ public Response<?> apply(ProjectResource resource, List<Input> inputs) throws RestApiException {
+
+ List<Response<?>> allResponses = new ArrayList<>();
+ for (Input input : inputs) {
+ Response<?> res = fetchAction.apply(resource, input);
+ allResponses.add(res);
+ }
+
+ return Response.ok(allResponses);
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BearerAuthenticationFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BearerAuthenticationFilter.java
index 68dac6a..64a0bd8 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BearerAuthenticationFilter.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/BearerAuthenticationFilter.java
@@ -125,7 +125,9 @@
return (requestURI.contains(pluginName)
&& (requestURI.endsWith(String.format("/%s~apply-object", pluginName))
|| requestURI.endsWith(String.format("/%s~apply-objects", pluginName))
+ || requestURI.endsWith(String.format("/%s~batch-apply-object", pluginName))
|| requestURI.endsWith(String.format("/%s~fetch", pluginName))
+ || requestURI.endsWith(String.format("/%s~batch-fetch", pluginName))
|| requestURI.endsWith(String.format("/%s~delete-project", pluginName))
|| requestURI.contains(String.format("/%s/init-project/", pluginName))))
|| (requestURI.matches(String.format(".*/projects/[^/]+/%s~HEAD", pluginName))
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java
index e49c8b6..3150fe1 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java
@@ -23,7 +23,6 @@
import com.google.gerrit.extensions.restapi.RestApiException;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.git.GitRepositoryManager;
-import com.google.gerrit.server.permissions.PermissionBackend;
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectState;
@@ -51,7 +50,6 @@
private final DynamicItem<EventDispatcher> eventDispatcher;
private final ProjectCache projectCache;
private final SourcesCollection sourcesCollection;
- private final PermissionBackend permissionBackend;
private final GitRepositoryManager gitManager;
@Inject
@@ -59,19 +57,29 @@
PullReplicationStateLogger fetchStateLog,
ProjectCache projectCache,
SourcesCollection sourcesCollection,
- PermissionBackend permissionBackend,
DynamicItem<EventDispatcher> eventDispatcher,
LocalGitRepositoryManagerProvider gitManagerProvider) {
this.fetchStateLog = fetchStateLog;
this.projectCache = projectCache;
this.eventDispatcher = eventDispatcher;
this.sourcesCollection = sourcesCollection;
- this.permissionBackend = permissionBackend;
this.gitManager = gitManagerProvider.get();
}
public void deleteRef(Project.NameKey name, String refName, String sourceLabel)
throws IOException, RestApiException {
+ Source source =
+ sourcesCollection
+ .getByRemoteName(sourceLabel)
+ .orElseThrow(
+ () ->
+ new IllegalStateException(
+ String.format("Could not find URI for %s remote", sourceLabel)));
+ if (!source.isMirror()) {
+ repLog.info(
+ "Ignoring ref {} deletion from project {}, as mirror option is false", refName, name);
+ return;
+ }
try {
repLog.info("Delete ref from {} for project {}, ref name {}", sourceLabel, name, refName);
Optional<ProjectState> projectState = projectCache.get(name);
@@ -85,13 +93,6 @@
return;
}
- Source source =
- sourcesCollection
- .getByRemoteName(sourceLabel)
- .orElseThrow(
- () ->
- new IllegalStateException(
- String.format("Could not find URI for %s remote", sourceLabel)));
URIish sourceUri = source.getURI(name);
try {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionAction.java
index adb333c..6a21104 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionAction.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionAction.java
@@ -14,6 +14,8 @@
package com.googlesource.gerrit.plugins.replication.pull.api;
+import static com.googlesource.gerrit.plugins.replication.pull.PullReplicationLogger.repLog;
+
import com.google.gerrit.extensions.api.access.PluginPermission;
import com.google.gerrit.extensions.restapi.AuthException;
import com.google.gerrit.extensions.restapi.BadRequestException;
@@ -28,9 +30,12 @@
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
-import com.googlesource.gerrit.plugins.replication.LocalFS;
+import com.googlesource.gerrit.plugins.deleteproject.cache.CacheDeleteHandler;
+import com.googlesource.gerrit.plugins.deleteproject.fs.RepositoryDelete;
import com.googlesource.gerrit.plugins.replication.pull.GerritConfigOps;
+import java.io.IOException;
import java.util.Optional;
+import org.eclipse.jgit.errors.RepositoryNotFoundException;
import org.eclipse.jgit.transport.URIish;
@Singleton
@@ -44,15 +49,21 @@
private final Provider<CurrentUser> userProvider;
private final GerritConfigOps gerritConfigOps;
private final PermissionBackend permissionBackend;
+ private final RepositoryDelete repositoryDelete;
+ private final CacheDeleteHandler cacheDeleteHandler;
@Inject
ProjectDeletionAction(
GerritConfigOps gerritConfigOps,
PermissionBackend permissionBackend,
- Provider<CurrentUser> userProvider) {
+ Provider<CurrentUser> userProvider,
+ RepositoryDelete repositoryDelete,
+ CacheDeleteHandler cacheDeleteHandler) {
this.gerritConfigOps = gerritConfigOps;
this.permissionBackend = permissionBackend;
this.userProvider = userProvider;
+ this.repositoryDelete = repositoryDelete;
+ this.cacheDeleteHandler = cacheDeleteHandler;
}
@Override
@@ -69,11 +80,24 @@
gerritConfigOps.getGitRepositoryURI(String.format("%s.git", projectResource.getName()));
if (maybeRepoURI.isPresent()) {
- if (new LocalFS(maybeRepoURI.get()).deleteProject(projectResource.getNameKey())) {
+ try {
+ // reuse repo deletion logic from delete-project plugin, as it can successfully delete
+ // the git directories hosted on nfs.
+ repositoryDelete.execute(projectResource.getNameKey());
+ // delete the project from the local project cache, otherwise future ops
+ // will fail as the replica will think that the project still exists locally.
+ cacheDeleteHandler.delete(projectResource.getProjectState().getProject());
+ repLog.info(
+ "Deleted local repository {} and removed it from the local project cache",
+ projectResource.getName());
return Response.ok();
+ } catch (RepositoryNotFoundException e) {
+ throw new ResourceNotFoundException(
+ String.format("Repository %s not found", projectResource.getName()), e);
+ } catch (IOException e) {
+ throw new UnprocessableEntityException(
+ String.format("Could not delete project %s", projectResource.getName()));
}
- throw new UnprocessableEntityException(
- String.format("Could not delete project %s", projectResource.getName()));
}
throw new ResourceNotFoundException(
String.format("Could not compute URI for repo: %s", projectResource.getName()));
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java
index 4ed3160..6c24cbc 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectInitializationAction.java
@@ -20,6 +20,7 @@
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static javax.servlet.http.HttpServletResponse.SC_CONFLICT;
import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
import com.google.common.base.Strings;
import com.google.common.flogger.FluentLogger;
@@ -28,6 +29,7 @@
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.restapi.AuthException;
import com.google.gerrit.extensions.restapi.BadRequestException;
+import com.google.gerrit.extensions.restapi.ResourceNotFoundException;
import com.google.gerrit.extensions.restapi.Url;
import com.google.gerrit.index.project.ProjectIndexer;
import com.google.gerrit.server.CurrentUser;
@@ -130,13 +132,16 @@
logExceptionAndUpdateResponse(httpServletResponse, e, SC_BAD_REQUEST, gitRepositoryName);
} catch (RefUpdateException | MissingParentObjectException e) {
logExceptionAndUpdateResponse(httpServletResponse, e, SC_CONFLICT, gitRepositoryName);
+ } catch (ResourceNotFoundException e) {
+ logExceptionAndUpdateResponse(
+ httpServletResponse, e, SC_INTERNAL_SERVER_ERROR, gitRepositoryName);
} catch (AuthException | PermissionBackendException e) {
logExceptionAndUpdateResponse(httpServletResponse, e, SC_FORBIDDEN, gitRepositoryName);
}
}
public boolean initProject(String gitRepositoryName)
- throws AuthException, PermissionBackendException {
+ throws AuthException, PermissionBackendException, IOException {
if (initProject(gitRepositoryName, true)) {
repLog.info("Init project API from {}", gitRepositoryName);
return true;
@@ -147,7 +152,7 @@
private boolean initProjectWithConfiguration(
HttpServletRequest httpServletRequest, String gitRepositoryName)
throws AuthException, PermissionBackendException, IOException, BadRequestException,
- MissingParentObjectException, RefUpdateException {
+ MissingParentObjectException, RefUpdateException, ResourceNotFoundException {
RevisionsInput input = PayloadSerDes.parseRevisionsInput(httpServletRequest);
validateInput(input);
@@ -184,7 +189,7 @@
}
private boolean initProject(String gitRepositoryName, boolean needsProjectReindexing)
- throws AuthException, PermissionBackendException {
+ throws AuthException, PermissionBackendException, IOException {
// When triggered internally(for example by consuming stream events) user is not provided
// and internal user is returned. Project creation should be always allowed for internal user.
if (!userProvider.get().isInternalUser()) {
@@ -199,7 +204,7 @@
Project.NameKey projectNameKey = Project.NameKey.parse(gitRepositoryName);
if (localFS.createProject(projectNameKey, RefNames.HEAD)) {
if (needsProjectReindexing) {
- projectIndexer.index(projectNameKey);
+ projectCache.onCreateProject(projectNameKey);
}
return true;
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationEndpoints.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationEndpoints.java
index fc97945..9fdf5d9 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationEndpoints.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationEndpoints.java
@@ -33,7 +33,10 @@
@UsedAt(PLUGIN_MULTI_SITE)
public static final String APPLY_OBJECTS_API_ENDPOINT = "apply-objects";
+ public static final String BATCH_APPLY_OBJECT_API_ENDPOINT = "batch-apply-object";
+
public static final String FETCH_ENDPOINT = "fetch";
+ public static final String BATCH_FETCH_ENDPOINT = "batch-fetch";
public static final String INIT_PROJECT_ENDPOINT = "init-project";
public static final String DELETE_PROJECT_ENDPOINT = "delete-project";
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java
index f501e90..86fba22 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java
@@ -18,8 +18,10 @@
import static com.googlesource.gerrit.plugins.replication.pull.api.HttpServletOps.checkAcceptHeader;
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static javax.servlet.http.HttpServletResponse.SC_CONFLICT;
+import static javax.servlet.http.HttpServletResponse.SC_CREATED;
import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
-import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
import static javax.servlet.http.HttpServletResponse.SC_UNAUTHORIZED;
import com.google.common.flogger.FluentLogger;
@@ -36,22 +38,31 @@
import com.google.gerrit.extensions.restapi.UnprocessableEntityException;
import com.google.gerrit.httpd.AllRequestFilter;
import com.google.gerrit.httpd.restapi.RestApiServlet;
+import com.google.gerrit.json.OutputFormat;
import com.google.gerrit.server.AnonymousUser;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectResource;
import com.google.gerrit.server.project.ProjectState;
+import com.google.gson.Gson;
import com.google.gson.JsonParseException;
+import com.google.gson.stream.JsonReader;
import com.google.gson.stream.MalformedJsonException;
import com.google.inject.Inject;
import com.google.inject.Provider;
+import com.google.inject.TypeLiteral;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.Input;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionsInput;
import com.googlesource.gerrit.plugins.replication.pull.api.exception.UnauthorizedAuthException;
import com.googlesource.gerrit.plugins.replication.pull.api.util.PayloadSerDes;
+import java.io.BufferedReader;
+import java.io.EOFException;
import java.io.IOException;
+import java.io.PrintWriter;
+import java.lang.reflect.Type;
+import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Optional;
@@ -68,24 +79,27 @@
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final Pattern projectNameInGerritUrl = Pattern.compile(".*/projects/([^/]+)/.*");
- private static final Pattern projectNameInitProjectUrl =
- Pattern.compile(".*/init-project/([^/]+.git)");
private FetchAction fetchAction;
+ private BatchFetchAction batchFetchAction;
private ApplyObjectAction applyObjectAction;
private ApplyObjectsAction applyObjectsAction;
+ private BatchApplyObjectAction batchApplyObjectAction;
private ProjectInitializationAction projectInitializationAction;
private UpdateHeadAction updateHEADAction;
private ProjectDeletionAction projectDeletionAction;
private ProjectCache projectCache;
+ private Gson gson;
private String pluginName;
private final Provider<CurrentUser> currentUserProvider;
@Inject
public PullReplicationFilter(
FetchAction fetchAction,
+ BatchFetchAction batchFetchAction,
ApplyObjectAction applyObjectAction,
ApplyObjectsAction applyObjectsAction,
+ BatchApplyObjectAction batchApplyObjectAction,
ProjectInitializationAction projectInitializationAction,
UpdateHeadAction updateHEADAction,
ProjectDeletionAction projectDeletionAction,
@@ -93,13 +107,16 @@
@PluginName String pluginName,
Provider<CurrentUser> currentUserProvider) {
this.fetchAction = fetchAction;
+ this.batchFetchAction = batchFetchAction;
this.applyObjectAction = applyObjectAction;
this.applyObjectsAction = applyObjectsAction;
+ this.batchApplyObjectAction = batchApplyObjectAction;
this.projectInitializationAction = projectInitializationAction;
this.updateHEADAction = updateHEADAction;
this.projectDeletionAction = projectDeletionAction;
this.projectCache = projectCache;
this.pluginName = pluginName;
+ this.gson = OutputFormat.JSON.newGsonBuilder().create();
this.currentUserProvider = currentUserProvider;
}
@@ -117,12 +134,18 @@
if (isFetchAction(httpRequest)) {
failIfcurrentUserIsAnonymous();
PayloadSerDes.writeResponse(httpResponse, doFetch(httpRequest));
+ } else if (isBatchFetchAction(httpRequest)) {
+ failIfcurrentUserIsAnonymous();
+ PayloadSerDes.writeResponse(httpResponse, doBatchFetch(httpRequest));
} else if (isApplyObjectAction(httpRequest)) {
failIfcurrentUserIsAnonymous();
PayloadSerDes.writeResponse(httpResponse, doApplyObject(httpRequest));
} else if (isApplyObjectsAction(httpRequest)) {
failIfcurrentUserIsAnonymous();
PayloadSerDes.writeResponse(httpResponse, doApplyObjects(httpRequest));
+ } else if (isBatchApplyObjectsAction(httpRequest)) {
+ failIfcurrentUserIsAnonymous();
+ PayloadSerDes.writeResponse(httpResponse, doBatchApplyObject(httpRequest));
} else if (isInitProjectAction(httpRequest)) {
failIfcurrentUserIsAnonymous();
if (!checkAcceptHeader(httpRequest, httpResponse)) {
@@ -159,7 +182,7 @@
httpRequest, httpResponse, SC_CONFLICT, e.getMessage(), e.caching(), e);
} catch (ResourceNotFoundException e) {
RestApiServlet.replyError(
- httpRequest, httpResponse, SC_INTERNAL_SERVER_ERROR, e.getMessage(), e.caching(), e);
+ httpRequest, httpResponse, SC_NOT_FOUND, e.getMessage(), e.caching(), e);
} catch (NoSuchElementException e) {
RestApiServlet.replyError(
httpRequest, httpResponse, SC_BAD_REQUEST, "Project name not present in the url", e);
@@ -205,6 +228,17 @@
}
@SuppressWarnings("unchecked")
+ private Response<Map<String, Object>> doBatchApplyObject(HttpServletRequest httpRequest)
+ throws RestApiException, IOException {
+ TypeLiteral<List<RevisionInput>> collectionType = new TypeLiteral<>() {};
+ List<RevisionInput> inputs = readJson(httpRequest, collectionType.getType());
+ IdString id = getProjectName(httpRequest).get();
+
+ return (Response<Map<String, Object>>)
+ batchApplyObjectAction.apply(parseProjectResource(id), inputs);
+ }
+
+ @SuppressWarnings("unchecked")
private Response<String> doUpdateHEAD(HttpServletRequest httpRequest) throws Exception {
HeadInput input = PayloadSerDes.parseHeadInput(httpRequest);
IdString id = getProjectName(httpRequest).get();
@@ -237,15 +271,58 @@
return new ProjectResource(project.get(), currentUserProvider.get());
}
- /**
- * Return project name from request URI. Request URI format:
- * /a/projects/<project_name>/pull-replication~apply-object
- *
- * @param req
- * @return project name
- */
- private Optional<IdString> getInitProjectName(HttpServletRequest req) {
- return extractProjectName(req, projectNameInitProjectUrl);
+ @SuppressWarnings("unchecked")
+ private Response<Map<String, Object>> doBatchFetch(HttpServletRequest httpRequest)
+ throws IOException, RestApiException {
+ TypeLiteral<List<Input>> collectionType = new TypeLiteral<>() {};
+ List<Input> inputs = readJson(httpRequest, collectionType.getType());
+ IdString id = getProjectName(httpRequest).get();
+
+ return (Response<Map<String, Object>>) batchFetchAction.apply(parseProjectResource(id), inputs);
+ }
+
+ private <T> void writeResponse(HttpServletResponse httpResponse, Response<T> response)
+ throws IOException {
+ String responseJson = gson.toJson(response);
+ if (response.statusCode() == SC_OK || response.statusCode() == SC_CREATED) {
+
+ httpResponse.setContentType("application/json");
+ httpResponse.setStatus(response.statusCode());
+ PrintWriter writer = httpResponse.getWriter();
+ writer.print(new String(RestApiServlet.JSON_MAGIC));
+ writer.print(responseJson);
+ } else {
+ httpResponse.sendError(response.statusCode(), responseJson);
+ }
+ }
+
+ private <T> T readJson(HttpServletRequest httpRequest, Type typeToken)
+ throws IOException, BadRequestException {
+
+ try (BufferedReader br = httpRequest.getReader();
+ JsonReader json = new JsonReader(br)) {
+ try {
+ json.setLenient(true);
+
+ try {
+ json.peek();
+ } catch (EOFException e) {
+ throw new BadRequestException("Expected JSON object", e);
+ }
+
+ return gson.fromJson(json, typeToken);
+ } finally {
+ try {
+ // Reader.close won't consume the rest of the input. Explicitly consume the request
+ // body.
+ br.skip(Long.MAX_VALUE);
+ } catch (Exception e) {
+ // ignore, e.g. trying to consume the rest of the input may fail if the request was
+ // cancelled
+ logger.atFine().withCause(e).log("Exception during the parsing of the request json");
+ }
+ }
+ }
}
private Optional<IdString> getProjectName(HttpServletRequest req) {
@@ -275,10 +352,22 @@
.endsWith(String.format("/%s~" + APPLY_OBJECTS_API_ENDPOINT, pluginName));
}
+ private boolean isBatchApplyObjectsAction(HttpServletRequest httpRequest) {
+ return httpRequest
+ .getRequestURI()
+ .endsWith(String.format("/%s~" + BATCH_APPLY_OBJECT_API_ENDPOINT, pluginName));
+ }
+
private boolean isFetchAction(HttpServletRequest httpRequest) {
return httpRequest.getRequestURI().endsWith(String.format("/%s~" + FETCH_ENDPOINT, pluginName));
}
+ private boolean isBatchFetchAction(HttpServletRequest httpRequest) {
+ return httpRequest
+ .getRequestURI()
+ .endsWith(String.format("/%s~" + BATCH_FETCH_ENDPOINT, pluginName));
+ }
+
private boolean isInitProjectAction(HttpServletRequest httpRequest) {
return httpRequest
.getRequestURI()
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/BatchApplyObjectData.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/BatchApplyObjectData.java
new file mode 100644
index 0000000..7a613c0
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/BatchApplyObjectData.java
@@ -0,0 +1,45 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.api.data;
+
+import com.google.auto.value.AutoValue;
+import java.util.Optional;
+
+@AutoValue
+public abstract class BatchApplyObjectData {
+
+ public static BatchApplyObjectData create(
+ String refName, Optional<RevisionData> revisionData, boolean isDelete)
+ throws IllegalArgumentException {
+ if (isDelete && revisionData.isPresent()) {
+ throw new IllegalArgumentException(
+ "DELETE ref-updates cannot be associated with a RevisionData");
+ }
+ return new AutoValue_BatchApplyObjectData(refName, revisionData, isDelete);
+ }
+
+ public abstract String refName();
+
+ public abstract Optional<RevisionData> revisionData();
+
+ public abstract boolean isDelete();
+
+ @Override
+ public String toString() {
+ return String.format(
+ "%s:%s isDelete=%s",
+ refName(), revisionData().map(RevisionData::toString).orElse("ABSENT"), isDelete());
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java
index 38d5d57..28c74c9 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java
@@ -19,10 +19,10 @@
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.Project.NameKey;
import com.googlesource.gerrit.plugins.replication.pull.Source;
+import com.googlesource.gerrit.plugins.replication.pull.api.data.BatchApplyObjectData;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData;
import java.io.IOException;
import java.util.List;
-import org.apache.http.client.ClientProtocolException;
import org.eclipse.jgit.transport.URIish;
public interface FetchApiClient {
@@ -36,15 +36,25 @@
String refName,
URIish targetUri,
long startTimeNanos,
- boolean forceAsyncCall)
- throws ClientProtocolException, IOException;
+ boolean forceAsyncFetch)
+ throws IOException;
default HttpResult callFetch(Project.NameKey project, String refName, URIish targetUri)
- throws ClientProtocolException, IOException {
+ throws IOException {
return callFetch(
project, refName, targetUri, MILLISECONDS.toNanos(System.currentTimeMillis()), false);
}
+ HttpResult callBatchFetch(
+ Project.NameKey project, List<String> refsInBatch, URIish targetUri, long startTimeNanos)
+ throws IOException;
+
+ default HttpResult callBatchFetch(
+ Project.NameKey project, List<String> refsInBatch, URIish targetUri) throws IOException {
+ return callBatchFetch(
+ project, refsInBatch, targetUri, MILLISECONDS.toNanos(System.currentTimeMillis()));
+ }
+
/**
* Replicates the creation of a project, including the configuration stored in refs/meta/config.
*
@@ -73,7 +83,14 @@
boolean isDelete,
RevisionData revisionData,
URIish targetUri)
- throws ClientProtocolException, IOException;
+ throws IOException;
+
+ HttpResult callBatchSendObject(
+ NameKey project,
+ List<BatchApplyObjectData> batchApplyObjects,
+ long eventCreatedOn,
+ URIish targetUri)
+ throws IOException;
HttpResult callSendObjects(
NameKey project,
@@ -81,5 +98,5 @@
long eventCreatedOn,
List<RevisionData> revisionData,
URIish targetUri)
- throws ClientProtocolException, IOException;
+ throws IOException;
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
index 5579415..8531b0a 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
@@ -14,8 +14,10 @@
package com.googlesource.gerrit.plugins.replication.pull.client;
+import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES;
import static java.util.Objects.requireNonNull;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
import com.google.common.base.Strings;
import com.google.common.flogger.FluentLogger;
@@ -36,6 +38,7 @@
import com.googlesource.gerrit.plugins.replication.pull.BearerTokenProvider;
import com.googlesource.gerrit.plugins.replication.pull.Source;
import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics;
+import com.googlesource.gerrit.plugins.replication.pull.api.data.BatchApplyObjectData;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionsInput;
@@ -43,7 +46,9 @@
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
+import java.util.stream.Collectors;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpResponse;
import org.apache.http.ParseException;
@@ -141,13 +146,81 @@
"{\"label\":\"%s\", \"ref_name\": \"%s\", \"async\":%s}",
instanceId, refName, callAsync),
StandardCharsets.UTF_8));
- post.addHeader(new BasicHeader("Content-Type", "application/json"));
+ post.addHeader(new BasicHeader(CONTENT_TYPE, "application/json"));
post.addHeader(
PullReplicationApiRequestMetrics.HTTP_HEADER_X_START_TIME_NANOS,
Long.toString(startTimeNanos));
return executeRequest(post, bearerTokenProvider.get(), targetUri);
}
+ private Map<Boolean, List<String>> partitionRefsToAsyncAndSync(List<String> refsInBatch) {
+ return refsInBatch.stream()
+ .collect(
+ Collectors.partitioningBy(
+ refName -> !syncRefsFilter.match(refName),
+ Collectors.mapping(
+ refName ->
+ String.format(
+ "{\"label\":\"%s\", \"ref_name\": \"%s\", \"async\":%s}",
+ instanceId, refName, !syncRefsFilter.match(refName)),
+ Collectors.toList())));
+ }
+
+ @Override
+ public HttpResult callBatchFetch(
+ NameKey project, List<String> refsInBatch, URIish targetUri, long startTimeNanos)
+ throws IOException {
+ Map<Boolean, List<String>> refsPartitionedInAsyncAndSync =
+ partitionRefsToAsyncAndSync(refsInBatch);
+ boolean async = true;
+ List<String> asyncRefs = refsPartitionedInAsyncAndSync.get(async);
+ List<String> syncRefs = refsPartitionedInAsyncAndSync.get(!async);
+
+ if (asyncRefs.isEmpty() && syncRefs.isEmpty()) {
+ throw new IllegalArgumentException(
+ "At least one ref should be provided during a batch-fetch operation");
+ }
+
+ String url = formatUrl(targetUri.toString(), project, "batch-fetch");
+
+ if (asyncRefs.isEmpty()) {
+ HttpPost syncPost =
+ createPostRequest(url, "[" + String.join(",", syncRefs) + "]", startTimeNanos);
+ return executeRequest(syncPost, bearerTokenProvider.get(), targetUri);
+ }
+ if (syncRefs.isEmpty()) {
+ HttpPost asyncPost =
+ createPostRequest(url, "[" + String.join(",", asyncRefs) + "]", startTimeNanos);
+ return executeRequest(asyncPost, bearerTokenProvider.get(), targetUri);
+ }
+
+ // first execute for async refs, then for sync
+ HttpPost asyncPost =
+ createPostRequest(url, "[" + String.join(",", asyncRefs) + "]", startTimeNanos);
+ HttpResult asyncResult = executeRequest(asyncPost, bearerTokenProvider.get(), targetUri);
+
+ if (asyncResult.isSuccessful()) {
+ HttpPost syncPost =
+ createPostRequest(
+ url,
+ "[" + String.join(",", syncRefs) + "]",
+ MILLISECONDS.toNanos(System.currentTimeMillis()));
+ return executeRequest(syncPost, bearerTokenProvider.get(), targetUri);
+ }
+
+ return asyncResult;
+ }
+
+ private HttpPost createPostRequest(String url, String msgBody, long startTimeNanos) {
+ HttpPost post = new HttpPost(url);
+ post.setEntity(new StringEntity(msgBody, StandardCharsets.UTF_8));
+ post.addHeader(new BasicHeader(CONTENT_TYPE, "application/json"));
+ post.addHeader(
+ PullReplicationApiRequestMetrics.HTTP_HEADER_X_START_TIME_NANOS,
+ Long.toString(startTimeNanos));
+ return post;
+ }
+
/* (non-Javadoc)
* @see com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient#initProject(com.google.gerrit.entities.Project.NameKey, org.eclipse.jgit.transport.URIish, long, java.util.List<com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData>)
*/
@@ -171,7 +244,7 @@
HttpPut put = new HttpPut(url);
put.setEntity(new StringEntity(GSON.toJson(input)));
put.addHeader(new BasicHeader("Accept", MediaType.ANY_TEXT_TYPE.toString()));
- put.addHeader(new BasicHeader("Content-Type", MediaType.JSON_UTF_8.toString()));
+ put.addHeader(new BasicHeader(CONTENT_TYPE, MediaType.JSON_UTF_8.toString()));
return executeRequest(put, bearerTokenProvider.get(), uri);
}
@@ -196,7 +269,7 @@
HttpPut req = new HttpPut(url);
req.setEntity(
new StringEntity(String.format("{\"ref\": \"%s\"}", newHead), StandardCharsets.UTF_8));
- req.addHeader(new BasicHeader("Content-Type", MediaType.JSON_UTF_8.toString()));
+ req.addHeader(new BasicHeader(CONTENT_TYPE, MediaType.JSON_UTF_8.toString()));
return executeRequest(req, bearerTokenProvider.get(), apiUri);
}
@@ -211,7 +284,7 @@
boolean isDelete,
@Nullable RevisionData revisionData,
URIish targetUri)
- throws ClientProtocolException, IOException {
+ throws IOException {
if (!isDelete) {
requireNonNull(
@@ -225,7 +298,33 @@
HttpPost post = new HttpPost(url);
post.setEntity(new StringEntity(GSON.toJson(input)));
- post.addHeader(new BasicHeader("Content-Type", MediaType.JSON_UTF_8.toString()));
+ post.addHeader(new BasicHeader(CONTENT_TYPE, MediaType.JSON_UTF_8.toString()));
+ return executeRequest(post, bearerTokenProvider.get(), targetUri);
+ }
+
+ @Override
+ public HttpResult callBatchSendObject(
+ NameKey project,
+ List<BatchApplyObjectData> batchedRefs,
+ long eventCreatedOn,
+ URIish targetUri)
+ throws IOException {
+ List<RevisionInput> inputs =
+ batchedRefs.stream()
+ .map(
+ batchApplyObject ->
+ new RevisionInput(
+ instanceId,
+ batchApplyObject.refName(),
+ eventCreatedOn,
+ batchApplyObject.revisionData().orElse(null)))
+ .collect(Collectors.toList());
+
+ String url = formatUrl(targetUri.toString(), project, "batch-apply-object");
+
+ HttpPost post = new HttpPost(url);
+ post.setEntity(new StringEntity(GSON.toJson(inputs)));
+ post.addHeader(new BasicHeader(CONTENT_TYPE, MediaType.JSON_UTF_8.toString()));
return executeRequest(post, bearerTokenProvider.get(), targetUri);
}
@@ -236,7 +335,7 @@
long eventCreatedOn,
List<RevisionData> revisionData,
URIish targetUri)
- throws ClientProtocolException, IOException {
+ throws IOException {
if (revisionData.size() == 1) {
return callSendObject(
project, refName, eventCreatedOn, false, revisionData.get(0), targetUri);
@@ -249,7 +348,7 @@
String url = formatUrl(targetUri.toString(), project, "apply-objects");
HttpPost post = new HttpPost(url);
post.setEntity(new StringEntity(GSON.toJson(input)));
- post.addHeader(new BasicHeader("Content-Type", MediaType.JSON_UTF_8.toString()));
+ post.addHeader(new BasicHeader(CONTENT_TYPE, MediaType.JSON_UTF_8.toString()));
return executeRequest(post, bearerTokenProvider.get(), targetUri);
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultUtils.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultUtils.java
new file mode 100644
index 0000000..257e7e4
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultUtils.java
@@ -0,0 +1,43 @@
+//
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.client;
+
+import com.google.gerrit.entities.Project;
+import java.util.Optional;
+
+public class HttpResultUtils {
+
+ public static String status(Optional<HttpResult> maybeResult) {
+ return maybeResult.map(HttpResult::toString).orElse("unknown");
+ }
+
+ public static boolean isSuccessful(Optional<HttpResult> maybeResult) {
+ return maybeResult.map(HttpResult::isSuccessful).orElse(false);
+ }
+
+ public static boolean isProjectMissing(
+ Optional<HttpResult> maybeResult, Project.NameKey project) {
+ return maybeResult.map(r -> r.isProjectMissing(project)).orElse(false);
+ }
+
+ public static boolean isParentObjectMissing(Optional<HttpResult> maybeResult) {
+ return maybeResult.map(HttpResult::isParentObjectMissing).orElse(false);
+ }
+
+ public static String errorMsg(Optional<HttpResult> maybeResult) {
+ return maybeResult.flatMap(HttpResult::getMessage).orElse("unknown");
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/EventsBrokerMessageConsumer.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/EventsBrokerMessageConsumer.java
index a21a48c..a9b4945 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/EventsBrokerMessageConsumer.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/EventsBrokerMessageConsumer.java
@@ -19,7 +19,6 @@
import com.gerritforge.gerrit.eventbroker.BrokerApi;
import com.gerritforge.gerrit.eventbroker.ExtendedBrokerApi;
-import com.google.common.flogger.FluentLogger;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.extensions.registration.DynamicItem;
@@ -29,11 +28,10 @@
import com.google.inject.Inject;
import com.google.inject.name.Named;
import com.googlesource.gerrit.plugins.replication.pull.ShutdownState;
+import java.io.IOException;
import java.util.function.Consumer;
public class EventsBrokerMessageConsumer implements Consumer<Event>, LifecycleListener {
-
- private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private final DynamicItem<BrokerApi> eventsBrokerDi;
private final StreamEventListener eventListener;
private final ShutdownState shutdownState;
@@ -60,7 +58,7 @@
try {
eventListener.fetchRefsForEvent(event);
if (shutdownState.isShuttingDown()) stop();
- } catch (AuthException | PermissionBackendException e) {
+ } catch (AuthException | PermissionBackendException | IOException e) {
throw new EventRejectedException(event, e);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandler.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandler.java
deleted file mode 100644
index 9ee1e8e..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandler.java
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright (C) 2021 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.replication.pull.event;
-
-import com.google.common.flogger.FluentLogger;
-import com.google.gerrit.entities.Change;
-import com.google.gerrit.entities.Project;
-import com.google.gerrit.entities.RefNames;
-import com.google.gerrit.server.events.Event;
-import com.google.gerrit.server.events.EventListener;
-import com.google.gerrit.server.index.change.ChangeIndexer;
-import com.google.inject.Inject;
-import com.googlesource.gerrit.plugins.replication.pull.Context;
-import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent;
-import com.googlesource.gerrit.plugins.replication.pull.ReplicationState;
-
-public class FetchRefReplicatedEventHandler implements EventListener {
- private static final FluentLogger logger = FluentLogger.forEnclosingClass();
- private ChangeIndexer changeIndexer;
-
- @Inject
- FetchRefReplicatedEventHandler(ChangeIndexer changeIndexer) {
- this.changeIndexer = changeIndexer;
- }
-
- @Override
- public void onEvent(Event event) {
- if (event instanceof FetchRefReplicatedEvent && isLocalEvent()) {
- FetchRefReplicatedEvent fetchRefReplicatedEvent = (FetchRefReplicatedEvent) event;
- if (!RefNames.isNoteDbMetaRef(fetchRefReplicatedEvent.getRefName())
- || !fetchRefReplicatedEvent
- .getStatus()
- .equals(ReplicationState.RefFetchResult.SUCCEEDED.toString())) {
- return;
- }
-
- Project.NameKey projectNameKey = fetchRefReplicatedEvent.getProjectNameKey();
- logger.atFine().log(
- "Indexing ref '%s' for project %s",
- fetchRefReplicatedEvent.getRefName(), projectNameKey.get());
- Change.Id changeId = Change.Id.fromRef(fetchRefReplicatedEvent.getRefName());
- if (changeId != null) {
- changeIndexer.indexAsync(projectNameKey, changeId);
- } else {
- logger.atWarning().log(
- "Couldn't get changeId from refName. Skipping indexing of change %s for project %s",
- fetchRefReplicatedEvent.getRefName(), projectNameKey.get());
- }
- }
- }
-
- private boolean isLocalEvent() {
- return Context.isLocalEvent();
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventModule.java
deleted file mode 100644
index 675563a..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventModule.java
+++ /dev/null
@@ -1,27 +0,0 @@
-// Copyright (C) 2021 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.replication.pull.event;
-
-import com.google.gerrit.extensions.registration.DynamicSet;
-import com.google.gerrit.lifecycle.LifecycleModule;
-import com.google.gerrit.server.events.EventListener;
-
-public class FetchRefReplicatedEventModule extends LifecycleModule {
-
- @Override
- protected void configure() {
- DynamicSet.bind(binder(), EventListener.class).to(FetchRefReplicatedEventHandler.class);
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
index b14d4a1..4c621ac 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
@@ -95,7 +95,7 @@
public void onEvent(Event event) {
try {
fetchRefsForEvent(event);
- } catch (AuthException | PermissionBackendException e) {
+ } catch (AuthException | PermissionBackendException | IOException e) {
logger.atSevere().withCause(e).log(
"This is the event handler of Gerrit's event-bus. It isn't"
+ "supposed to throw any exception, otherwise the other handlers "
@@ -103,7 +103,8 @@
}
}
- public void fetchRefsForEvent(Event event) throws AuthException, PermissionBackendException {
+ public void fetchRefsForEvent(Event event)
+ throws AuthException, PermissionBackendException, IOException {
if (instanceId.equals(event.instanceId) || !shouldReplicateProject(event)) {
return;
}
@@ -153,7 +154,7 @@
projectCreatedEvent.instanceId,
projectCreatedEvent.getProjectNameKey(),
metrics);
- } catch (AuthException | PermissionBackendException e) {
+ } catch (AuthException | PermissionBackendException | IOException e) {
logger.atSevere().withCause(e).log(
"Cannot initialise project:%s", projectCreatedEvent.projectName);
throw e;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java
index 0ed262d..70b4795 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java
@@ -15,6 +15,8 @@
package com.googlesource.gerrit.plugins.replication.pull.fetch;
import com.google.gerrit.entities.Project;
+import com.google.gerrit.extensions.restapi.IdString;
+import com.google.gerrit.extensions.restapi.ResourceNotFoundException;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.replication.pull.LocalGitRepositoryManagerProvider;
@@ -23,6 +25,7 @@
import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingLatestPatchSetException;
import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException;
import java.io.IOException;
+import org.eclipse.jgit.errors.RepositoryNotFoundException;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectInserter;
import org.eclipse.jgit.lib.RefUpdate;
@@ -44,7 +47,8 @@
}
public RefUpdateState apply(Project.NameKey name, RefSpec refSpec, RevisionData[] revisionsData)
- throws MissingParentObjectException, IOException, MissingLatestPatchSetException {
+ throws MissingParentObjectException, IOException, ResourceNotFoundException,
+ MissingLatestPatchSetException {
try (Repository git = gitManager.openRepository(name)) {
ObjectId refHead = null;
@@ -97,6 +101,8 @@
RefUpdate.Result result = ru.update();
return new RefUpdateState(refSpec.getSource(), result);
}
+ } catch (RepositoryNotFoundException e) {
+ throw new ResourceNotFoundException(IdString.fromDecoded(name.get()), e);
}
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java
index 1845230..039c395 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/JGitFetch.java
@@ -25,7 +25,10 @@
import org.eclipse.jgit.errors.TransportException;
import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.lib.Repository;
-import org.eclipse.jgit.transport.*;
+import org.eclipse.jgit.transport.FetchResult;
+import org.eclipse.jgit.transport.RefSpec;
+import org.eclipse.jgit.transport.Transport;
+import org.eclipse.jgit.transport.URIish;
public class JGitFetch implements Fetch {
URIish uri;
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 61d62b4..99d8b78 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -591,6 +591,21 @@
By default, replicates without matching, i.e. replicates
everything from all remotes.
+remote.NAME.enableBatchedRefs
+: Choose whether the batch-apply-object endpoint is enabled.
+ If you set this to `true`, then there will be a single call
+ to the batch-apply-object endpoint with all the refs from
+ the batch ref update included. The default behaviour means
+ one call to the apply object(s) endpoint per ref.
+
+ *NOTE*: the default value is only needed for backwards
+ compatibility to allow migrating transparently to the
+ latest pull-replication plugin version. Once the migration is
+ over, this value should be set to `true` to leverage the
+ performance improvements introduced by the `batch-apply-object` API.
+
+ By default, false.
+
Directory `replication`
--------------------
The optional directory `$site_path/etc/replication` contains Git-style
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FakeGitReferenceUpdatedEvent.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FakeGitReferenceUpdatedEvent.java
deleted file mode 100644
index 1472be2..0000000
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FakeGitReferenceUpdatedEvent.java
+++ /dev/null
@@ -1,37 +0,0 @@
-// Copyright (C) 2020 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.replication.pull;
-
-import com.google.common.base.Suppliers;
-import com.google.gerrit.entities.Project;
-import com.google.gerrit.server.data.RefUpdateAttribute;
-import com.google.gerrit.server.events.RefUpdatedEvent;
-
-public class FakeGitReferenceUpdatedEvent extends RefUpdatedEvent {
- FakeGitReferenceUpdatedEvent(
- Project.NameKey project,
- String ref,
- String oldObjectId,
- String newObjectId,
- String instanceId) {
- RefUpdateAttribute upd = new RefUpdateAttribute();
- upd.newRev = newObjectId;
- upd.oldRev = oldObjectId;
- upd.project = project.get();
- upd.refName = ref;
- this.refUpdate = Suppliers.ofInstance(upd);
- this.instanceId = instanceId;
- }
-}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchOneTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchOneTest.java
index dd9f98e..3445604 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchOneTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchOneTest.java
@@ -16,7 +16,13 @@
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.anyList;
+import static org.mockito.Mockito.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
import com.google.gerrit.acceptance.TestMetricMaker;
import com.google.gerrit.entities.Project;
@@ -29,7 +35,11 @@
import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory;
import com.googlesource.gerrit.plugins.replication.pull.fetch.InexistentRefTransportException;
import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState;
-import java.util.*;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
import java.util.concurrent.Callable;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationBatchRefUpdatedAsyncIT.java
similarity index 90%
rename from src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java
rename to src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationBatchRefUpdatedAsyncIT.java
index 0ebd0ae..a72a97e 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationBatchRefUpdatedAsyncIT.java
@@ -29,10 +29,15 @@
sysModule =
"com.googlesource.gerrit.plugins.replication.pull.PullReplicationITAbstract$PullReplicationTestModule",
httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule")
-public class PullReplicationAsyncIT extends PullReplicationITAbstract {
+public class PullReplicationBatchRefUpdatedAsyncIT extends PullReplicationITAbstract {
@Inject private SitePaths sitePaths;
@Override
+ protected boolean useBatchRefUpdateEvent() {
+ return true;
+ }
+
+ @Override
public void setUpTestPlugin() throws Exception {
FileBasedConfig config =
new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationBatchRefUpdatedIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationBatchRefUpdatedIT.java
new file mode 100644
index 0000000..2e727d1
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationBatchRefUpdatedIT.java
@@ -0,0 +1,33 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import com.google.gerrit.acceptance.SkipProjectClone;
+import com.google.gerrit.acceptance.TestPlugin;
+import com.google.gerrit.acceptance.UseLocalDisk;
+
+@SkipProjectClone
+@UseLocalDisk
+@TestPlugin(
+ name = "pull-replication",
+ sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule",
+ httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule")
+public class PullReplicationBatchRefUpdatedIT extends PullReplicationITBase {
+
+ @Override
+ protected boolean useBatchRefUpdateEvent() {
+ return true;
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigBase.java
similarity index 72%
rename from src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigIT.java
rename to src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigBase.java
index 0721dd2..4349cde 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigBase.java
@@ -17,26 +17,16 @@
import static com.google.common.truth.Truth.assertThat;
import static java.util.stream.Collectors.toList;
-import com.google.common.flogger.FluentLogger;
-import com.google.gerrit.acceptance.LightweightPluginDaemonTest;
import com.google.gerrit.acceptance.PushOneCommit.Result;
-import com.google.gerrit.acceptance.SkipProjectClone;
-import com.google.gerrit.acceptance.TestPlugin;
-import com.google.gerrit.acceptance.UseLocalDisk;
import com.google.gerrit.acceptance.config.GerritConfig;
-import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
-import com.google.gerrit.entities.Project;
import com.google.gerrit.extensions.api.projects.BranchInput;
-import com.google.gerrit.server.config.SitePaths;
-import com.google.inject.Inject;
+import com.google.gerrit.server.events.ProjectEvent;
import com.googlesource.gerrit.plugins.replication.AutoReloadConfigDecorator;
+import java.io.IOException;
import java.nio.file.Files;
-import java.nio.file.Path;
-import java.time.Duration;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
-import java.util.function.Supplier;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
@@ -45,33 +35,14 @@
import org.eclipse.jgit.util.FS;
import org.junit.Test;
-@SkipProjectClone
-@UseLocalDisk
-@TestPlugin(
- name = "pull-replication",
- sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule",
- httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule")
-public class PullReplicationFanoutConfigIT extends LightweightPluginDaemonTest {
- private static final Optional<String> ALL_PROJECTS = Optional.empty();
- private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+abstract class PullReplicationFanoutConfigBase extends PullReplicationSetupBase {
private static final int TEST_REPLICATION_DELAY = 60;
- private static final Duration TEST_TIMEOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 2);
- private static final String TEST_REPLICATION_SUFFIX = "suffix1";
- private static final String TEST_REPLICATION_REMOTE = "remote1";
- @Inject private SitePaths sitePaths;
- @Inject private ProjectOperations projectOperations;
- private Path gitPath;
- private FileBasedConfig config;
private FileBasedConfig remoteConfig;
- private FileBasedConfig secureConfig;
@Override
public void setUpTestPlugin() throws Exception {
gitPath = sitePaths.site_path.resolve("git");
-
- config =
- new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED);
remoteConfig =
new FileBasedConfig(
sitePaths
@@ -80,17 +51,8 @@
.toFile(),
FS.DETECTED);
- setReplicationSource(
- TEST_REPLICATION_REMOTE); // Simulates a full replication.config initialization
-
setRemoteConfig(TEST_REPLICATION_SUFFIX, ALL_PROJECTS);
-
- secureConfig =
- new FileBasedConfig(sitePaths.etc_dir.resolve("secure.config").toFile(), FS.DETECTED);
- setReplicationCredentials(TEST_REPLICATION_REMOTE, admin.username(), admin.httpPassword());
- secureConfig.save();
-
- super.setUpTestPlugin();
+ super.setUpTestPlugin(false);
}
@Test
@@ -103,8 +65,8 @@
String sourceRef = pushResult.getPatchSet().refName();
ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class);
- FakeGitReferenceUpdatedEvent event =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent event =
+ generateUpdateEvent(
project,
sourceRef,
ObjectId.zeroId().getName(),
@@ -139,8 +101,8 @@
RevCommit sourceCommit = pushResult.getCommit();
final String sourceRef = pushResult.getPatchSet().refName();
ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class);
- FakeGitReferenceUpdatedEvent event =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent event =
+ generateUpdateEvent(
project,
sourceRef,
ObjectId.zeroId().getName(),
@@ -172,8 +134,8 @@
ReplicationQueue pullReplicationQueue =
plugin.getSysInjector().getInstance(ReplicationQueue.class);
- FakeGitReferenceUpdatedEvent event =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent event =
+ generateUpdateEvent(
project,
newBranch,
ObjectId.zeroId().getName(),
@@ -250,37 +212,31 @@
waitUntil(() -> sources.getAll().size() == 1);
}
- private Ref getRef(Repository repo, String branchName) throws Exception {
- return repo.getRefDatabase().exactRef(branchName);
+ @Override
+ protected void setReplicationSource(
+ String remoteName, List<String> replicaSuffixes, Optional<String> project)
+ throws IOException {
+ setReplicationSource(remoteName);
}
- private Ref checkedGetRef(Repository repo, String branchName) {
- try {
- return repo.getRefDatabase().exactRef(branchName);
- } catch (Exception e) {
- logger.atSevere().withCause(e).log("failed to get ref %s in repo %s", branchName, repo);
- return null;
- }
- }
-
- private void setReplicationSource(String remoteName) throws Exception {
+ private void setReplicationSource(String remoteName) throws IOException {
config.setBoolean("gerrit", null, "autoReload", true);
config.save();
}
- private void setRemoteConfig(String replicaSuffix, Optional<String> project) throws Exception {
+ private void setRemoteConfig(String replicaSuffix, Optional<String> project) throws IOException {
setRemoteConfig(remoteConfig, replicaSuffix, project);
}
private void setRemoteConfig(
FileBasedConfig remoteConfig, String replicaSuffix, Optional<String> project)
- throws Exception {
+ throws IOException {
setRemoteConfig(remoteConfig, Arrays.asList(replicaSuffix), project);
}
private void setRemoteConfig(
FileBasedConfig remoteConfig, List<String> replicaSuffixes, Optional<String> project)
- throws Exception {
+ throws IOException {
List<String> replicaUrls =
replicaSuffixes.stream()
.map(suffix -> gitPath.resolve("${name}" + suffix + ".git").toString())
@@ -293,23 +249,4 @@
project.ifPresent(prj -> remoteConfig.setString("remote", null, "projects", prj));
remoteConfig.save();
}
-
- private void setReplicationCredentials(String remoteName, String username, String password)
- throws Exception {
- secureConfig.setString("remote", remoteName, "username", username);
- secureConfig.setString("remote", remoteName, "password", password);
- secureConfig.save();
- }
-
- private void waitUntil(Supplier<Boolean> waitCondition) throws Exception {
- WaitUtil.waitUntil(waitCondition, TEST_TIMEOUT);
- }
-
- private <T> T getInstance(Class<T> classObj) {
- return plugin.getSysInjector().getInstance(classObj);
- }
-
- private Project.NameKey createTestProject(String name) throws Exception {
- return projectOperations.newProject().name(name).create();
- }
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigBatchRefUpdateEventIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigBatchRefUpdateEventIT.java
new file mode 100644
index 0000000..7babf34
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigBatchRefUpdateEventIT.java
@@ -0,0 +1,33 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import com.google.gerrit.acceptance.SkipProjectClone;
+import com.google.gerrit.acceptance.TestPlugin;
+import com.google.gerrit.acceptance.UseLocalDisk;
+
+@SkipProjectClone
+@UseLocalDisk
+@TestPlugin(
+ name = "pull-replication",
+ sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule",
+ httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule")
+public class PullReplicationFanoutConfigBatchRefUpdateEventIT
+ extends PullReplicationFanoutConfigBase {
+
+ @Override
+ protected boolean useBatchRefUpdateEvent() {
+ return true;
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigRefUpdatedEventIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigRefUpdatedEventIT.java
new file mode 100644
index 0000000..16b0b02
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigRefUpdatedEventIT.java
@@ -0,0 +1,32 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import com.google.gerrit.acceptance.SkipProjectClone;
+import com.google.gerrit.acceptance.TestPlugin;
+import com.google.gerrit.acceptance.UseLocalDisk;
+
+@SkipProjectClone
+@UseLocalDisk
+@TestPlugin(
+ name = "pull-replication",
+ sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule",
+ httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule")
+public class PullReplicationFanoutConfigRefUpdatedEventIT extends PullReplicationFanoutConfigBase {
+
+ @Override
+ protected boolean useBatchRefUpdateEvent() {
+ return false;
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationITAbstract.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationITAbstract.java
index a99f759..130bb41 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationITAbstract.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationITAbstract.java
@@ -37,6 +37,7 @@
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.events.Event;
import com.google.gerrit.server.events.EventListener;
+import com.google.gerrit.server.events.ProjectEvent;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.googlesource.gerrit.plugins.replication.AutoReloadConfigDecorator;
@@ -165,8 +166,8 @@
String sourceRef = pushResult.getPatchSet().refName();
ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class);
- FakeGitReferenceUpdatedEvent event =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent event =
+ generateUpdateEvent(
project,
sourceRef,
ObjectId.zeroId().getName(),
@@ -213,8 +214,8 @@
ReplicationQueue pullReplicationQueue =
plugin.getSysInjector().getInstance(ReplicationQueue.class);
- FakeGitReferenceUpdatedEvent event =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent event =
+ generateUpdateEvent(
project,
newBranch,
ObjectId.zeroId().getName(),
@@ -245,8 +246,8 @@
ReplicationQueue pullReplicationQueue =
plugin.getSysInjector().getInstance(ReplicationQueue.class);
- FakeGitReferenceUpdatedEvent event =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent event =
+ generateUpdateEvent(
project,
newBranch,
ObjectId.zeroId().getName(),
@@ -292,8 +293,8 @@
ReplicationQueue pullReplicationQueue =
plugin.getSysInjector().getInstance(ReplicationQueue.class);
- FakeGitReferenceUpdatedEvent event =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent event =
+ generateUpdateEvent(
project,
newBranch,
ObjectId.zeroId().getName(),
@@ -322,8 +323,8 @@
assertThat(pushedRefs).hasSize(1);
assertThat(pushedRefs.iterator().next().getStatus()).isEqualTo(Status.OK);
- FakeGitReferenceUpdatedEvent forcedPushEvent =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent forcedPushEvent =
+ generateUpdateEvent(
project,
newBranch,
branchRevision,
@@ -367,8 +368,8 @@
String sourceRef = pushResult.getPatchSet().refName();
ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class);
- FakeGitReferenceUpdatedEvent event =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent event =
+ generateUpdateEvent(
project,
sourceRef,
ObjectId.zeroId().getName(),
@@ -414,8 +415,8 @@
ReplicationQueue pullReplicationQueue =
plugin.getSysInjector().getInstance(ReplicationQueue.class);
- FakeGitReferenceUpdatedEvent event =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent event =
+ generateUpdateEvent(
project,
newBranch,
ObjectId.zeroId().getName(),
@@ -524,8 +525,8 @@
String sourceRef = pushResult.getPatchSet().refName();
ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class);
- FakeGitReferenceUpdatedEvent event =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent event =
+ generateUpdateEvent(
project,
sourceRef,
ObjectId.zeroId().getName(),
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationITBase.java
similarity index 93%
rename from src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java
rename to src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationITBase.java
index 5717a8b..0754870 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationITBase.java
@@ -21,8 +21,6 @@
import static com.google.gerrit.server.group.SystemGroupBackend.REGISTERED_USERS;
import com.google.gerrit.acceptance.PushOneCommit.Result;
-import com.google.gerrit.acceptance.SkipProjectClone;
-import com.google.gerrit.acceptance.TestPlugin;
import com.google.gerrit.acceptance.UseLocalDisk;
import com.google.gerrit.acceptance.config.GerritConfig;
import com.google.gerrit.entities.Permission;
@@ -34,6 +32,7 @@
import com.google.gerrit.extensions.events.HeadUpdatedListener;
import com.google.gerrit.extensions.events.ProjectDeletedListener;
import com.google.gerrit.extensions.restapi.RestApiException;
+import com.google.gerrit.server.events.ProjectEvent;
import com.googlesource.gerrit.plugins.replication.AutoReloadConfigDecorator;
import com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient;
import java.io.IOException;
@@ -54,14 +53,7 @@
import org.junit.Ignore;
import org.junit.Test;
-@SkipProjectClone
-@UseLocalDisk
-@TestPlugin(
- name = "pull-replication",
- sysModule =
- "com.googlesource.gerrit.plugins.replication.pull.PullReplicationITAbstract$PullReplicationTestModule",
- httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule")
-public class PullReplicationIT extends PullReplicationSetupBase {
+abstract class PullReplicationITBase extends PullReplicationSetupBase {
@Override
protected void setReplicationSource(
@@ -94,8 +86,8 @@
String sourceRef = pushResult.getPatchSet().refName();
ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class);
- FakeGitReferenceUpdatedEvent event =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent event =
+ generateUpdateEvent(
project,
sourceRef,
ObjectId.zeroId().getName(),
@@ -127,8 +119,8 @@
ReplicationQueue pullReplicationQueue =
plugin.getSysInjector().getInstance(ReplicationQueue.class);
- FakeGitReferenceUpdatedEvent event =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent event =
+ generateUpdateEvent(
project,
newBranch,
ObjectId.zeroId().getName(),
@@ -170,8 +162,8 @@
ReplicationQueue pullReplicationQueue =
plugin.getSysInjector().getInstance(ReplicationQueue.class);
- FakeGitReferenceUpdatedEvent event =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent event =
+ generateUpdateEvent(
project,
newBranch,
ObjectId.zeroId().getName(),
@@ -196,8 +188,8 @@
assertThat(pushedRefs).hasSize(1);
assertThat(pushedRefs.iterator().next().getStatus()).isEqualTo(Status.OK);
- FakeGitReferenceUpdatedEvent forcedPushEvent =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent forcedPushEvent =
+ generateUpdateEvent(
project,
newBranch,
branchRevision,
@@ -235,8 +227,8 @@
String sourceRef = pushResult.getPatchSet().refName();
ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class);
- FakeGitReferenceUpdatedEvent event =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent event =
+ generateUpdateEvent(
project,
sourceRef,
ObjectId.zeroId().getName(),
@@ -276,8 +268,8 @@
ReplicationQueue pullReplicationQueue =
plugin.getSysInjector().getInstance(ReplicationQueue.class);
- FakeGitReferenceUpdatedEvent event =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent event =
+ generateUpdateEvent(
project,
newBranch,
ObjectId.zeroId().getName(),
@@ -390,8 +382,8 @@
String sourceRef = pushResult.getPatchSet().refName();
ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class);
- FakeGitReferenceUpdatedEvent event =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent event =
+ generateUpdateEvent(
project,
sourceRef,
ObjectId.zeroId().getName(),
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationRefUpdatedAsyncIT.java
similarity index 90%
copy from src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java
copy to src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationRefUpdatedAsyncIT.java
index 0ebd0ae..fba8783 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationRefUpdatedAsyncIT.java
@@ -29,10 +29,15 @@
sysModule =
"com.googlesource.gerrit.plugins.replication.pull.PullReplicationITAbstract$PullReplicationTestModule",
httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule")
-public class PullReplicationAsyncIT extends PullReplicationITAbstract {
+public class PullReplicationRefUpdatedAsyncIT extends PullReplicationITAbstract {
@Inject private SitePaths sitePaths;
@Override
+ protected boolean useBatchRefUpdateEvent() {
+ return false;
+ }
+
+ @Override
public void setUpTestPlugin() throws Exception {
FileBasedConfig config =
new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationRefUpdatedIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationRefUpdatedIT.java
new file mode 100644
index 0000000..6e7c369
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationRefUpdatedIT.java
@@ -0,0 +1,33 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import com.google.gerrit.acceptance.SkipProjectClone;
+import com.google.gerrit.acceptance.TestPlugin;
+import com.google.gerrit.acceptance.UseLocalDisk;
+
+@SkipProjectClone
+@UseLocalDisk
+@TestPlugin(
+ name = "pull-replication",
+ sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule",
+ httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule")
+public class PullReplicationRefUpdatedIT extends PullReplicationITBase {
+
+ @Override
+ protected boolean useBatchRefUpdateEvent() {
+ return false;
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationSetupBase.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationSetupBase.java
index fea576a..9cb2dc9 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationSetupBase.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationSetupBase.java
@@ -16,13 +16,20 @@
import static java.util.stream.Collectors.toList;
+import com.google.common.base.Suppliers;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.acceptance.LightweightPluginDaemonTest;
import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
+import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.Project.NameKey;
import com.google.gerrit.extensions.events.ProjectDeletedListener;
import com.google.gerrit.extensions.registration.DynamicSet;
import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.data.AccountAttribute;
+import com.google.gerrit.server.data.RefUpdateAttribute;
+import com.google.gerrit.server.events.BatchRefUpdateEvent;
+import com.google.gerrit.server.events.ProjectEvent;
+import com.google.gerrit.server.events.RefUpdatedEvent;
import com.google.inject.Inject;
import java.io.File;
import java.io.IOException;
@@ -53,6 +60,22 @@
protected FileBasedConfig config;
protected FileBasedConfig secureConfig;
+ protected abstract boolean useBatchRefUpdateEvent();
+
+ protected ProjectEvent generateUpdateEvent(
+ Project.NameKey project,
+ String ref,
+ String oldObjectId,
+ String newObjectId,
+ String instanceId) {
+
+ if (useBatchRefUpdateEvent()) {
+ return generateBatchRefUpdateEvent(project, ref, oldObjectId, newObjectId, instanceId);
+ }
+
+ return generateRefUpdateEvent(project, ref, oldObjectId, newObjectId, instanceId);
+ }
+
protected void setUpTestPlugin(boolean loadExisting) throws Exception {
gitPath = sitePaths.site_path.resolve("git");
@@ -72,6 +95,9 @@
setReplicationCredentials(TEST_REPLICATION_REMOTE, admin.username(), admin.httpPassword());
secureConfig.save();
+ cfg.setBoolean(
+ "event", "stream-events", "enableBatchRefUpdatedEvents", useBatchRefUpdateEvent());
+
super.setUpTestPlugin();
}
@@ -119,4 +145,38 @@
List<String> replicaSuffixes, Function<String, String> toURL) {
return replicaSuffixes.stream().map(suffix -> toURL.apply(suffix)).collect(toList());
}
+
+ private BatchRefUpdateEvent generateBatchRefUpdateEvent(
+ Project.NameKey project,
+ String ref,
+ String oldObjectId,
+ String newObjectId,
+ String instanceId) {
+ RefUpdateAttribute upd = new RefUpdateAttribute();
+ upd.newRev = newObjectId;
+ upd.oldRev = oldObjectId;
+ upd.project = project.get();
+ upd.refName = ref;
+ BatchRefUpdateEvent event =
+ new BatchRefUpdateEvent(
+ project,
+ Suppliers.ofInstance(List.of(upd)),
+ Suppliers.ofInstance(new AccountAttribute(admin.id().get())));
+ event.instanceId = instanceId;
+ return event;
+ }
+
+ private ProjectEvent generateRefUpdateEvent(
+ NameKey project, String ref, String oldObjectId, String newObjectId, String instanceId) {
+ RefUpdateAttribute upd = new RefUpdateAttribute();
+ upd.newRev = newObjectId;
+ upd.oldRev = oldObjectId;
+ upd.project = project.get();
+ upd.refName = ref;
+ RefUpdatedEvent event = new RefUpdatedEvent();
+ event.refUpdate = Suppliers.ofInstance(upd);
+ event.submitter = Suppliers.ofInstance(new AccountAttribute(admin.id().get()));
+ event.instanceId = instanceId;
+ return event;
+ }
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolBase.java
similarity index 94%
rename from src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolIT.java
rename to src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolBase.java
index e55e383..2e95ef1 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolBase.java
@@ -21,6 +21,7 @@
import com.google.gerrit.acceptance.TestPlugin;
import com.google.gerrit.acceptance.UseLocalDisk;
import com.google.gerrit.acceptance.config.GerritConfig;
+import com.google.gerrit.server.events.ProjectEvent;
import java.io.IOException;
import java.util.List;
import java.util.Optional;
@@ -36,7 +37,8 @@
name = "pull-replication",
sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule",
httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule")
-public class PullReplicationWithGitHttpTransportProtocolIT extends PullReplicationSetupBase {
+public abstract class PullReplicationWithGitHttpTransportProtocolBase
+ extends PullReplicationSetupBase {
@Override
protected void setReplicationSource(
@@ -80,8 +82,8 @@
String sourceRef = pushResult.getPatchSet().refName();
ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class);
- FakeGitReferenceUpdatedEvent event =
- new FakeGitReferenceUpdatedEvent(
+ ProjectEvent event =
+ generateUpdateEvent(
project,
sourceRef,
ObjectId.zeroId().getName(),
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolBatchRefUpdatedIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolBatchRefUpdatedIT.java
new file mode 100644
index 0000000..8c8ca37
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolBatchRefUpdatedIT.java
@@ -0,0 +1,34 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import com.google.gerrit.acceptance.SkipProjectClone;
+import com.google.gerrit.acceptance.TestPlugin;
+import com.google.gerrit.acceptance.UseLocalDisk;
+
+@SkipProjectClone
+@UseLocalDisk
+@TestPlugin(
+ name = "pull-replication",
+ sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule",
+ httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule")
+public class PullReplicationWithGitHttpTransportProtocolBatchRefUpdatedIT
+ extends PullReplicationWithGitHttpTransportProtocolBase {
+
+ @Override
+ protected boolean useBatchRefUpdateEvent() {
+ return true;
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolRefUpdatedIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolRefUpdatedIT.java
new file mode 100644
index 0000000..5f3c7b6
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationWithGitHttpTransportProtocolRefUpdatedIT.java
@@ -0,0 +1,34 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import com.google.gerrit.acceptance.SkipProjectClone;
+import com.google.gerrit.acceptance.TestPlugin;
+import com.google.gerrit.acceptance.UseLocalDisk;
+
+@SkipProjectClone
+@UseLocalDisk
+@TestPlugin(
+ name = "pull-replication",
+ sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule",
+ httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule")
+public class PullReplicationWithGitHttpTransportProtocolRefUpdatedIT
+ extends PullReplicationWithGitHttpTransportProtocolBase {
+
+ @Override
+ protected boolean useBatchRefUpdateEvent() {
+ return false;
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
index 3d4d531..e3eb65b 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
@@ -26,7 +26,7 @@
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
import com.google.common.base.Suppliers;
@@ -35,12 +35,13 @@
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.api.changes.NotifyHandling;
-import com.google.gerrit.extensions.common.AccountInfo;
import com.google.gerrit.extensions.events.ProjectDeletedListener;
import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.metrics.DisabledMetricMaker;
import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.data.AccountAttribute;
import com.google.gerrit.server.data.RefUpdateAttribute;
+import com.google.gerrit.server.events.BatchRefUpdateEvent;
import com.google.gerrit.server.events.Event;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.events.RefUpdatedEvent;
@@ -48,6 +49,7 @@
import com.google.inject.Provider;
import com.googlesource.gerrit.plugins.replication.ReplicationConfig;
import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig;
+import com.googlesource.gerrit.plugins.replication.pull.api.data.BatchApplyObjectData;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData;
import com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient;
import com.googlesource.gerrit.plugins.replication.pull.client.FetchRestApiClient;
@@ -55,10 +57,13 @@
import com.googlesource.gerrit.plugins.replication.pull.filter.ApplyObjectsRefsFilter;
import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter;
import java.io.IOException;
+import java.net.URISyntaxException;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
+import java.util.stream.Collectors;
+import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.transport.URIish;
@@ -78,6 +83,10 @@
private static final String FOREIGN_INSTANCE_ID = "any other instance id";
private static final String TEST_REF_NAME = "refs/meta/heads/anyref";
+ private static final Project.NameKey PROJECT = Project.nameKey("defaultProject");
+ private static final String NEW_OBJECT_ID =
+ ObjectId.fromString("3c1ddc050d7906adb0e29bc3bc46af8749b2f63b").getName();
+
@Mock private WorkQueue wq;
@Mock private Source source;
@Mock private SourcesCollection sourceCollection;
@@ -86,23 +95,27 @@
@Mock ReplicationStateListeners sl;
@Mock FetchRestApiClient fetchRestApiClient;
@Mock FetchApiClient.Factory fetchClientFactory;
- @Mock AccountInfo accountInfo;
+ @Mock AccountAttribute accountAttribute;
@Mock RevisionReader revReader;
@Mock RevisionData revisionData;
@Mock HttpResult successfulHttpResult;
@Mock HttpResult fetchHttpResult;
+ @Mock HttpResult batchFetchHttpResult;
@Mock RevisionData revisionDataWithParents;
List<ObjectId> revisionDataParentObjectIds;
@Mock HttpResult httpResult;
+ @Mock HttpResult batchHttpResult;
@Mock ApplyObjectsRefsFilter applyObjectsRefsFilter;
+
+ @Mock Config config;
ApplyObjectMetrics applyObjectMetrics;
- FetchReplicationMetrics fetchMetrics;
ReplicationQueueMetrics queueMetrics;
ShutdownState shutdownState;
@Captor ArgumentCaptor<String> stringCaptor;
@Captor ArgumentCaptor<Project.NameKey> projectNameKeyCaptor;
@Captor ArgumentCaptor<List<RevisionData>> revisionsDataCaptor;
+ @Captor ArgumentCaptor<List<BatchApplyObjectData>> batchRefsCaptor;
private ExcludedRefsFilter refsFilter;
private ReplicationQueue objectUnderTest;
@@ -121,8 +134,11 @@
when(source.wouldFetchRef(anyString())).thenReturn(true);
ImmutableList<String> apis = ImmutableList.of("http://localhost:18080");
when(source.getApis()).thenReturn(apis);
+ when(source.enableBatchedRefs()).thenReturn(true);
when(sourceCollection.getAll()).thenReturn(Lists.newArrayList(source));
when(rd.get()).thenReturn(sourceCollection);
+ when(config.getBoolean("event", "stream-events", "enableBatchRefUpdatedEvents", false))
+ .thenReturn(true);
lenient()
.when(revReader.read(any(), any(), anyString(), eq(0)))
.thenReturn(Optional.of(revisionData));
@@ -148,16 +164,24 @@
lenient()
.when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any()))
.thenReturn(httpResult);
- when(fetchRestApiClient.callFetch(any(), anyString(), any())).thenReturn(fetchHttpResult);
+ lenient()
+ .when(fetchRestApiClient.callBatchSendObject(any(), any(), anyLong(), any()))
+ .thenReturn(batchHttpResult);
+ when(fetchRestApiClient.callFetch(any(), anyString(), any(), anyLong(), anyBoolean()))
+ .thenReturn(fetchHttpResult);
+ when(fetchRestApiClient.callBatchFetch(any(), any(), any())).thenReturn(batchFetchHttpResult);
when(fetchRestApiClient.initProject(any(), any(), anyLong(), any()))
.thenReturn(successfulHttpResult);
when(successfulHttpResult.isSuccessful()).thenReturn(true);
when(httpResult.isSuccessful()).thenReturn(true);
+ lenient().when(batchHttpResult.isSuccessful()).thenReturn(true);
+ lenient().when(fetchHttpResult.isSuccessful()).thenReturn(true);
+ when(batchFetchHttpResult.isSuccessful()).thenReturn(true);
when(httpResult.isProjectMissing(any())).thenReturn(false);
+ when(batchHttpResult.isProjectMissing(any())).thenReturn(false);
when(applyObjectsRefsFilter.match(any())).thenReturn(false);
applyObjectMetrics = new ApplyObjectMetrics("pull-replication", new DisabledMetricMaker());
- fetchMetrics = new FetchReplicationMetrics("pull-replication", new DisabledMetricMaker());
queueMetrics = new ReplicationQueueMetrics("pull-replication", new DisabledMetricMaker());
shutdownState = new ShutdownState();
@@ -171,38 +195,65 @@
refsFilter,
() -> revReader,
applyObjectMetrics,
- fetchMetrics,
queueMetrics,
LOCAL_INSTANCE_ID,
+ config,
applyObjectsRefsFilter,
shutdownState);
}
@Test
- public void shouldCallSendObjectWhenMetaRef() throws Exception {
+ public void shouldCallBatchSendObjectWhenMetaRef() throws Exception {
+ Event event = generateBatchRefUpdateEvent("refs/changes/01/1/meta");
+ objectUnderTest.start();
+ objectUnderTest.onEvent(event);
+
+ verify(fetchRestApiClient).callBatchSendObject(any(), any(), anyLong(), any());
+ }
+
+ @Test
+ public void shouldCallSendObjectWhenMetaRefAndRefUpdateEvent() throws Exception {
+ when(config.getBoolean("event", "stream-events", "enableBatchRefUpdatedEvents", false))
+ .thenReturn(false);
+
+ objectUnderTest =
+ new ReplicationQueue(
+ wq,
+ rd,
+ dis,
+ sl,
+ fetchClientFactory,
+ refsFilter,
+ () -> revReader,
+ applyObjectMetrics,
+ queueMetrics,
+ LOCAL_INSTANCE_ID,
+ config,
+ applyObjectsRefsFilter,
+ shutdownState);
+
Event event = new TestEvent("refs/changes/01/1/meta");
objectUnderTest.start();
objectUnderTest.onEvent(event);
- verify(fetchRestApiClient).callSendObjects(any(), anyString(), anyLong(), any(), any());
+ verify(fetchRestApiClient).callBatchSendObject(any(), any(), anyLong(), any());
}
@Test
public void shouldIgnoreEventWhenIsNotLocalInstanceId() throws Exception {
- Event event = new TestEvent();
+ Event event = generateBatchRefUpdateEvent(TEST_REF_NAME);
event.instanceId = FOREIGN_INSTANCE_ID;
objectUnderTest.start();
objectUnderTest.onEvent(event);
- verify(fetchRestApiClient, never())
- .callSendObjects(any(), anyString(), anyLong(), any(), any());
+ verify(fetchRestApiClient, never()).callBatchSendObject(any(), any(), anyLong(), any());
}
@Test
public void shouldCallInitProjectWhenProjectIsMissing() throws Exception {
- Event event = new TestEvent("refs/changes/01/1/meta");
- when(httpResult.isSuccessful()).thenReturn(false);
- when(httpResult.isProjectMissing(any())).thenReturn(true);
+ Event event = generateBatchRefUpdateEvent("refs/changes/01/1/meta");
+ when(batchHttpResult.isSuccessful()).thenReturn(false);
+ when(batchHttpResult.isProjectMissing(any())).thenReturn(true);
when(source.isCreateMissingRepositories()).thenReturn(true);
objectUnderTest.start();
@@ -212,11 +263,24 @@
}
@Test
+ public void shouldCallSendObjectReorderingRefsHavingMetaAtTheEnd() throws Exception {
+ Event event = generateBatchRefUpdateEvent("refs/changes/01/1/meta", "refs/changes/01/1/1");
+ objectUnderTest.start();
+ objectUnderTest.onEvent(event);
+ verifySendObjectOrdering("refs/changes/01/1/1", "refs/changes/01/1/meta");
+ }
+
+ @Test
+ public void shouldCallSendObjectKeepingMetaAtTheEnd() throws Exception {
+ Event event = generateBatchRefUpdateEvent("refs/changes/01/1/1", "refs/changes/01/1/meta");
+ objectUnderTest.start();
+ objectUnderTest.onEvent(event);
+ verifySendObjectOrdering("refs/changes/01/1/1", "refs/changes/01/1/meta");
+ }
+
+ @Test
public void shouldNotCallInitProjectWhenReplicateNewRepositoriesNotSet() throws Exception {
Event event = new TestEvent("refs/changes/01/1/meta");
- when(httpResult.isSuccessful()).thenReturn(false);
- when(httpResult.isProjectMissing(any())).thenReturn(true);
- when(source.isCreateMissingRepositories()).thenReturn(false);
objectUnderTest.start();
objectUnderTest.onEvent(event);
@@ -227,10 +291,12 @@
@Test
public void shouldNotCallInitProjectWhenProjectWithoutConfiguration() throws Exception {
Event event = new TestEvent("refs/changes/01/1/meta");
- when(httpResult.isSuccessful()).thenReturn(false);
- when(httpResult.isProjectMissing(any())).thenReturn(true);
- when(source.isCreateMissingRepositories()).thenReturn(true);
- when(revReader.read(any(), any(), eq(RefNames.REFS_CONFIG), anyInt()))
+ lenient().when(httpResult.isSuccessful()).thenReturn(false);
+
+ lenient().when(httpResult.isProjectMissing(any())).thenReturn(true);
+ lenient().when(source.isCreateMissingRepositories()).thenReturn(true);
+ lenient()
+ .when(revReader.read(any(), any(), eq(RefNames.REFS_CONFIG), anyInt()))
.thenReturn(Optional.empty());
objectUnderTest.start();
@@ -240,105 +306,199 @@
}
@Test
- public void shouldCallSendObjectWhenPatchSetRef() throws Exception {
- Event event = new TestEvent("refs/changes/01/1/1");
+ public void shouldCallBatchSendObjectWhenPatchSetRefAndRefUpdateEvent() throws Exception {
+ when(config.getBoolean("event", "stream-events", "enableBatchRefUpdatedEvents", false))
+ .thenReturn(false);
+
+ objectUnderTest =
+ new ReplicationQueue(
+ wq,
+ rd,
+ dis,
+ sl,
+ fetchClientFactory,
+ refsFilter,
+ () -> revReader,
+ applyObjectMetrics,
+ queueMetrics,
+ LOCAL_INSTANCE_ID,
+ config,
+ applyObjectsRefsFilter,
+ shutdownState);
+ }
+
+ @Test
+ public void shouldCallBatchSendObjectWhenPatchSetRef() throws Exception {
+ Event event = generateBatchRefUpdateEvent("refs/changes/01/1/1");
objectUnderTest.start();
objectUnderTest.onEvent(event);
- verify(fetchRestApiClient).callSendObjects(any(), anyString(), anyLong(), any(), any());
+ verify(fetchRestApiClient).callBatchSendObject(any(), any(), anyLong(), any());
}
@Test
public void shouldFallbackToCallFetchWhenIOException() throws Exception {
- RefUpdatedEvent event = new TestEvent("refs/changes/01/1/meta");
+ BatchRefUpdateEvent event = generateBatchRefUpdateEvent("refs/changes/01/1/meta");
+
objectUnderTest.start();
when(revReader.read(any(), any(), anyString(), anyInt())).thenThrow(IOException.class);
objectUnderTest.onEvent(event);
- verifyFallbackToRestApiClientFetchAsync(event);
+ verify(fetchRestApiClient).callBatchFetch(any(), any(), any());
}
@Test
- public void shouldFallbackToCallFetchWhenLargeRef() throws Exception {
- RefUpdatedEvent event = new TestEvent("refs/changes/01/1/1");
+ public void shouldFallbackToCallBatchFetchWhenLargeRef() throws Exception {
+ Event event = generateBatchRefUpdateEvent("refs/changes/01/1/1");
+
objectUnderTest.start();
when(revReader.read(any(), any(), anyString(), anyInt())).thenReturn(Optional.empty());
objectUnderTest.onEvent(event);
- verifyFallbackToRestApiClientFetchAsync(event);
+ verify(fetchRestApiClient).callBatchFetch(any(), any(), any());
}
@Test
- public void shouldFallbackToCallFetchWhenParentObjectIsMissing() throws Exception {
- RefUpdatedEvent event = new TestEvent("refs/changes/01/1/1");
+ public void
+ shouldFallbackToCallBatchFetchWhenParentObjectIsMissingAndRefDoesntMatchApplyObjectsRefsFilter()
+ throws Exception {
+ Event event = generateBatchRefUpdateEvent("refs/changes/01/1/1");
objectUnderTest.start();
- when(httpResult.isSuccessful()).thenReturn(false);
- when(httpResult.isParentObjectMissing()).thenReturn(true);
- when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any()))
- .thenReturn(httpResult);
+ when(batchHttpResult.isSuccessful()).thenReturn(false);
+ when(batchHttpResult.isParentObjectMissing()).thenReturn(true);
objectUnderTest.onEvent(event);
- verifyFallbackToRestApiClientFetchAsync(event);
+ verify(fetchRestApiClient).callBatchFetch(any(), any(), any());
+ }
+
+ @Test
+ public void
+ shouldFallbackToApplyObjectsForEachRefWhenParentObjectIsMissingAndRefMatchesApplyObjectsRefFilter()
+ throws Exception {
+ Event event = generateBatchRefUpdateEvent("refs/changes/01/1/1", "refs/changes/02/1/1");
+ objectUnderTest.start();
+
+ when(batchHttpResult.isSuccessful()).thenReturn(false);
+ when(batchHttpResult.isParentObjectMissing()).thenReturn(true);
+ when(applyObjectsRefsFilter.match(any())).thenReturn(true);
+
+ objectUnderTest.onEvent(event);
+
+ verify(fetchRestApiClient, times(2))
+ .callSendObjects(any(), anyString(), anyLong(), any(), any());
+ verify(fetchRestApiClient, never()).callFetch(any(), anyString(), any());
+ }
+
+ @Test
+ public void shouldFallbackToCallBatchFetchWhenParentObjectNotMissingButApplyObjectFails()
+ throws Exception {
+ BatchRefUpdateEvent event = generateBatchRefUpdateEvent("refs/changes/01/1/1");
+ objectUnderTest.start();
+
+ when(batchHttpResult.isSuccessful()).thenReturn(false);
+ when(batchHttpResult.isParentObjectMissing()).thenReturn(false);
+ lenient().when(httpResult.isSuccessful()).thenReturn(false);
+ lenient().when(httpResult.isParentObjectMissing()).thenReturn(false);
+
+ objectUnderTest.onEvent(event);
+
+ verify(fetchRestApiClient).callBatchFetch(any(), any(), any());
}
@Test
public void shouldFallbackToApplyAllParentObjectsWhenParentObjectIsMissingOnMetaRef()
throws Exception {
- Event event = new TestEvent("refs/changes/01/1/meta");
+ Event event = generateBatchRefUpdateEvent("refs/changes/01/1/meta");
objectUnderTest.start();
- when(httpResult.isSuccessful()).thenReturn(false, true);
- when(httpResult.isParentObjectMissing()).thenReturn(true, false);
- when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any()))
- .thenReturn(httpResult);
+ when(batchHttpResult.isSuccessful()).thenReturn(false);
+ when(batchHttpResult.isParentObjectMissing()).thenReturn(true);
objectUnderTest.onEvent(event);
- verify(fetchRestApiClient, times(2))
+ verify(fetchRestApiClient, times(1))
.callSendObjects(any(), anyString(), anyLong(), revisionsDataCaptor.capture(), any());
List<List<RevisionData>> revisionsDataValues = revisionsDataCaptor.getAllValues();
- assertThat(revisionsDataValues).hasSize(2);
+ assertThat(revisionsDataValues).hasSize(1);
List<RevisionData> firstRevisionsValues = revisionsDataValues.get(0);
- assertThat(firstRevisionsValues).hasSize(1);
+ assertThat(firstRevisionsValues).hasSize(1 + revisionDataParentObjectIds.size());
assertThat(firstRevisionsValues).contains(revisionData);
-
- List<RevisionData> secondRevisionsValues = revisionsDataValues.get(1);
- assertThat(secondRevisionsValues).hasSize(1 + revisionDataParentObjectIds.size());
}
@Test
public void shouldFallbackToApplyAllParentObjectsWhenParentObjectIsMissingOnAllowedRefs()
throws Exception {
String refName = "refs/tags/test-tag";
- Event event = new TestEvent(refName);
+ Event event = generateBatchRefUpdateEvent(refName);
objectUnderTest.start();
- when(httpResult.isSuccessful()).thenReturn(false, true);
- when(httpResult.isParentObjectMissing()).thenReturn(true, false);
- when(fetchRestApiClient.callSendObjects(any(), anyString(), anyLong(), any(), any()))
- .thenReturn(httpResult);
+ when(batchHttpResult.isSuccessful()).thenReturn(false);
+ when(batchHttpResult.isParentObjectMissing()).thenReturn(true);
when(applyObjectsRefsFilter.match(refName)).thenReturn(true);
objectUnderTest.onEvent(event);
- verify(fetchRestApiClient, times(2))
+ verify(fetchRestApiClient, times(1))
.callSendObjects(any(), anyString(), anyLong(), revisionsDataCaptor.capture(), any());
List<List<RevisionData>> revisionsDataValues = revisionsDataCaptor.getAllValues();
- assertThat(revisionsDataValues).hasSize(2);
+ assertThat(revisionsDataValues).hasSize(1);
List<RevisionData> firstRevisionsValues = revisionsDataValues.get(0);
- assertThat(firstRevisionsValues).hasSize(1);
+ assertThat(firstRevisionsValues).hasSize(1 + revisionDataParentObjectIds.size());
assertThat(firstRevisionsValues).contains(revisionData);
+ }
- List<RevisionData> secondRevisionsValues = revisionsDataValues.get(1);
- assertThat(secondRevisionsValues).hasSize(1 + revisionDataParentObjectIds.size());
+ @Test
+ public void shouldCallSendObjectsIfBatchedRefsNotEnabledAtSource() throws Exception {
+ Event event = generateBatchRefUpdateEvent("refs/changes/01/1/1");
+ when(source.enableBatchedRefs()).thenReturn(false);
+ objectUnderTest.start();
+ objectUnderTest.onEvent(event);
+
+ verify(fetchRestApiClient, never()).callBatchSendObject(any(), any(), anyLong(), any());
+ verify(fetchRestApiClient).callSendObjects(any(), anyString(), anyLong(), any(), any());
+ }
+
+ @Test
+ public void shouldCallFetchIfBatchedRefsNotEnabledAtSource() throws Exception {
+ BatchRefUpdateEvent event = generateBatchRefUpdateEvent("refs/changes/01/1/1");
+ when(source.enableBatchedRefs()).thenReturn(false);
+ when(httpResult.isSuccessful()).thenReturn(false);
+ when(httpResult.isParentObjectMissing()).thenReturn(false);
+
+ objectUnderTest.start();
+ objectUnderTest.onEvent(event);
+
+ verify(fetchRestApiClient, never()).callBatchFetch(any(), any(), any());
+ verifyFallbackToRestApiClientFetchAsync(event);
+ }
+
+ @Test
+ public void shouldCallBatchFetchForAllTheRefsInTheBatchIfApplyObjectFails()
+ throws Exception, URISyntaxException {
+ Event event = generateBatchRefUpdateEvent("refs/changes/01/1/1", "refs/changes/02/1/1");
+ when(batchHttpResult.isSuccessful()).thenReturn(false);
+ when(batchHttpResult.isParentObjectMissing()).thenReturn(true);
+ when(applyObjectsRefsFilter.match(any())).thenReturn(true, true);
+ when(httpResult.isSuccessful()).thenReturn(true, false);
+
+ objectUnderTest.start();
+ objectUnderTest.onEvent(event);
+
+ verify(fetchRestApiClient, times(2))
+ .callSendObjects(any(), anyString(), anyLong(), any(), any());
+ verify(fetchRestApiClient)
+ .callBatchFetch(
+ PROJECT,
+ List.of("refs/changes/01/1/1", "refs/changes/02/1/1"),
+ new URIish("http://localhost:18080"));
}
@Test
@@ -360,15 +520,15 @@
refsFilter,
() -> revReader,
applyObjectMetrics,
- fetchMetrics,
queueMetrics,
LOCAL_INSTANCE_ID,
+ config,
applyObjectsRefsFilter,
shutdownState);
- Event event = new TestEvent("refs/multi-site/version");
+ Event event = generateBatchRefUpdateEvent("refs/multi-site/version");
objectUnderTest.onEvent(event);
- verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo);
+ verifyNoInteractions(wq, rd, dis, sl, fetchClientFactory, accountAttribute);
}
@Test
@@ -379,10 +539,10 @@
@Test
public void shouldSkipEventWhenStarredChangesRef() {
- Event event = new TestEvent("refs/starred-changes/41/2941/1000000");
+ Event event = generateBatchRefUpdateEvent("refs/starred-changes/41/2941/1000000");
objectUnderTest.onEvent(event);
- verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo);
+ verifyNoInteractions(wq, rd, dis, sl, fetchClientFactory, accountAttribute);
}
@Test
@@ -445,6 +605,36 @@
return createTempDirectory(prefix);
}
+ private BatchRefUpdateEvent generateBatchRefUpdateEvent(String... refs) {
+ List<RefUpdateAttribute> refUpdates =
+ Arrays.stream(refs)
+ .map(
+ ref -> {
+ RefUpdateAttribute upd = new RefUpdateAttribute();
+ upd.newRev = NEW_OBJECT_ID;
+ upd.oldRev = ObjectId.zeroId().getName();
+ upd.project = PROJECT.get();
+ upd.refName = ref;
+ return upd;
+ })
+ .collect(Collectors.toList());
+
+ BatchRefUpdateEvent event =
+ new BatchRefUpdateEvent(
+ PROJECT, Suppliers.ofInstance(refUpdates), Suppliers.ofInstance(accountAttribute));
+ event.instanceId = LOCAL_INSTANCE_ID;
+ return event;
+ }
+
+ private void verifySendObjectOrdering(String firstRef, String secondRef) throws Exception {
+ verify(fetchRestApiClient)
+ .callBatchSendObject(any(), batchRefsCaptor.capture(), anyLong(), any());
+ List<BatchApplyObjectData> batchRefs = batchRefsCaptor.getValue();
+
+ assertThat(batchRefs.get(0).refName()).isEqualTo(firstRef);
+ assertThat(batchRefs.get(1).refName()).isEqualTo(secondRef);
+ }
+
private class TestEvent extends RefUpdatedEvent {
public TestEvent() {
@@ -487,11 +677,12 @@
}
}
- private void verifyFallbackToRestApiClientFetchAsync(RefUpdatedEvent event) throws IOException {
+ private void verifyFallbackToRestApiClientFetchAsync(BatchRefUpdateEvent event)
+ throws IOException {
verify(fetchRestApiClient)
.callFetch(
eq(event.getProjectNameKey()),
- eq(event.getRefName()),
+ eq(event.getRefNames().get(0)),
any(URIish.class),
any(Long.class),
eq(FetchRestApiClient.FORCE_ASYNC));
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java
index 96bd68e..8e7dc35 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java
@@ -33,7 +33,7 @@
import com.google.gerrit.extensions.api.changes.ReviewInput.CommentInput;
import com.google.gerrit.extensions.client.Comment;
import com.google.gerrit.extensions.config.FactoryModule;
-import com.google.gerrit.server.notedb.Sequences;
+import com.google.gerrit.server.Sequence;
import com.google.inject.Scopes;
import com.googlesource.gerrit.plugins.replication.ReplicationConfig;
import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig;
@@ -248,7 +248,7 @@
@Test
public void shouldNotReadRefsSequences() throws Exception {
createChange().assertOkStatus();
- String refName = RefNames.REFS_SEQUENCES + Sequences.NAME_CHANGES;
+ String refName = RefNames.REFS_SEQUENCES + Sequence.NAME_CHANGES;
Optional<RevisionData> revisionDataOption =
refObjectId(refName).flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId, 0));
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java
index 245003a..a4fc41b 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java
@@ -27,6 +27,7 @@
import com.google.gerrit.extensions.restapi.AuthException;
import com.google.gerrit.extensions.restapi.BadRequestException;
import com.google.gerrit.extensions.restapi.ResourceConflictException;
+import com.google.gerrit.extensions.restapi.ResourceNotFoundException;
import com.google.gerrit.extensions.restapi.Response;
import com.google.gerrit.server.project.ProjectResource;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData;
@@ -210,6 +211,18 @@
applyObjectAction.apply(projectResource, inputParams);
}
+ @Test(expected = ResourceNotFoundException.class)
+ public void shouldRethrowResourceNotFoundException() throws Exception {
+ RevisionInput inputParams =
+ new RevisionInput(label, refName, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData());
+
+ doThrow(new ResourceNotFoundException("test_projects"))
+ .when(applyObjectCommand)
+ .applyObject(any(), anyString(), any(), anyString(), anyLong());
+
+ applyObjectAction.apply(projectResource, inputParams);
+ }
+
private RevisionData createSampleRevisionData() {
RevisionObjectData commitData =
new RevisionObjectData(
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchApplyObjectActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchApplyObjectActionTest.java
new file mode 100644
index 0000000..d8dabf1
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchApplyObjectActionTest.java
@@ -0,0 +1,214 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.api;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.apache.http.HttpStatus.SC_OK;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.common.collect.Lists;
+import com.google.gerrit.extensions.restapi.MergeConflictException;
+import com.google.gerrit.extensions.restapi.Response;
+import com.google.gerrit.extensions.restapi.RestApiException;
+import com.google.gerrit.server.project.ProjectResource;
+import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData;
+import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput;
+import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData;
+import java.util.Collections;
+import java.util.List;
+import javax.servlet.http.HttpServletResponse;
+import org.eclipse.jgit.lib.Constants;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.mockito.stubbing.Answer;
+
+@RunWith(MockitoJUnitRunner.class)
+public class BatchApplyObjectActionTest {
+
+ private static final long DUMMY_EVENT_TIMESTAMP = 1684875939;
+
+ private BatchApplyObjectAction batchApplyObjectAction;
+ private static final String LABEL = "instance-2-label";
+ private static final String REF_NAME = "refs/heads/master";
+ private static final String REF_META_NAME = "refs/meta/version";
+ private static final String SAMPLE_COMMIT_OBJECT_ID = "9f8d52853089a3cf00c02ff7bd0817bd4353a95a";
+ private static final String SAMPLE_TREE_OBJECT_ID = "4b825dc642cb6eb9a060e54bf8d69288fbee4904";
+
+ private static final String SAMPLE_COMMIT_CONTENT =
+ "tree "
+ + SAMPLE_TREE_OBJECT_ID
+ + "\n"
+ + "parent 20eb48d28be86dc88fb4bef747f08de0fbefe12d\n"
+ + "author Gerrit User 1000000 <1000000@69ec38f0-350e-4d9c-96d4-bc956f2faaac> 1610471611 +0100\n"
+ + "committer Gerrit Code Review <root@maczech-XPS-15> 1610471611 +0100\n"
+ + "\n"
+ + "Update patch set 1\n"
+ + "\n"
+ + "Change has been successfully merged by Administrator\n"
+ + "\n"
+ + "Patch-set: 1\n"
+ + "Status: merged\n"
+ + "Tag: autogenerated:gerrit:merged\n"
+ + "Reviewer: Gerrit User 1000000 <1000000@69ec38f0-350e-4d9c-96d4-bc956f2faaac>\n"
+ + "Label: SUBM=+1\n"
+ + "Submission-id: 1904-1610471611558-783c0a2f\n"
+ + "Submitted-with: OK\n"
+ + "Submitted-with: OK: Code-Review: Gerrit User 1000000 <1000000@69ec38f0-350e-4d9c-96d4-bc956f2faaac>";
+
+ @Mock private ApplyObjectAction applyObjectAction;
+ @Mock private ProjectResource projectResource;
+
+ @Before
+ public void setup() {
+ batchApplyObjectAction = new BatchApplyObjectAction(applyObjectAction);
+ }
+
+ @Test
+ public void shouldDelegateToApplyObjectActionForEveryRevision() throws RestApiException {
+ RevisionInput first =
+ new RevisionInput(LABEL, REF_NAME, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData());
+ RevisionInput second =
+ new RevisionInput(LABEL, "foo", DUMMY_EVENT_TIMESTAMP, createSampleRevisionData());
+
+ batchApplyObjectAction.apply(projectResource, List.of(first, second));
+
+ verify(applyObjectAction).apply(projectResource, first);
+ verify(applyObjectAction).apply(projectResource, second);
+ }
+
+ @Test
+ public void shouldReturnOkResponseCodeWhenAllRevisionsAreProcessedSuccessfully()
+ throws RestApiException {
+ RevisionInput first =
+ new RevisionInput(LABEL, REF_NAME, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData());
+ RevisionInput second =
+ new RevisionInput(LABEL, "foo", DUMMY_EVENT_TIMESTAMP, createSampleRevisionData());
+
+ when(applyObjectAction.apply(projectResource, first))
+ .thenAnswer((Answer<Response<?>>) invocation -> Response.created(first));
+ when(applyObjectAction.apply(projectResource, second))
+ .thenAnswer((Answer<Response<?>>) invocation -> Response.created(second));
+
+ Response<?> response = batchApplyObjectAction.apply(projectResource, List.of(first, second));
+
+ assertThat(response.statusCode()).isEqualTo(SC_OK);
+ }
+
+ @Test
+ public void shouldReturnAListWithAllTheRevisionsInResponseBodyOnSuccess()
+ throws RestApiException {
+ RevisionInput first =
+ new RevisionInput(LABEL, REF_NAME, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData());
+ RevisionInput second =
+ new RevisionInput(LABEL, "foo", DUMMY_EVENT_TIMESTAMP, createSampleRevisionData());
+ Response<?> firstResponse = Response.created(first);
+ Response<?> secondResponse = Response.created(second);
+
+ when(applyObjectAction.apply(projectResource, first))
+ .thenAnswer((Answer<Response<?>>) invocation -> firstResponse);
+ when(applyObjectAction.apply(projectResource, second))
+ .thenAnswer((Answer<Response<?>>) invocation -> secondResponse);
+
+ Response<?> response = batchApplyObjectAction.apply(projectResource, List.of(first, second));
+
+ assertThat((List<Response<?>>) response.value())
+ .isEqualTo(List.of(firstResponse, secondResponse));
+ }
+
+ @Test
+ public void shouldAcceptAMixOfCreatesAndDeletes() throws RestApiException {
+ RevisionInput delete = new RevisionInput(LABEL, REF_NAME, DUMMY_EVENT_TIMESTAMP, null);
+ RevisionInput create =
+ new RevisionInput(LABEL, "foo", DUMMY_EVENT_TIMESTAMP, createSampleRevisionData());
+ Response<?> deleteResponse = Response.withStatusCode(HttpServletResponse.SC_NO_CONTENT, "");
+ Response<?> createResponse = Response.created(create);
+
+ when(applyObjectAction.apply(projectResource, delete))
+ .thenAnswer((Answer<Response<?>>) invocation -> deleteResponse);
+ when(applyObjectAction.apply(projectResource, create))
+ .thenAnswer((Answer<Response<?>>) invocation -> createResponse);
+
+ Response<?> response = batchApplyObjectAction.apply(projectResource, List.of(delete, create));
+
+ assertThat((List<Response<?>>) response.value())
+ .isEqualTo(List.of(deleteResponse, createResponse));
+ }
+
+ @Test
+ public void shouldReturnOneOkCodeEvenIfInputContainsBothCreatesAndDeletes()
+ throws RestApiException {
+ RevisionInput create =
+ new RevisionInput(LABEL, REF_NAME, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData());
+ RevisionInput delete = new RevisionInput(LABEL, REF_META_NAME, DUMMY_EVENT_TIMESTAMP + 1, null);
+
+ List<RevisionInput> inputs = List.of(create, delete);
+
+ Response<?> response = batchApplyObjectAction.apply(projectResource, inputs);
+
+ assertThat(response.statusCode()).isEqualTo(SC_OK);
+ }
+
+ @Test(expected = RestApiException.class)
+ public void shouldThrowARestApiExceptionIfProcessingFailsForAnyOfTheRevisions()
+ throws RestApiException {
+ RevisionInput good =
+ new RevisionInput(LABEL, REF_NAME, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData());
+ RevisionInput bad =
+ new RevisionInput(LABEL, "bad", DUMMY_EVENT_TIMESTAMP, createSampleRevisionData());
+
+ when(applyObjectAction.apply(projectResource, good))
+ .thenAnswer((Answer<Response<?>>) invocation -> Response.created(good));
+ when(applyObjectAction.apply(projectResource, bad))
+ .thenThrow(new MergeConflictException("BOOM"));
+
+ batchApplyObjectAction.apply(projectResource, List.of(good, bad));
+ }
+
+ @Test
+ public void shouldStopProcessingWhenAFailureOccurs() throws RestApiException {
+ RevisionInput good =
+ new RevisionInput(LABEL, REF_NAME, DUMMY_EVENT_TIMESTAMP, createSampleRevisionData());
+ RevisionInput bad =
+ new RevisionInput(LABEL, "bad", DUMMY_EVENT_TIMESTAMP, createSampleRevisionData());
+
+ when(applyObjectAction.apply(projectResource, bad))
+ .thenThrow(new MergeConflictException("BOOM"));
+
+ try {
+ batchApplyObjectAction.apply(projectResource, List.of(bad, good));
+ } catch (MergeConflictException e) {
+ verify(applyObjectAction, never()).apply(projectResource, good);
+ }
+ }
+
+ private RevisionData createSampleRevisionData() {
+ RevisionObjectData commitData =
+ new RevisionObjectData(
+ SAMPLE_COMMIT_OBJECT_ID, Constants.OBJ_COMMIT, SAMPLE_COMMIT_CONTENT.getBytes());
+ RevisionObjectData treeData =
+ new RevisionObjectData(SAMPLE_TREE_OBJECT_ID, Constants.OBJ_TREE, new byte[] {});
+ return createSampleRevisionData(commitData, treeData);
+ }
+
+ private RevisionData createSampleRevisionData(
+ RevisionObjectData commitData, RevisionObjectData treeData) {
+ return new RevisionData(Collections.emptyList(), commitData, treeData, Lists.newArrayList());
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchActionTest.java
new file mode 100644
index 0000000..738815a
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BatchFetchActionTest.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.googlesource.gerrit.plugins.replication.pull.api;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.apache.http.HttpStatus.SC_OK;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.gerrit.extensions.restapi.MergeConflictException;
+import com.google.gerrit.extensions.restapi.Response;
+import com.google.gerrit.extensions.restapi.RestApiException;
+import com.google.gerrit.server.project.ProjectResource;
+import java.util.List;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.mockito.stubbing.Answer;
+
+@RunWith(MockitoJUnitRunner.class)
+public class BatchFetchActionTest {
+
+ BatchFetchAction batchFetchAction;
+ String label = "instance-2-label";
+ String master = "refs/heads/master";
+ String test = "refs/heads/test";
+
+ @Mock ProjectResource projectResource;
+ @Mock FetchAction fetchAction;
+
+ @Before
+ public void setup() {
+ batchFetchAction = new BatchFetchAction(fetchAction);
+ }
+
+ @Test
+ public void shouldDelegateToFetchActionForEveryFetchInput() throws RestApiException {
+ FetchAction.Input first = createInput(master);
+ FetchAction.Input second = createInput(test);
+
+ batchFetchAction.apply(projectResource, List.of(first, second));
+
+ verify(fetchAction).apply(projectResource, first);
+ verify(fetchAction).apply(projectResource, second);
+ }
+
+ @Test
+ public void shouldReturnOkResponseCodeWhenAllInputsAreProcessedSuccessfully()
+ throws RestApiException {
+ FetchAction.Input first = createInput(master);
+ FetchAction.Input second = createInput(test);
+
+ when(fetchAction.apply(any(), any()))
+ .thenAnswer((Answer<Response<?>>) invocation -> Response.accepted("some-url"));
+ Response<?> response = batchFetchAction.apply(projectResource, List.of(first, second));
+
+ assertThat(response.statusCode()).isEqualTo(SC_OK);
+ }
+
+ @Test
+ public void shouldReturnAListWithAllResponsesOnSuccess() throws RestApiException {
+ FetchAction.Input first = createInput(master);
+ FetchAction.Input second = createInput(test);
+ String masterUrl = "master-url";
+ String testUrl = "test-url";
+ Response.Accepted firstResponse = Response.accepted(masterUrl);
+ Response.Accepted secondResponse = Response.accepted(testUrl);
+
+ when(fetchAction.apply(projectResource, first))
+ .thenAnswer((Answer<Response<?>>) invocation -> firstResponse);
+ when(fetchAction.apply(projectResource, second))
+ .thenAnswer((Answer<Response<?>>) invocation -> secondResponse);
+ Response<?> response = batchFetchAction.apply(projectResource, List.of(first, second));
+
+ assertThat((List<Response<?>>) response.value())
+ .isEqualTo(List.of(firstResponse, secondResponse));
+ }
+
+ @Test
+ public void shouldReturnAMixOfSyncAndAsyncResponses() throws RestApiException {
+ FetchAction.Input async = createInput(master);
+ FetchAction.Input sync = createInput(test);
+ String masterUrl = "master-url";
+ Response.Accepted asyncResponse = Response.accepted(masterUrl);
+ Response<?> syncResponse = Response.created(sync);
+
+ when(fetchAction.apply(projectResource, async))
+ .thenAnswer((Answer<Response<?>>) invocation -> asyncResponse);
+ when(fetchAction.apply(projectResource, sync))
+ .thenAnswer((Answer<Response<?>>) invocation -> syncResponse);
+ Response<?> response = batchFetchAction.apply(projectResource, List.of(async, sync));
+
+ assertThat((List<Response<?>>) response.value())
+ .isEqualTo(List.of(asyncResponse, syncResponse));
+ }
+
+ @Test(expected = RestApiException.class)
+ public void shouldThrowRestApiExceptionWhenProcessingFailsForAnInput() throws RestApiException {
+ FetchAction.Input first = createInput(master);
+ FetchAction.Input second = createInput(test);
+ String masterUrl = "master-url";
+
+ when(fetchAction.apply(projectResource, first))
+ .thenAnswer((Answer<Response<?>>) invocation -> Response.accepted(masterUrl));
+ when(fetchAction.apply(projectResource, second)).thenThrow(new MergeConflictException("BOOM"));
+
+ batchFetchAction.apply(projectResource, List.of(first, second));
+ }
+
+ private FetchAction.Input createInput(String refName) {
+ FetchAction.Input input = new FetchAction.Input();
+ input.label = label;
+ input.refName = refName;
+ return input;
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BearerAuthenticationFilterTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BearerAuthenticationFilterTest.java
index 95b86cc..ea5149b 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BearerAuthenticationFilterTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/BearerAuthenticationFilterTest.java
@@ -85,6 +85,11 @@
}
@Test
+ public void shouldAuthenticateWhenBatchFetch() throws Exception {
+ authenticateAndFilter("any-prefix/pull-replication~batch-fetch", NO_QUERY_PARAMETERS);
+ }
+
+ @Test
public void shouldAuthenticateWhenApplyObject() throws Exception {
authenticateAndFilter("any-prefix/pull-replication~apply-object", NO_QUERY_PARAMETERS);
}
@@ -95,6 +100,11 @@
}
@Test
+ public void shouldAuthenticateWhenBatchApplyObject() throws Exception {
+ authenticateAndFilter("any-prefix/pull-replication~batch-apply-object", NO_QUERY_PARAMETERS);
+ }
+
+ @Test
public void shouldAuthenticateWhenDeleteProject() throws Exception {
authenticateAndFilter("any-prefix/pull-replication~delete-project", NO_QUERY_PARAMETERS);
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java
index fc1b02c..40912ac 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java
@@ -19,6 +19,7 @@
import static org.mockito.Mockito.anyString;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
import com.google.gerrit.entities.Project;
@@ -27,7 +28,6 @@
import com.google.gerrit.server.events.Event;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.git.LocalDiskRepositoryManager;
-import com.google.gerrit.server.permissions.PermissionBackend;
import com.google.gerrit.server.permissions.PermissionBackend.ForProject;
import com.google.gerrit.server.permissions.PermissionBackend.ForRef;
import com.google.gerrit.server.permissions.PermissionBackend.WithUser;
@@ -41,7 +41,6 @@
import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject;
import java.util.Optional;
import org.eclipse.jgit.lib.Ref;
-import org.eclipse.jgit.lib.RefDatabase;
import org.eclipse.jgit.lib.RefUpdate;
import org.eclipse.jgit.lib.RefUpdate.Result;
import org.eclipse.jgit.lib.Repository;
@@ -71,7 +70,6 @@
@Mock private ProjectState projectState;
@Mock private SourcesCollection sourceCollection;
@Mock private Source source;
- @Mock private PermissionBackend permissionBackend;
@Mock private WithUser currentUser;
@Mock private ForProject forProject;
@Mock private ForRef forRef;
@@ -79,7 +77,6 @@
@Mock private RefUpdate refUpdate;
@Mock private Repository repository;
@Mock private Ref currentRef;
- @Mock private RefDatabase refDb;
@Captor ArgumentCaptor<Event> eventCaptor;
private DeleteRefCommand objectUnderTest;
@@ -93,8 +90,7 @@
when(source.getURI(TEST_PROJECT_NAME)).thenReturn(TEST_REMOTE_URI);
when(gitManager.openRepository(any())).thenReturn(repository);
when(repository.updateRef(any())).thenReturn(refUpdate);
- when(repository.getRefDatabase()).thenReturn(refDb);
- when(refDb.exactRef(anyString())).thenReturn(currentRef);
+ when(repository.exactRef(anyString())).thenReturn(currentRef);
when(refUpdate.delete()).thenReturn(Result.FORCED);
objectUnderTest =
@@ -102,13 +98,14 @@
fetchStateLog,
projectCache,
sourceCollection,
- permissionBackend,
eventDispatcherDataItem,
new LocalGitRepositoryManagerProvider(gitManager));
}
@Test
public void shouldSendEventWhenDeletingRef() throws Exception {
+ when(source.isMirror()).thenReturn(true);
+
objectUnderTest.deleteRef(TEST_PROJECT_NAME, TEST_REF_NAME, TEST_SOURCE_LABEL);
verify(eventDispatcher).postEvent(eventCaptor.capture());
@@ -120,9 +117,18 @@
}
@Test
- public void shouldHandleNonExistingRef() throws Exception {
- when(refDb.exactRef(anyString())).thenReturn(null);
+ public void shouldNotSendNotSendEventWhenMirroringIsDisabled() throws Exception {
+ when(source.isMirror()).thenReturn(false);
+ objectUnderTest.deleteRef(TEST_PROJECT_NAME, TEST_REF_NAME, TEST_SOURCE_LABEL);
+
+ verifyNoInteractions(eventDispatcher);
+ }
+
+ @Test
+ public void shouldHandleNonExistingRef() throws Exception {
+ when(source.isMirror()).thenReturn(true);
+ when(repository.exactRef(anyString())).thenReturn(null);
objectUnderTest.deleteRef(TEST_PROJECT_NAME, NON_EXISTING_REF_NAME, TEST_SOURCE_LABEL);
verify(eventDispatcher, never()).postEvent(any());
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionActionIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionActionIT.java
index 7c7846c..3bd9d35 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionActionIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ProjectDeletionActionIT.java
@@ -14,6 +14,7 @@
package com.googlesource.gerrit.plugins.replication.pull.api;
+import static com.google.common.truth.Truth.assertThat;
import static com.google.gerrit.acceptance.testsuite.project.TestProjectUpdate.allowCapability;
import com.google.gerrit.acceptance.config.GerritConfig;
@@ -181,6 +182,50 @@
assertHttpResponseCode(HttpServletResponse.SC_OK));
}
+ @Test
+ @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId")
+ public void shouldRemoveFromTheProjectCacheWhenProjectIsSuccessfullyDeleted() throws Exception {
+ String testProjectName = project.get();
+ url = getURLWithAuthenticationPrefix(testProjectName);
+ assertThat(projectCache.get(project).isPresent()).isTrue();
+
+ httpClientFactory
+ .create(source)
+ .execute(
+ withBasicAuthenticationAsAdmin(createDeleteRequest()),
+ assertHttpResponseCode(HttpServletResponse.SC_OK));
+ assertThat(projectCache.get(project).isPresent()).isFalse();
+ }
+
+ @Test
+ @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId")
+ @GerritConfig(name = "container.replica", value = "true")
+ public void shouldRemoveFromtheReplicaProjectCacheWhenProjectIsSuccessfullyDeletedFromTheReplica()
+ throws Exception {
+ String testProjectName = project.get();
+ url = getURLWithAuthenticationPrefix(testProjectName);
+ assertThat(projectCache.get(project).isPresent()).isTrue();
+
+ httpClientFactory
+ .create(source)
+ .execute(
+ withBasicAuthenticationAsAdmin(createDeleteRequest()),
+ assertHttpResponseCode(HttpServletResponse.SC_OK));
+ assertThat(projectCache.get(project).isPresent()).isFalse();
+ }
+
+ @Test
+ @GerritConfig(name = "container.replica", value = "true")
+ @GerritConfig(name = "gerrit.instanceId", value = "testInstanceId")
+ public void shouldNotRemoveFromTheReplicaCacheIfAProjectCannotBeDeleted() throws Exception {
+ assertThat(projectCache.get(project).isPresent()).isTrue();
+ httpClientFactory
+ .create(source)
+ .execute(
+ createDeleteRequest(), assertHttpResponseCode(HttpServletResponse.SC_UNAUTHORIZED));
+ assertThat(projectCache.get(project).isPresent()).isTrue();
+ }
+
@Override
protected String getURLWithAuthenticationPrefix(String projectName) {
return String.format(
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilterTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilterTest.java
index 9492d49..34885b2 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilterTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilterTest.java
@@ -8,11 +8,14 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.internal.verification.VerificationModeFactory.atLeastOnce;
-import static org.mockito.internal.verification.VerificationModeFactory.times;
import com.google.common.net.MediaType;
import com.google.gerrit.entities.Project;
-import com.google.gerrit.extensions.restapi.*;
+import com.google.gerrit.extensions.restapi.AuthException;
+import com.google.gerrit.extensions.restapi.ResourceConflictException;
+import com.google.gerrit.extensions.restapi.ResourceNotFoundException;
+import com.google.gerrit.extensions.restapi.Response;
+import com.google.gerrit.extensions.restapi.UnprocessableEntityException;
import com.google.gerrit.server.AnonymousUser;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.IdentifiedUser;
@@ -20,7 +23,11 @@
import com.google.gerrit.server.project.ProjectResource;
import com.google.gerrit.server.project.ProjectState;
import com.google.inject.util.Providers;
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
import javax.servlet.FilterChain;
@@ -39,8 +46,10 @@
@Mock HttpServletResponse response;
@Mock FilterChain filterChain;
@Mock private FetchAction fetchAction;
+ @Mock private BatchFetchAction batchFetchAction;
@Mock private ApplyObjectAction applyObjectAction;
@Mock private ApplyObjectsAction applyObjectsAction;
+ @Mock private BatchApplyObjectAction batchApplyObjectAction;
@Mock private ProjectInitializationAction projectInitializationAction;
@Mock private UpdateHeadAction updateHEADAction;
@Mock private ProjectDeletionAction projectDeletionAction;
@@ -55,12 +64,17 @@
private final String PROJECT_NAME_GIT = "some-project.git";
private final String FETCH_URI =
String.format("any-prefix/projects/%s/%s~fetch", PROJECT_NAME, PLUGIN_NAME);
+ private final String BATCH_FETCH_URI =
+ String.format("any-prefix/projects/%s/%s~batch-fetch", PROJECT_NAME, PLUGIN_NAME);
private final String APPLY_OBJECT_URI =
String.format("any-prefix/projects/%s/%s~apply-object", PROJECT_NAME, PLUGIN_NAME);
private final String APPLY_OBJECTS_URI =
String.format("any-prefix/projects/%s/%s~apply-objects", PROJECT_NAME, PLUGIN_NAME);
private final String HEAD_URI =
String.format("any-prefix/projects/%s/%s~HEAD", PROJECT_NAME, PLUGIN_NAME);
+
+ private final String BATCH_APPLY_OBJECT_URI =
+ String.format("any-prefix/projects/%s/%s~batch-apply-object", PROJECT_NAME, PLUGIN_NAME);
private final String DELETE_PROJECT_URI =
String.format("any-prefix/projects/%s/%s~delete-project", PROJECT_NAME, PLUGIN_NAME);
private final String INIT_PROJECT_URI =
@@ -75,8 +89,10 @@
private PullReplicationFilter createPullReplicationFilter(CurrentUser currentUser) {
return new PullReplicationFilter(
fetchAction,
+ batchFetchAction,
applyObjectAction,
applyObjectsAction,
+ batchApplyObjectAction,
projectInitializationAction,
updateHEADAction,
projectDeletionAction,
@@ -124,6 +140,31 @@
}
@Test
+ public void shouldFilterBatchFetchAction() throws Exception {
+ byte[] payloadBatchFetch =
+ ("[{"
+ + "\"label\":\"Replication\", "
+ + "\"ref_name\": \"refs/heads/master\", "
+ + "\"async\":false"
+ + "},"
+ + "{"
+ + "\"label\":\"Replication\", "
+ + "\"ref_name\": \"refs/heads/test\", "
+ + "\"async\":false"
+ + "}]")
+ .getBytes(StandardCharsets.UTF_8);
+
+ defineBehaviours(payloadBatchFetch, BATCH_FETCH_URI);
+ when(batchFetchAction.apply(any(), any())).thenReturn(OK_RESPONSE);
+
+ PullReplicationFilter pullReplicationFilter = createPullReplicationFilter();
+ pullReplicationFilter.doFilter(request, response, filterChain);
+
+ verifyBehaviours();
+ verify(batchFetchAction).apply(any(ProjectResource.class), any());
+ }
+
+ @Test
public void shouldFilterApplyObjectAction() throws Exception {
byte[] payloadApplyObject =
@@ -206,7 +247,6 @@
final PullReplicationFilter pullReplicationFilter = createPullReplicationFilter();
pullReplicationFilter.doFilter(request, response, filterChain);
- verify(request, times(7)).getRequestURI();
verify(projectCache).get(Project.nameKey(PROJECT_NAME));
verify(projectDeletionAction).apply(any(ProjectResource.class), any());
verify(response).getWriter();
@@ -249,7 +289,7 @@
}
@Test
- public void shouldBe500WhenResourceNotFound() throws Exception {
+ public void shouldBe404WhenResourceNotFound() throws Exception {
when(request.getRequestURI()).thenReturn(DELETE_PROJECT_URI);
when(request.getMethod()).thenReturn("DELETE");
when(projectCache.get(Project.nameKey(PROJECT_NAME))).thenReturn(Optional.of(projectState));
@@ -260,7 +300,7 @@
final PullReplicationFilter pullReplicationFilter = createPullReplicationFilter();
pullReplicationFilter.doFilter(request, response, filterChain);
- verify(response).setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ verify(response).setStatus(HttpServletResponse.SC_NOT_FOUND);
}
@Test
@@ -346,4 +386,33 @@
verify(response).setStatus(HttpServletResponse.SC_BAD_REQUEST);
}
+
+ @Test
+ public void shouldFilterBatchApplyObjectAction() throws Exception {
+
+ byte[] payloadApplyObject =
+ ("[{\"label\":\"Replication\",\"ref_name\":\"refs/heads/foo\","
+ + "\"revision_data\":{"
+ + "\"commit_object\":{\"type\":1,\"content\":\"some-content\"},"
+ + "\"tree_object\":{\"type\":2,\"content\":\"some-content\"},"
+ + "\"blobs\":[]}"
+ + "},"
+ + "{\"label\":\"Replication\",\"ref_name\":\"refs/heads/bar\","
+ + "\"revision_data\":{"
+ + "\"commit_object\":{\"type\":1,\"content\":\"some-content\"},"
+ + "\"tree_object\":{\"type\":2,\"content\":\"some-content\"},"
+ + "\"blobs\":[]}"
+ + "}]")
+ .getBytes(StandardCharsets.UTF_8);
+
+ defineBehaviours(payloadApplyObject, BATCH_APPLY_OBJECT_URI);
+
+ when(batchApplyObjectAction.apply(any(), any())).thenReturn(OK_RESPONSE);
+
+ PullReplicationFilter pullReplicationFilter = createPullReplicationFilter();
+ pullReplicationFilter.doFilter(request, response, filterChain);
+
+ verifyBehaviours();
+ verify(batchApplyObjectAction).apply(any(ProjectResource.class), any());
+ }
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/data/BatchApplyObjectDataTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/data/BatchApplyObjectDataTest.java
new file mode 100644
index 0000000..bf74b56
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/data/BatchApplyObjectDataTest.java
@@ -0,0 +1,32 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.api.data;
+
+import java.util.Optional;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class BatchApplyObjectDataTest {
+
+ @Mock private RevisionData revisionData;
+
+ @Test(expected = IllegalArgumentException.class)
+ public void shouldFailIfRevisionDataIsPresentForADelete() {
+ BatchApplyObjectData.create("foo", Optional.of(revisionData), true);
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java
index 65ccdfb..b061de0 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientBase.java
@@ -31,12 +31,16 @@
import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig;
import com.googlesource.gerrit.plugins.replication.pull.BearerTokenProvider;
import com.googlesource.gerrit.plugins.replication.pull.Source;
+import com.googlesource.gerrit.plugins.replication.pull.api.data.BatchApplyObjectData;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData;
import com.googlesource.gerrit.plugins.replication.pull.filter.SyncRefsFilter;
import java.io.InputStreamReader;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
import org.apache.http.Header;
import org.apache.http.HttpHeaders;
import org.apache.http.client.methods.HttpDelete;
@@ -176,6 +180,25 @@
}
@Test
+ public void shouldCallBatchFetchEndpoint() throws Exception {
+
+ objectUnderTest.callBatchFetch(
+ Project.nameKey("test_repo"),
+ List.of(refName, RefNames.REFS_HEADS + "test"),
+ new URIish(api));
+
+ verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any());
+
+ HttpPost httpPost = httpPostCaptor.getValue();
+ assertThat(httpPost.getURI().getHost()).isEqualTo("gerrit-host");
+ assertThat(httpPost.getURI().getPath())
+ .isEqualTo(
+ String.format(
+ "%s/projects/test_repo/pull-replication~batch-fetch", urlAuthenticationPrefix()));
+ assertAuthentication(httpPost);
+ }
+
+ @Test
public void shouldByDefaultCallSyncFetchForAllRefs() throws Exception {
objectUnderTest.callFetch(Project.nameKey("test_repo"), refName, new URIish(api));
@@ -212,6 +235,41 @@
}
@Test
+ public void shouldCallAsyncBatchFetchForAllRefs() throws Exception {
+
+ when(config.getStringList("replication", null, "syncRefs"))
+ .thenReturn(new String[] {"NO_SYNC_REFS"});
+ syncRefsFilter = new SyncRefsFilter(replicationConfig);
+ objectUnderTest =
+ new FetchRestApiClient(
+ credentials,
+ httpClientFactory,
+ replicationConfig,
+ syncRefsFilter,
+ pluginName,
+ instanceId,
+ bearerTokenProvider,
+ source);
+
+ String testRef = RefNames.REFS_HEADS + "test";
+ List<String> refs = List.of(refName, testRef);
+ objectUnderTest.callBatchFetch(Project.nameKey("test_repo"), refs, new URIish(api));
+
+ verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any());
+
+ HttpPost httpPost = httpPostCaptor.getValue();
+ String expectedPayload =
+ "[{\"label\":\"Replication\", \"ref_name\": \""
+ + refName
+ + "\", \"async\":true},"
+ + "{\"label\":\"Replication\", \"ref_name\": \""
+ + refs.get(1)
+ + "\", \"async\":true}"
+ + "]";
+ assertThat(readPayload(httpPost)).isEqualTo(expectedPayload);
+ }
+
+ @Test
public void shouldCallSyncFetchOnlyForMetaRef() throws Exception {
String metaRefName = "refs/changes/01/101/meta";
String expectedMetaRefPayload =
@@ -243,6 +301,33 @@
}
@Test
+ public void shouldCallSyncBatchFetchOnlyForMetaRef() throws Exception {
+ String metaRefName = "refs/changes/01/101/meta";
+ String expectedMetaRefPayload =
+ "[{\"label\":\"Replication\", \"ref_name\": \"" + metaRefName + "\", \"async\":false}]";
+
+ when(config.getStringList("replication", null, "syncRefs"))
+ .thenReturn(new String[] {"^refs\\/changes\\/.*\\/meta"});
+ syncRefsFilter = new SyncRefsFilter(replicationConfig);
+ objectUnderTest =
+ new FetchRestApiClient(
+ credentials,
+ httpClientFactory,
+ replicationConfig,
+ syncRefsFilter,
+ pluginName,
+ instanceId,
+ bearerTokenProvider,
+ source);
+
+ objectUnderTest.callBatchFetch(
+ Project.nameKey("test_repo"), List.of(metaRefName), new URIish(api));
+ verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any());
+ HttpPost httpPost = httpPostCaptor.getValue();
+ assertThat(readPayload(httpPost)).isEqualTo(expectedMetaRefPayload);
+ }
+
+ @Test
public void shouldCallFetchEndpointWithPayload() throws Exception {
objectUnderTest.callFetch(Project.nameKey("test_repo"), refName, new URIish(api));
@@ -254,6 +339,92 @@
}
@Test
+ public void shouldCallBatchFetchEndpointWithPayload() throws Exception {
+
+ String testRef = RefNames.REFS_HEADS + "test";
+ List<String> refs = List.of(refName, testRef);
+ objectUnderTest.callBatchFetch(Project.nameKey("test_repo"), refs, new URIish(api));
+
+ verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any());
+
+ HttpPost httpPost = httpPostCaptor.getValue();
+ String expectedPayload =
+ "[{\"label\":\"Replication\", \"ref_name\": \""
+ + refName
+ + "\", \"async\":false},"
+ + "{\"label\":\"Replication\", \"ref_name\": \""
+ + refs.get(1)
+ + "\", \"async\":false}"
+ + "]";
+ assertThat(readPayload(httpPost)).isEqualTo(expectedPayload);
+ }
+
+ @Test
+ public void shouldExecuteOneFetchCallForAsyncAndOneForSyncRefsDuringBatchFetch()
+ throws Exception {
+
+ when(config.getStringList("replication", null, "syncRefs"))
+ .thenReturn(new String[] {"^refs\\/heads\\/test"});
+ syncRefsFilter = new SyncRefsFilter(replicationConfig);
+ String testRef = RefNames.REFS_HEADS + "test";
+ List<String> refs = List.of(refName, testRef);
+ objectUnderTest =
+ new FetchRestApiClient(
+ credentials,
+ httpClientFactory,
+ replicationConfig,
+ syncRefsFilter,
+ pluginName,
+ instanceId,
+ bearerTokenProvider,
+ source);
+ objectUnderTest.callBatchFetch(Project.nameKey("test_repo"), refs, new URIish(api));
+
+ verify(httpClient, times(2)).execute(httpPostCaptor.capture(), any());
+
+ List<HttpPost> httpPosts = httpPostCaptor.getAllValues();
+ String expectedSyncPayload =
+ "[{\"label\":\"Replication\", \"ref_name\": \""
+ + refs.get(1)
+ + "\", \"async\":false}"
+ + "]";
+ String expectedAsyncPayload =
+ "[{\"label\":\"Replication\", \"ref_name\": \"" + refName + "\", \"async\":true}]";
+
+ assertThat(readPayload(httpPosts.get(0))).isEqualTo(expectedAsyncPayload);
+ assertThat(readPayload(httpPosts.get(1))).isEqualTo(expectedSyncPayload);
+ }
+
+ @Test
+ public void shouldNotExecuteSyncFetchCallWhenAsyncCallFailsDuringBatchFetch() throws Exception {
+ when(config.getStringList("replication", null, "syncRefs"))
+ .thenReturn(new String[] {"^refs\\/heads\\/test"});
+ when(httpClient.execute(any(), any())).thenReturn(new HttpResult(500, Optional.of("BOOM")));
+ syncRefsFilter = new SyncRefsFilter(replicationConfig);
+ String testRef = RefNames.REFS_HEADS + "test";
+ List<String> refs = List.of(refName, testRef);
+ objectUnderTest =
+ new FetchRestApiClient(
+ credentials,
+ httpClientFactory,
+ replicationConfig,
+ syncRefsFilter,
+ pluginName,
+ instanceId,
+ bearerTokenProvider,
+ source);
+ objectUnderTest.callBatchFetch(Project.nameKey("test_repo"), refs, new URIish(api));
+
+ verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any());
+
+ HttpPost httpPost = httpPostCaptor.getValue();
+ String expectedAsyncPayload =
+ "[{\"label\":\"Replication\", \"ref_name\": \"" + refName + "\", \"async\":true}]";
+
+ assertThat(readPayload(httpPost)).isEqualTo(expectedAsyncPayload);
+ }
+
+ @Test
public void shouldSetContentTypeHeader() throws Exception {
objectUnderTest.callFetch(Project.nameKey("test_repo"), refName, new URIish(api));
@@ -266,6 +437,18 @@
}
@Test
+ public void shouldSetContentTypeHeaderInBatchFetch() throws Exception {
+
+ objectUnderTest.callBatchFetch(Project.nameKey("test_repo"), List.of(refName), new URIish(api));
+
+ verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any());
+
+ HttpPost httpPost = httpPostCaptor.getValue();
+ assertThat(httpPost.getLastHeader("Content-Type").getValue())
+ .isEqualTo(expectedHeader.getValue());
+ }
+
+ @Test
public void shouldCallSendObjectEndpoint() throws Exception {
objectUnderTest.callSendObject(
@@ -449,19 +632,139 @@
assertAuthentication(httpPut);
}
+ @Test
+ public void shouldCallBatchSendObjectEndpoint() throws Exception {
+
+ List<BatchApplyObjectData> batchApplyObjects = new ArrayList<>();
+ batchApplyObjects.add(
+ BatchApplyObjectData.create(refName, Optional.of(createSampleRevisionData("a")), false));
+
+ objectUnderTest.callBatchSendObject(
+ Project.nameKey("test_repo"), batchApplyObjects, eventCreatedOn, new URIish(api));
+
+ verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any());
+
+ HttpPost httpPost = httpPostCaptor.getValue();
+ assertThat(httpPost.getURI().getHost()).isEqualTo("gerrit-host");
+ assertThat(httpPost.getURI().getPath())
+ .isEqualTo(
+ String.format(
+ "%s/projects/test_repo/pull-replication~batch-apply-object",
+ urlAuthenticationPrefix()));
+ assertAuthentication(httpPost);
+ }
+
+ @Test
+ public void shouldCallBatchApplyObjectEndpointWithAListOfRefsInPayload() throws Exception {
+ List<BatchApplyObjectData> batchApplyObjects = new ArrayList<>();
+ RevisionData revisionA = createSampleRevisionData("a");
+ RevisionData revisionB = createSampleRevisionData("b");
+ String refNameB = "refs/heads/b";
+ batchApplyObjects.add(BatchApplyObjectData.create(refName, Optional.of(revisionA), false));
+ batchApplyObjects.add(BatchApplyObjectData.create(refNameB, Optional.of(revisionB), false));
+
+ objectUnderTest.callBatchSendObject(
+ Project.nameKey("test_repo"), batchApplyObjects, eventCreatedOn, new URIish(api));
+
+ verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any());
+
+ HttpPost httpPost = httpPostCaptor.getValue();
+
+ String expectedSendObjectsPayload =
+ "[{\"label\":\"Replication\",\"ref_name\":\""
+ + refName
+ + "\",\"event_created_on\":"
+ + eventCreatedOn
+ + ",\"revision_data\":{\"commit_object\":{\"sha1\":\""
+ + revisionA.getCommitObject().getSha1()
+ + "\",\"type\":1,\"content\":\"Y29tbWl0YWNvbnRlbnQ\\u003d\"},\"tree_object\":{\"sha1\":\""
+ + revisionA.getTreeObject().getSha1()
+ + "\",\"type\":2,\"content\":\"dHJlZWFjb250ZW50\"},\"blobs\":[{\"sha1\":\""
+ + revisionA.getBlobs().get(0).getSha1()
+ + "\",\"type\":3,\"content\":\"YmxvYmFjb250ZW50\"}]}},"
+ + "{\"label\":\"Replication\",\"ref_name\":\""
+ + refNameB
+ + "\",\"event_created_on\":"
+ + eventCreatedOn
+ + ",\"revision_data\":{\"commit_object\":{\"sha1\":\""
+ + revisionB.getCommitObject().getSha1()
+ + "\",\"type\":1,\"content\":\"Y29tbWl0YmNvbnRlbnQ\\u003d\"},\"tree_object\":{\"sha1\":\""
+ + revisionB.getTreeObject().getSha1()
+ + "\",\"type\":2,\"content\":\"dHJlZWJjb250ZW50\"},\"blobs\":[{\"sha1\":\""
+ + revisionB.getBlobs().get(0).getSha1()
+ + "\",\"type\":3,\"content\":\"YmxvYmJjb250ZW50\"}]}}]";
+ assertThat(readPayload(httpPost)).isEqualTo(expectedSendObjectsPayload);
+ }
+
+ @Test
+ public void shouldCallBatchApplyObjectEndpointWithNoRevisionDataForDeletes() throws Exception {
+ List<BatchApplyObjectData> batchApplyObjects = new ArrayList<>();
+ batchApplyObjects.add(BatchApplyObjectData.create(refName, Optional.empty(), true));
+
+ objectUnderTest.callBatchSendObject(
+ Project.nameKey("test_repo"), batchApplyObjects, eventCreatedOn, new URIish(api));
+
+ verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any());
+
+ HttpPost httpPost = httpPostCaptor.getValue();
+
+ String expectedSendObjectsPayload =
+ "[{\"label\":\"Replication\",\"ref_name\":\""
+ + refName
+ + "\",\"event_created_on\":"
+ + eventCreatedOn
+ + "}]";
+ assertThat(readPayload(httpPost)).isEqualTo(expectedSendObjectsPayload);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void shouldThrowExceptionIfDeleteFlagIsSetButRevisionDataIsPresentForBatchSendEndpoint()
+ throws Exception {
+ List<BatchApplyObjectData> batchApplyObjects = new ArrayList<>();
+ batchApplyObjects.add(
+ BatchApplyObjectData.create(refName, Optional.of(createSampleRevisionData()), true));
+
+ objectUnderTest.callBatchSendObject(
+ Project.nameKey("test_repo"), batchApplyObjects, eventCreatedOn, new URIish(api));
+ }
+
public String readPayload(HttpPost entity) throws Exception {
ByteBuffer buf = IO.readWholeStream(entity.getEntity().getContent(), 1024);
return RawParseUtils.decode(buf.array(), buf.arrayOffset(), buf.limit()).trim();
}
- private RevisionData createSampleRevisionData() {
+ private RevisionData createSampleRevisionData(String prefix) {
+ String commitPrefix = "commit" + prefix;
+ String treePrefix = "tree" + prefix;
+ String blobPrefix = "blob" + prefix;
+ return createSampleRevisionData(
+ commitPrefix,
+ commitPrefix + "content",
+ treePrefix,
+ treePrefix + "content",
+ blobPrefix,
+ blobPrefix + "content");
+ }
+
+ private RevisionData createSampleRevisionData(
+ String commitObjectId,
+ String commitContent,
+ String treeObjectId,
+ String treeContent,
+ String blobObjectId,
+ String blobContent) {
RevisionObjectData commitData =
- new RevisionObjectData(commitObjectId, Constants.OBJ_COMMIT, commitObject.getBytes());
+ new RevisionObjectData(commitObjectId, Constants.OBJ_COMMIT, commitContent.getBytes());
RevisionObjectData treeData =
- new RevisionObjectData(treeObjectId, Constants.OBJ_TREE, treeObject.getBytes());
+ new RevisionObjectData(treeObjectId, Constants.OBJ_TREE, treeContent.getBytes());
RevisionObjectData blobData =
- new RevisionObjectData(blobObjectId, Constants.OBJ_BLOB, blobObject.getBytes());
+ new RevisionObjectData(blobObjectId, Constants.OBJ_BLOB, blobContent.getBytes());
return new RevisionData(
Collections.emptyList(), commitData, treeData, Lists.newArrayList(blobData));
}
+
+ private RevisionData createSampleRevisionData() {
+ return createSampleRevisionData(
+ commitObjectId, commitObject, treeObjectId, treeObject, blobObjectId, blobObject);
+ }
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientWithBasicAuthenticationTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientWithBasicAuthenticationTest.java
index b4d4e7a..165ed80 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientWithBasicAuthenticationTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientWithBasicAuthenticationTest.java
@@ -16,7 +16,10 @@
import static com.google.common.truth.Truth.assertThat;
import static javax.servlet.http.HttpServletResponse.SC_CREATED;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
import com.googlesource.gerrit.plugins.replication.pull.filter.SyncRefsFilter;
import java.util.Optional;
@@ -36,7 +39,7 @@
@Before
public void setup() throws Exception {
when(bearerTokenProvider.get()).thenReturn(Optional.empty());
- when(credentialProvider.supports(any()))
+ when(credentialProvider.supports(any(CredentialItem[].class)))
.thenAnswer(
new Answer<Boolean>() {
@@ -51,7 +54,7 @@
}
});
- when(credentialProvider.get(any(), any(CredentialItem.class))).thenReturn(true);
+ when(credentialProvider.get(any(), any(CredentialItem[].class))).thenReturn(true);
when(credentials.create(anyString())).thenReturn(credentialProvider);
when(replicationConfig.getConfig()).thenReturn(config);
when(config.getStringList("replication", null, "syncRefs")).thenReturn(new String[0]);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientWithBearerTokenTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientWithBearerTokenTest.java
index fe50b5c..9c662a9 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientWithBearerTokenTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientWithBearerTokenTest.java
@@ -16,7 +16,9 @@
import static com.google.common.truth.Truth.assertThat;
import static javax.servlet.http.HttpServletResponse.SC_CREATED;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
import com.googlesource.gerrit.plugins.replication.pull.filter.SyncRefsFilter;
import java.util.Optional;
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultTest.java
index d82f3a5..3f73729 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultTest.java
@@ -35,6 +35,7 @@
{HttpServletResponse.SC_ACCEPTED, true},
{HttpServletResponse.SC_NO_CONTENT, true},
{HttpServletResponse.SC_BAD_REQUEST, false},
+ {HttpServletResponse.SC_NOT_FOUND, false},
{HttpServletResponse.SC_CONFLICT, false}
});
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandlerTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandlerTest.java
deleted file mode 100644
index c4c35ed..0000000
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandlerTest.java
+++ /dev/null
@@ -1,136 +0,0 @@
-// Copyright (C) 2021 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.replication.pull.event;
-
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-import com.google.gerrit.entities.Change;
-import com.google.gerrit.entities.Project;
-import com.google.gerrit.server.index.change.ChangeIndexer;
-import com.googlesource.gerrit.plugins.replication.pull.Context;
-import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent;
-import com.googlesource.gerrit.plugins.replication.pull.ReplicationState;
-import org.eclipse.jgit.lib.RefUpdate;
-import org.eclipse.jgit.transport.URIish;
-import org.junit.Before;
-import org.junit.Test;
-
-public class FetchRefReplicatedEventHandlerTest {
- private ChangeIndexer changeIndexerMock;
- private FetchRefReplicatedEventHandler fetchRefReplicatedEventHandler;
- private static URIish sourceUri;
-
- @Before
- public void setUp() throws Exception {
- changeIndexerMock = mock(ChangeIndexer.class);
- fetchRefReplicatedEventHandler = new FetchRefReplicatedEventHandler(changeIndexerMock);
- sourceUri = new URIish("git://aSourceNode/testProject.git");
- }
-
- @Test
- public void onEventShouldIndexExistingChange() {
- Project.NameKey projectNameKey = Project.nameKey("testProject");
- String ref = "refs/changes/41/41/meta";
- Change.Id changeId = Change.Id.fromRef(ref);
- try {
- Context.setLocalEvent(true);
- fetchRefReplicatedEventHandler.onEvent(
- new FetchRefReplicatedEvent(
- projectNameKey.get(),
- ref,
- sourceUri,
- ReplicationState.RefFetchResult.SUCCEEDED,
- RefUpdate.Result.FAST_FORWARD));
- verify(changeIndexerMock, times(1)).indexAsync(eq(projectNameKey), eq(changeId));
- } finally {
- Context.unsetLocalEvent();
- }
- }
-
- @Test
- public void onEventShouldNotIndexIfNotLocalEvent() {
- Project.NameKey projectNameKey = Project.nameKey("testProject");
- String ref = "refs/changes/41/41/meta";
- Change.Id changeId = Change.Id.fromRef(ref);
- fetchRefReplicatedEventHandler.onEvent(
- new FetchRefReplicatedEvent(
- projectNameKey.get(),
- ref,
- sourceUri,
- ReplicationState.RefFetchResult.SUCCEEDED,
- RefUpdate.Result.FAST_FORWARD));
- verify(changeIndexerMock, never()).index(eq(projectNameKey), eq(changeId));
- }
-
- @Test
- public void onEventShouldIndexOnlyMetaRef() {
- Project.NameKey projectNameKey = Project.nameKey("testProject");
- String ref = "refs/changes/41/41/1";
- Change.Id changeId = Change.Id.fromRef(ref);
- fetchRefReplicatedEventHandler.onEvent(
- new FetchRefReplicatedEvent(
- projectNameKey.get(),
- ref,
- sourceUri,
- ReplicationState.RefFetchResult.SUCCEEDED,
- RefUpdate.Result.FAST_FORWARD));
- verify(changeIndexerMock, never()).index(eq(projectNameKey), eq(changeId));
- }
-
- @Test
- public void onEventShouldNotIndexMissingChange() {
- fetchRefReplicatedEventHandler.onEvent(
- new FetchRefReplicatedEvent(
- Project.nameKey("testProject").get(),
- "invalidRef",
- sourceUri,
- ReplicationState.RefFetchResult.SUCCEEDED,
- RefUpdate.Result.FAST_FORWARD));
- verify(changeIndexerMock, never()).index(any(), any());
- }
-
- @Test
- public void onEventShouldNotIndexFailingChange() {
- Project.NameKey projectNameKey = Project.nameKey("testProject");
- String ref = "refs/changes/41/41/meta";
- fetchRefReplicatedEventHandler.onEvent(
- new FetchRefReplicatedEvent(
- projectNameKey.get(),
- ref,
- sourceUri,
- ReplicationState.RefFetchResult.FAILED,
- RefUpdate.Result.FAST_FORWARD));
- verify(changeIndexerMock, never()).index(any(), any());
- }
-
- @Test
- public void onEventShouldNotIndexNotAttemptedChange() {
- Project.NameKey projectNameKey = Project.nameKey("testProject");
- String ref = "refs/changes/41/41/meta";
- fetchRefReplicatedEventHandler.onEvent(
- new FetchRefReplicatedEvent(
- projectNameKey.get(),
- ref,
- sourceUri,
- ReplicationState.RefFetchResult.NOT_ATTEMPTED,
- RefUpdate.Result.FAST_FORWARD));
- verify(changeIndexerMock, never()).index(any(), any());
- }
-}