Merge branch 'stable-3.3' into stable-3.4
* stable-3.3:
Do not rely on System.nanoTime for E2E metrics
Introduce the apply-objects REST-API for the whole '/meta' chain
Fix the processing of an empty HTML response body from REST-API
Fix issue with ref deletion and global-refdb
Fix issue with fetching all refs after project creation
Always fallback to fetch when ApplyObject REST-API fails
Log the reason why a ref object wasn't loaded by RevisionReader
Consider any HTTP 2xx response code from REST-API as success
Return NO_CONTENT when removing a ref through ApplyObject
Introduce E2E fetch REST-API metrics
Fix ApplyObjectIT.shouldApplyRefMetaObject test for apply object
Introduce E2E apply object REST-API metrics
Add missing @Override to parseRemotes
Add more logging for the apply object REST-API
Support ApplyObject of non-commit refs
Change-Id: I721852cb091ad12c2fc94cb124b01968b0bad78d
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectMetrics.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectMetrics.java
index 78b6ddb..78745bb 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectMetrics.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectMetrics.java
@@ -26,25 +26,33 @@
@Singleton
public class ApplyObjectMetrics {
private final Timer1<String> executionTime;
+ private final Timer1<String> end2EndTime;
@Inject
ApplyObjectMetrics(@PluginName String pluginName, MetricMaker metricMaker) {
- Field<String> SOURCE_FIELD =
+ Field<String> field =
Field.ofString(
- "source",
+ "pull_replication",
(metadataBuilder, fieldValue) ->
metadataBuilder
.pluginName(pluginName)
- .addPluginMetadata(PluginMetadata.create("source", fieldValue)))
+ .addPluginMetadata(PluginMetadata.create("pull_replication", fieldValue)))
.build();
-
executionTime =
metricMaker.newTimer(
"apply_object_latency",
new Description("Time spent applying object from remote source.")
.setCumulative()
.setUnit(Description.Units.MILLISECONDS),
- SOURCE_FIELD);
+ field);
+
+ end2EndTime =
+ metricMaker.newTimer(
+ "apply_object_end_2_end_latency",
+ new Description("Time spent for e2e replication with the apply object REST API")
+ .setCumulative()
+ .setUnit(Description.Units.MILLISECONDS),
+ field);
}
/**
@@ -56,4 +64,14 @@
public Timer1.Context<String> start(String name) {
return executionTime.start(name);
}
+
+ /**
+ * Start the replication latency timer from a source.
+ *
+ * @param name the source name.
+ * @return the timer context.
+ */
+ public Timer1.Context<String> startEnd2End(String name) {
+ return end2EndTime.start(name);
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchAll.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchAll.java
index bcd86d7..42310ff 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchAll.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchAll.java
@@ -21,6 +21,7 @@
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.googlesource.gerrit.plugins.replication.ReplicationFilter;
+import java.util.Optional;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.eclipse.jgit.transport.URIish;
@@ -91,7 +92,7 @@
for (Source cfg : sources.getAll()) {
if (cfg.wouldFetchProject(project)) {
for (URIish uri : cfg.getURIs(project, urlMatch)) {
- cfg.schedule(project, FetchOne.ALL_REFS, uri, state, replicationType);
+ cfg.schedule(project, FetchOne.ALL_REFS, uri, state, replicationType, Optional.empty());
}
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java
index 9b2f905..4fad8f9 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java
@@ -32,6 +32,7 @@
import com.google.gerrit.server.util.IdGenerator;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
+import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics;
import com.googlesource.gerrit.plugins.replication.pull.fetch.Fetch;
import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory;
import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState;
@@ -69,7 +70,8 @@
static final String ID_KEY = "fetchOneId";
interface Factory {
- FetchOne create(Project.NameKey d, URIish u);
+ FetchOne create(
+ Project.NameKey d, URIish u, Optional<PullReplicationApiRequestMetrics> apiRequestMetrics);
}
private final GitRepositoryManager gitManager;
@@ -94,6 +96,7 @@
private final FetchReplicationMetrics metrics;
private final AtomicBoolean canceledWhileRunning;
private final FetchFactory fetchFactory;
+ private final Optional<PullReplicationApiRequestMetrics> apiRequestMetrics;
@Inject
FetchOne(
@@ -106,7 +109,8 @@
FetchReplicationMetrics m,
FetchFactory fetchFactory,
@Assisted Project.NameKey d,
- @Assisted URIish u) {
+ @Assisted URIish u,
+ @Assisted Optional<PullReplicationApiRequestMetrics> apiRequestMetrics) {
gitManager = grm;
pool = s;
config = c.getRemoteConfig();
@@ -122,6 +126,7 @@
canceledWhileRunning = new AtomicBoolean(false);
this.fetchFactory = fetchFactory;
maxRetries = s.getMaxRetries();
+ this.apiRequestMetrics = apiRequestMetrics;
}
@Override
@@ -299,12 +304,17 @@
git = gitManager.openRepository(projectName);
runImpl();
long elapsed = NANOSECONDS.toMillis(context.stop());
+ Optional<Long> elapsedEnd2End =
+ apiRequestMetrics
+ .flatMap(metrics -> metrics.stop(config.getName()))
+ .map(NANOSECONDS::toMillis);
repLog.info(
- "Replication from {} completed in {}ms, {}ms delay, {} retries",
+ "Replication from {} completed in {}ms, {}ms delay, {} retries{}",
uri,
elapsed,
delay,
- retryCount);
+ retryCount,
+ elapsedEnd2End.map(el -> String.format(", E2E %dms", el)).orElse(""));
} catch (RepositoryNotFoundException e) {
stateLog.error(
"Cannot replicate " + projectName + "; Local repository error: " + e.getMessage(),
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchReplicationMetrics.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchReplicationMetrics.java
index e952252..22bb073 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchReplicationMetrics.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchReplicationMetrics.java
@@ -23,10 +23,12 @@
import com.google.gerrit.server.logging.PluginMetadata;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+import java.util.concurrent.TimeUnit;
@Singleton
public class FetchReplicationMetrics {
private final Timer1<String> executionTime;
+ private final Timer1<String> end2EndExecutionTime;
private final Histogram1<String> executionDelay;
private final Histogram1<String> executionRetries;
@@ -34,11 +36,11 @@
FetchReplicationMetrics(@PluginName String pluginName, MetricMaker metricMaker) {
Field<String> SOURCE_FIELD =
Field.ofString(
- "source",
+ "pull_replication",
(metadataBuilder, fieldValue) ->
metadataBuilder
.pluginName(pluginName)
- .addPluginMetadata(PluginMetadata.create("source", fieldValue)))
+ .addPluginMetadata(PluginMetadata.create("pull_replication", fieldValue)))
.build();
executionTime =
@@ -49,6 +51,14 @@
.setUnit(Description.Units.MILLISECONDS),
SOURCE_FIELD);
+ end2EndExecutionTime =
+ metricMaker.newTimer(
+ "replication_end_2_end_latency",
+ new Description("Time spent end-2-end fetching from remote source.")
+ .setCumulative()
+ .setUnit(Description.Units.MILLISECONDS),
+ SOURCE_FIELD);
+
executionDelay =
metricMaker.newHistogram(
"replication_delay",
@@ -77,6 +87,26 @@
}
/**
+ * Start the end-to-end replication latency timer from a source.
+ *
+ * @param name the source name.
+ * @return the timer context.
+ */
+ public Timer1.Context<String> startEnd2End(String name) {
+ return end2EndExecutionTime.start(name);
+ }
+
+ /**
+ * Record the end-to-end replication latency timer from a source.
+ *
+ * @param name the source name.
+ * @param metricNanos the timer value in nanos
+ */
+ public void recordEnd2End(String name, long metricNanos) {
+ end2EndExecutionTime.record(name, metricNanos, TimeUnit.NANOSECONDS);
+ }
+
+ /**
* Record the replication delay and retry metrics for a source.
*
* @param name the source name.
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
index cfe01e2..f3e38fd 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
@@ -15,14 +15,17 @@
package com.googlesource.gerrit.plugins.replication.pull;
import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Queues;
import com.google.gerrit.entities.Project;
import com.google.gerrit.entities.Project.NameKey;
+import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.events.GitReferenceUpdatedListener;
import com.google.gerrit.extensions.events.HeadUpdatedListener;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.extensions.events.ProjectDeletedListener;
import com.google.gerrit.extensions.registration.DynamicItem;
+import com.google.gerrit.metrics.Timer1.Context;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.git.WorkQueue;
import com.google.inject.Inject;
@@ -36,7 +39,10 @@
import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter;
import java.io.IOException;
import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashSet;
+import java.util.List;
import java.util.Optional;
import java.util.Queue;
import java.util.Set;
@@ -46,7 +52,11 @@
import java.util.concurrent.TimeoutException;
import java.util.function.Consumer;
import org.apache.http.client.ClientProtocolException;
+import org.eclipse.jgit.errors.CorruptObjectException;
+import org.eclipse.jgit.errors.IncorrectObjectTypeException;
import org.eclipse.jgit.errors.InvalidObjectIdException;
+import org.eclipse.jgit.errors.MissingObjectException;
+import org.eclipse.jgit.errors.RepositoryNotFoundException;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.transport.URIish;
import org.slf4j.Logger;
@@ -74,7 +84,9 @@
private FetchApiClient.Factory fetchClientFactory;
private Integer fetchCallsTimeout;
private ExcludedRefsFilter refsFilter;
- private RevisionReader revisionReader;
+ private Provider<RevisionReader> revReaderProvider;
+ private final ApplyObjectMetrics applyObjectMetrics;
+ private final FetchReplicationMetrics fetchMetrics;
@Inject
ReplicationQueue(
@@ -84,7 +96,9 @@
ReplicationStateListeners sl,
FetchApiClient.Factory fetchClientFactory,
ExcludedRefsFilter refsFilter,
- RevisionReader revReader) {
+ Provider<RevisionReader> revReaderProvider,
+ ApplyObjectMetrics applyObjectMetrics,
+ FetchReplicationMetrics fetchMetrics) {
workQueue = wq;
dispatcher = dis;
sources = rd;
@@ -92,7 +106,9 @@
beforeStartupEventsQueue = Queues.newConcurrentLinkedQueue();
this.fetchClientFactory = fetchClientFactory;
this.refsFilter = refsFilter;
- this.revisionReader = revReader;
+ this.revReaderProvider = revReaderProvider;
+ this.applyObjectMetrics = applyObjectMetrics;
+ this.fetchMetrics = fetchMetrics;
}
@Override
@@ -225,9 +241,19 @@
CallFunction call = getCallFunction(project, objectId, refName, isDelete, state);
return (source) -> {
+ boolean callSuccessful;
try {
- call.call(source);
- } catch (MissingParentObjectException e) {
+ callSuccessful = call.call(source);
+ } catch (Exception e) {
+ repLog.warn(
+ String.format(
+ "Failed to apply object %s on project %s:%s, falling back to git fetch",
+ objectId.name(), project, refName),
+ e);
+ callSuccessful = false;
+ }
+
+ if (!callSuccessful) {
callFetch(source, project, refName, state);
}
};
@@ -244,10 +270,18 @@
}
try {
- Optional<RevisionData> revisionData = revisionReader.read(project, objectId, refName);
+ Optional<RevisionData> revisionData =
+ revReaderProvider.get().read(project, objectId, refName, 0);
+ repLog.info(
+ "RevisionData is {} for {}:{}",
+ revisionData.map(RevisionData::toString).orElse("ABSENT"),
+ project,
+ refName);
+
if (revisionData.isPresent()) {
return ((source) ->
- callSendObject(source, project, refName, isDelete, revisionData.get(), state));
+ callSendObject(
+ source, project, refName, isDelete, Arrays.asList(revisionData.get()), state));
}
} catch (InvalidObjectIdException | IOException e) {
stateLog.error(
@@ -261,94 +295,189 @@
return (source) -> callFetch(source, project, refName, state);
}
- private void callSendObject(
+ private boolean callSendObject(
Source source,
Project.NameKey project,
String refName,
boolean isDelete,
- RevisionData revision,
+ List<RevisionData> revision,
ReplicationState state)
throws MissingParentObjectException {
+ boolean resultIsSuccessful = true;
if (source.wouldFetchProject(project) && source.wouldFetchRef(refName)) {
for (String apiUrl : source.getApis()) {
try {
URIish uri = new URIish(apiUrl);
FetchApiClient fetchClient = fetchClientFactory.create(source);
+ repLog.info(
+ "Pull replication REST API apply object to {} for {}:{} - {}",
+ apiUrl,
+ project,
+ refName,
+ revision);
+ Context<String> apiTimer = applyObjectMetrics.startEnd2End(source.getRemoteConfigName());
+ HttpResult result =
+ isDelete
+ ? fetchClient.callSendObject(project, refName, isDelete, null, uri)
+ : fetchClient.callSendObjects(project, refName, revision, uri);
+ boolean resultSuccessful = result.isSuccessful();
+ repLog.info(
+ "Pull replication REST API apply object to {} COMPLETED for {}:{} - {}, HTTP Result:"
+ + " {} - time:{} ms",
+ apiUrl,
+ project,
+ refName,
+ revision,
+ result,
+ apiTimer.stop() / 1000000.0);
- HttpResult result = fetchClient.callSendObject(project, refName, isDelete, revision, uri);
- if (isProjectMissing(result, project) && source.isCreateMissingRepositories()) {
+ if (!resultSuccessful
+ && result.isProjectMissing(project)
+ && source.isCreateMissingRepositories()) {
result = initProject(project, uri, fetchClient, result);
+ repLog.info("Missing project {} created, HTTP Result:{}", project, result);
}
- if (!result.isSuccessful()) {
- repLog.warn(
- String.format(
- "Pull replication rest api apply object call failed. Endpoint url: %s, reason:%s",
- apiUrl, result.getMessage().orElse("unknown")));
+
+ if (!resultSuccessful) {
if (result.isParentObjectMissing()) {
+
+ if (RefNames.isNoteDbMetaRef(refName) && revision.size() == 1) {
+ List<RevisionData> allRevisions =
+ fetchWholeMetaHistory(project, refName, revision.get(0));
+ repLog.info(
+ "Pull replication REST API apply object to {} for {}:{} - {}",
+ apiUrl,
+ project,
+ refName,
+ allRevisions);
+ return callSendObject(source, project, refName, isDelete, allRevisions, state);
+ }
+
throw new MissingParentObjectException(
project, refName, source.getRemoteConfigName());
}
}
+
+ resultIsSuccessful &= resultSuccessful;
} catch (URISyntaxException e) {
+ repLog.warn(
+ "Pull replication REST API apply object to {} *FAILED* for {}:{} - {}",
+ apiUrl,
+ project,
+ refName,
+ revision,
+ e);
stateLog.error(String.format("Cannot parse pull replication api url:%s", apiUrl), state);
+ resultIsSuccessful = false;
} catch (IOException e) {
+ repLog.warn(
+ "Pull replication REST API apply object to {} *FAILED* for {}:{} - {}",
+ apiUrl,
+ project,
+ refName,
+ revision,
+ e);
stateLog.error(
String.format(
- "Exception during the pull replication fetch rest api call. Endpoint url:%s, message:%s",
+ "Exception during the pull replication fetch rest api call. Endpoint url:%s,"
+ + " message:%s",
apiUrl, e.getMessage()),
e,
state);
+ resultIsSuccessful = false;
}
}
}
+
+ return resultIsSuccessful;
}
- private void callFetch(
+ private List<RevisionData> fetchWholeMetaHistory(
+ NameKey project, String refName, RevisionData revision)
+ throws RepositoryNotFoundException, MissingObjectException, IncorrectObjectTypeException,
+ CorruptObjectException, IOException {
+ RevisionReader revisionReader = revReaderProvider.get();
+ Optional<RevisionData> revisionDataWithParents =
+ revisionReader.read(project, refName, Integer.MAX_VALUE);
+
+ ImmutableList.Builder<RevisionData> revisionDataBuilder = ImmutableList.builder();
+ List<ObjectId> parentObjectIds =
+ revisionDataWithParents
+ .map(RevisionData::getParentObjetIds)
+ .orElse(Collections.emptyList());
+ for (ObjectId parentObjectId : parentObjectIds) {
+ revisionReader.read(project, parentObjectId, refName, 0).ifPresent(revisionDataBuilder::add);
+ }
+
+ revisionDataBuilder.add(revision);
+
+ return revisionDataBuilder.build();
+ }
+
+ private boolean callFetch(
Source source, Project.NameKey project, String refName, ReplicationState state) {
+ boolean resultIsSuccessful = true;
if (source.wouldFetchProject(project) && source.wouldFetchRef(refName)) {
for (String apiUrl : source.getApis()) {
try {
URIish uri = new URIish(apiUrl);
FetchApiClient fetchClient = fetchClientFactory.create(source);
- HttpResult result = fetchClient.callFetch(project, refName, uri);
- if (isProjectMissing(result, project) && source.isCreateMissingRepositories()) {
+ repLog.info("Pull replication REST API fetch to {} for {}:{}", apiUrl, project, refName);
+ Context<String> timer = fetchMetrics.startEnd2End(source.getRemoteConfigName());
+ HttpResult result = fetchClient.callFetch(project, refName, uri, timer.getStartTime());
+ long elapsedMs = TimeUnit.NANOSECONDS.toMillis(timer.stop());
+ boolean resultSuccessful = result.isSuccessful();
+ repLog.info(
+ "Pull replication REST API fetch to {} COMPLETED for {}:{}, HTTP Result:"
+ + " {} - time:{} ms",
+ apiUrl,
+ project,
+ refName,
+ result,
+ elapsedMs);
+ if (!resultSuccessful
+ && result.isProjectMissing(project)
+ && source.isCreateMissingRepositories()) {
result = initProject(project, uri, fetchClient, result);
}
- if (!result.isSuccessful()) {
+ if (!resultSuccessful) {
stateLog.warn(
String.format(
"Pull replication rest api fetch call failed. Endpoint url: %s, reason:%s",
apiUrl, result.getMessage().orElse("unknown")),
state);
}
+
+ resultIsSuccessful &= result.isSuccessful();
} catch (URISyntaxException e) {
stateLog.error(String.format("Cannot parse pull replication api url:%s", apiUrl), state);
+ resultIsSuccessful = false;
} catch (Exception e) {
stateLog.error(
String.format(
- "Exception during the pull replication fetch rest api call. Endpoint url:%s, message:%s",
+ "Exception during the pull replication fetch rest api call. Endpoint url:%s,"
+ + " message:%s",
apiUrl, e.getMessage()),
e,
state);
+ resultIsSuccessful = false;
}
}
}
+
+ return resultIsSuccessful;
}
public boolean retry(int attempt, int maxRetries) {
return maxRetries == 0 || attempt < maxRetries;
}
- private Boolean isProjectMissing(HttpResult result, Project.NameKey project) {
- return !result.isSuccessful() && result.isProjectMissing(project);
- }
-
private HttpResult initProject(
Project.NameKey project, URIish uri, FetchApiClient fetchClient, HttpResult result)
throws IOException, ClientProtocolException {
HttpResult initProjectResult = fetchClient.initProject(project, uri);
if (initProjectResult.isSuccessful()) {
- result = fetchClient.callFetch(project, "refs/*", uri);
+ result = fetchClient.callFetch(project, FetchOne.ALL_REFS, uri);
} else {
String errorMessage = initProjectResult.getMessage().map(e -> " - Error: " + e).orElse("");
repLog.error("Cannot create project " + project + errorMessage);
@@ -399,6 +528,6 @@
@FunctionalInterface
private interface CallFunction {
- void call(Source source) throws MissingParentObjectException;
+ boolean call(Source source) throws MissingParentObjectException;
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java
index 468c5fc..db46b23 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java
@@ -17,6 +17,7 @@
import static com.googlesource.gerrit.plugins.replication.pull.PullReplicationLogger.repLog;
import com.google.common.collect.Lists;
+import com.google.gerrit.common.Nullable;
import com.google.gerrit.entities.Project;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.inject.Inject;
@@ -24,6 +25,9 @@
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Optional;
import org.eclipse.jgit.diff.DiffEntry;
@@ -36,6 +40,7 @@
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
+import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevTree;
@@ -45,8 +50,11 @@
public class RevisionReader {
private static final String CONFIG_MAX_API_PAYLOAD_SIZE = "maxApiPayloadSize";
private static final Long DEFAULT_MAX_PAYLOAD_SIZE_IN_BYTES = 10000L;
+ static final String CONFIG_MAX_API_HISTORY_DEPTH = "maxApiHistoryDepth";
+ private static final int DEFAULT_MAX_API_HISTORY_DEPTH = 128;
private GitRepositoryManager gitRepositoryManager;
private Long maxRefSize;
+ private final int maxDepth;
@Inject
public RevisionReader(GitRepositoryManager gitRepositoryManager, ReplicationConfig cfg) {
@@ -54,17 +62,49 @@
this.maxRefSize =
cfg.getConfig()
.getLong("replication", CONFIG_MAX_API_PAYLOAD_SIZE, DEFAULT_MAX_PAYLOAD_SIZE_IN_BYTES);
+ this.maxDepth =
+ cfg.getConfig()
+ .getInt("replication", CONFIG_MAX_API_HISTORY_DEPTH, DEFAULT_MAX_API_HISTORY_DEPTH);
}
- public Optional<RevisionData> read(Project.NameKey project, ObjectId objectId, String refName)
+ public Optional<RevisionData> read(
+ Project.NameKey project, String refName, int maxParentObjectIds)
+ throws RepositoryNotFoundException, MissingObjectException, IncorrectObjectTypeException,
+ CorruptObjectException, IOException {
+ return read(project, null, refName, maxParentObjectIds);
+ }
+
+ public Optional<RevisionData> read(
+ Project.NameKey project,
+ @Nullable ObjectId refObjectId,
+ String refName,
+ int maxParentObjectIds)
throws MissingObjectException, IncorrectObjectTypeException, CorruptObjectException,
RepositoryNotFoundException, IOException {
try (Repository git = gitRepositoryManager.openRepository(project)) {
Long totalRefSize = 0l;
+ Ref ref = git.exactRef(refName);
+ if (ref == null) {
+ return Optional.empty();
+ }
+
+ ObjectId objectId = refObjectId == null ? ref.getObjectId() : refObjectId;
+
ObjectLoader commitLoader = git.open(objectId);
totalRefSize += commitLoader.getSize();
- verifySize(totalRefSize, commitLoader);
+ verifySize(project, refName, objectId, totalRefSize, commitLoader);
+
+ if (commitLoader.getType() == Constants.OBJ_BLOB) {
+ return Optional.of(
+ new RevisionData(
+ Collections.emptyList(),
+ null,
+ null,
+ Arrays.asList(
+ new RevisionObjectData(
+ objectId.name(), Constants.OBJ_BLOB, commitLoader.getCachedBytes()))));
+ }
if (commitLoader.getType() != Constants.OBJ_COMMIT) {
repLog.trace(
@@ -77,28 +117,34 @@
RevCommit commit = RevCommit.parse(commitLoader.getCachedBytes());
RevisionObjectData commitRev =
- new RevisionObjectData(commit.getType(), commitLoader.getCachedBytes());
+ new RevisionObjectData(objectId.name(), commit.getType(), commitLoader.getCachedBytes());
RevTree tree = commit.getTree();
- ObjectLoader treeLoader = git.open(commit.getTree().toObjectId());
+ ObjectId treeObjectId = commit.getTree().toObjectId();
+ ObjectLoader treeLoader = git.open(treeObjectId);
totalRefSize += treeLoader.getSize();
- verifySize(totalRefSize, treeLoader);
+ verifySize(project, refName, treeObjectId, totalRefSize, treeLoader);
RevisionObjectData treeRev =
- new RevisionObjectData(tree.getType(), treeLoader.getCachedBytes());
+ new RevisionObjectData(treeObjectId.name(), tree.getType(), treeLoader.getCachedBytes());
List<RevisionObjectData> blobs = Lists.newLinkedList();
try (TreeWalk walk = new TreeWalk(git)) {
if (commit.getParentCount() > 0) {
List<DiffEntry> diffEntries = readDiffs(git, commit, tree, walk);
- blobs = readBlobs(git, totalRefSize, diffEntries);
+ blobs = readBlobs(project, refName, git, totalRefSize, diffEntries);
} else {
walk.setRecursive(true);
walk.addTree(tree);
- blobs = readBlobs(git, totalRefSize, walk);
+ blobs = readBlobs(project, refName, git, totalRefSize, walk);
}
}
- return Optional.of(new RevisionData(commitRev, treeRev, blobs));
+
+ List<ObjectId> parentObjectIds =
+ getParentObjectIds(git, commit.getParents(), 0, Math.min(maxDepth, maxParentObjectIds));
+ Collections.reverse(parentObjectIds);
+
+ return Optional.of(new RevisionData(parentObjectIds, commitRev, treeRev, blobs));
} catch (LargeObjectException e) {
repLog.trace(
"Ref {} size for project {} is greater than configured '{}'",
@@ -109,6 +155,32 @@
}
}
+ private List<ObjectId> getParentObjectIds(
+ Repository git, RevCommit[] commit, int parentsDepth, int maxParentObjectIds)
+ throws MissingObjectException, IncorrectObjectTypeException, IOException {
+ if (commit == null || commit.length == 0) {
+ return Collections.emptyList();
+ }
+
+ ArrayList<ObjectId> parentObjectIds = new ArrayList<>();
+ for (RevCommit revCommit : commit) {
+ if (parentsDepth < maxParentObjectIds) {
+ parentObjectIds.add(revCommit.getId());
+ parentsDepth++;
+
+ ObjectLoader ol = git.open(revCommit.getId(), Constants.OBJ_COMMIT);
+ RevCommit[] commitParents = RevCommit.parse(ol.getCachedBytes()).getParents();
+
+ List<ObjectId> nestedParentObjectIds =
+ getParentObjectIds(git, commitParents, parentsDepth, maxParentObjectIds);
+ parentObjectIds.addAll(nestedParentObjectIds);
+ parentsDepth += nestedParentObjectIds.size();
+ }
+ }
+
+ return parentObjectIds;
+ }
+
private List<DiffEntry> readDiffs(Repository git, RevCommit commit, RevTree tree, TreeWalk walk)
throws MissingObjectException, IncorrectObjectTypeException, CorruptObjectException,
IOException {
@@ -117,7 +189,8 @@
return DiffEntry.scan(walk, true);
}
- private List<RevisionObjectData> readBlobs(Repository git, Long totalRefSize, TreeWalk walk)
+ private List<RevisionObjectData> readBlobs(
+ Project.NameKey projectName, String refName, Repository git, Long totalRefSize, TreeWalk walk)
throws MissingObjectException, IncorrectObjectTypeException, CorruptObjectException,
IOException {
List<RevisionObjectData> blobs = Lists.newLinkedList();
@@ -125,26 +198,33 @@
ObjectId objectId = walk.getObjectId(0);
ObjectLoader objectLoader = git.open(objectId);
totalRefSize += objectLoader.getSize();
- verifySize(totalRefSize, objectLoader);
+ verifySize(projectName, refName, objectId, totalRefSize, objectLoader);
RevisionObjectData rev =
- new RevisionObjectData(objectLoader.getType(), objectLoader.getCachedBytes());
+ new RevisionObjectData(
+ objectId.name(), objectLoader.getType(), objectLoader.getCachedBytes());
blobs.add(rev);
}
return blobs;
}
private List<RevisionObjectData> readBlobs(
- Repository git, Long totalRefSize, List<DiffEntry> diffEntries)
+ Project.NameKey projectName,
+ String refName,
+ Repository git,
+ Long totalRefSize,
+ List<DiffEntry> diffEntries)
throws MissingObjectException, IOException {
List<RevisionObjectData> blobs = Lists.newLinkedList();
for (DiffEntry diffEntry : diffEntries) {
if (!ChangeType.DELETE.equals(diffEntry.getChangeType())) {
- ObjectLoader objectLoader = git.open(diffEntry.getNewId().toObjectId());
+ ObjectId diffObjectId = diffEntry.getNewId().toObjectId();
+ ObjectLoader objectLoader = git.open(diffObjectId);
totalRefSize += objectLoader.getSize();
- verifySize(totalRefSize, objectLoader);
+ verifySize(projectName, refName, diffObjectId, totalRefSize, objectLoader);
RevisionObjectData rev =
- new RevisionObjectData(objectLoader.getType(), objectLoader.getCachedBytes());
+ new RevisionObjectData(
+ diffObjectId.name(), objectLoader.getType(), objectLoader.getCachedBytes());
blobs.add(rev);
}
}
@@ -159,9 +239,44 @@
return parentCommit.getTree();
}
- private void verifySize(Long totalRefSize, ObjectLoader loader) throws LargeObjectException {
- if (loader.isLarge() || totalRefSize > maxRefSize) {
- throw new LargeObjectException();
+ private void verifySize(
+ Project.NameKey projectName,
+ String refName,
+ ObjectId objectId,
+ Long totalRefSize,
+ ObjectLoader loader)
+ throws LargeObjectException {
+ if (loader.isLarge()) {
+ repLog.warn(
+ "Objects associated with {}:{} ({}) are too big to fit into the object loader's memory",
+ projectName,
+ refName,
+ objectTypeToString(loader.getType()));
+ throw new LargeObjectException(objectId);
+ }
+
+ if (totalRefSize > maxRefSize) {
+ repLog.warn(
+ "Objects associated with {}:{} ({}) use {} bytes, over the maximum limit of {} bytes",
+ projectName,
+ refName,
+ objectTypeToString(loader.getType()),
+ totalRefSize,
+ maxRefSize);
+ throw new LargeObjectException(objectId);
+ }
+ }
+
+ private static String objectTypeToString(int type) {
+ switch (type) {
+ case Constants.OBJ_BLOB:
+ return "BLOB";
+ case Constants.OBJ_COMMIT:
+ return "COMMIT";
+ case Constants.OBJ_TREE:
+ return "TREE";
+ default:
+ return "type:" + type;
}
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
index 71ec666..6bf4c21 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
@@ -61,6 +61,7 @@
import com.google.inject.servlet.RequestScoped;
import com.googlesource.gerrit.plugins.replication.RemoteSiteUser;
import com.googlesource.gerrit.plugins.replication.ReplicationFilter;
+import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics;
import com.googlesource.gerrit.plugins.replication.pull.fetch.BatchFetchClient;
import com.googlesource.gerrit.plugins.replication.pull.fetch.CGitFetch;
import com.googlesource.gerrit.plugins.replication.pull.fetch.CGitFetchValidator;
@@ -392,9 +393,10 @@
Project.NameKey project,
String ref,
ReplicationState state,
- ReplicationType replicationType) {
+ ReplicationType replicationType,
+ Optional<PullReplicationApiRequestMetrics> apiRequestMetrics) {
URIish uri = getURI(project);
- return schedule(project, ref, uri, state, replicationType);
+ return schedule(project, ref, uri, state, replicationType, apiRequestMetrics);
}
public Future<?> schedule(
@@ -402,7 +404,8 @@
String ref,
URIish uri,
ReplicationState state,
- ReplicationType replicationType) {
+ ReplicationType replicationType,
+ Optional<PullReplicationApiRequestMetrics> apiRequestMetrics) {
repLog.info("scheduling replication {}:{} => {}", uri, ref, project);
if (!shouldReplicate(project, ref, state)) {
@@ -438,7 +441,7 @@
FetchOne e = pending.get(uri);
Future<?> f = CompletableFuture.completedFuture(null);
if (e == null) {
- e = opFactory.create(project, uri);
+ e = opFactory.create(project, uri, apiRequestMetrics);
addRef(e, ref);
e.addState(ref, state);
pending.put(uri, e);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfigParser.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfigParser.java
index 6dfed44..a8799c2 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfigParser.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfigParser.java
@@ -35,6 +35,7 @@
/* (non-Javadoc)
* @see com.googlesource.gerrit.plugins.replication.ConfigParser#parseRemotes(org.eclipse.jgit.lib.Config)
*/
+ @Override
public List<RemoteConfiguration> parseRemotes(Config config) throws ConfigInvalidException {
if (config.getSections().isEmpty()) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java
index 5132f41..04cc9e8 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java
@@ -14,6 +14,8 @@
package com.googlesource.gerrit.plugins.replication.pull.api;
+import static com.googlesource.gerrit.plugins.replication.pull.PullReplicationLogger.repLog;
+
import com.google.common.base.Strings;
import com.google.gerrit.extensions.restapi.AuthException;
import com.google.gerrit.extensions.restapi.BadRequestException;
@@ -29,19 +31,20 @@
import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException;
import java.io.IOException;
import java.util.Objects;
+import javax.servlet.http.HttpServletResponse;
public class ApplyObjectAction implements RestModifyView<ProjectResource, RevisionInput> {
- private final ApplyObjectCommand command;
+ private final ApplyObjectCommand applyObjectCommand;
private final DeleteRefCommand deleteRefCommand;
private final FetchPreconditions preConditions;
@Inject
public ApplyObjectAction(
- ApplyObjectCommand command,
+ ApplyObjectCommand applyObjectCommand,
DeleteRefCommand deleteRefCommand,
FetchPreconditions preConditions) {
- this.command = command;
+ this.applyObjectCommand = applyObjectCommand;
this.deleteRefCommand = deleteRefCommand;
this.preConditions = preConditions;
}
@@ -50,42 +53,78 @@
public Response<?> apply(ProjectResource resource, RevisionInput input) throws RestApiException {
if (!preConditions.canCallFetchApi()) {
- throw new AuthException("not allowed to call fetch command");
+ throw new AuthException("Not allowed to call fetch command");
}
+ if (Strings.isNullOrEmpty(input.getLabel())) {
+ throw new BadRequestException("Source label cannot be null or empty");
+ }
+ if (Strings.isNullOrEmpty(input.getRefName())) {
+ throw new BadRequestException("Ref-update refname cannot be null or empty");
+ }
+
try {
- if (Strings.isNullOrEmpty(input.getLabel())) {
- throw new BadRequestException("Source label cannot be null or empty");
- }
- if (Strings.isNullOrEmpty(input.getRefName())) {
- throw new BadRequestException("Ref-update refname cannot be null or empty");
- }
+ repLog.info(
+ "Apply object API from {} for {}:{} - {}",
+ resource.getNameKey(),
+ input.getLabel(),
+ input.getRefName(),
+ input.getRevisionData());
if (Objects.isNull(input.getRevisionData())) {
deleteRefCommand.deleteRef(resource.getNameKey(), input.getRefName(), input.getLabel());
- return Response.created(input);
+ repLog.info(
+ "Apply object API - REF DELETED - from {} for {}:{} - {}",
+ resource.getNameKey(),
+ input.getLabel(),
+ input.getRefName(),
+ input.getRevisionData());
+ return Response.withStatusCode(HttpServletResponse.SC_NO_CONTENT, "");
}
- if (Objects.isNull(input.getRevisionData().getCommitObject())
- || Objects.isNull(input.getRevisionData().getCommitObject().getContent())
- || input.getRevisionData().getCommitObject().getContent().length == 0
- || Objects.isNull(input.getRevisionData().getCommitObject().getType())) {
- throw new BadRequestException("Ref-update commit object cannot be null or empty");
+ try {
+ input.validate();
+ } catch (IllegalArgumentException e) {
+ BadRequestException bre =
+ new BadRequestException("Ref-update with invalid input: " + e.getMessage(), e);
+ repLog.error(
+ "Apply object API *FAILED* from {} for {}:{} - {}",
+ input.getLabel(),
+ resource.getNameKey(),
+ input.getRefName(),
+ input.getRevisionData(),
+ bre);
+ throw bre;
}
- if (Objects.isNull(input.getRevisionData().getTreeObject())
- || Objects.isNull(input.getRevisionData().getTreeObject().getContent())
- || Objects.isNull(input.getRevisionData().getTreeObject().getType())) {
- throw new BadRequestException("Ref-update tree object cannot be null");
- }
-
- command.applyObject(
+ applyObjectCommand.applyObject(
resource.getNameKey(), input.getRefName(), input.getRevisionData(), input.getLabel());
return Response.created(input);
} catch (MissingParentObjectException e) {
+ repLog.error(
+ "Apply object API *FAILED* from {} for {}:{} - {}",
+ input.getLabel(),
+ resource.getNameKey(),
+ input.getRefName(),
+ input.getRevisionData(),
+ e);
throw new ResourceConflictException(e.getMessage(), e);
} catch (NumberFormatException | IOException e) {
+ repLog.error(
+ "Apply object API *FAILED* from {} for {}:{} - {}",
+ input.getLabel(),
+ resource.getNameKey(),
+ input.getRefName(),
+ input.getRevisionData(),
+ e);
throw RestApiException.wrap(e.getMessage(), e);
} catch (RefUpdateException e) {
+ repLog.error(
+ "Apply object API *FAILED* from {} for {}:{} - {}",
+ input.getLabel(),
+ resource.getNameKey(),
+ input.getRefName(),
+ input.getRevisionData(),
+ e);
throw new UnprocessableEntityException(e.getMessage());
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java
index 276aa16..b27d8b7 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java
@@ -37,6 +37,7 @@
import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject;
import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Set;
import org.eclipse.jgit.lib.RefUpdate;
import org.eclipse.jgit.transport.RefSpec;
@@ -70,12 +71,25 @@
}
public void applyObject(
- Project.NameKey name, String refName, RevisionData revisionData, String sourceLabel)
+ Project.NameKey name, String refName, RevisionData revisionsData, String sourceLabel)
+ throws IOException, RefUpdateException, MissingParentObjectException {
+ applyObjects(name, refName, new RevisionData[] {revisionsData}, sourceLabel);
+ }
+
+ public void applyObjects(
+ Project.NameKey name, String refName, RevisionData[] revisionsData, String sourceLabel)
throws IOException, RefUpdateException, MissingParentObjectException {
- repLog.info("Apply object from {} for project {}, ref name {}", sourceLabel, name, refName);
+ repLog.info(
+ "Apply object from {} for {}:{} - {}",
+ sourceLabel,
+ name,
+ refName,
+ Arrays.toString(revisionsData));
Timer1.Context<String> context = metrics.start(sourceLabel);
- RefUpdateState refUpdateState = applyObject.apply(name, new RefSpec(refName), revisionData);
+
+ RefUpdateState refUpdateState = applyObject.apply(name, new RefSpec(refName), revisionsData);
+
long elapsed = NANOSECONDS.toMillis(context.stop());
try {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectsAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectsAction.java
new file mode 100644
index 0000000..a1e1f5b
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectsAction.java
@@ -0,0 +1,131 @@
+// Copyright (C) 2022 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.api;
+
+import static com.googlesource.gerrit.plugins.replication.pull.PullReplicationLogger.repLog;
+
+import com.google.common.base.Strings;
+import com.google.gerrit.extensions.restapi.AuthException;
+import com.google.gerrit.extensions.restapi.BadRequestException;
+import com.google.gerrit.extensions.restapi.ResourceConflictException;
+import com.google.gerrit.extensions.restapi.Response;
+import com.google.gerrit.extensions.restapi.RestApiException;
+import com.google.gerrit.extensions.restapi.RestModifyView;
+import com.google.gerrit.extensions.restapi.UnprocessableEntityException;
+import com.google.gerrit.server.project.ProjectResource;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionsInput;
+import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException;
+import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+import javax.servlet.http.HttpServletResponse;
+
+public class ApplyObjectsAction implements RestModifyView<ProjectResource, RevisionsInput> {
+
+ private final ApplyObjectCommand command;
+ private final DeleteRefCommand deleteRefCommand;
+ private final FetchPreconditions preConditions;
+
+ @Inject
+ public ApplyObjectsAction(
+ ApplyObjectCommand command,
+ DeleteRefCommand deleteRefCommand,
+ FetchPreconditions preConditions) {
+ this.command = command;
+ this.deleteRefCommand = deleteRefCommand;
+ this.preConditions = preConditions;
+ }
+
+ @Override
+ public Response<?> apply(ProjectResource resource, RevisionsInput input) throws RestApiException {
+ if (!preConditions.canCallFetchApi()) {
+ throw new AuthException("not allowed to call fetch command");
+ }
+
+ try {
+ if (Strings.isNullOrEmpty(input.getLabel())) {
+ throw new BadRequestException("Source label cannot be null or empty");
+ }
+ if (Strings.isNullOrEmpty(input.getRefName())) {
+ throw new BadRequestException("Ref-update refname cannot be null or empty");
+ }
+
+ repLog.info(
+ "Apply object API from {} for {}:{} - {}",
+ resource.getNameKey(),
+ input.getLabel(),
+ input.getRefName(),
+ Arrays.toString(input.getRevisionsData()));
+
+ if (Objects.isNull(input.getRevisionsData())) {
+ deleteRefCommand.deleteRef(resource.getNameKey(), input.getRefName(), input.getLabel());
+ repLog.info(
+ "Apply object API - REF DELETED - from {} for {}:{}",
+ resource.getNameKey(),
+ input.getLabel(),
+ input.getRefName());
+ return Response.withStatusCode(HttpServletResponse.SC_NO_CONTENT, "");
+ }
+
+ try {
+ input.validate();
+ } catch (IllegalArgumentException e) {
+ BadRequestException bre =
+ new BadRequestException("Ref-update with invalid input: " + e.getMessage(), e);
+ repLog.error(
+ "Apply object API *FAILED* from {} for {}:{} - {}",
+ input.getLabel(),
+ resource.getNameKey(),
+ input.getRefName(),
+ Arrays.toString(input.getRevisionsData()),
+ bre);
+ throw bre;
+ }
+
+ command.applyObjects(
+ resource.getNameKey(), input.getRefName(), input.getRevisionsData(), input.getLabel());
+ return Response.created(input);
+ } catch (MissingParentObjectException e) {
+ repLog.error(
+ "Apply object API *FAILED* from {} for {}:{} - {}",
+ input.getLabel(),
+ resource.getNameKey(),
+ input.getRefName(),
+ Arrays.toString(input.getRevisionsData()),
+ e);
+ throw new ResourceConflictException(e.getMessage(), e);
+ } catch (NumberFormatException | IOException e) {
+ repLog.error(
+ "Apply object API *FAILED* from {} for {}:{} - {}",
+ input.getLabel(),
+ resource.getNameKey(),
+ input.getRefName(),
+ Arrays.toString(input.getRevisionsData()),
+ e);
+ throw RestApiException.wrap(e.getMessage(), e);
+ } catch (RefUpdateException e) {
+ repLog.error(
+ "Apply object API *FAILED* from {} for {}:{} - {}",
+ input.getLabel(),
+ resource.getNameKey(),
+ input.getRefName(),
+ Arrays.toString(input.getRevisionsData()),
+ e);
+ throw new UnprocessableEntityException(e.getMessage());
+ }
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java
index 0c5e016..2a3a79d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java
@@ -19,41 +19,52 @@
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.entities.Project;
import com.google.gerrit.extensions.registration.DynamicItem;
-import com.google.gerrit.extensions.restapi.ResourceConflictException;
import com.google.gerrit.extensions.restapi.ResourceNotFoundException;
import com.google.gerrit.extensions.restapi.RestApiException;
import com.google.gerrit.server.events.EventDispatcher;
+import com.google.gerrit.server.git.LocalDiskRepositoryManager;
+import com.google.gerrit.server.permissions.PermissionBackend;
import com.google.gerrit.server.permissions.PermissionBackendException;
+import com.google.gerrit.server.permissions.RefPermission;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectState;
-import com.google.gerrit.server.restapi.project.DeleteRef;
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.replication.pull.Context;
import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent;
import com.googlesource.gerrit.plugins.replication.pull.PullReplicationStateLogger;
import com.googlesource.gerrit.plugins.replication.pull.ReplicationState;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState;
import java.io.IOException;
import java.util.Optional;
+import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.RefUpdate;
+import org.eclipse.jgit.lib.Repository;
public class DeleteRefCommand {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private final PullReplicationStateLogger fetchStateLog;
- private final DeleteRef deleteRef;
+ private final ApplyObject applyObject;
private final DynamicItem<EventDispatcher> eventDispatcher;
private final ProjectCache projectCache;
+ private final PermissionBackend permissionBackend;
+ private final LocalDiskRepositoryManager gitManager;
@Inject
public DeleteRefCommand(
PullReplicationStateLogger fetchStateLog,
ProjectCache projectCache,
- DeleteRef deleteRef,
- DynamicItem<EventDispatcher> eventDispatcher) {
+ ApplyObject applyObject,
+ PermissionBackend permissionBackend,
+ DynamicItem<EventDispatcher> eventDispatcher,
+ LocalDiskRepositoryManager gitManager) {
this.fetchStateLog = fetchStateLog;
this.projectCache = projectCache;
- this.deleteRef = deleteRef;
+ this.applyObject = applyObject;
this.eventDispatcher = eventDispatcher;
+ this.permissionBackend = permissionBackend;
+ this.gitManager = gitManager;
}
public void deleteRef(Project.NameKey name, String refName, String sourceLabel)
@@ -66,8 +77,15 @@
}
try {
+ projectState.get().checkStatePermitsWrite();
+ permissionBackend
+ .currentUser()
+ .project(projectState.get().getNameKey())
+ .ref(refName)
+ .check(RefPermission.DELETE);
+
Context.setLocalEvent(true);
- deleteRef.deleteSingleRef(projectState.get(), refName);
+ deleteRef(name, refName);
eventDispatcher
.get()
@@ -83,7 +101,7 @@
"Unexpected error while trying to delete ref '%s' on project %s and notifying it",
refName, name);
throw RestApiException.wrap(e.getMessage(), e);
- } catch (ResourceConflictException e) {
+ } catch (IOException e) {
eventDispatcher
.get()
.postEvent(
@@ -110,4 +128,18 @@
throw RestApiException.wrap(e.getMessage(), e);
}
}
+
+ private RefUpdateState deleteRef(Project.NameKey name, String refName) throws IOException {
+
+ try (Repository repository = gitManager.openRepository(name)) {
+ RefUpdate.Result result;
+ RefUpdate u = repository.updateRef(refName);
+ u.setExpectedOldObjectId(repository.exactRef(refName).getObjectId());
+ u.setNewObjectId(ObjectId.zeroId());
+ u.setForceUpdate(true);
+
+ result = u.delete();
+ return new RefUpdateState(refName, result);
+ }
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java
index 2c9583a..fdb4f8f 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java
@@ -104,7 +104,10 @@
@SuppressWarnings("unchecked")
WorkQueue.Task<Void> task =
(WorkQueue.Task<Void>)
- workQueue.getDefaultQueue().submit(fetchJobFactory.create(project, input));
+ workQueue
+ .getDefaultQueue()
+ .submit(
+ fetchJobFactory.create(project, input, PullReplicationApiRequestMetrics.get()));
Optional<String> url =
urlFormatter
.get()
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java
index e1ac9ac..3a502ef 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java
@@ -54,19 +54,28 @@
this.eventDispatcher = eventDispatcher;
}
- public void fetchAsync(Project.NameKey name, String label, String refName)
+ public void fetchAsync(
+ Project.NameKey name,
+ String label,
+ String refName,
+ PullReplicationApiRequestMetrics apiRequestMetrics)
throws InterruptedException, ExecutionException, RemoteConfigurationMissingException,
TimeoutException {
- fetch(name, label, refName, ASYNC);
+ fetch(name, label, refName, ASYNC, Optional.of(apiRequestMetrics));
}
public void fetchSync(Project.NameKey name, String label, String refName)
throws InterruptedException, ExecutionException, RemoteConfigurationMissingException,
TimeoutException {
- fetch(name, label, refName, SYNC);
+ fetch(name, label, refName, SYNC, Optional.empty());
}
- private void fetch(Project.NameKey name, String label, String refName, ReplicationType fetchType)
+ private void fetch(
+ Project.NameKey name,
+ String label,
+ String refName,
+ ReplicationType fetchType,
+ Optional<PullReplicationApiRequestMetrics> apiRequestMetrics)
throws InterruptedException, ExecutionException, RemoteConfigurationMissingException,
TimeoutException {
ReplicationState state =
@@ -82,7 +91,7 @@
try {
state.markAllFetchTasksScheduled();
- Future<?> future = source.get().schedule(name, refName, state, fetchType);
+ Future<?> future = source.get().schedule(name, refName, state, fetchType, apiRequestMetrics);
future.get(source.get().getTimeout(), TimeUnit.SECONDS);
} catch (ExecutionException
| IllegalStateException
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchJob.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchJob.java
index f533734..9478d5c 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchJob.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchJob.java
@@ -26,25 +26,31 @@
private static final FluentLogger log = FluentLogger.forEnclosingClass();
public interface Factory {
- FetchJob create(Project.NameKey project, FetchAction.Input input);
+ FetchJob create(
+ Project.NameKey project, FetchAction.Input input, PullReplicationApiRequestMetrics metrics);
}
private FetchCommand command;
private Project.NameKey project;
private FetchAction.Input input;
+ private final PullReplicationApiRequestMetrics metrics;
@Inject
public FetchJob(
- FetchCommand command, @Assisted Project.NameKey project, @Assisted FetchAction.Input input) {
+ FetchCommand command,
+ @Assisted Project.NameKey project,
+ @Assisted FetchAction.Input input,
+ PullReplicationApiRequestMetrics metrics) {
this.command = command;
this.project = project;
this.input = input;
+ this.metrics = metrics;
}
@Override
public void run() {
try {
- command.fetchAsync(project, input.label, input.refName);
+ command.fetchAsync(project, input.label, input.refName, metrics);
} catch (InterruptedException
| ExecutionException
| RemoteConfigurationMissingException
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/HttpModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/HttpModule.java
index b2ef28d..b140cb4 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/HttpModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/HttpModule.java
@@ -38,5 +38,9 @@
} else {
serveRegex("/init-project/.*$").with(ProjectInitializationAction.class);
}
+
+ DynamicSet.bind(binder(), AllRequestFilter.class)
+ .to(PullReplicationApiMetricsFilter.class)
+ .in(Scopes.SINGLETON);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiMetricsFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiMetricsFilter.java
new file mode 100644
index 0000000..3858db2
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiMetricsFilter.java
@@ -0,0 +1,53 @@
+// Copyright (C) 2022 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.api;
+
+import com.google.gerrit.httpd.AllRequestFilter;
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
+import java.io.IOException;
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+@Singleton
+public class PullReplicationApiMetricsFilter extends AllRequestFilter {
+ private final Provider<PullReplicationApiRequestMetrics> apiRequestMetrics;
+
+ @Inject
+ public PullReplicationApiMetricsFilter(
+ Provider<PullReplicationApiRequestMetrics> apiRequestMetrics) {
+ this.apiRequestMetrics = apiRequestMetrics;
+ }
+
+ @Override
+ public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain)
+ throws IOException, ServletException {
+ if (!(request instanceof HttpServletRequest) || !(response instanceof HttpServletResponse)) {
+ chain.doFilter(request, response);
+ return;
+ }
+
+ PullReplicationApiRequestMetrics requestMetrics = apiRequestMetrics.get();
+ requestMetrics.start((HttpServletRequest) request);
+ PullReplicationApiRequestMetrics.set(requestMetrics);
+
+ chain.doFilter(request, response);
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiModule.java
index e7b3a7f..d1d28a6 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiModule.java
@@ -31,6 +31,7 @@
bind(UpdateHeadAction.class).in(Scopes.SINGLETON);
post(PROJECT_KIND, "fetch").to(FetchAction.class);
post(PROJECT_KIND, "apply-object").to(ApplyObjectAction.class);
+ post(PROJECT_KIND, "apply-objects").to(ApplyObjectsAction.class);
delete(PROJECT_KIND, "delete-project").to(ProjectDeletionAction.class);
put(PROJECT_KIND, "HEAD").to(UpdateHeadAction.class);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiRequestMetrics.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiRequestMetrics.java
new file mode 100644
index 0000000..8e4e43f
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiRequestMetrics.java
@@ -0,0 +1,82 @@
+// Copyright (C) 2022 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.api;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+import com.google.gerrit.server.events.Event;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.replication.pull.FetchReplicationMetrics;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.servlet.http.HttpServletRequest;
+
+public class PullReplicationApiRequestMetrics {
+ private static final ThreadLocal<PullReplicationApiRequestMetrics> localApiRequestMetrics =
+ new ThreadLocal<>();
+
+ public static final String HTTP_HEADER_X_START_TIME_NANOS = "X-StartTimeNanos";
+
+ private Optional<Long> startTimeNanos = Optional.empty();
+ private final AtomicBoolean initialised = new AtomicBoolean();
+ private final FetchReplicationMetrics metrics;
+
+ public static PullReplicationApiRequestMetrics get() {
+ return localApiRequestMetrics.get();
+ }
+
+ public static void set(PullReplicationApiRequestMetrics metrics) {
+ localApiRequestMetrics.set(metrics);
+ }
+
+ @Inject
+ public PullReplicationApiRequestMetrics(FetchReplicationMetrics metrics) {
+ this.metrics = metrics;
+ }
+
+ public void start(HttpServletRequest req) {
+ if (!initialised.compareAndSet(false, true)) {
+ throw new IllegalStateException("PullReplicationApiRequestMetrics already initialised");
+ }
+
+ startTimeNanos =
+ Optional.ofNullable(req.getHeader(HTTP_HEADER_X_START_TIME_NANOS))
+ .map(Long::parseLong)
+ /* Adjust with the system's nanotime for preventing negative execution times
+ * due to a clock skew between the client and the server timestamp.
+ */
+ .map(nanoTime -> Math.min(currentTimeNanos(), nanoTime));
+ }
+
+ public void start(Event event) {
+ if (!initialised.compareAndSet(false, true)) {
+ throw new IllegalStateException("PullReplicationApiRequestMetrics already initialised");
+ }
+ startTimeNanos = Optional.of(event.eventCreatedOn * 1000 * 1000 * 1000);
+ }
+
+ public Optional<Long> stop(String replicationSourceName) {
+ return startTimeNanos.map(
+ start -> {
+ long elapsed = currentTimeNanos() - start;
+ metrics.recordEnd2End(replicationSourceName, elapsed);
+ return elapsed;
+ });
+ }
+
+ private long currentTimeNanos() {
+ return MILLISECONDS.toNanos(System.currentTimeMillis());
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java
index 4af2bf7..0a9b266 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java
@@ -55,6 +55,7 @@
import com.google.inject.TypeLiteral;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.Input;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput;
+import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionsInput;
import com.googlesource.gerrit.plugins.replication.pull.api.exception.InitProjectException;
import java.io.BufferedReader;
import java.io.EOFException;
@@ -75,6 +76,7 @@
private FetchAction fetchAction;
private ApplyObjectAction applyObjectAction;
+ private ApplyObjectsAction applyObjectsAction;
private ProjectInitializationAction projectInitializationAction;
private UpdateHeadAction updateHEADAction;
private ProjectDeletionAction projectDeletionAction;
@@ -87,6 +89,7 @@
public PullReplicationFilter(
FetchAction fetchAction,
ApplyObjectAction applyObjectAction,
+ ApplyObjectsAction applyObjectsAction,
ProjectInitializationAction projectInitializationAction,
UpdateHeadAction updateHEADAction,
ProjectDeletionAction projectDeletionAction,
@@ -95,6 +98,7 @@
@PluginName String pluginName) {
this.fetchAction = fetchAction;
this.applyObjectAction = applyObjectAction;
+ this.applyObjectsAction = applyObjectsAction;
this.projectInitializationAction = projectInitializationAction;
this.updateHEADAction = updateHEADAction;
this.projectDeletionAction = projectDeletionAction;
@@ -127,6 +131,12 @@
} else {
httpResponse.sendError(SC_UNAUTHORIZED);
}
+ } else if (isApplyObjectsAction(httpRequest)) {
+ if (userProvider.get().isIdentifiedUser()) {
+ writeResponse(httpResponse, doApplyObjects(httpRequest));
+ } else {
+ httpResponse.sendError(SC_UNAUTHORIZED);
+ }
} else if (isInitProjectAction(httpRequest)) {
if (userProvider.get().isIdentifiedUser()) {
if (!checkAcceptHeader(httpRequest, httpResponse)) {
@@ -199,6 +209,16 @@
}
@SuppressWarnings("unchecked")
+ private Response<Map<String, Object>> doApplyObjects(HttpServletRequest httpRequest)
+ throws RestApiException, IOException, PermissionBackendException {
+ RevisionsInput input = readJson(httpRequest, TypeLiteral.get(RevisionsInput.class));
+ IdString id = getProjectName(httpRequest);
+ ProjectResource projectResource = projectsCollection.parse(TopLevelResource.INSTANCE, id);
+
+ return (Response<Map<String, Object>>) applyObjectsAction.apply(projectResource, input);
+ }
+
+ @SuppressWarnings("unchecked")
private Response<String> doUpdateHEAD(HttpServletRequest httpRequest) throws Exception {
HeadInput input = readJson(httpRequest, TypeLiteral.get(HeadInput.class));
ProjectResource projectResource =
@@ -293,6 +313,10 @@
return httpRequest.getRequestURI().endsWith("pull-replication~apply-object");
}
+ private boolean isApplyObjectsAction(HttpServletRequest httpRequest) {
+ return httpRequest.getRequestURI().endsWith("pull-replication~apply-objects");
+ }
+
private boolean isFetchAction(HttpServletRequest httpRequest) {
return httpRequest.getRequestURI().endsWith("pull-replication~fetch");
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionData.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionData.java
index bcd4e05..ffe98da 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionData.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionData.java
@@ -15,8 +15,11 @@
package com.googlesource.gerrit.plugins.replication.pull.api.data;
import java.util.List;
+import org.eclipse.jgit.lib.ObjectId;
public class RevisionData {
+ private transient List<ObjectId> parentObjectIds;
+
private RevisionObjectData commitObject;
private RevisionObjectData treeObject;
@@ -24,14 +27,20 @@
private List<RevisionObjectData> blobs;
public RevisionData(
+ List<ObjectId> parentObjectIds,
RevisionObjectData commitObject,
RevisionObjectData treeObject,
List<RevisionObjectData> blobs) {
+ this.parentObjectIds = parentObjectIds;
this.commitObject = commitObject;
this.treeObject = treeObject;
this.blobs = blobs;
}
+ public List<ObjectId> getParentObjetIds() {
+ return parentObjectIds;
+ }
+
public RevisionObjectData getCommitObject() {
return commitObject;
}
@@ -43,4 +52,15 @@
public List<RevisionObjectData> getBlobs() {
return blobs;
}
+
+ @Override
+ public String toString() {
+ return "{"
+ + (commitObject != null ? "commitObject=" + commitObject : "")
+ + " "
+ + (treeObject != null ? "treeObject=" + treeObject : "")
+ + " "
+ + (blobs != null && !blobs.isEmpty() ? "blobs=" + blobs : "")
+ + "}";
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java
index bc3e218..c18e11d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java
@@ -14,6 +14,10 @@
package com.googlesource.gerrit.plugins.replication.pull.api.data;
+import java.util.List;
+import java.util.Objects;
+import org.eclipse.jgit.lib.Constants;
+
public class RevisionInput {
private String label;
@@ -38,4 +42,44 @@
public RevisionData getRevisionData() {
return revisionData;
}
+
+ public void validate() {
+ validate(refName, revisionData);
+ }
+
+ static void validate(String refName, RevisionData revisionData) {
+ // Non-heads refs can point to non-commit objects
+ if (!refName.startsWith(Constants.R_HEADS)
+ && Objects.isNull(revisionData.getCommitObject())
+ && Objects.isNull(revisionData.getTreeObject())) {
+
+ List<RevisionObjectData> blobs = revisionData.getBlobs();
+
+ if (Objects.isNull(blobs) || blobs.isEmpty()) {
+ throw new IllegalArgumentException(
+ "Ref " + refName + " cannot have a null or empty list of BLOBs associated");
+ }
+
+ if (blobs.size() > 1) {
+ throw new IllegalArgumentException("Ref " + refName + " has more than one BLOB associated");
+ }
+
+ return;
+ }
+
+ if (Objects.isNull(revisionData.getCommitObject())
+ || Objects.isNull(revisionData.getCommitObject().getContent())
+ || revisionData.getCommitObject().getContent().length == 0
+ || Objects.isNull(revisionData.getCommitObject().getType())) {
+ throw new IllegalArgumentException(
+ "Commit object for ref " + refName + " cannot be null or empty");
+ }
+
+ if (Objects.isNull(revisionData.getTreeObject())
+ || Objects.isNull(revisionData.getTreeObject().getContent())
+ || Objects.isNull(revisionData.getTreeObject().getType())) {
+ throw new IllegalArgumentException(
+ "Ref-update tree object for ref " + refName + " cannot be null");
+ }
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionObjectData.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionObjectData.java
index 02ba06c..b21f495 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionObjectData.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionObjectData.java
@@ -15,12 +15,15 @@
package com.googlesource.gerrit.plugins.replication.pull.api.data;
import java.util.Base64;
+import org.eclipse.jgit.lib.Constants;
public class RevisionObjectData {
+ private final String sha1;
private final Integer type;
private final String content;
- public RevisionObjectData(int type, byte[] content) {
+ public RevisionObjectData(String sha1, int type, byte[] content) {
+ this.sha1 = sha1;
this.type = type;
this.content = content == null ? "" : Base64.getEncoder().encodeToString(content);
}
@@ -32,4 +35,29 @@
public byte[] getContent() {
return Base64.getDecoder().decode(content);
}
+
+ public String getSha1() {
+ return sha1;
+ }
+
+ @Override
+ public String toString() {
+ String typeStr;
+ switch (type) {
+ case Constants.OBJ_BLOB:
+ typeStr = "BLOB";
+ break;
+ case Constants.OBJ_COMMIT:
+ typeStr = "COMMIT";
+ break;
+ case Constants.OBJ_TREE:
+ typeStr = "TREE";
+ break;
+ default:
+ typeStr = "type:" + type;
+ break;
+ }
+
+ return sha1 + " (" + typeStr + ")";
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionsInput.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionsInput.java
new file mode 100644
index 0000000..2361f6b
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionsInput.java
@@ -0,0 +1,60 @@
+// Copyright (C) 2022 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.api.data;
+
+import java.util.Arrays;
+
+public class RevisionsInput {
+ private String label;
+
+ private String refName;
+
+ private RevisionData[] revisionsData;
+
+ public RevisionsInput(String label, String refName, RevisionData[] revisionsData) {
+ this.label = label;
+ this.refName = refName;
+ this.revisionsData = revisionsData;
+ }
+
+ public String getLabel() {
+ return label;
+ }
+
+ public String getRefName() {
+ return refName;
+ }
+
+ public RevisionData[] getRevisionsData() {
+ return revisionsData;
+ }
+
+ public void validate() {
+ for (RevisionData revisionData : revisionsData) {
+ RevisionInput.validate(refName, revisionData);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "RevisionsInput { "
+ + label
+ + ":"
+ + refName
+ + " - "
+ + Arrays.toString(revisionsData)
+ + "}";
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java
index 476a35b..6000eb9 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java
@@ -14,10 +14,13 @@
package com.googlesource.gerrit.plugins.replication.pull.client;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
import com.google.gerrit.entities.Project;
import com.googlesource.gerrit.plugins.replication.pull.Source;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData;
import java.io.IOException;
+import java.util.List;
import org.apache.http.client.ClientProtocolException;
import org.eclipse.jgit.transport.URIish;
@@ -27,9 +30,15 @@
FetchApiClient create(Source source);
}
- HttpResult callFetch(Project.NameKey project, String refName, URIish targetUri)
+ HttpResult callFetch(
+ Project.NameKey project, String refName, URIish targetUri, long startTimeNanos)
throws ClientProtocolException, IOException;
+ default HttpResult callFetch(Project.NameKey project, String refName, URIish targetUri)
+ throws ClientProtocolException, IOException {
+ return callFetch(project, refName, targetUri, MILLISECONDS.toNanos(System.currentTimeMillis()));
+ }
+
HttpResult initProject(Project.NameKey project, URIish uri) throws IOException;
HttpResult deleteProject(Project.NameKey project, URIish apiUri) throws IOException;
@@ -43,4 +52,8 @@
RevisionData revisionData,
URIish targetUri)
throws ClientProtocolException, IOException;
+
+ HttpResult callSendObjects(
+ Project.NameKey project, String refName, List<RevisionData> revisionData, URIish targetUri)
+ throws ClientProtocolException, IOException;
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
index f1d486d..9b33c81 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
@@ -23,6 +23,7 @@
import com.google.common.net.MediaType;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.entities.Project;
+import com.google.gerrit.entities.Project.NameKey;
import com.google.gerrit.extensions.annotations.PluginName;
import com.google.gerrit.extensions.restapi.Url;
import com.google.gerrit.server.config.GerritInstanceId;
@@ -33,11 +34,14 @@
import com.googlesource.gerrit.plugins.replication.CredentialsFactory;
import com.googlesource.gerrit.plugins.replication.ReplicationConfig;
import com.googlesource.gerrit.plugins.replication.pull.Source;
+import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput;
+import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionsInput;
import com.googlesource.gerrit.plugins.replication.pull.filter.SyncRefsFilter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
+import java.util.List;
import java.util.Optional;
import org.apache.http.HttpResponse;
import org.apache.http.ParseException;
@@ -100,7 +104,8 @@
* @see com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient#callFetch(com.google.gerrit.entities.Project.NameKey, java.lang.String, org.eclipse.jgit.transport.URIish)
*/
@Override
- public HttpResult callFetch(Project.NameKey project, String refName, URIish targetUri)
+ public HttpResult callFetch(
+ Project.NameKey project, String refName, URIish targetUri, long startTimeNanos)
throws ClientProtocolException, IOException {
String url =
String.format(
@@ -115,6 +120,9 @@
instanceId, refName, callAsync),
StandardCharsets.UTF_8));
post.addHeader(new BasicHeader("Content-Type", "application/json"));
+ post.addHeader(
+ PullReplicationApiRequestMetrics.HTTP_HEADER_X_START_TIME_NANOS,
+ Long.toString(startTimeNanos));
return httpClientFactory.create(source).execute(post, this, getContext(targetUri));
}
@@ -179,10 +187,7 @@
}
RevisionInput input = new RevisionInput(instanceId, refName, revisionData);
- String url =
- String.format(
- "%s/a/projects/%s/%s~apply-object",
- targetUri.toString(), Url.encode(project.get()), pluginName);
+ String url = formatUrl(project, targetUri, "apply-object");
HttpPost post = new HttpPost(url);
post.setEntity(new StringEntity(GSON.toJson(input)));
@@ -190,6 +195,32 @@
return httpClientFactory.create(source).execute(post, this, getContext(targetUri));
}
+ @Override
+ public HttpResult callSendObjects(
+ NameKey project, String refName, List<RevisionData> revisionData, URIish targetUri)
+ throws ClientProtocolException, IOException {
+ if (revisionData.size() == 1) {
+ return callSendObject(project, refName, false, revisionData.get(0), targetUri);
+ }
+
+ RevisionData[] inputData = new RevisionData[revisionData.size()];
+ RevisionsInput input = new RevisionsInput(instanceId, refName, revisionData.toArray(inputData));
+
+ String url = formatUrl(project, targetUri, "apply-objects");
+ HttpPost post = new HttpPost(url);
+ post.setEntity(new StringEntity(GSON.toJson(input)));
+ post.addHeader(new BasicHeader("Content-Type", MediaType.JSON_UTF_8.toString()));
+ return httpClientFactory.create(source).execute(post, this, getContext(targetUri));
+ }
+
+ private String formatUrl(Project.NameKey project, URIish targetUri, String api) {
+ String url =
+ String.format(
+ "%s/a/projects/%s/%s~%s",
+ targetUri.toString(), Url.encode(project.get()), pluginName, api);
+ return url;
+ }
+
private void requireNull(Object object, String string) {
if (object != null) {
throw new IllegalArgumentException(string);
@@ -198,13 +229,20 @@
@Override
public HttpResult handleResponse(HttpResponse response) {
- Optional<String> responseBody = Optional.empty();
- try {
- responseBody = Optional.ofNullable(EntityUtils.toString(response.getEntity()));
- } catch (ParseException | IOException e) {
- logger.atSevere().withCause(e).log("Unable get response body from %s", response.toString());
- }
+ Optional<String> responseBody =
+ Optional.ofNullable(response.getEntity())
+ .flatMap(
+ body -> {
+ try {
+ return Optional.of(EntityUtils.toString(body));
+ } catch (ParseException | IOException e) {
+ logger.atSevere().withCause(e).log(
+ "Unable get response body from %s", response.toString());
+ return Optional.empty();
+ }
+ });
+
return new HttpResult(response.getStatusLine().getStatusCode(), responseBody);
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResult.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResult.java
index bd164df..ec9d65f 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResult.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResult.java
@@ -15,9 +15,6 @@
package com.googlesource.gerrit.plugins.replication.pull.client;
import static javax.servlet.http.HttpServletResponse.SC_CONFLICT;
-import static javax.servlet.http.HttpServletResponse.SC_CREATED;
-import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
-import static javax.servlet.http.HttpServletResponse.SC_OK;
import com.google.gerrit.entities.Project;
import java.util.Optional;
@@ -36,7 +33,7 @@
}
public boolean isSuccessful() {
- return responseCode == SC_CREATED || responseCode == SC_NO_CONTENT || responseCode == SC_OK;
+ return responseCode / 100 == 2; // Any 2xx response code is a success
}
public boolean isProjectMissing(Project.NameKey projectName) {
@@ -47,4 +44,11 @@
public boolean isParentObjectMissing() {
return responseCode == SC_CONFLICT;
}
+
+ @Override
+ public String toString() {
+ return isSuccessful()
+ ? "OK"
+ : "FAILED" + ", status=" + responseCode + message.map(s -> " '" + s + "'").orElse("");
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
index 76e3cae..0f092e5 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListener.java
@@ -30,11 +30,13 @@
import com.google.gerrit.server.git.WorkQueue;
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.inject.Inject;
+import com.google.inject.Provider;
import com.googlesource.gerrit.plugins.replication.pull.FetchOne;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchJob;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchJob.Factory;
import com.googlesource.gerrit.plugins.replication.pull.api.ProjectInitializationAction;
+import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics;
import org.eclipse.jgit.lib.ObjectId;
public class StreamEventListener implements EventListener {
@@ -45,17 +47,20 @@
private ProjectInitializationAction projectInitializationAction;
private Factory fetchJobFactory;
+ private final Provider<PullReplicationApiRequestMetrics> metricsProvider;
@Inject
public StreamEventListener(
@Nullable @GerritInstanceId String instanceId,
ProjectInitializationAction projectInitializationAction,
WorkQueue workQueue,
- FetchJob.Factory fetchJobFactory) {
+ FetchJob.Factory fetchJobFactory,
+ Provider<PullReplicationApiRequestMetrics> metricsProvider) {
this.instanceId = instanceId;
this.projectInitializationAction = projectInitializationAction;
this.workQueue = workQueue;
this.fetchJobFactory = fetchJobFactory;
+ this.metricsProvider = metricsProvider;
requireNonNull(
Strings.emptyToNull(this.instanceId), "gerrit.instanceId cannot be null or empty");
@@ -64,13 +69,16 @@
@Override
public void onEvent(Event event) {
if (!instanceId.equals(event.instanceId)) {
+ PullReplicationApiRequestMetrics metrics = metricsProvider.get();
+ metrics.start(event);
if (event instanceof RefUpdatedEvent) {
RefUpdatedEvent refUpdatedEvent = (RefUpdatedEvent) event;
if (!isProjectDelete(refUpdatedEvent)) {
fetchRefsAsync(
refUpdatedEvent.getRefName(),
refUpdatedEvent.instanceId,
- refUpdatedEvent.getProjectNameKey());
+ refUpdatedEvent.getProjectNameKey(),
+ metrics);
}
}
if (event instanceof ProjectCreatedEvent) {
@@ -80,7 +88,8 @@
fetchRefsAsync(
FetchOne.ALL_REFS,
projectCreatedEvent.instanceId,
- projectCreatedEvent.getProjectNameKey());
+ projectCreatedEvent.getProjectNameKey(),
+ metrics);
} catch (AuthException | PermissionBackendException e) {
logger.atSevere().withCause(e).log(
"Cannot initialise project:%s", projectCreatedEvent.projectName);
@@ -94,11 +103,15 @@
&& ObjectId.zeroId().equals(ObjectId.fromString(event.refUpdate.get().newRev));
}
- protected void fetchRefsAsync(String refName, String sourceInstanceId, NameKey projectNameKey) {
+ protected void fetchRefsAsync(
+ String refName,
+ String sourceInstanceId,
+ NameKey projectNameKey,
+ PullReplicationApiRequestMetrics metrics) {
FetchAction.Input input = new FetchAction.Input();
input.refName = refName;
input.label = sourceInstanceId;
- workQueue.getDefaultQueue().submit(fetchJobFactory.create(projectNameKey, input));
+ workQueue.getDefaultQueue().submit(fetchJobFactory.create(projectNameKey, input, metrics));
}
private String getProjectRepositoryName(ProjectCreatedEvent projectCreatedEvent) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java
index 03362bf..2bb1caf 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java
@@ -41,35 +41,51 @@
this.gitManager = gitManager;
}
- public RefUpdateState apply(Project.NameKey name, RefSpec refSpec, RevisionData revisionData)
+ public RefUpdateState apply(Project.NameKey name, RefSpec refSpec, RevisionData[] revisionsData)
throws MissingParentObjectException, IOException {
try (Repository git = gitManager.openRepository(name)) {
- ObjectId newObjectID = null;
+ ObjectId refHead = null;
+ RefUpdate ru = git.updateRef(refSpec.getSource());
try (ObjectInserter oi = git.newObjectInserter()) {
- RevisionObjectData commitObject = revisionData.getCommitObject();
- RevCommit commit = RevCommit.parse(commitObject.getContent());
- for (RevCommit parent : commit.getParents()) {
- if (!git.getObjectDatabase().has(parent.getId())) {
- throw new MissingParentObjectException(name, refSpec.getSource(), parent.getId());
+ for (RevisionData revisionData : revisionsData) {
+
+ ObjectId newObjectID = null;
+ RevisionObjectData commitObject = revisionData.getCommitObject();
+
+ if (commitObject != null) {
+ RevCommit commit = RevCommit.parse(commitObject.getContent());
+ for (RevCommit parent : commit.getParents()) {
+ if (!git.getObjectDatabase().has(parent.getId())) {
+ throw new MissingParentObjectException(name, refSpec.getSource(), parent.getId());
+ }
+ }
+ refHead = newObjectID = oi.insert(commitObject.getType(), commitObject.getContent());
+
+ RevisionObjectData treeObject = revisionData.getTreeObject();
+ oi.insert(treeObject.getType(), treeObject.getContent());
+ }
+
+ for (RevisionObjectData rev : revisionData.getBlobs()) {
+ ObjectId blobObjectId = oi.insert(rev.getType(), rev.getContent());
+ if (newObjectID == null) {
+ newObjectID = blobObjectId;
+ }
+ refHead = newObjectID;
+ }
+
+ oi.flush();
+
+ if (commitObject == null) {
+ // Non-commits must be forced as they do not have a graph associated
+ ru.setForceUpdate(true);
}
}
- newObjectID = oi.insert(commitObject.getType(), commitObject.getContent());
- RevisionObjectData treeObject = revisionData.getTreeObject();
- oi.insert(treeObject.getType(), treeObject.getContent());
-
- for (RevisionObjectData rev : revisionData.getBlobs()) {
- oi.insert(rev.getType(), rev.getContent());
- }
-
- oi.flush();
+ ru.setNewObjectId(refHead);
+ RefUpdate.Result result = ru.update();
+ return new RefUpdateState(refSpec.getSource(), result);
}
- RefUpdate ru = git.updateRef(refSpec.getSource());
- ru.setNewObjectId(newObjectID);
- RefUpdate.Result result = ru.update();
-
- return new RefUpdateState(refSpec.getSource(), result);
}
}
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java
new file mode 100644
index 0000000..58be58e
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java
@@ -0,0 +1,43 @@
+// Copyright (C) 2022 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull;
+
+import com.google.gerrit.acceptance.SkipProjectClone;
+import com.google.gerrit.acceptance.TestPlugin;
+import com.google.gerrit.acceptance.UseLocalDisk;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.inject.Inject;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
+
+@SkipProjectClone
+@UseLocalDisk
+@TestPlugin(
+ name = "pull-replication",
+ sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule",
+ httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule")
+public class PullReplicationAsyncIT extends PullReplicationIT {
+ @Inject private SitePaths sitePaths;
+
+ @Override
+ public void setUpTestPlugin() throws Exception {
+ FileBasedConfig config =
+ new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED);
+ config.setString("replication", null, "syncRefs", "^$");
+ config.save();
+
+ super.setUpTestPlugin(true);
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java
index 13b3460..62f42c3 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java
@@ -51,6 +51,7 @@
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.googlesource.gerrit.plugins.replication.AutoReloadConfigDecorator;
+import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.time.Duration;
@@ -79,11 +80,12 @@
@UseLocalDisk
@TestPlugin(
name = "pull-replication",
- sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule")
+ sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule",
+ httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule")
public class PullReplicationIT extends LightweightPluginDaemonTest {
private static final Optional<String> ALL_PROJECTS = Optional.empty();
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
- private static final int TEST_REPLICATION_DELAY = 60;
+ private static final int TEST_REPLICATION_DELAY = 1;
private static final Duration TEST_TIMEOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 2000);
private static final String TEST_REPLICATION_SUFFIX = "suffix1";
private static final String TEST_REPLICATION_REMOTE = "remote1";
@@ -97,10 +99,17 @@
@Override
public void setUpTestPlugin() throws Exception {
+ setUpTestPlugin(false);
+ }
+
+ protected void setUpTestPlugin(boolean loadExisting) throws Exception {
gitPath = sitePaths.site_path.resolve("git");
- config =
- new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED);
+ File configFile = sitePaths.etc_dir.resolve("replication.config").toFile();
+ config = new FileBasedConfig(configFile, FS.DETECTED);
+ if (loadExisting && configFile.exists()) {
+ config.load();
+ }
setReplicationSource(
TEST_REPLICATION_REMOTE,
TEST_REPLICATION_SUFFIX,
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
index 4e831bc..48e0e71 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
@@ -17,9 +17,12 @@
import static com.google.common.truth.Truth.assertThat;
import static java.nio.file.Files.createTempDirectory;
import static org.mockito.ArgumentMatchers.anyBoolean;
-import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyInt;
+import static org.mockito.Mockito.anyLong;
import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -35,6 +38,7 @@
import com.google.gerrit.extensions.events.GitReferenceUpdatedListener.Event;
import com.google.gerrit.extensions.events.ProjectDeletedListener;
import com.google.gerrit.extensions.registration.DynamicItem;
+import com.google.gerrit.metrics.DisabledMetricMaker;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.git.WorkQueue;
@@ -49,6 +53,8 @@
import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter;
import java.io.IOException;
import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
import java.util.Optional;
import org.apache.http.client.ClientProtocolException;
import org.eclipse.jgit.errors.LargeObjectException;
@@ -78,10 +84,17 @@
@Mock AccountInfo accountInfo;
@Mock RevisionReader revReader;
@Mock RevisionData revisionData;
+ @Mock HttpResult successfulHttpResult;
+ @Mock HttpResult fetchHttpResult;
+ @Mock RevisionData revisionDataWithParents;
+ List<ObjectId> revisionDataParentObjectIds;
@Mock HttpResult httpResult;
+ ApplyObjectMetrics applyObjectMetrics;
+ FetchReplicationMetrics fetchMetrics;
@Captor ArgumentCaptor<String> stringCaptor;
@Captor ArgumentCaptor<Project.NameKey> projectNameKeyCaptor;
+ @Captor ArgumentCaptor<List<RevisionData>> revisionsDataCaptor;
private ExcludedRefsFilter refsFilter;
private ReplicationQueue objectUnderTest;
@@ -102,16 +115,51 @@
when(source.getApis()).thenReturn(apis);
when(sourceCollection.getAll()).thenReturn(Lists.newArrayList(source));
when(rd.get()).thenReturn(sourceCollection);
- when(revReader.read(any(), any(), anyString())).thenReturn(Optional.of(revisionData));
+ lenient()
+ .when(revReader.read(any(), any(), anyString(), eq(0)))
+ .thenReturn(Optional.of(revisionData));
+ lenient().when(revReader.read(any(), anyString(), eq(0))).thenReturn(Optional.of(revisionData));
+ lenient()
+ .when(revReader.read(any(), any(), anyString(), eq(Integer.MAX_VALUE)))
+ .thenReturn(Optional.of(revisionDataWithParents));
+ lenient()
+ .when(revReader.read(any(), anyString(), eq(Integer.MAX_VALUE)))
+ .thenReturn(Optional.of(revisionDataWithParents));
+ revisionDataParentObjectIds =
+ Arrays.asList(
+ ObjectId.fromString("9f8d52853089a3cf00c02ff7bd0817bd4353a95a"),
+ ObjectId.fromString("b5d7bcf1d1c5b0f0726d10a16c8315f06f900bfb"));
+ when(revisionDataWithParents.getParentObjetIds()).thenReturn(revisionDataParentObjectIds);
+
when(fetchClientFactory.create(any())).thenReturn(fetchRestApiClient);
- when(fetchRestApiClient.callSendObject(any(), anyString(), anyBoolean(), any(), any()))
+ lenient()
+ .when(fetchRestApiClient.callSendObject(any(), anyString(), anyBoolean(), any(), any()))
.thenReturn(httpResult);
- when(fetchRestApiClient.callFetch(any(), anyString(), any())).thenReturn(httpResult);
+ lenient()
+ .when(fetchRestApiClient.callSendObjects(any(), anyString(), any(), any()))
+ .thenReturn(httpResult);
+ when(fetchRestApiClient.callFetch(any(), anyString(), any(), anyLong()))
+ .thenReturn(fetchHttpResult);
+ when(fetchRestApiClient.initProject(any(), any())).thenReturn(successfulHttpResult);
+ when(successfulHttpResult.isSuccessful()).thenReturn(true);
when(httpResult.isSuccessful()).thenReturn(true);
+ when(fetchHttpResult.isSuccessful()).thenReturn(true);
when(httpResult.isProjectMissing(any())).thenReturn(false);
+ applyObjectMetrics = new ApplyObjectMetrics("pull-replication", new DisabledMetricMaker());
+ fetchMetrics = new FetchReplicationMetrics("pull-replication", new DisabledMetricMaker());
+
objectUnderTest =
- new ReplicationQueue(wq, rd, dis, sl, fetchClientFactory, refsFilter, revReader);
+ new ReplicationQueue(
+ wq,
+ rd,
+ dis,
+ sl,
+ fetchClientFactory,
+ refsFilter,
+ () -> revReader,
+ applyObjectMetrics,
+ fetchMetrics);
}
@Test
@@ -120,7 +168,7 @@
objectUnderTest.start();
objectUnderTest.onGitReferenceUpdated(event);
- verify(fetchRestApiClient).callSendObject(any(), anyString(), eq(false), any(), any());
+ verify(fetchRestApiClient).callSendObjects(any(), anyString(), any(), any());
}
@Test
@@ -155,7 +203,7 @@
objectUnderTest.start();
objectUnderTest.onGitReferenceUpdated(event);
- verify(fetchRestApiClient).callSendObject(any(), anyString(), eq(false), any(), any());
+ verify(fetchRestApiClient).callSendObjects(any(), anyString(), any(), any());
}
@Test
@@ -164,11 +212,11 @@
Event event = new TestEvent("refs/changes/01/1/meta");
objectUnderTest.start();
- when(revReader.read(any(), any(), anyString())).thenThrow(IOException.class);
+ when(revReader.read(any(), any(), anyString(), anyInt())).thenThrow(IOException.class);
objectUnderTest.onGitReferenceUpdated(event);
- verify(fetchRestApiClient).callFetch(any(), anyString(), any());
+ verify(fetchRestApiClient).callFetch(any(), anyString(), any(), anyLong());
}
@Test
@@ -177,27 +225,53 @@
Event event = new TestEvent("refs/changes/01/1/1");
objectUnderTest.start();
- when(revReader.read(any(), any(), anyString())).thenReturn(Optional.empty());
+ when(revReader.read(any(), any(), anyString(), anyInt())).thenReturn(Optional.empty());
objectUnderTest.onGitReferenceUpdated(event);
- verify(fetchRestApiClient).callFetch(any(), anyString(), any());
+ verify(fetchRestApiClient).callFetch(any(), anyString(), any(), anyLong());
}
@Test
public void shouldFallbackToCallFetchWhenParentObjectIsMissing()
throws ClientProtocolException, IOException {
- Event event = new TestEvent("refs/changes/01/1/meta");
+ Event event = new TestEvent("refs/changes/01/1/1");
objectUnderTest.start();
when(httpResult.isSuccessful()).thenReturn(false);
when(httpResult.isParentObjectMissing()).thenReturn(true);
- when(fetchRestApiClient.callSendObject(any(), anyString(), eq(false), any(), any()))
+ when(fetchRestApiClient.callSendObjects(any(), anyString(), any(), any()))
.thenReturn(httpResult);
objectUnderTest.onGitReferenceUpdated(event);
- verify(fetchRestApiClient).callFetch(any(), anyString(), any());
+ verify(fetchRestApiClient).callFetch(any(), anyString(), any(), anyLong());
+ }
+
+ @Test
+ public void shouldFallbackToApplyAllParentObjectsWhenParentObjectIsMissingOnMetaRef()
+ throws ClientProtocolException, IOException {
+ Event event = new TestEvent("refs/changes/01/1/meta");
+ objectUnderTest.start();
+
+ when(httpResult.isSuccessful()).thenReturn(false, true);
+ when(httpResult.isParentObjectMissing()).thenReturn(true, false);
+ when(fetchRestApiClient.callSendObjects(any(), anyString(), any(), any()))
+ .thenReturn(httpResult);
+
+ objectUnderTest.onGitReferenceUpdated(event);
+
+ verify(fetchRestApiClient, times(2))
+ .callSendObjects(any(), anyString(), revisionsDataCaptor.capture(), any());
+ List<List<RevisionData>> revisionsDataValues = revisionsDataCaptor.getAllValues();
+ assertThat(revisionsDataValues).hasSize(2);
+
+ List<RevisionData> firstRevisionsValues = revisionsDataValues.get(0);
+ assertThat(firstRevisionsValues).hasSize(1);
+ assertThat(firstRevisionsValues).contains(revisionData);
+
+ List<RevisionData> secondRevisionsValues = revisionsDataValues.get(1);
+ assertThat(secondRevisionsValues).hasSize(1 + revisionDataParentObjectIds.size());
}
@Test
@@ -242,7 +316,16 @@
refsFilter = new ExcludedRefsFilter(replicationConfig);
objectUnderTest =
- new ReplicationQueue(wq, rd, dis, sl, fetchClientFactory, refsFilter, revReader);
+ new ReplicationQueue(
+ wq,
+ rd,
+ dis,
+ sl,
+ fetchClientFactory,
+ refsFilter,
+ () -> revReader,
+ applyObjectMetrics,
+ fetchMetrics);
Event event = new TestEvent("refs/multi-site/version");
objectUnderTest.onGitReferenceUpdated(event);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java
index e300613..1d8520f 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java
@@ -24,12 +24,14 @@
import com.google.gerrit.acceptance.TestPlugin;
import com.google.gerrit.acceptance.UseLocalDisk;
import com.google.gerrit.entities.Change;
+import com.google.gerrit.entities.Change.Id;
import com.google.gerrit.entities.Patch;
import com.google.gerrit.entities.RefNames;
import com.google.gerrit.extensions.api.changes.ReviewInput;
import com.google.gerrit.extensions.api.changes.ReviewInput.CommentInput;
import com.google.gerrit.extensions.client.Comment;
import com.google.gerrit.extensions.config.FactoryModule;
+import com.google.gerrit.extensions.restapi.RestApiException;
import com.google.gerrit.server.notedb.Sequences;
import com.google.inject.Scopes;
import com.googlesource.gerrit.plugins.replication.ReplicationConfig;
@@ -38,11 +40,13 @@
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData;
import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject;
import java.io.IOException;
+import java.util.List;
import java.util.Optional;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.junit.Before;
import org.junit.Test;
@@ -53,9 +57,12 @@
public class RevisionReaderIT extends LightweightPluginDaemonTest {
RevisionReader objectUnderTest;
+ ReplicationFileBasedConfig replicationConfig;
+
@Before
public void setup() {
objectUnderTest = plugin.getSysInjector().getInstance(RevisionReader.class);
+ replicationConfig = plugin.getSysInjector().getInstance(ReplicationFileBasedConfig.class);
}
@Test
@@ -64,7 +71,7 @@
String refName = RefNames.changeMetaRef(pushResult.getChange().getId());
Optional<RevisionData> revisionDataOption =
- refObjectId(refName).flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId));
+ refObjectId(refName).flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId, 0));
assertThat(revisionDataOption.isPresent()).isTrue();
RevisionData revisionData = revisionDataOption.get();
@@ -80,9 +87,65 @@
assertThat(revisionData.getBlobs()).isEmpty();
}
- private Optional<RevisionData> readRevisionFromObjectUnderTest(String refName, ObjectId objId) {
+ @Test
+ public void shouldReadRefMetaObjectWithMaxNumberOfParents() throws Exception {
+ int numberOfParents = 3;
+ setReplicationConfig(numberOfParents);
+ Result pushResult = createChange();
+ Id changeId = pushResult.getChange().getId();
+ String refName = RefNames.changeMetaRef(pushResult.getChange().getId());
+
+ addMultipleComments(numberOfParents, changeId);
+
+ Optional<RevisionData> revisionDataOption =
+ refObjectId(refName)
+ .flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId, numberOfParents));
+
+ assertThat(revisionDataOption.isPresent()).isTrue();
+ List<ObjectId> parentObjectIds = revisionDataOption.get().getParentObjetIds();
+ assertThat(parentObjectIds).hasSize(numberOfParents);
+ }
+
+ @Test
+ public void shouldReadRefMetaObjectLimitedToMaxNumberOfParents() throws Exception {
+ int numberOfParents = 3;
+ setReplicationConfig(numberOfParents);
+ Result pushResult = createChange();
+ Id changeId = pushResult.getChange().getId();
+ String refName = RefNames.changeMetaRef(pushResult.getChange().getId());
+
+ addMultipleComments(numberOfParents + 1, changeId);
+
+ Optional<RevisionData> revisionDataOption =
+ refObjectId(refName)
+ .flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId, numberOfParents));
+
+ assertThat(revisionDataOption.isPresent()).isTrue();
+ List<ObjectId> parentObjectIds = revisionDataOption.get().getParentObjetIds();
+ assertThat(parentObjectIds).hasSize(numberOfParents);
+ }
+
+ private void addMultipleComments(int numberOfParents, Id changeId) throws RestApiException {
+ for (int i = 0; i < numberOfParents; i++) {
+ addComment(changeId);
+ }
+ }
+
+ private void setReplicationConfig(int numberOfParents) throws IOException {
+ FileBasedConfig config = (FileBasedConfig) replicationConfig.getConfig();
+ config.setInt(
+ "replication", null, RevisionReader.CONFIG_MAX_API_HISTORY_DEPTH, numberOfParents);
+ config.save();
+ }
+
+ private void addComment(Id changeId) throws RestApiException {
+ gApi.changes().id(changeId.get()).current().review(new ReviewInput().message("foo"));
+ }
+
+ private Optional<RevisionData> readRevisionFromObjectUnderTest(
+ String refName, ObjectId objId, int maxParentsDepth) {
try {
- return objectUnderTest.read(project, objId, refName);
+ return objectUnderTest.read(project, objId, refName, maxParentsDepth);
} catch (Exception e) {
throw new IllegalStateException(e);
}
@@ -107,7 +170,7 @@
gApi.changes().id(changeId.get()).current().review(reviewInput);
Optional<RevisionData> revisionDataOption =
- refObjectId(refName).flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId));
+ refObjectId(refName).flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId, 0));
assertThat(revisionDataOption.isPresent()).isTrue();
RevisionData revisionData = revisionDataOption.get();
@@ -131,7 +194,7 @@
createChange().assertOkStatus();
String refName = RefNames.REFS_SEQUENCES + Sequences.NAME_CHANGES;
Optional<RevisionData> revisionDataOption =
- refObjectId(refName).flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId));
+ refObjectId(refName).flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId, 0));
Truth8.assertThat(revisionDataOption).isEmpty();
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ActionITBase.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ActionITBase.java
index e6f788a..7cef485 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ActionITBase.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ActionITBase.java
@@ -149,7 +149,8 @@
protected Optional<RevisionData> createRevisionData(NameKey projectName, String refName)
throws Exception {
try (Repository repository = repoManager.openRepository(projectName)) {
- return revisionReader.read(projectName, repository.exactRef(refName).getObjectId(), refName);
+ return revisionReader.read(
+ projectName, repository.exactRef(refName).getObjectId(), refName, 0);
}
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java
index 56e1429..814ba76 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java
@@ -35,6 +35,9 @@
import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException;
import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.junit.Before;
@@ -49,11 +52,18 @@
String label = "instance-2-label";
String url = "file:///gerrit-host/instance-1/git/${name}.git";
String refName = "refs/heads/master";
+ String refMetaName = "refs/meta/version";
String location = "http://gerrit-host/a/config/server/tasks/08d173e9";
int taskId = 1234;
+ private String sampleCommitObjectId = "9f8d52853089a3cf00c02ff7bd0817bd4353a95a";
+ private String sampleTreeObjectId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904";
+ private String sampleBlobObjectId = "b5d7bcf1d1c5b0f0726d10a16c8315f06f900bfb";
+
private String sampleCommitContent =
- "tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n"
+ "tree "
+ + sampleTreeObjectId
+ + "\n"
+ "parent 20eb48d28be86dc88fb4bef747f08de0fbefe12d\n"
+ "author Gerrit User 1000000 <1000000@69ec38f0-350e-4d9c-96d4-bc956f2faaac> 1610471611 +0100\n"
+ "committer Gerrit Code Review <root@maczech-XPS-15> 1610471611 +0100\n"
@@ -92,6 +102,21 @@
assertThat(response.statusCode()).isEqualTo(SC_CREATED);
}
+ @Test
+ public void shouldReturnCreatedResponseCodeForBlob() throws RestApiException {
+ byte[] blobData = "foo".getBytes(StandardCharsets.UTF_8);
+ RevisionInput inputParams =
+ new RevisionInput(
+ label,
+ refMetaName,
+ createSampleRevisionDataBlob(
+ new RevisionObjectData(sampleBlobObjectId, Constants.OBJ_BLOB, blobData)));
+
+ Response<?> response = applyObjectAction.apply(projectResource, inputParams);
+
+ assertThat(response.statusCode()).isEqualTo(SC_CREATED);
+ }
+
@SuppressWarnings("cast")
@Test
public void shouldReturnSourceUrlAndrefNameAsAResponseBody() throws Exception {
@@ -131,8 +156,10 @@
@Test(expected = BadRequestException.class)
public void shouldThrowBadRequestExceptionWhenMissingCommitObjectData() throws Exception {
- RevisionObjectData commitData = new RevisionObjectData(Constants.OBJ_COMMIT, null);
- RevisionObjectData treeData = new RevisionObjectData(Constants.OBJ_TREE, new byte[] {});
+ RevisionObjectData commitData =
+ new RevisionObjectData(sampleCommitObjectId, Constants.OBJ_COMMIT, null);
+ RevisionObjectData treeData =
+ new RevisionObjectData(sampleTreeObjectId, Constants.OBJ_TREE, new byte[] {});
RevisionInput inputParams =
new RevisionInput(label, refName, createSampleRevisionData(commitData, treeData));
@@ -142,7 +169,8 @@
@Test(expected = BadRequestException.class)
public void shouldThrowBadRequestExceptionWhenMissingTreeObject() throws Exception {
RevisionObjectData commitData =
- new RevisionObjectData(Constants.OBJ_COMMIT, sampleCommitContent.getBytes());
+ new RevisionObjectData(
+ sampleCommitObjectId, Constants.OBJ_COMMIT, sampleCommitContent.getBytes());
RevisionInput inputParams =
new RevisionInput(label, refName, createSampleRevisionData(commitData, null));
@@ -175,13 +203,19 @@
private RevisionData createSampleRevisionData() {
RevisionObjectData commitData =
- new RevisionObjectData(Constants.OBJ_COMMIT, sampleCommitContent.getBytes());
- RevisionObjectData treeData = new RevisionObjectData(Constants.OBJ_TREE, new byte[] {});
+ new RevisionObjectData(
+ sampleCommitObjectId, Constants.OBJ_COMMIT, sampleCommitContent.getBytes());
+ RevisionObjectData treeData =
+ new RevisionObjectData(sampleTreeObjectId, Constants.OBJ_TREE, new byte[] {});
return createSampleRevisionData(commitData, treeData);
}
private RevisionData createSampleRevisionData(
RevisionObjectData commitData, RevisionObjectData treeData) {
- return new RevisionData(commitData, treeData, Lists.newArrayList());
+ return new RevisionData(Collections.emptyList(), commitData, treeData, Lists.newArrayList());
+ }
+
+ private RevisionData createSampleRevisionDataBlob(RevisionObjectData blob) {
+ return new RevisionData(Collections.emptyList(), null, null, Arrays.asList(blob));
}
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommandTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommandTest.java
index 51051c0..d73a6e7 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommandTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommandTest.java
@@ -38,6 +38,7 @@
import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject;
import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState;
import java.io.IOException;
+import java.util.Collections;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.RefUpdate;
import org.junit.Before;
@@ -55,6 +56,10 @@
private static final NameKey TEST_PROJECT_NAME = Project.nameKey("test-project");
private static final String TEST_REMOTE_NAME = "test-remote-name";
+ private String sampleCommitObjectId = "9f8d52853089a3cf00c02ff7bd0817bd4353a95a";
+ private String sampleTreeObjectId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904";
+ private String sampleBlobObjectId = "b5d7bcf1d1c5b0f0726d10a16c8315f06f900bfb";
+
@Mock private PullReplicationStateLogger fetchStateLog;
@Mock private ApplyObject applyObject;
@Mock private ApplyObjectMetrics metrics;
@@ -93,8 +98,10 @@
}
private RevisionData createSampleRevisionData() {
- RevisionObjectData commitData = new RevisionObjectData(Constants.OBJ_COMMIT, new byte[] {});
- RevisionObjectData treeData = new RevisionObjectData(Constants.OBJ_TREE, new byte[] {});
- return new RevisionData(commitData, treeData, Lists.newArrayList());
+ RevisionObjectData commitData =
+ new RevisionObjectData(sampleCommitObjectId, Constants.OBJ_COMMIT, new byte[] {});
+ RevisionObjectData treeData =
+ new RevisionObjectData(sampleTreeObjectId, Constants.OBJ_TREE, new byte[] {});
+ return new RevisionData(Collections.emptyList(), commitData, treeData, Lists.newArrayList());
}
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java
index daf2001..4415a4b 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java
@@ -16,6 +16,7 @@
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyString;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -24,12 +25,22 @@
import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.server.events.Event;
import com.google.gerrit.server.events.EventDispatcher;
+import com.google.gerrit.server.git.LocalDiskRepositoryManager;
+import com.google.gerrit.server.permissions.PermissionBackend;
+import com.google.gerrit.server.permissions.PermissionBackend.ForProject;
+import com.google.gerrit.server.permissions.PermissionBackend.ForRef;
+import com.google.gerrit.server.permissions.PermissionBackend.WithUser;
import com.google.gerrit.server.project.ProjectCache;
import com.google.gerrit.server.project.ProjectState;
-import com.google.gerrit.server.restapi.project.DeleteRef;
import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent;
import com.googlesource.gerrit.plugins.replication.pull.PullReplicationStateLogger;
+import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject;
import java.util.Optional;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.RefDatabase;
+import org.eclipse.jgit.lib.RefUpdate;
+import org.eclipse.jgit.lib.RefUpdate.Result;
+import org.eclipse.jgit.lib.Repository;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -48,19 +59,42 @@
@Mock private DynamicItem<EventDispatcher> eventDispatcherDataItem;
@Mock private EventDispatcher eventDispatcher;
@Mock private ProjectCache projectCache;
- @Mock private DeleteRef deleteRef;
+ @Mock private ApplyObject applyObject;
@Mock private ProjectState projectState;
+ @Mock private PermissionBackend permissionBackend;
+ @Mock private WithUser currentUser;
+ @Mock private ForProject forProject;
+ @Mock private ForRef forRef;
+ @Mock private LocalDiskRepositoryManager gitManager;
+ @Mock private RefUpdate refUpdate;
+ @Mock private Repository repository;
+ @Mock private Ref currentRef;
+ @Mock private RefDatabase refDb;
@Captor ArgumentCaptor<Event> eventCaptor;
private DeleteRefCommand objectUnderTest;
@Before
- public void setup() {
+ public void setup() throws Exception {
when(eventDispatcherDataItem.get()).thenReturn(eventDispatcher);
when(projectCache.get(any())).thenReturn(Optional.of(projectState));
+ when(permissionBackend.currentUser()).thenReturn(currentUser);
+ when(currentUser.project(any())).thenReturn(forProject);
+ when(forProject.ref(any())).thenReturn(forRef);
+ when(gitManager.openRepository(any())).thenReturn(repository);
+ when(repository.updateRef(any())).thenReturn(refUpdate);
+ when(repository.getRefDatabase()).thenReturn(refDb);
+ when(refDb.exactRef(anyString())).thenReturn(currentRef);
+ when(refUpdate.delete()).thenReturn(Result.FORCED);
objectUnderTest =
- new DeleteRefCommand(fetchStateLog, projectCache, deleteRef, eventDispatcherDataItem);
+ new DeleteRefCommand(
+ fetchStateLog,
+ projectCache,
+ applyObject,
+ permissionBackend,
+ eventDispatcherDataItem,
+ gitManager);
}
@Test
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionTest.java
index 8fd4b78..ce0b9d3 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionTest.java
@@ -67,7 +67,7 @@
@Before
public void setup() {
- when(fetchJobFactory.create(any(), any())).thenReturn(fetchJob);
+ when(fetchJobFactory.create(any(), any(), any())).thenReturn(fetchJob);
when(workQueue.getDefaultQueue()).thenReturn(exceutorService);
when(urlFormatter.getRestUrl(anyString())).thenReturn(Optional.of(location));
when(exceutorService.submit(any(Runnable.class)))
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java
index e1ad565..9af2d10 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java
@@ -35,6 +35,7 @@
import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection;
import com.googlesource.gerrit.plugins.replication.pull.api.exception.RemoteConfigurationMissingException;
import java.net.URISyntaxException;
+import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
@@ -56,6 +57,7 @@
@Mock Source source;
@Mock SourcesCollection sources;
@Mock DynamicItem<EventDispatcher> eventDispatcher;
+ @Mock PullReplicationApiRequestMetrics apiRequestMetrics;
@SuppressWarnings("rawtypes")
@Mock
@@ -76,7 +78,7 @@
when(fetchReplicationStateFactory.create(any())).thenReturn(state);
when(source.getRemoteConfigName()).thenReturn(label);
when(sources.getAll()).thenReturn(Lists.newArrayList(source));
- when(source.schedule(eq(projectName), eq(REF_NAME_TO_FETCH), eq(state), any()))
+ when(source.schedule(eq(projectName), eq(REF_NAME_TO_FETCH), eq(state), any(), any()))
.thenReturn(CompletableFuture.completedFuture(null));
objectUnderTest =
new FetchCommand(fetchReplicationStateFactory, fetchStateLog, sources, eventDispatcher);
@@ -88,16 +90,18 @@
TimeoutException {
objectUnderTest.fetchSync(projectName, label, REF_NAME_TO_FETCH);
- verify(source, times(1)).schedule(projectName, REF_NAME_TO_FETCH, state, SYNC);
+ verify(source, times(1))
+ .schedule(projectName, REF_NAME_TO_FETCH, state, SYNC, Optional.empty());
}
@Test
public void shouldScheduleRefFetchWithDelay()
throws InterruptedException, ExecutionException, RemoteConfigurationMissingException,
TimeoutException {
- objectUnderTest.fetchAsync(projectName, label, REF_NAME_TO_FETCH);
+ objectUnderTest.fetchAsync(projectName, label, REF_NAME_TO_FETCH, apiRequestMetrics);
- verify(source, times(1)).schedule(projectName, REF_NAME_TO_FETCH, state, ASYNC);
+ verify(source, times(1))
+ .schedule(projectName, REF_NAME_TO_FETCH, state, ASYNC, Optional.of(apiRequestMetrics));
}
@Test
@@ -106,7 +110,8 @@
TimeoutException {
objectUnderTest.fetchSync(projectName, label, REF_NAME_TO_FETCH);
- verify(source, times(1)).schedule(projectName, REF_NAME_TO_FETCH, state, SYNC);
+ verify(source, times(1))
+ .schedule(projectName, REF_NAME_TO_FETCH, state, SYNC, Optional.empty());
verify(state, times(1)).markAllFetchTasksScheduled();
}
@@ -123,7 +128,8 @@
public void shouldUpdateStateWhenInterruptedException()
throws InterruptedException, ExecutionException, TimeoutException {
when(future.get(anyLong(), eq(TimeUnit.SECONDS))).thenThrow(new InterruptedException());
- when(source.schedule(projectName, REF_NAME_TO_FETCH, state, SYNC)).thenReturn(future);
+ when(source.schedule(projectName, REF_NAME_TO_FETCH, state, SYNC, Optional.empty()))
+ .thenReturn(future);
InterruptedException e =
assertThrows(
@@ -138,7 +144,8 @@
throws InterruptedException, ExecutionException, TimeoutException {
when(future.get(anyLong(), eq(TimeUnit.SECONDS)))
.thenThrow(new ExecutionException(new Exception()));
- when(source.schedule(projectName, REF_NAME_TO_FETCH, state, SYNC)).thenReturn(future);
+ when(source.schedule(projectName, REF_NAME_TO_FETCH, state, SYNC, Optional.empty()))
+ .thenReturn(future);
ExecutionException e =
assertThrows(
@@ -152,7 +159,8 @@
public void shouldUpdateStateWhenTimeoutException()
throws InterruptedException, ExecutionException, TimeoutException {
when(future.get(anyLong(), eq(TimeUnit.SECONDS))).thenThrow(new TimeoutException());
- when(source.schedule(projectName, REF_NAME_TO_FETCH, state, SYNC)).thenReturn(future);
+ when(source.schedule(projectName, REF_NAME_TO_FETCH, state, SYNC, Optional.empty()))
+ .thenReturn(future);
TimeoutException e =
assertThrows(
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java
index f0b8b99..00904c7 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java
@@ -38,6 +38,7 @@
import java.io.InputStreamReader;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
+import java.util.Collections;
import java.util.Optional;
import org.apache.http.Header;
import org.apache.http.client.ClientProtocolException;
@@ -89,10 +90,22 @@
Header expectedHeader = new BasicHeader("Content-Type", "application/json");
SyncRefsFilter syncRefsFilter;
+ String commitObjectId = "9f8d52853089a3cf00c02ff7bd0817bd4353a95a";
+ String treeObjectId = "77814d216a6cab2ddb9f2877fbbd0febdc0fa608";
+ String blobObjectId = "bb383f5249c68a4cc8c82bdd1228b4a8883ff6e8";
+
String expectedSendObjectPayload =
- "{\"label\":\"Replication\",\"ref_name\":\"refs/heads/master\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"dHJlZSA3NzgxNGQyMTZhNmNhYjJkZGI5ZjI4NzdmYmJkMGZlYmRjMGZhNjA4CnBhcmVudCA5ODNmZjFhM2NmNzQ3MjVhNTNhNWRlYzhkMGMwNjEyMjEyOGY1YThkCmF1dGhvciBHZXJyaXQgVXNlciAxMDAwMDAwIDwxMDAwMDAwQDY5ZWMzOGYwLTM1MGUtNGQ5Yy05NmQ0LWJjOTU2ZjJmYWFhYz4gMTYxMDU3ODY0OCArMDEwMApjb21taXR0ZXIgR2Vycml0IENvZGUgUmV2aWV3IDxyb290QG1hY3plY2gtWFBTLTE1PiAxNjEwNTc4NjQ4ICswMTAwCgpVcGRhdGUgcGF0Y2ggc2V0IDEKClBhdGNoIFNldCAxOgoKKDEgY29tbWVudCkKClBhdGNoLXNldDogMQo\\u003d\"},\"tree_object\":{\"type\":2,\"content\":\"MTAwNjQ0IGJsb2IgYmIzODNmNTI0OWM2OGE0Y2M4YzgyYmRkMTIyOGI0YTg4ODNmZjZlOCAgICBmNzVhNjkwMDRhOTNiNGNjYzhjZTIxNWMxMjgwODYzNmMyYjc1Njc1\"},\"blobs\":[{\"type\":3,\"content\":\"ewogICJjb21tZW50cyI6IFsKICAgIHsKICAgICAgImtleSI6IHsKICAgICAgICAidXVpZCI6ICI5MGI1YWJmZl80ZjY3NTI2YSIsCiAgICAgICAgImZpbGVuYW1lIjogIi9DT01NSVRfTVNHIiwKICAgICAgICAicGF0Y2hTZXRJZCI6IDEKICAgICAgfSwKICAgICAgImxpbmVOYnIiOiA5LAogICAgICAiYXV0aG9yIjogewogICAgICAgICJpZCI6IDEwMDAwMDAKICAgICAgfSwKICAgICAgIndyaXR0ZW5PbiI6ICIyMDIxLTAxLTEzVDIyOjU3OjI4WiIsCiAgICAgICJzaWRlIjogMSwKICAgICAgIm1lc3NhZ2UiOiAidGVzdCBjb21tZW50IiwKICAgICAgInJhbmdlIjogewogICAgICAgICJzdGFydExpbmUiOiA5LAogICAgICAgICJzdGFydENoYXIiOiAyMSwKICAgICAgICAiZW5kTGluZSI6IDksCiAgICAgICAgImVuZENoYXIiOiAzNAogICAgICB9LAogICAgICAicmV2SWQiOiAiZjc1YTY5MDA0YTkzYjRjY2M4Y2UyMTVjMTI4MDg2MzZjMmI3NTY3NSIsCiAgICAgICJzZXJ2ZXJJZCI6ICI2OWVjMzhmMC0zNTBlLTRkOWMtOTZkNC1iYzk1NmYyZmFhYWMiLAogICAgICAidW5yZXNvbHZlZCI6IHRydWUKICAgIH0KICBdCn0\\u003d\"}]}}";
+ "{\"label\":\"Replication\",\"ref_name\":\"refs/heads/master\",\"revision_data\":{\"commit_object\":{\"sha1\":\""
+ + commitObjectId
+ + "\",\"type\":1,\"content\":\"dHJlZSA3NzgxNGQyMTZhNmNhYjJkZGI5ZjI4NzdmYmJkMGZlYmRjMGZhNjA4CnBhcmVudCA5ODNmZjFhM2NmNzQ3MjVhNTNhNWRlYzhkMGMwNjEyMjEyOGY1YThkCmF1dGhvciBHZXJyaXQgVXNlciAxMDAwMDAwIDwxMDAwMDAwQDY5ZWMzOGYwLTM1MGUtNGQ5Yy05NmQ0LWJjOTU2ZjJmYWFhYz4gMTYxMDU3ODY0OCArMDEwMApjb21taXR0ZXIgR2Vycml0IENvZGUgUmV2aWV3IDxyb290QG1hY3plY2gtWFBTLTE1PiAxNjEwNTc4NjQ4ICswMTAwCgpVcGRhdGUgcGF0Y2ggc2V0IDEKClBhdGNoIFNldCAxOgoKKDEgY29tbWVudCkKClBhdGNoLXNldDogMQo\\u003d\"},\"tree_object\":{\"sha1\":\""
+ + treeObjectId
+ + "\",\"type\":2,\"content\":\"MTAwNjQ0IGJsb2IgYmIzODNmNTI0OWM2OGE0Y2M4YzgyYmRkMTIyOGI0YTg4ODNmZjZlOCAgICBmNzVhNjkwMDRhOTNiNGNjYzhjZTIxNWMxMjgwODYzNmMyYjc1Njc1\"},\"blobs\":[{\"sha1\":\""
+ + blobObjectId
+ + "\",\"type\":3,\"content\":\"ewogICJjb21tZW50cyI6IFsKICAgIHsKICAgICAgImtleSI6IHsKICAgICAgICAidXVpZCI6ICI5MGI1YWJmZl80ZjY3NTI2YSIsCiAgICAgICAgImZpbGVuYW1lIjogIi9DT01NSVRfTVNHIiwKICAgICAgICAicGF0Y2hTZXRJZCI6IDEKICAgICAgfSwKICAgICAgImxpbmVOYnIiOiA5LAogICAgICAiYXV0aG9yIjogewogICAgICAgICJpZCI6IDEwMDAwMDAKICAgICAgfSwKICAgICAgIndyaXR0ZW5PbiI6ICIyMDIxLTAxLTEzVDIyOjU3OjI4WiIsCiAgICAgICJzaWRlIjogMSwKICAgICAgIm1lc3NhZ2UiOiAidGVzdCBjb21tZW50IiwKICAgICAgInJhbmdlIjogewogICAgICAgICJzdGFydExpbmUiOiA5LAogICAgICAgICJzdGFydENoYXIiOiAyMSwKICAgICAgICAiZW5kTGluZSI6IDksCiAgICAgICAgImVuZENoYXIiOiAzNAogICAgICB9LAogICAgICAicmV2SWQiOiAiZjc1YTY5MDA0YTkzYjRjY2M4Y2UyMTVjMTI4MDg2MzZjMmI3NTY3NSIsCiAgICAgICJzZXJ2ZXJJZCI6ICI2OWVjMzhmMC0zNTBlLTRkOWMtOTZkNC1iYzk1NmYyZmFhYWMiLAogICAgICAidW5yZXNvbHZlZCI6IHRydWUKICAgIH0KICBdCn0\\u003d\"}]}}";
String commitObject =
- "tree 77814d216a6cab2ddb9f2877fbbd0febdc0fa608\n"
+ "tree "
+ + treeObjectId
+ + "\n"
+ "parent 983ff1a3cf74725a53a5dec8d0c06122128f5a8d\n"
+ "author Gerrit User 1000000 <1000000@69ec38f0-350e-4d9c-96d4-bc956f2faaac> 1610578648 +0100\n"
+ "committer Gerrit Code Review <root@maczech-XPS-15> 1610578648 +0100\n"
@@ -105,7 +118,7 @@
+ "\n"
+ "Patch-set: 1\n";
String treeObject =
- "100644 blob bb383f5249c68a4cc8c82bdd1228b4a8883ff6e8 f75a69004a93b4ccc8ce215c12808636c2b75675";
+ "100644 blob " + blobObjectId + " f75a69004a93b4ccc8ce215c12808636c2b75675";
String blobObject =
"{\n"
+ " \"comments\": [\n"
@@ -460,9 +473,12 @@
private RevisionData createSampleRevisionData() {
RevisionObjectData commitData =
- new RevisionObjectData(Constants.OBJ_COMMIT, commitObject.getBytes());
- RevisionObjectData treeData = new RevisionObjectData(Constants.OBJ_TREE, treeObject.getBytes());
- RevisionObjectData blobData = new RevisionObjectData(Constants.OBJ_BLOB, blobObject.getBytes());
- return new RevisionData(commitData, treeData, Lists.newArrayList(blobData));
+ new RevisionObjectData(commitObjectId, Constants.OBJ_COMMIT, commitObject.getBytes());
+ RevisionObjectData treeData =
+ new RevisionObjectData(treeObjectId, Constants.OBJ_TREE, treeObject.getBytes());
+ RevisionObjectData blobData =
+ new RevisionObjectData(blobObjectId, Constants.OBJ_BLOB, blobObject.getBytes());
+ return new RevisionData(
+ Collections.emptyList(), commitData, treeData, Lists.newArrayList(blobData));
}
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultTest.java
new file mode 100644
index 0000000..d82f3a5
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultTest.java
@@ -0,0 +1,55 @@
+// Copyright (C) 2022 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.client;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import java.util.Arrays;
+import java.util.Optional;
+import javax.servlet.http.HttpServletResponse;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class HttpResultTest {
+
+ @Parameterized.Parameters(name = "HTTP Status = {0} is successful: {1}")
+ public static Iterable<Object[]> data() {
+ return Arrays.asList(
+ new Object[][] {
+ {HttpServletResponse.SC_OK, true},
+ {HttpServletResponse.SC_CREATED, true},
+ {HttpServletResponse.SC_ACCEPTED, true},
+ {HttpServletResponse.SC_NO_CONTENT, true},
+ {HttpServletResponse.SC_BAD_REQUEST, false},
+ {HttpServletResponse.SC_CONFLICT, false}
+ });
+ }
+
+ private Integer httpStatus;
+ private boolean isSuccessful;
+
+ public HttpResultTest(Integer httpStatus, Boolean isSuccessful) {
+ this.httpStatus = httpStatus;
+ this.isSuccessful = isSuccessful;
+ }
+
+ @Test
+ public void httpResultIsSuccessful() {
+ HttpResult httpResult = new HttpResult(httpStatus, Optional.empty());
+ assertThat(httpResult.isSuccessful()).isEqualTo(isSuccessful);
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java
index 655575f..c673011 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/StreamEventListenerTest.java
@@ -34,6 +34,7 @@
import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.Input;
import com.googlesource.gerrit.plugins.replication.pull.api.FetchJob;
import com.googlesource.gerrit.plugins.replication.pull.api.ProjectInitializationAction;
+import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics;
import java.util.concurrent.ScheduledExecutorService;
import org.eclipse.jgit.lib.ObjectId;
import org.junit.Before;
@@ -58,16 +59,18 @@
@Mock private FetchJob fetchJob;
@Mock private FetchJob.Factory fetchJobFactory;
@Captor ArgumentCaptor<Input> inputCaptor;
+ @Mock private PullReplicationApiRequestMetrics metrics;
private StreamEventListener objectUnderTest;
@Before
public void setup() {
when(workQueue.getDefaultQueue()).thenReturn(executor);
- when(fetchJobFactory.create(eq(Project.nameKey(TEST_PROJECT)), any())).thenReturn(fetchJob);
+ when(fetchJobFactory.create(eq(Project.nameKey(TEST_PROJECT)), any(), any()))
+ .thenReturn(fetchJob);
objectUnderTest =
new StreamEventListener(
- INSTANCE_ID, projectInitializationAction, workQueue, fetchJobFactory);
+ INSTANCE_ID, projectInitializationAction, workQueue, fetchJobFactory, () -> metrics);
}
@Test
@@ -107,7 +110,7 @@
objectUnderTest.onEvent(event);
- verify(fetchJobFactory).create(eq(Project.nameKey(TEST_PROJECT)), inputCaptor.capture());
+ verify(fetchJobFactory).create(eq(Project.nameKey(TEST_PROJECT)), inputCaptor.capture(), any());
Input input = inputCaptor.getValue();
assertThat(input.label).isEqualTo(REMOTE_INSTANCE_ID);
@@ -136,7 +139,7 @@
objectUnderTest.onEvent(event);
- verify(fetchJobFactory).create(eq(Project.nameKey(TEST_PROJECT)), inputCaptor.capture());
+ verify(fetchJobFactory).create(eq(Project.nameKey(TEST_PROJECT)), inputCaptor.capture(), any());
Input input = inputCaptor.getValue();
assertThat(input.label).isEqualTo(REMOTE_INSTANCE_ID);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObjectIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObjectIT.java
index e2a162e..161830b 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObjectIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObjectIT.java
@@ -79,14 +79,36 @@
Optional<RevisionData> revisionData =
reader.read(
- Project.nameKey(testRepoProjectName), pushResult.getCommit().toObjectId(), refName);
+ Project.nameKey(testRepoProjectName), pushResult.getCommit().toObjectId(), refName, 0);
RefSpec refSpec = new RefSpec(refName);
- objectUnderTest.apply(project, refSpec, revisionData.get());
+ objectUnderTest.apply(project, refSpec, toArray(revisionData));
try (Repository repo = repoManager.openRepository(project);
TestRepository<Repository> testRepo = new TestRepository<>(repo); ) {
Optional<RevisionData> newRevisionData =
- reader.read(project, repo.exactRef(refName).getObjectId(), refName);
+ reader.read(project, repo.exactRef(refName).getObjectId(), refName, 0);
+ compareObjects(revisionData.get(), newRevisionData);
+ testRepo.fsck();
+ }
+ }
+
+ @Test
+ public void shouldApplyRefSequencesChanges() throws Exception {
+ String testRepoProjectName = project + TEST_REPLICATION_SUFFIX;
+ testRepo = cloneProject(createTestProject(testRepoProjectName));
+
+ createChange();
+ String seqChangesRef = RefNames.REFS_SEQUENCES + "changes";
+
+ Optional<RevisionData> revisionData = reader.read(allProjects, seqChangesRef, 0);
+
+ RefSpec refSpec = new RefSpec(seqChangesRef);
+ objectUnderTest.apply(project, refSpec, toArray(revisionData));
+ try (Repository repo = repoManager.openRepository(project);
+ TestRepository<Repository> testRepo = new TestRepository<>(repo); ) {
+
+ Optional<RevisionData> newRevisionData =
+ reader.read(project, repo.exactRef(seqChangesRef).getObjectId(), seqChangesRef, 0);
compareObjects(revisionData.get(), newRevisionData);
testRepo.fsck();
}
@@ -105,8 +127,8 @@
NameKey testRepoKey = Project.nameKey(testRepoProjectName);
try (Repository repo = repoManager.openRepository(testRepoKey)) {
Optional<RevisionData> revisionData =
- reader.read(testRepoKey, repo.exactRef(refName).getObjectId(), refName);
- objectUnderTest.apply(project, refSpec, revisionData.get());
+ reader.read(testRepoKey, repo.exactRef(refName).getObjectId(), refName, 0);
+ objectUnderTest.apply(project, refSpec, toArray(revisionData));
}
ReviewInput reviewInput = new ReviewInput();
@@ -117,12 +139,12 @@
try (Repository repo = repoManager.openRepository(project);
TestRepository<Repository> testRepo = new TestRepository<>(repo)) {
Optional<RevisionData> revisionDataWithComment =
- reader.read(testRepoKey, repo.exactRef(refName).getObjectId(), refName);
+ reader.read(testRepoKey, repo.exactRef(refName).getObjectId(), refName, 0);
- objectUnderTest.apply(project, refSpec, revisionDataWithComment.get());
+ objectUnderTest.apply(project, refSpec, toArray(revisionDataWithComment));
Optional<RevisionData> newRevisionData =
- reader.read(project, repo.exactRef(refName).getObjectId(), refName);
+ reader.read(project, repo.exactRef(refName).getObjectId(), refName, 0);
compareObjects(revisionDataWithComment.get(), newRevisionData);
@@ -147,12 +169,12 @@
gApi.changes().id(changeId.get()).current().review(reviewInput);
Optional<RevisionData> revisionData =
- reader.read(createTestProject, repo.exactRef(refName).getObjectId(), refName);
+ reader.read(createTestProject, repo.exactRef(refName).getObjectId(), refName, 0);
RefSpec refSpec = new RefSpec(refName);
assertThrows(
MissingParentObjectException.class,
- () -> objectUnderTest.apply(project, refSpec, revisionData.get()));
+ () -> objectUnderTest.apply(project, refSpec, toArray(revisionData)));
}
}
@@ -173,6 +195,9 @@
}
private void compareContent(RevisionObjectData expected, RevisionObjectData actual) {
+ if (expected == actual) {
+ return;
+ }
assertThat(actual.getType()).isEqualTo(expected.getType());
assertThat(Bytes.asList(actual.getContent()))
.containsExactlyElementsIn(Bytes.asList(expected.getContent()))
@@ -205,4 +230,10 @@
bind(ApplyObject.class);
}
}
+
+ private RevisionData[] toArray(Optional<RevisionData> optional) {
+ ImmutableList.Builder<RevisionData> listBuilder = ImmutableList.builder();
+ optional.ifPresent(listBuilder::add);
+ return listBuilder.build().toArray(new RevisionData[1]);
+ }
}
