Merge branch 'stable-3.2' into stable-3.3 * stable-3.2: Do not rely on System.nanoTime for E2E metrics Introduce the apply-objects REST-API for the whole '/meta' chain Fix the processing of an empty HTML response body from REST-API Fix issue with ref deletion and global-refdb Fix issue with fetching all refs after project creation Always fallback to fetch when ApplyObject REST-API fails Log the reason why a ref object wasn't loaded by RevisionReader Consider any HTTP 2xx response code from REST-API as success Return NO_CONTENT when removing a ref through ApplyObject Introduce E2E fetch REST-API metrics Change-Id: I58f27ad1977921598e991b35360d016ec125fe13
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchAll.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchAll.java index bcd86d7..42310ff 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchAll.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchAll.java
@@ -21,6 +21,7 @@ import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; import com.googlesource.gerrit.plugins.replication.ReplicationFilter; +import java.util.Optional; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import org.eclipse.jgit.transport.URIish; @@ -91,7 +92,7 @@ for (Source cfg : sources.getAll()) { if (cfg.wouldFetchProject(project)) { for (URIish uri : cfg.getURIs(project, urlMatch)) { - cfg.schedule(project, FetchOne.ALL_REFS, uri, state, replicationType); + cfg.schedule(project, FetchOne.ALL_REFS, uri, state, replicationType, Optional.empty()); } } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java index cee52b5..cdf573f 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchOne.java
@@ -32,6 +32,7 @@ import com.google.gerrit.server.util.IdGenerator; import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; +import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics; import com.googlesource.gerrit.plugins.replication.pull.fetch.Fetch; import com.googlesource.gerrit.plugins.replication.pull.fetch.FetchFactory; import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState; @@ -69,7 +70,8 @@ static final String ID_KEY = "fetchOneId"; interface Factory { - FetchOne create(Project.NameKey d, URIish u); + FetchOne create( + Project.NameKey d, URIish u, Optional<PullReplicationApiRequestMetrics> apiRequestMetrics); } private final GitRepositoryManager gitManager; @@ -94,6 +96,7 @@ private final FetchReplicationMetrics metrics; private final AtomicBoolean canceledWhileRunning; private final FetchFactory fetchFactory; + private final Optional<PullReplicationApiRequestMetrics> apiRequestMetrics; @Inject FetchOne( @@ -106,7 +109,8 @@ FetchReplicationMetrics m, FetchFactory fetchFactory, @Assisted Project.NameKey d, - @Assisted URIish u) { + @Assisted URIish u, + @Assisted Optional<PullReplicationApiRequestMetrics> apiRequestMetrics) { gitManager = grm; pool = s; config = c.getRemoteConfig(); @@ -122,6 +126,7 @@ canceledWhileRunning = new AtomicBoolean(false); this.fetchFactory = fetchFactory; maxRetries = s.getMaxRetries(); + this.apiRequestMetrics = apiRequestMetrics; } @Override @@ -299,12 +304,17 @@ git = gitManager.openRepository(projectName); runImpl(); long elapsed = NANOSECONDS.toMillis(context.stop()); + Optional<Long> elapsedEnd2End = + apiRequestMetrics + .flatMap(metrics -> metrics.stop(config.getName())) + .map(NANOSECONDS::toMillis); repLog.info( - "Replication from {} completed in {}ms, {}ms delay, {} retries", + "Replication from {} completed in {}ms, {}ms delay, {} retries{}", uri, elapsed, delay, - retryCount); + retryCount, + elapsedEnd2End.map(el -> String.format(", E2E %dms", el)).orElse("")); } catch (RepositoryNotFoundException e) { stateLog.error( "Cannot replicate " + projectName + "; Local repository error: " + e.getMessage(),
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchReplicationMetrics.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchReplicationMetrics.java index e952252..22bb073 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchReplicationMetrics.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchReplicationMetrics.java
@@ -23,10 +23,12 @@ import com.google.gerrit.server.logging.PluginMetadata; import com.google.inject.Inject; import com.google.inject.Singleton; +import java.util.concurrent.TimeUnit; @Singleton public class FetchReplicationMetrics { private final Timer1<String> executionTime; + private final Timer1<String> end2EndExecutionTime; private final Histogram1<String> executionDelay; private final Histogram1<String> executionRetries; @@ -34,11 +36,11 @@ FetchReplicationMetrics(@PluginName String pluginName, MetricMaker metricMaker) { Field<String> SOURCE_FIELD = Field.ofString( - "source", + "pull_replication", (metadataBuilder, fieldValue) -> metadataBuilder .pluginName(pluginName) - .addPluginMetadata(PluginMetadata.create("source", fieldValue))) + .addPluginMetadata(PluginMetadata.create("pull_replication", fieldValue))) .build(); executionTime = @@ -49,6 +51,14 @@ .setUnit(Description.Units.MILLISECONDS), SOURCE_FIELD); + end2EndExecutionTime = + metricMaker.newTimer( + "replication_end_2_end_latency", + new Description("Time spent end-2-end fetching from remote source.") + .setCumulative() + .setUnit(Description.Units.MILLISECONDS), + SOURCE_FIELD); + executionDelay = metricMaker.newHistogram( "replication_delay", @@ -77,6 +87,26 @@ } /** + * Start the end-to-end replication latency timer from a source. + * + * @param name the source name. + * @return the timer context. + */ + public Timer1.Context<String> startEnd2End(String name) { + return end2EndExecutionTime.start(name); + } + + /** + * Record the end-to-end replication latency timer from a source. + * + * @param name the source name. + * @param metricNanos the timer value in nanos + */ + public void recordEnd2End(String name, long metricNanos) { + end2EndExecutionTime.record(name, metricNanos, TimeUnit.NANOSECONDS); + } + + /** * Record the replication delay and retry metrics for a source. * * @param name the source name.
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java index 6393c08..f3e38fd 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
@@ -15,9 +15,11 @@ package com.googlesource.gerrit.plugins.replication.pull; import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Queues; import com.google.gerrit.entities.Project; import com.google.gerrit.entities.Project.NameKey; +import com.google.gerrit.entities.RefNames; import com.google.gerrit.extensions.events.GitReferenceUpdatedListener; import com.google.gerrit.extensions.events.HeadUpdatedListener; import com.google.gerrit.extensions.events.LifecycleListener; @@ -37,7 +39,10 @@ import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter; import java.io.IOException; import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Optional; import java.util.Queue; import java.util.Set; @@ -47,7 +52,11 @@ import java.util.concurrent.TimeoutException; import java.util.function.Consumer; import org.apache.http.client.ClientProtocolException; +import org.eclipse.jgit.errors.CorruptObjectException; +import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.InvalidObjectIdException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.errors.RepositoryNotFoundException; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.transport.URIish; import org.slf4j.Logger; @@ -75,8 +84,9 @@ private FetchApiClient.Factory fetchClientFactory; private Integer fetchCallsTimeout; private ExcludedRefsFilter refsFilter; - private RevisionReader revisionReader; + private Provider<RevisionReader> revReaderProvider; private final ApplyObjectMetrics applyObjectMetrics; + private final FetchReplicationMetrics fetchMetrics; @Inject ReplicationQueue( @@ -86,8 +96,9 @@ ReplicationStateListeners sl, FetchApiClient.Factory fetchClientFactory, ExcludedRefsFilter refsFilter, - RevisionReader revReader, - ApplyObjectMetrics applyObjectMetrics) { + Provider<RevisionReader> revReaderProvider, + ApplyObjectMetrics applyObjectMetrics, + FetchReplicationMetrics fetchMetrics) { workQueue = wq; dispatcher = dis; sources = rd; @@ -95,8 +106,9 @@ beforeStartupEventsQueue = Queues.newConcurrentLinkedQueue(); this.fetchClientFactory = fetchClientFactory; this.refsFilter = refsFilter; - this.revisionReader = revReader; + this.revReaderProvider = revReaderProvider; this.applyObjectMetrics = applyObjectMetrics; + this.fetchMetrics = fetchMetrics; } @Override @@ -229,9 +241,19 @@ CallFunction call = getCallFunction(project, objectId, refName, isDelete, state); return (source) -> { + boolean callSuccessful; try { - call.call(source); - } catch (MissingParentObjectException e) { + callSuccessful = call.call(source); + } catch (Exception e) { + repLog.warn( + String.format( + "Failed to apply object %s on project %s:%s, falling back to git fetch", + objectId.name(), project, refName), + e); + callSuccessful = false; + } + + if (!callSuccessful) { callFetch(source, project, refName, state); } }; @@ -248,7 +270,8 @@ } try { - Optional<RevisionData> revisionData = revisionReader.read(project, objectId, refName); + Optional<RevisionData> revisionData = + revReaderProvider.get().read(project, objectId, refName, 0); repLog.info( "RevisionData is {} for {}:{}", revisionData.map(RevisionData::toString).orElse("ABSENT"), @@ -257,7 +280,8 @@ if (revisionData.isPresent()) { return ((source) -> - callSendObject(source, project, refName, isDelete, revisionData.get(), state)); + callSendObject( + source, project, refName, isDelete, Arrays.asList(revisionData.get()), state)); } } catch (InvalidObjectIdException | IOException e) { stateLog.error( @@ -271,14 +295,15 @@ return (source) -> callFetch(source, project, refName, state); } - private void callSendObject( + private boolean callSendObject( Source source, Project.NameKey project, String refName, boolean isDelete, - RevisionData revision, + List<RevisionData> revision, ReplicationState state) throws MissingParentObjectException { + boolean resultIsSuccessful = true; if (source.wouldFetchProject(project) && source.wouldFetchRef(refName)) { for (String apiUrl : source.getApis()) { try { @@ -291,7 +316,11 @@ refName, revision); Context<String> apiTimer = applyObjectMetrics.startEnd2End(source.getRemoteConfigName()); - HttpResult result = fetchClient.callSendObject(project, refName, isDelete, revision, uri); + HttpResult result = + isDelete + ? fetchClient.callSendObject(project, refName, isDelete, null, uri) + : fetchClient.callSendObjects(project, refName, revision, uri); + boolean resultSuccessful = result.isSuccessful(); repLog.info( "Pull replication REST API apply object to {} COMPLETED for {}:{} - {}, HTTP Result:" + " {} - time:{} ms", @@ -302,17 +331,34 @@ result, apiTimer.stop() / 1000000.0); - if (isProjectMissing(result, project) && source.isCreateMissingRepositories()) { + if (!resultSuccessful + && result.isProjectMissing(project) + && source.isCreateMissingRepositories()) { result = initProject(project, uri, fetchClient, result); repLog.info("Missing project {} created, HTTP Result:{}", project, result); } - if (!result.isSuccessful()) { + if (!resultSuccessful) { if (result.isParentObjectMissing()) { + + if (RefNames.isNoteDbMetaRef(refName) && revision.size() == 1) { + List<RevisionData> allRevisions = + fetchWholeMetaHistory(project, refName, revision.get(0)); + repLog.info( + "Pull replication REST API apply object to {} for {}:{} - {}", + apiUrl, + project, + refName, + allRevisions); + return callSendObject(source, project, refName, isDelete, allRevisions, state); + } + throw new MissingParentObjectException( project, refName, source.getRemoteConfigName()); } } + + resultIsSuccessful &= resultSuccessful; } catch (URISyntaxException e) { repLog.warn( "Pull replication REST API apply object to {} *FAILED* for {}:{} - {}", @@ -322,6 +368,7 @@ revision, e); stateLog.error(String.format("Cannot parse pull replication api url:%s", apiUrl), state); + resultIsSuccessful = false; } catch (IOException e) { repLog.warn( "Pull replication REST API apply object to {} *FAILED* for {}:{} - {}", @@ -337,31 +384,74 @@ apiUrl, e.getMessage()), e, state); + resultIsSuccessful = false; } } } + + return resultIsSuccessful; } - private void callFetch( + private List<RevisionData> fetchWholeMetaHistory( + NameKey project, String refName, RevisionData revision) + throws RepositoryNotFoundException, MissingObjectException, IncorrectObjectTypeException, + CorruptObjectException, IOException { + RevisionReader revisionReader = revReaderProvider.get(); + Optional<RevisionData> revisionDataWithParents = + revisionReader.read(project, refName, Integer.MAX_VALUE); + + ImmutableList.Builder<RevisionData> revisionDataBuilder = ImmutableList.builder(); + List<ObjectId> parentObjectIds = + revisionDataWithParents + .map(RevisionData::getParentObjetIds) + .orElse(Collections.emptyList()); + for (ObjectId parentObjectId : parentObjectIds) { + revisionReader.read(project, parentObjectId, refName, 0).ifPresent(revisionDataBuilder::add); + } + + revisionDataBuilder.add(revision); + + return revisionDataBuilder.build(); + } + + private boolean callFetch( Source source, Project.NameKey project, String refName, ReplicationState state) { + boolean resultIsSuccessful = true; if (source.wouldFetchProject(project) && source.wouldFetchRef(refName)) { for (String apiUrl : source.getApis()) { try { URIish uri = new URIish(apiUrl); FetchApiClient fetchClient = fetchClientFactory.create(source); - HttpResult result = fetchClient.callFetch(project, refName, uri); - if (isProjectMissing(result, project) && source.isCreateMissingRepositories()) { + repLog.info("Pull replication REST API fetch to {} for {}:{}", apiUrl, project, refName); + Context<String> timer = fetchMetrics.startEnd2End(source.getRemoteConfigName()); + HttpResult result = fetchClient.callFetch(project, refName, uri, timer.getStartTime()); + long elapsedMs = TimeUnit.NANOSECONDS.toMillis(timer.stop()); + boolean resultSuccessful = result.isSuccessful(); + repLog.info( + "Pull replication REST API fetch to {} COMPLETED for {}:{}, HTTP Result:" + + " {} - time:{} ms", + apiUrl, + project, + refName, + result, + elapsedMs); + if (!resultSuccessful + && result.isProjectMissing(project) + && source.isCreateMissingRepositories()) { result = initProject(project, uri, fetchClient, result); } - if (!result.isSuccessful()) { + if (!resultSuccessful) { stateLog.warn( String.format( "Pull replication rest api fetch call failed. Endpoint url: %s, reason:%s", apiUrl, result.getMessage().orElse("unknown")), state); } + + resultIsSuccessful &= result.isSuccessful(); } catch (URISyntaxException e) { stateLog.error(String.format("Cannot parse pull replication api url:%s", apiUrl), state); + resultIsSuccessful = false; } catch (Exception e) { stateLog.error( String.format( @@ -370,25 +460,24 @@ apiUrl, e.getMessage()), e, state); + resultIsSuccessful = false; } } } + + return resultIsSuccessful; } public boolean retry(int attempt, int maxRetries) { return maxRetries == 0 || attempt < maxRetries; } - private Boolean isProjectMissing(HttpResult result, Project.NameKey project) { - return !result.isSuccessful() && result.isProjectMissing(project); - } - private HttpResult initProject( Project.NameKey project, URIish uri, FetchApiClient fetchClient, HttpResult result) throws IOException, ClientProtocolException { HttpResult initProjectResult = fetchClient.initProject(project, uri); if (initProjectResult.isSuccessful()) { - result = fetchClient.callFetch(project, "refs/*", uri); + result = fetchClient.callFetch(project, FetchOne.ALL_REFS, uri); } else { String errorMessage = initProjectResult.getMessage().map(e -> " - Error: " + e).orElse(""); repLog.error("Cannot create project " + project + errorMessage); @@ -439,6 +528,6 @@ @FunctionalInterface private interface CallFunction { - void call(Source source) throws MissingParentObjectException; + boolean call(Source source) throws MissingParentObjectException; } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java index 92cbd95..db46b23 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java
@@ -25,7 +25,9 @@ import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; import org.eclipse.jgit.diff.DiffEntry; @@ -48,8 +50,11 @@ public class RevisionReader { private static final String CONFIG_MAX_API_PAYLOAD_SIZE = "maxApiPayloadSize"; private static final Long DEFAULT_MAX_PAYLOAD_SIZE_IN_BYTES = 10000L; + static final String CONFIG_MAX_API_HISTORY_DEPTH = "maxApiHistoryDepth"; + private static final int DEFAULT_MAX_API_HISTORY_DEPTH = 128; private GitRepositoryManager gitRepositoryManager; private Long maxRefSize; + private final int maxDepth; @Inject public RevisionReader(GitRepositoryManager gitRepositoryManager, ReplicationConfig cfg) { @@ -57,16 +62,23 @@ this.maxRefSize = cfg.getConfig() .getLong("replication", CONFIG_MAX_API_PAYLOAD_SIZE, DEFAULT_MAX_PAYLOAD_SIZE_IN_BYTES); - } - - public Optional<RevisionData> read(Project.NameKey project, String refName) - throws RepositoryNotFoundException, MissingObjectException, IncorrectObjectTypeException, - CorruptObjectException, IOException { - return read(project, null, refName); + this.maxDepth = + cfg.getConfig() + .getInt("replication", CONFIG_MAX_API_HISTORY_DEPTH, DEFAULT_MAX_API_HISTORY_DEPTH); } public Optional<RevisionData> read( - Project.NameKey project, @Nullable ObjectId refObjectId, String refName) + Project.NameKey project, String refName, int maxParentObjectIds) + throws RepositoryNotFoundException, MissingObjectException, IncorrectObjectTypeException, + CorruptObjectException, IOException { + return read(project, null, refName, maxParentObjectIds); + } + + public Optional<RevisionData> read( + Project.NameKey project, + @Nullable ObjectId refObjectId, + String refName, + int maxParentObjectIds) throws MissingObjectException, IncorrectObjectTypeException, CorruptObjectException, RepositoryNotFoundException, IOException { try (Repository git = gitRepositoryManager.openRepository(project)) { @@ -81,15 +93,17 @@ ObjectLoader commitLoader = git.open(objectId); totalRefSize += commitLoader.getSize(); - verifySize(totalRefSize, commitLoader); + verifySize(project, refName, objectId, totalRefSize, commitLoader); if (commitLoader.getType() == Constants.OBJ_BLOB) { return Optional.of( new RevisionData( + Collections.emptyList(), null, null, Arrays.asList( - new RevisionObjectData(Constants.OBJ_BLOB, commitLoader.getCachedBytes())))); + new RevisionObjectData( + objectId.name(), Constants.OBJ_BLOB, commitLoader.getCachedBytes())))); } if (commitLoader.getType() != Constants.OBJ_COMMIT) { @@ -103,28 +117,34 @@ RevCommit commit = RevCommit.parse(commitLoader.getCachedBytes()); RevisionObjectData commitRev = - new RevisionObjectData(commit.getType(), commitLoader.getCachedBytes()); + new RevisionObjectData(objectId.name(), commit.getType(), commitLoader.getCachedBytes()); RevTree tree = commit.getTree(); - ObjectLoader treeLoader = git.open(commit.getTree().toObjectId()); + ObjectId treeObjectId = commit.getTree().toObjectId(); + ObjectLoader treeLoader = git.open(treeObjectId); totalRefSize += treeLoader.getSize(); - verifySize(totalRefSize, treeLoader); + verifySize(project, refName, treeObjectId, totalRefSize, treeLoader); RevisionObjectData treeRev = - new RevisionObjectData(tree.getType(), treeLoader.getCachedBytes()); + new RevisionObjectData(treeObjectId.name(), tree.getType(), treeLoader.getCachedBytes()); List<RevisionObjectData> blobs = Lists.newLinkedList(); try (TreeWalk walk = new TreeWalk(git)) { if (commit.getParentCount() > 0) { List<DiffEntry> diffEntries = readDiffs(git, commit, tree, walk); - blobs = readBlobs(git, totalRefSize, diffEntries); + blobs = readBlobs(project, refName, git, totalRefSize, diffEntries); } else { walk.setRecursive(true); walk.addTree(tree); - blobs = readBlobs(git, totalRefSize, walk); + blobs = readBlobs(project, refName, git, totalRefSize, walk); } } - return Optional.of(new RevisionData(commitRev, treeRev, blobs)); + + List<ObjectId> parentObjectIds = + getParentObjectIds(git, commit.getParents(), 0, Math.min(maxDepth, maxParentObjectIds)); + Collections.reverse(parentObjectIds); + + return Optional.of(new RevisionData(parentObjectIds, commitRev, treeRev, blobs)); } catch (LargeObjectException e) { repLog.trace( "Ref {} size for project {} is greater than configured '{}'", @@ -135,6 +155,32 @@ } } + private List<ObjectId> getParentObjectIds( + Repository git, RevCommit[] commit, int parentsDepth, int maxParentObjectIds) + throws MissingObjectException, IncorrectObjectTypeException, IOException { + if (commit == null || commit.length == 0) { + return Collections.emptyList(); + } + + ArrayList<ObjectId> parentObjectIds = new ArrayList<>(); + for (RevCommit revCommit : commit) { + if (parentsDepth < maxParentObjectIds) { + parentObjectIds.add(revCommit.getId()); + parentsDepth++; + + ObjectLoader ol = git.open(revCommit.getId(), Constants.OBJ_COMMIT); + RevCommit[] commitParents = RevCommit.parse(ol.getCachedBytes()).getParents(); + + List<ObjectId> nestedParentObjectIds = + getParentObjectIds(git, commitParents, parentsDepth, maxParentObjectIds); + parentObjectIds.addAll(nestedParentObjectIds); + parentsDepth += nestedParentObjectIds.size(); + } + } + + return parentObjectIds; + } + private List<DiffEntry> readDiffs(Repository git, RevCommit commit, RevTree tree, TreeWalk walk) throws MissingObjectException, IncorrectObjectTypeException, CorruptObjectException, IOException { @@ -143,7 +189,8 @@ return DiffEntry.scan(walk, true); } - private List<RevisionObjectData> readBlobs(Repository git, Long totalRefSize, TreeWalk walk) + private List<RevisionObjectData> readBlobs( + Project.NameKey projectName, String refName, Repository git, Long totalRefSize, TreeWalk walk) throws MissingObjectException, IncorrectObjectTypeException, CorruptObjectException, IOException { List<RevisionObjectData> blobs = Lists.newLinkedList(); @@ -151,26 +198,33 @@ ObjectId objectId = walk.getObjectId(0); ObjectLoader objectLoader = git.open(objectId); totalRefSize += objectLoader.getSize(); - verifySize(totalRefSize, objectLoader); + verifySize(projectName, refName, objectId, totalRefSize, objectLoader); RevisionObjectData rev = - new RevisionObjectData(objectLoader.getType(), objectLoader.getCachedBytes()); + new RevisionObjectData( + objectId.name(), objectLoader.getType(), objectLoader.getCachedBytes()); blobs.add(rev); } return blobs; } private List<RevisionObjectData> readBlobs( - Repository git, Long totalRefSize, List<DiffEntry> diffEntries) + Project.NameKey projectName, + String refName, + Repository git, + Long totalRefSize, + List<DiffEntry> diffEntries) throws MissingObjectException, IOException { List<RevisionObjectData> blobs = Lists.newLinkedList(); for (DiffEntry diffEntry : diffEntries) { if (!ChangeType.DELETE.equals(diffEntry.getChangeType())) { - ObjectLoader objectLoader = git.open(diffEntry.getNewId().toObjectId()); + ObjectId diffObjectId = diffEntry.getNewId().toObjectId(); + ObjectLoader objectLoader = git.open(diffObjectId); totalRefSize += objectLoader.getSize(); - verifySize(totalRefSize, objectLoader); + verifySize(projectName, refName, diffObjectId, totalRefSize, objectLoader); RevisionObjectData rev = - new RevisionObjectData(objectLoader.getType(), objectLoader.getCachedBytes()); + new RevisionObjectData( + diffObjectId.name(), objectLoader.getType(), objectLoader.getCachedBytes()); blobs.add(rev); } } @@ -185,9 +239,44 @@ return parentCommit.getTree(); } - private void verifySize(Long totalRefSize, ObjectLoader loader) throws LargeObjectException { - if (loader.isLarge() || totalRefSize > maxRefSize) { - throw new LargeObjectException(); + private void verifySize( + Project.NameKey projectName, + String refName, + ObjectId objectId, + Long totalRefSize, + ObjectLoader loader) + throws LargeObjectException { + if (loader.isLarge()) { + repLog.warn( + "Objects associated with {}:{} ({}) are too big to fit into the object loader's memory", + projectName, + refName, + objectTypeToString(loader.getType())); + throw new LargeObjectException(objectId); + } + + if (totalRefSize > maxRefSize) { + repLog.warn( + "Objects associated with {}:{} ({}) use {} bytes, over the maximum limit of {} bytes", + projectName, + refName, + objectTypeToString(loader.getType()), + totalRefSize, + maxRefSize); + throw new LargeObjectException(objectId); + } + } + + private static String objectTypeToString(int type) { + switch (type) { + case Constants.OBJ_BLOB: + return "BLOB"; + case Constants.OBJ_COMMIT: + return "COMMIT"; + case Constants.OBJ_TREE: + return "TREE"; + default: + return "type:" + type; } } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java index 71ec666..6bf4c21 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
@@ -61,6 +61,7 @@ import com.google.inject.servlet.RequestScoped; import com.googlesource.gerrit.plugins.replication.RemoteSiteUser; import com.googlesource.gerrit.plugins.replication.ReplicationFilter; +import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics; import com.googlesource.gerrit.plugins.replication.pull.fetch.BatchFetchClient; import com.googlesource.gerrit.plugins.replication.pull.fetch.CGitFetch; import com.googlesource.gerrit.plugins.replication.pull.fetch.CGitFetchValidator; @@ -392,9 +393,10 @@ Project.NameKey project, String ref, ReplicationState state, - ReplicationType replicationType) { + ReplicationType replicationType, + Optional<PullReplicationApiRequestMetrics> apiRequestMetrics) { URIish uri = getURI(project); - return schedule(project, ref, uri, state, replicationType); + return schedule(project, ref, uri, state, replicationType, apiRequestMetrics); } public Future<?> schedule( @@ -402,7 +404,8 @@ String ref, URIish uri, ReplicationState state, - ReplicationType replicationType) { + ReplicationType replicationType, + Optional<PullReplicationApiRequestMetrics> apiRequestMetrics) { repLog.info("scheduling replication {}:{} => {}", uri, ref, project); if (!shouldReplicate(project, ref, state)) { @@ -438,7 +441,7 @@ FetchOne e = pending.get(uri); Future<?> f = CompletableFuture.completedFuture(null); if (e == null) { - e = opFactory.create(project, uri); + e = opFactory.create(project, uri, apiRequestMetrics); addRef(e, ref); e.addState(ref, state); pending.put(uri, e);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java index 442f585..04cc9e8 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java
@@ -31,19 +31,20 @@ import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException; import java.io.IOException; import java.util.Objects; +import javax.servlet.http.HttpServletResponse; public class ApplyObjectAction implements RestModifyView<ProjectResource, RevisionInput> { - private final ApplyObjectCommand command; + private final ApplyObjectCommand applyObjectCommand; private final DeleteRefCommand deleteRefCommand; private final FetchPreconditions preConditions; @Inject public ApplyObjectAction( - ApplyObjectCommand command, + ApplyObjectCommand applyObjectCommand, DeleteRefCommand deleteRefCommand, FetchPreconditions preConditions) { - this.command = command; + this.applyObjectCommand = applyObjectCommand; this.deleteRefCommand = deleteRefCommand; this.preConditions = preConditions; } @@ -52,16 +53,16 @@ public Response<?> apply(ProjectResource resource, RevisionInput input) throws RestApiException { if (!preConditions.canCallFetchApi()) { - throw new AuthException("not allowed to call fetch command"); + throw new AuthException("Not allowed to call fetch command"); } - try { - if (Strings.isNullOrEmpty(input.getLabel())) { - throw new BadRequestException("Source label cannot be null or empty"); - } - if (Strings.isNullOrEmpty(input.getRefName())) { - throw new BadRequestException("Ref-update refname cannot be null or empty"); - } + if (Strings.isNullOrEmpty(input.getLabel())) { + throw new BadRequestException("Source label cannot be null or empty"); + } + if (Strings.isNullOrEmpty(input.getRefName())) { + throw new BadRequestException("Ref-update refname cannot be null or empty"); + } + try { repLog.info( "Apply object API from {} for {}:{} - {}", resource.getNameKey(), @@ -77,7 +78,7 @@ input.getLabel(), input.getRefName(), input.getRevisionData()); - return Response.created(input); + return Response.withStatusCode(HttpServletResponse.SC_NO_CONTENT, ""); } try { @@ -95,7 +96,7 @@ throw bre; } - command.applyObject( + applyObjectCommand.applyObject( resource.getNameKey(), input.getRefName(), input.getRevisionData(), input.getLabel()); return Response.created(input); } catch (MissingParentObjectException e) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java index ff231d8..b27d8b7 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java
@@ -37,6 +37,7 @@ import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject; import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState; import java.io.IOException; +import java.util.Arrays; import java.util.Set; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.transport.RefSpec; @@ -70,12 +71,25 @@ } public void applyObject( - Project.NameKey name, String refName, RevisionData revisionData, String sourceLabel) + Project.NameKey name, String refName, RevisionData revisionsData, String sourceLabel) + throws IOException, RefUpdateException, MissingParentObjectException { + applyObjects(name, refName, new RevisionData[] {revisionsData}, sourceLabel); + } + + public void applyObjects( + Project.NameKey name, String refName, RevisionData[] revisionsData, String sourceLabel) throws IOException, RefUpdateException, MissingParentObjectException { - repLog.info("Apply object from {} for {}:{} - {}", sourceLabel, name, refName, revisionData); + repLog.info( + "Apply object from {} for {}:{} - {}", + sourceLabel, + name, + refName, + Arrays.toString(revisionsData)); Timer1.Context<String> context = metrics.start(sourceLabel); - RefUpdateState refUpdateState = applyObject.apply(name, new RefSpec(refName), revisionData); + + RefUpdateState refUpdateState = applyObject.apply(name, new RefSpec(refName), revisionsData); + long elapsed = NANOSECONDS.toMillis(context.stop()); try {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectsAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectsAction.java new file mode 100644 index 0000000..a1e1f5b --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectsAction.java
@@ -0,0 +1,131 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api; + +import static com.googlesource.gerrit.plugins.replication.pull.PullReplicationLogger.repLog; + +import com.google.common.base.Strings; +import com.google.gerrit.extensions.restapi.AuthException; +import com.google.gerrit.extensions.restapi.BadRequestException; +import com.google.gerrit.extensions.restapi.ResourceConflictException; +import com.google.gerrit.extensions.restapi.Response; +import com.google.gerrit.extensions.restapi.RestApiException; +import com.google.gerrit.extensions.restapi.RestModifyView; +import com.google.gerrit.extensions.restapi.UnprocessableEntityException; +import com.google.gerrit.server.project.ProjectResource; +import com.google.inject.Inject; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionsInput; +import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException; +import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException; +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; +import javax.servlet.http.HttpServletResponse; + +public class ApplyObjectsAction implements RestModifyView<ProjectResource, RevisionsInput> { + + private final ApplyObjectCommand command; + private final DeleteRefCommand deleteRefCommand; + private final FetchPreconditions preConditions; + + @Inject + public ApplyObjectsAction( + ApplyObjectCommand command, + DeleteRefCommand deleteRefCommand, + FetchPreconditions preConditions) { + this.command = command; + this.deleteRefCommand = deleteRefCommand; + this.preConditions = preConditions; + } + + @Override + public Response<?> apply(ProjectResource resource, RevisionsInput input) throws RestApiException { + if (!preConditions.canCallFetchApi()) { + throw new AuthException("not allowed to call fetch command"); + } + + try { + if (Strings.isNullOrEmpty(input.getLabel())) { + throw new BadRequestException("Source label cannot be null or empty"); + } + if (Strings.isNullOrEmpty(input.getRefName())) { + throw new BadRequestException("Ref-update refname cannot be null or empty"); + } + + repLog.info( + "Apply object API from {} for {}:{} - {}", + resource.getNameKey(), + input.getLabel(), + input.getRefName(), + Arrays.toString(input.getRevisionsData())); + + if (Objects.isNull(input.getRevisionsData())) { + deleteRefCommand.deleteRef(resource.getNameKey(), input.getRefName(), input.getLabel()); + repLog.info( + "Apply object API - REF DELETED - from {} for {}:{}", + resource.getNameKey(), + input.getLabel(), + input.getRefName()); + return Response.withStatusCode(HttpServletResponse.SC_NO_CONTENT, ""); + } + + try { + input.validate(); + } catch (IllegalArgumentException e) { + BadRequestException bre = + new BadRequestException("Ref-update with invalid input: " + e.getMessage(), e); + repLog.error( + "Apply object API *FAILED* from {} for {}:{} - {}", + input.getLabel(), + resource.getNameKey(), + input.getRefName(), + Arrays.toString(input.getRevisionsData()), + bre); + throw bre; + } + + command.applyObjects( + resource.getNameKey(), input.getRefName(), input.getRevisionsData(), input.getLabel()); + return Response.created(input); + } catch (MissingParentObjectException e) { + repLog.error( + "Apply object API *FAILED* from {} for {}:{} - {}", + input.getLabel(), + resource.getNameKey(), + input.getRefName(), + Arrays.toString(input.getRevisionsData()), + e); + throw new ResourceConflictException(e.getMessage(), e); + } catch (NumberFormatException | IOException e) { + repLog.error( + "Apply object API *FAILED* from {} for {}:{} - {}", + input.getLabel(), + resource.getNameKey(), + input.getRefName(), + Arrays.toString(input.getRevisionsData()), + e); + throw RestApiException.wrap(e.getMessage(), e); + } catch (RefUpdateException e) { + repLog.error( + "Apply object API *FAILED* from {} for {}:{} - {}", + input.getLabel(), + resource.getNameKey(), + input.getRefName(), + Arrays.toString(input.getRevisionsData()), + e); + throw new UnprocessableEntityException(e.getMessage()); + } + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java index 0c5e016..2a3a79d 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommand.java
@@ -19,41 +19,52 @@ import com.google.common.flogger.FluentLogger; import com.google.gerrit.entities.Project; import com.google.gerrit.extensions.registration.DynamicItem; -import com.google.gerrit.extensions.restapi.ResourceConflictException; import com.google.gerrit.extensions.restapi.ResourceNotFoundException; import com.google.gerrit.extensions.restapi.RestApiException; import com.google.gerrit.server.events.EventDispatcher; +import com.google.gerrit.server.git.LocalDiskRepositoryManager; +import com.google.gerrit.server.permissions.PermissionBackend; import com.google.gerrit.server.permissions.PermissionBackendException; +import com.google.gerrit.server.permissions.RefPermission; import com.google.gerrit.server.project.ProjectCache; import com.google.gerrit.server.project.ProjectState; -import com.google.gerrit.server.restapi.project.DeleteRef; import com.google.inject.Inject; import com.googlesource.gerrit.plugins.replication.pull.Context; import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent; import com.googlesource.gerrit.plugins.replication.pull.PullReplicationStateLogger; import com.googlesource.gerrit.plugins.replication.pull.ReplicationState; +import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject; +import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState; import java.io.IOException; import java.util.Optional; +import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.RefUpdate; +import org.eclipse.jgit.lib.Repository; public class DeleteRefCommand { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private final PullReplicationStateLogger fetchStateLog; - private final DeleteRef deleteRef; + private final ApplyObject applyObject; private final DynamicItem<EventDispatcher> eventDispatcher; private final ProjectCache projectCache; + private final PermissionBackend permissionBackend; + private final LocalDiskRepositoryManager gitManager; @Inject public DeleteRefCommand( PullReplicationStateLogger fetchStateLog, ProjectCache projectCache, - DeleteRef deleteRef, - DynamicItem<EventDispatcher> eventDispatcher) { + ApplyObject applyObject, + PermissionBackend permissionBackend, + DynamicItem<EventDispatcher> eventDispatcher, + LocalDiskRepositoryManager gitManager) { this.fetchStateLog = fetchStateLog; this.projectCache = projectCache; - this.deleteRef = deleteRef; + this.applyObject = applyObject; this.eventDispatcher = eventDispatcher; + this.permissionBackend = permissionBackend; + this.gitManager = gitManager; } public void deleteRef(Project.NameKey name, String refName, String sourceLabel) @@ -66,8 +77,15 @@ } try { + projectState.get().checkStatePermitsWrite(); + permissionBackend + .currentUser() + .project(projectState.get().getNameKey()) + .ref(refName) + .check(RefPermission.DELETE); + Context.setLocalEvent(true); - deleteRef.deleteSingleRef(projectState.get(), refName); + deleteRef(name, refName); eventDispatcher .get() @@ -83,7 +101,7 @@ "Unexpected error while trying to delete ref '%s' on project %s and notifying it", refName, name); throw RestApiException.wrap(e.getMessage(), e); - } catch (ResourceConflictException e) { + } catch (IOException e) { eventDispatcher .get() .postEvent( @@ -110,4 +128,18 @@ throw RestApiException.wrap(e.getMessage(), e); } } + + private RefUpdateState deleteRef(Project.NameKey name, String refName) throws IOException { + + try (Repository repository = gitManager.openRepository(name)) { + RefUpdate.Result result; + RefUpdate u = repository.updateRef(refName); + u.setExpectedOldObjectId(repository.exactRef(refName).getObjectId()); + u.setNewObjectId(ObjectId.zeroId()); + u.setForceUpdate(true); + + result = u.delete(); + return new RefUpdateState(refName, result); + } + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java index 2767f1d..48964db 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java
@@ -101,7 +101,10 @@ @SuppressWarnings("unchecked") WorkQueue.Task<Void> task = (WorkQueue.Task<Void>) - workQueue.getDefaultQueue().submit(new FetchJob(command, project, input)); + workQueue + .getDefaultQueue() + .submit( + new FetchJob(command, project, input, PullReplicationApiRequestMetrics.get())); Optional<String> url = urlFormatter .get() @@ -117,17 +120,23 @@ private FetchCommand command; private Project.NameKey project; private FetchAction.Input input; + private final PullReplicationApiRequestMetrics apiRequestMetrics; - public FetchJob(FetchCommand command, Project.NameKey project, FetchAction.Input input) { + public FetchJob( + FetchCommand command, + Project.NameKey project, + FetchAction.Input input, + PullReplicationApiRequestMetrics apiRequestMetrics) { this.command = command; this.project = project; this.input = input; + this.apiRequestMetrics = apiRequestMetrics; } @Override public void run() { try { - command.fetchAsync(project, input.label, input.refName); + command.fetchAsync(project, input.label, input.refName, apiRequestMetrics); } catch (InterruptedException | ExecutionException | RemoteConfigurationMissingException
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java index e1ac9ac..3a502ef 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java
@@ -54,19 +54,28 @@ this.eventDispatcher = eventDispatcher; } - public void fetchAsync(Project.NameKey name, String label, String refName) + public void fetchAsync( + Project.NameKey name, + String label, + String refName, + PullReplicationApiRequestMetrics apiRequestMetrics) throws InterruptedException, ExecutionException, RemoteConfigurationMissingException, TimeoutException { - fetch(name, label, refName, ASYNC); + fetch(name, label, refName, ASYNC, Optional.of(apiRequestMetrics)); } public void fetchSync(Project.NameKey name, String label, String refName) throws InterruptedException, ExecutionException, RemoteConfigurationMissingException, TimeoutException { - fetch(name, label, refName, SYNC); + fetch(name, label, refName, SYNC, Optional.empty()); } - private void fetch(Project.NameKey name, String label, String refName, ReplicationType fetchType) + private void fetch( + Project.NameKey name, + String label, + String refName, + ReplicationType fetchType, + Optional<PullReplicationApiRequestMetrics> apiRequestMetrics) throws InterruptedException, ExecutionException, RemoteConfigurationMissingException, TimeoutException { ReplicationState state = @@ -82,7 +91,7 @@ try { state.markAllFetchTasksScheduled(); - Future<?> future = source.get().schedule(name, refName, state, fetchType); + Future<?> future = source.get().schedule(name, refName, state, fetchType, apiRequestMetrics); future.get(source.get().getTimeout(), TimeUnit.SECONDS); } catch (ExecutionException | IllegalStateException
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/HttpModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/HttpModule.java index b2ef28d..b140cb4 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/HttpModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/HttpModule.java
@@ -38,5 +38,9 @@ } else { serveRegex("/init-project/.*$").with(ProjectInitializationAction.class); } + + DynamicSet.bind(binder(), AllRequestFilter.class) + .to(PullReplicationApiMetricsFilter.class) + .in(Scopes.SINGLETON); } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiMetricsFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiMetricsFilter.java new file mode 100644 index 0000000..3858db2 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiMetricsFilter.java
@@ -0,0 +1,53 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api; + +import com.google.gerrit.httpd.AllRequestFilter; +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.google.inject.Singleton; +import java.io.IOException; +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +@Singleton +public class PullReplicationApiMetricsFilter extends AllRequestFilter { + private final Provider<PullReplicationApiRequestMetrics> apiRequestMetrics; + + @Inject + public PullReplicationApiMetricsFilter( + Provider<PullReplicationApiRequestMetrics> apiRequestMetrics) { + this.apiRequestMetrics = apiRequestMetrics; + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + if (!(request instanceof HttpServletRequest) || !(response instanceof HttpServletResponse)) { + chain.doFilter(request, response); + return; + } + + PullReplicationApiRequestMetrics requestMetrics = apiRequestMetrics.get(); + requestMetrics.start((HttpServletRequest) request); + PullReplicationApiRequestMetrics.set(requestMetrics); + + chain.doFilter(request, response); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiModule.java index e7b3a7f..d1d28a6 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiModule.java
@@ -31,6 +31,7 @@ bind(UpdateHeadAction.class).in(Scopes.SINGLETON); post(PROJECT_KIND, "fetch").to(FetchAction.class); post(PROJECT_KIND, "apply-object").to(ApplyObjectAction.class); + post(PROJECT_KIND, "apply-objects").to(ApplyObjectsAction.class); delete(PROJECT_KIND, "delete-project").to(ProjectDeletionAction.class); put(PROJECT_KIND, "HEAD").to(UpdateHeadAction.class);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiRequestMetrics.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiRequestMetrics.java new file mode 100644 index 0000000..597b66f --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiRequestMetrics.java
@@ -0,0 +1,74 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import com.google.inject.Inject; +import com.googlesource.gerrit.plugins.replication.pull.FetchReplicationMetrics; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.servlet.http.HttpServletRequest; + +public class PullReplicationApiRequestMetrics { + private static final ThreadLocal<PullReplicationApiRequestMetrics> localApiRequestMetrics = + new ThreadLocal<>(); + + public static final String HTTP_HEADER_X_START_TIME_NANOS = "X-StartTimeNanos"; + + private Optional<Long> startTimeNanos; + private final AtomicBoolean initialised = new AtomicBoolean(); + private final FetchReplicationMetrics metrics; + + public static PullReplicationApiRequestMetrics get() { + return localApiRequestMetrics.get(); + } + + public static void set(PullReplicationApiRequestMetrics metrics) { + localApiRequestMetrics.set(metrics); + } + + @Inject + public PullReplicationApiRequestMetrics(FetchReplicationMetrics metrics) { + this.metrics = metrics; + } + + public void start(HttpServletRequest req) { + if (!initialised.compareAndSet(false, true)) { + throw new IllegalStateException("PullReplicationApiRequestMetrics already initialised"); + } + + startTimeNanos = + Optional.ofNullable(req.getHeader(HTTP_HEADER_X_START_TIME_NANOS)) + .map(Long::parseLong) + /* Adjust with the system's nanotime for preventing negative execution times + * due to a clock skew between the client and the server timestamp. + */ + .map(nanoTime -> Math.min(currentTimeNanos(), nanoTime)); + } + + public Optional<Long> stop(String replicationSourceName) { + return startTimeNanos.map( + start -> { + long elapsed = currentTimeNanos() - start; + metrics.recordEnd2End(replicationSourceName, elapsed); + return elapsed; + }); + } + + private long currentTimeNanos() { + return MILLISECONDS.toNanos(System.currentTimeMillis()); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java index 4af2bf7..0a9b266 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationFilter.java
@@ -55,6 +55,7 @@ import com.google.inject.TypeLiteral; import com.googlesource.gerrit.plugins.replication.pull.api.FetchAction.Input; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionsInput; import com.googlesource.gerrit.plugins.replication.pull.api.exception.InitProjectException; import java.io.BufferedReader; import java.io.EOFException; @@ -75,6 +76,7 @@ private FetchAction fetchAction; private ApplyObjectAction applyObjectAction; + private ApplyObjectsAction applyObjectsAction; private ProjectInitializationAction projectInitializationAction; private UpdateHeadAction updateHEADAction; private ProjectDeletionAction projectDeletionAction; @@ -87,6 +89,7 @@ public PullReplicationFilter( FetchAction fetchAction, ApplyObjectAction applyObjectAction, + ApplyObjectsAction applyObjectsAction, ProjectInitializationAction projectInitializationAction, UpdateHeadAction updateHEADAction, ProjectDeletionAction projectDeletionAction, @@ -95,6 +98,7 @@ @PluginName String pluginName) { this.fetchAction = fetchAction; this.applyObjectAction = applyObjectAction; + this.applyObjectsAction = applyObjectsAction; this.projectInitializationAction = projectInitializationAction; this.updateHEADAction = updateHEADAction; this.projectDeletionAction = projectDeletionAction; @@ -127,6 +131,12 @@ } else { httpResponse.sendError(SC_UNAUTHORIZED); } + } else if (isApplyObjectsAction(httpRequest)) { + if (userProvider.get().isIdentifiedUser()) { + writeResponse(httpResponse, doApplyObjects(httpRequest)); + } else { + httpResponse.sendError(SC_UNAUTHORIZED); + } } else if (isInitProjectAction(httpRequest)) { if (userProvider.get().isIdentifiedUser()) { if (!checkAcceptHeader(httpRequest, httpResponse)) { @@ -199,6 +209,16 @@ } @SuppressWarnings("unchecked") + private Response<Map<String, Object>> doApplyObjects(HttpServletRequest httpRequest) + throws RestApiException, IOException, PermissionBackendException { + RevisionsInput input = readJson(httpRequest, TypeLiteral.get(RevisionsInput.class)); + IdString id = getProjectName(httpRequest); + ProjectResource projectResource = projectsCollection.parse(TopLevelResource.INSTANCE, id); + + return (Response<Map<String, Object>>) applyObjectsAction.apply(projectResource, input); + } + + @SuppressWarnings("unchecked") private Response<String> doUpdateHEAD(HttpServletRequest httpRequest) throws Exception { HeadInput input = readJson(httpRequest, TypeLiteral.get(HeadInput.class)); ProjectResource projectResource = @@ -293,6 +313,10 @@ return httpRequest.getRequestURI().endsWith("pull-replication~apply-object"); } + private boolean isApplyObjectsAction(HttpServletRequest httpRequest) { + return httpRequest.getRequestURI().endsWith("pull-replication~apply-objects"); + } + private boolean isFetchAction(HttpServletRequest httpRequest) { return httpRequest.getRequestURI().endsWith("pull-replication~fetch"); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionData.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionData.java index 9c33d07..ffe98da 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionData.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionData.java
@@ -15,8 +15,11 @@ package com.googlesource.gerrit.plugins.replication.pull.api.data; import java.util.List; +import org.eclipse.jgit.lib.ObjectId; public class RevisionData { + private transient List<ObjectId> parentObjectIds; + private RevisionObjectData commitObject; private RevisionObjectData treeObject; @@ -24,14 +27,20 @@ private List<RevisionObjectData> blobs; public RevisionData( + List<ObjectId> parentObjectIds, RevisionObjectData commitObject, RevisionObjectData treeObject, List<RevisionObjectData> blobs) { + this.parentObjectIds = parentObjectIds; this.commitObject = commitObject; this.treeObject = treeObject; this.blobs = blobs; } + public List<ObjectId> getParentObjetIds() { + return parentObjectIds; + } + public RevisionObjectData getCommitObject() { return commitObject; }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java index 18fc27e..c18e11d 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java
@@ -44,6 +44,10 @@ } public void validate() { + validate(refName, revisionData); + } + + static void validate(String refName, RevisionData revisionData) { // Non-heads refs can point to non-commit objects if (!refName.startsWith(Constants.R_HEADS) && Objects.isNull(revisionData.getCommitObject())
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionObjectData.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionObjectData.java index 0962717..b21f495 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionObjectData.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionObjectData.java
@@ -18,10 +18,12 @@ import org.eclipse.jgit.lib.Constants; public class RevisionObjectData { + private final String sha1; private final Integer type; private final String content; - public RevisionObjectData(int type, byte[] content) { + public RevisionObjectData(String sha1, int type, byte[] content) { + this.sha1 = sha1; this.type = type; this.content = content == null ? "" : Base64.getEncoder().encodeToString(content); } @@ -34,17 +36,28 @@ return Base64.getDecoder().decode(content); } + public String getSha1() { + return sha1; + } + @Override public String toString() { + String typeStr; switch (type) { case Constants.OBJ_BLOB: - return "BLOB"; + typeStr = "BLOB"; + break; case Constants.OBJ_COMMIT: - return "COMMIT"; + typeStr = "COMMIT"; + break; case Constants.OBJ_TREE: - return "TREE"; + typeStr = "TREE"; + break; default: - return "type:" + type; + typeStr = "type:" + type; + break; } + + return sha1 + " (" + typeStr + ")"; } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionsInput.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionsInput.java new file mode 100644 index 0000000..2361f6b --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionsInput.java
@@ -0,0 +1,60 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api.data; + +import java.util.Arrays; + +public class RevisionsInput { + private String label; + + private String refName; + + private RevisionData[] revisionsData; + + public RevisionsInput(String label, String refName, RevisionData[] revisionsData) { + this.label = label; + this.refName = refName; + this.revisionsData = revisionsData; + } + + public String getLabel() { + return label; + } + + public String getRefName() { + return refName; + } + + public RevisionData[] getRevisionsData() { + return revisionsData; + } + + public void validate() { + for (RevisionData revisionData : revisionsData) { + RevisionInput.validate(refName, revisionData); + } + } + + @Override + public String toString() { + return "RevisionsInput { " + + label + + ":" + + refName + + " - " + + Arrays.toString(revisionsData) + + "}"; + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java index 476a35b..6000eb9 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchApiClient.java
@@ -14,10 +14,13 @@ package com.googlesource.gerrit.plugins.replication.pull.client; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + import com.google.gerrit.entities.Project; import com.googlesource.gerrit.plugins.replication.pull.Source; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; import java.io.IOException; +import java.util.List; import org.apache.http.client.ClientProtocolException; import org.eclipse.jgit.transport.URIish; @@ -27,9 +30,15 @@ FetchApiClient create(Source source); } - HttpResult callFetch(Project.NameKey project, String refName, URIish targetUri) + HttpResult callFetch( + Project.NameKey project, String refName, URIish targetUri, long startTimeNanos) throws ClientProtocolException, IOException; + default HttpResult callFetch(Project.NameKey project, String refName, URIish targetUri) + throws ClientProtocolException, IOException { + return callFetch(project, refName, targetUri, MILLISECONDS.toNanos(System.currentTimeMillis())); + } + HttpResult initProject(Project.NameKey project, URIish uri) throws IOException; HttpResult deleteProject(Project.NameKey project, URIish apiUri) throws IOException; @@ -43,4 +52,8 @@ RevisionData revisionData, URIish targetUri) throws ClientProtocolException, IOException; + + HttpResult callSendObjects( + Project.NameKey project, String refName, List<RevisionData> revisionData, URIish targetUri) + throws ClientProtocolException, IOException; }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java index f2f57cd..ed919de 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
@@ -23,6 +23,7 @@ import com.google.common.net.MediaType; import com.google.gerrit.common.Nullable; import com.google.gerrit.entities.Project; +import com.google.gerrit.entities.Project.NameKey; import com.google.gerrit.extensions.annotations.PluginName; import com.google.gerrit.extensions.restapi.Url; import com.google.gson.Gson; @@ -32,11 +33,14 @@ import com.googlesource.gerrit.plugins.replication.CredentialsFactory; import com.googlesource.gerrit.plugins.replication.ReplicationConfig; import com.googlesource.gerrit.plugins.replication.pull.Source; +import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiRequestMetrics; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionsInput; import com.googlesource.gerrit.plugins.replication.pull.filter.SyncRefsFilter; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.List; import java.util.Optional; import org.apache.http.HttpResponse; import org.apache.http.ParseException; @@ -95,7 +99,8 @@ * @see com.googlesource.gerrit.plugins.replication.pull.client.FetchApiClient#callFetch(com.google.gerrit.entities.Project.NameKey, java.lang.String, org.eclipse.jgit.transport.URIish) */ @Override - public HttpResult callFetch(Project.NameKey project, String refName, URIish targetUri) + public HttpResult callFetch( + Project.NameKey project, String refName, URIish targetUri, long startTimeNanos) throws ClientProtocolException, IOException { String url = String.format( @@ -110,6 +115,9 @@ instanceLabel, refName, callAsync), StandardCharsets.UTF_8)); post.addHeader(new BasicHeader("Content-Type", "application/json")); + post.addHeader( + PullReplicationApiRequestMetrics.HTTP_HEADER_X_START_TIME_NANOS, + Long.toString(startTimeNanos)); return httpClientFactory.create(source).execute(post, this, getContext(targetUri)); } @@ -174,10 +182,7 @@ } RevisionInput input = new RevisionInput(instanceLabel, refName, revisionData); - String url = - String.format( - "%s/a/projects/%s/%s~apply-object", - targetUri.toString(), Url.encode(project.get()), pluginName); + String url = formatUrl(project, targetUri, "apply-object"); HttpPost post = new HttpPost(url); post.setEntity(new StringEntity(GSON.toJson(input))); @@ -185,6 +190,33 @@ return httpClientFactory.create(source).execute(post, this, getContext(targetUri)); } + @Override + public HttpResult callSendObjects( + NameKey project, String refName, List<RevisionData> revisionData, URIish targetUri) + throws ClientProtocolException, IOException { + if (revisionData.size() == 1) { + return callSendObject(project, refName, false, revisionData.get(0), targetUri); + } + + RevisionData[] inputData = new RevisionData[revisionData.size()]; + RevisionsInput input = + new RevisionsInput(instanceLabel, refName, revisionData.toArray(inputData)); + + String url = formatUrl(project, targetUri, "apply-objects"); + HttpPost post = new HttpPost(url); + post.setEntity(new StringEntity(GSON.toJson(input))); + post.addHeader(new BasicHeader("Content-Type", MediaType.JSON_UTF_8.toString())); + return httpClientFactory.create(source).execute(post, this, getContext(targetUri)); + } + + private String formatUrl(Project.NameKey project, URIish targetUri, String api) { + String url = + String.format( + "%s/a/projects/%s/%s~%s", + targetUri.toString(), Url.encode(project.get()), pluginName, api); + return url; + } + private void requireNull(Object object, String string) { if (object != null) { throw new IllegalArgumentException(string); @@ -193,13 +225,20 @@ @Override public HttpResult handleResponse(HttpResponse response) { - Optional<String> responseBody = Optional.empty(); - try { - responseBody = Optional.ofNullable(EntityUtils.toString(response.getEntity())); - } catch (ParseException | IOException e) { - logger.atSevere().withCause(e).log("Unable get response body from %s", response.toString()); - } + Optional<String> responseBody = + Optional.ofNullable(response.getEntity()) + .flatMap( + body -> { + try { + return Optional.of(EntityUtils.toString(body)); + } catch (ParseException | IOException e) { + logger.atSevere().withCause(e).log( + "Unable get response body from %s", response.toString()); + return Optional.empty(); + } + }); + return new HttpResult(response.getStatusLine().getStatusCode(), responseBody); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResult.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResult.java index 3efcacf..ec9d65f 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResult.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResult.java
@@ -15,9 +15,6 @@ package com.googlesource.gerrit.plugins.replication.pull.client; import static javax.servlet.http.HttpServletResponse.SC_CONFLICT; -import static javax.servlet.http.HttpServletResponse.SC_CREATED; -import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT; -import static javax.servlet.http.HttpServletResponse.SC_OK; import com.google.gerrit.entities.Project; import java.util.Optional; @@ -36,7 +33,7 @@ } public boolean isSuccessful() { - return responseCode == SC_CREATED || responseCode == SC_NO_CONTENT || responseCode == SC_OK; + return responseCode / 100 == 2; // Any 2xx response code is a success } public boolean isProjectMissing(Project.NameKey projectName) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java index fca085b..2bb1caf 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java
@@ -41,47 +41,51 @@ this.gitManager = gitManager; } - public RefUpdateState apply(Project.NameKey name, RefSpec refSpec, RevisionData revisionData) + public RefUpdateState apply(Project.NameKey name, RefSpec refSpec, RevisionData[] revisionsData) throws MissingParentObjectException, IOException { try (Repository git = gitManager.openRepository(name)) { - ObjectId newObjectID = null; - RevisionObjectData commitObject = revisionData.getCommitObject(); - - try (ObjectInserter oi = git.newObjectInserter()) { - if (commitObject != null) { - RevCommit commit = RevCommit.parse(commitObject.getContent()); - for (RevCommit parent : commit.getParents()) { - if (!git.getObjectDatabase().has(parent.getId())) { - throw new MissingParentObjectException(name, refSpec.getSource(), parent.getId()); - } - } - newObjectID = oi.insert(commitObject.getType(), commitObject.getContent()); - - RevisionObjectData treeObject = revisionData.getTreeObject(); - oi.insert(treeObject.getType(), treeObject.getContent()); - } - - for (RevisionObjectData rev : revisionData.getBlobs()) { - ObjectId blobObjectId = oi.insert(rev.getType(), rev.getContent()); - if (newObjectID == null) { - newObjectID = blobObjectId; - } - } - - oi.flush(); - } + ObjectId refHead = null; RefUpdate ru = git.updateRef(refSpec.getSource()); - ru.setNewObjectId(newObjectID); + try (ObjectInserter oi = git.newObjectInserter()) { + for (RevisionData revisionData : revisionsData) { - if (commitObject == null) { - // Non-commits must be forced as they do not have a graph associated - ru.setForceUpdate(true); + ObjectId newObjectID = null; + RevisionObjectData commitObject = revisionData.getCommitObject(); + + if (commitObject != null) { + RevCommit commit = RevCommit.parse(commitObject.getContent()); + for (RevCommit parent : commit.getParents()) { + if (!git.getObjectDatabase().has(parent.getId())) { + throw new MissingParentObjectException(name, refSpec.getSource(), parent.getId()); + } + } + refHead = newObjectID = oi.insert(commitObject.getType(), commitObject.getContent()); + + RevisionObjectData treeObject = revisionData.getTreeObject(); + oi.insert(treeObject.getType(), treeObject.getContent()); + } + + for (RevisionObjectData rev : revisionData.getBlobs()) { + ObjectId blobObjectId = oi.insert(rev.getType(), rev.getContent()); + if (newObjectID == null) { + newObjectID = blobObjectId; + } + refHead = newObjectID; + } + + oi.flush(); + + if (commitObject == null) { + // Non-commits must be forced as they do not have a graph associated + ru.setForceUpdate(true); + } + } + + ru.setNewObjectId(refHead); + RefUpdate.Result result = ru.update(); + return new RefUpdateState(refSpec.getSource(), result); } - - RefUpdate.Result result = ru.update(); - - return new RefUpdateState(refSpec.getSource(), result); } } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java new file mode 100644 index 0000000..58be58e --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationAsyncIT.java
@@ -0,0 +1,43 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull; + +import com.google.gerrit.acceptance.SkipProjectClone; +import com.google.gerrit.acceptance.TestPlugin; +import com.google.gerrit.acceptance.UseLocalDisk; +import com.google.gerrit.server.config.SitePaths; +import com.google.inject.Inject; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.util.FS; + +@SkipProjectClone +@UseLocalDisk +@TestPlugin( + name = "pull-replication", + sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule", + httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule") +public class PullReplicationAsyncIT extends PullReplicationIT { + @Inject private SitePaths sitePaths; + + @Override + public void setUpTestPlugin() throws Exception { + FileBasedConfig config = + new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED); + config.setString("replication", null, "syncRefs", "^$"); + config.save(); + + super.setUpTestPlugin(true); + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java index cf94086..ec60793 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationIT.java
@@ -50,6 +50,7 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import com.googlesource.gerrit.plugins.replication.AutoReloadConfigDecorator; +import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.time.Duration; @@ -78,11 +79,12 @@ @UseLocalDisk @TestPlugin( name = "pull-replication", - sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule") + sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule", + httpModule = "com.googlesource.gerrit.plugins.replication.pull.api.HttpModule") public class PullReplicationIT extends LightweightPluginDaemonTest { private static final Optional<String> ALL_PROJECTS = Optional.empty(); private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private static final int TEST_REPLICATION_DELAY = 60; + private static final int TEST_REPLICATION_DELAY = 1; private static final Duration TEST_TIMEOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 2000); private static final String TEST_REPLICATION_SUFFIX = "suffix1"; private static final String TEST_REPLICATION_REMOTE = "remote1"; @@ -96,10 +98,17 @@ @Override public void setUpTestPlugin() throws Exception { + setUpTestPlugin(false); + } + + protected void setUpTestPlugin(boolean loadExisting) throws Exception { gitPath = sitePaths.site_path.resolve("git"); - config = - new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED); + File configFile = sitePaths.etc_dir.resolve("replication.config").toFile(); + config = new FileBasedConfig(configFile, FS.DETECTED); + if (loadExisting && configFile.exists()) { + config.load(); + } setReplicationSource( TEST_REPLICATION_REMOTE, TEST_REPLICATION_SUFFIX,
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java index 08d7e5a..48e0e71 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
@@ -17,9 +17,12 @@ import static com.google.common.truth.Truth.assertThat; import static java.nio.file.Files.createTempDirectory; import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyInt; +import static org.mockito.Mockito.anyLong; import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -50,6 +53,8 @@ import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter; import java.io.IOException; import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; import java.util.Optional; import org.apache.http.client.ClientProtocolException; import org.eclipse.jgit.errors.LargeObjectException; @@ -79,11 +84,17 @@ @Mock AccountInfo accountInfo; @Mock RevisionReader revReader; @Mock RevisionData revisionData; + @Mock HttpResult successfulHttpResult; + @Mock HttpResult fetchHttpResult; + @Mock RevisionData revisionDataWithParents; + List<ObjectId> revisionDataParentObjectIds; @Mock HttpResult httpResult; ApplyObjectMetrics applyObjectMetrics; + FetchReplicationMetrics fetchMetrics; @Captor ArgumentCaptor<String> stringCaptor; @Captor ArgumentCaptor<Project.NameKey> projectNameKeyCaptor; + @Captor ArgumentCaptor<List<RevisionData>> revisionsDataCaptor; private ExcludedRefsFilter refsFilter; private ReplicationQueue objectUnderTest; @@ -104,19 +115,51 @@ when(source.getApis()).thenReturn(apis); when(sourceCollection.getAll()).thenReturn(Lists.newArrayList(source)); when(rd.get()).thenReturn(sourceCollection); - when(revReader.read(any(), any(), anyString())).thenReturn(Optional.of(revisionData)); + lenient() + .when(revReader.read(any(), any(), anyString(), eq(0))) + .thenReturn(Optional.of(revisionData)); + lenient().when(revReader.read(any(), anyString(), eq(0))).thenReturn(Optional.of(revisionData)); + lenient() + .when(revReader.read(any(), any(), anyString(), eq(Integer.MAX_VALUE))) + .thenReturn(Optional.of(revisionDataWithParents)); + lenient() + .when(revReader.read(any(), anyString(), eq(Integer.MAX_VALUE))) + .thenReturn(Optional.of(revisionDataWithParents)); + revisionDataParentObjectIds = + Arrays.asList( + ObjectId.fromString("9f8d52853089a3cf00c02ff7bd0817bd4353a95a"), + ObjectId.fromString("b5d7bcf1d1c5b0f0726d10a16c8315f06f900bfb")); + when(revisionDataWithParents.getParentObjetIds()).thenReturn(revisionDataParentObjectIds); + when(fetchClientFactory.create(any())).thenReturn(fetchRestApiClient); - when(fetchRestApiClient.callSendObject(any(), anyString(), anyBoolean(), any(), any())) + lenient() + .when(fetchRestApiClient.callSendObject(any(), anyString(), anyBoolean(), any(), any())) .thenReturn(httpResult); - when(fetchRestApiClient.callFetch(any(), anyString(), any())).thenReturn(httpResult); + lenient() + .when(fetchRestApiClient.callSendObjects(any(), anyString(), any(), any())) + .thenReturn(httpResult); + when(fetchRestApiClient.callFetch(any(), anyString(), any(), anyLong())) + .thenReturn(fetchHttpResult); + when(fetchRestApiClient.initProject(any(), any())).thenReturn(successfulHttpResult); + when(successfulHttpResult.isSuccessful()).thenReturn(true); when(httpResult.isSuccessful()).thenReturn(true); + when(fetchHttpResult.isSuccessful()).thenReturn(true); when(httpResult.isProjectMissing(any())).thenReturn(false); applyObjectMetrics = new ApplyObjectMetrics("pull-replication", new DisabledMetricMaker()); + fetchMetrics = new FetchReplicationMetrics("pull-replication", new DisabledMetricMaker()); objectUnderTest = new ReplicationQueue( - wq, rd, dis, sl, fetchClientFactory, refsFilter, revReader, applyObjectMetrics); + wq, + rd, + dis, + sl, + fetchClientFactory, + refsFilter, + () -> revReader, + applyObjectMetrics, + fetchMetrics); } @Test @@ -125,7 +168,7 @@ objectUnderTest.start(); objectUnderTest.onGitReferenceUpdated(event); - verify(fetchRestApiClient).callSendObject(any(), anyString(), eq(false), any(), any()); + verify(fetchRestApiClient).callSendObjects(any(), anyString(), any(), any()); } @Test @@ -160,7 +203,7 @@ objectUnderTest.start(); objectUnderTest.onGitReferenceUpdated(event); - verify(fetchRestApiClient).callSendObject(any(), anyString(), eq(false), any(), any()); + verify(fetchRestApiClient).callSendObjects(any(), anyString(), any(), any()); } @Test @@ -169,11 +212,11 @@ Event event = new TestEvent("refs/changes/01/1/meta"); objectUnderTest.start(); - when(revReader.read(any(), any(), anyString())).thenThrow(IOException.class); + when(revReader.read(any(), any(), anyString(), anyInt())).thenThrow(IOException.class); objectUnderTest.onGitReferenceUpdated(event); - verify(fetchRestApiClient).callFetch(any(), anyString(), any()); + verify(fetchRestApiClient).callFetch(any(), anyString(), any(), anyLong()); } @Test @@ -182,27 +225,53 @@ Event event = new TestEvent("refs/changes/01/1/1"); objectUnderTest.start(); - when(revReader.read(any(), any(), anyString())).thenReturn(Optional.empty()); + when(revReader.read(any(), any(), anyString(), anyInt())).thenReturn(Optional.empty()); objectUnderTest.onGitReferenceUpdated(event); - verify(fetchRestApiClient).callFetch(any(), anyString(), any()); + verify(fetchRestApiClient).callFetch(any(), anyString(), any(), anyLong()); } @Test public void shouldFallbackToCallFetchWhenParentObjectIsMissing() throws ClientProtocolException, IOException { - Event event = new TestEvent("refs/changes/01/1/meta"); + Event event = new TestEvent("refs/changes/01/1/1"); objectUnderTest.start(); when(httpResult.isSuccessful()).thenReturn(false); when(httpResult.isParentObjectMissing()).thenReturn(true); - when(fetchRestApiClient.callSendObject(any(), anyString(), eq(false), any(), any())) + when(fetchRestApiClient.callSendObjects(any(), anyString(), any(), any())) .thenReturn(httpResult); objectUnderTest.onGitReferenceUpdated(event); - verify(fetchRestApiClient).callFetch(any(), anyString(), any()); + verify(fetchRestApiClient).callFetch(any(), anyString(), any(), anyLong()); + } + + @Test + public void shouldFallbackToApplyAllParentObjectsWhenParentObjectIsMissingOnMetaRef() + throws ClientProtocolException, IOException { + Event event = new TestEvent("refs/changes/01/1/meta"); + objectUnderTest.start(); + + when(httpResult.isSuccessful()).thenReturn(false, true); + when(httpResult.isParentObjectMissing()).thenReturn(true, false); + when(fetchRestApiClient.callSendObjects(any(), anyString(), any(), any())) + .thenReturn(httpResult); + + objectUnderTest.onGitReferenceUpdated(event); + + verify(fetchRestApiClient, times(2)) + .callSendObjects(any(), anyString(), revisionsDataCaptor.capture(), any()); + List<List<RevisionData>> revisionsDataValues = revisionsDataCaptor.getAllValues(); + assertThat(revisionsDataValues).hasSize(2); + + List<RevisionData> firstRevisionsValues = revisionsDataValues.get(0); + assertThat(firstRevisionsValues).hasSize(1); + assertThat(firstRevisionsValues).contains(revisionData); + + List<RevisionData> secondRevisionsValues = revisionsDataValues.get(1); + assertThat(secondRevisionsValues).hasSize(1 + revisionDataParentObjectIds.size()); } @Test @@ -248,7 +317,15 @@ objectUnderTest = new ReplicationQueue( - wq, rd, dis, sl, fetchClientFactory, refsFilter, revReader, applyObjectMetrics); + wq, + rd, + dis, + sl, + fetchClientFactory, + refsFilter, + () -> revReader, + applyObjectMetrics, + fetchMetrics); Event event = new TestEvent("refs/multi-site/version"); objectUnderTest.onGitReferenceUpdated(event);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java index e300613..1d8520f 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java
@@ -24,12 +24,14 @@ import com.google.gerrit.acceptance.TestPlugin; import com.google.gerrit.acceptance.UseLocalDisk; import com.google.gerrit.entities.Change; +import com.google.gerrit.entities.Change.Id; import com.google.gerrit.entities.Patch; import com.google.gerrit.entities.RefNames; import com.google.gerrit.extensions.api.changes.ReviewInput; import com.google.gerrit.extensions.api.changes.ReviewInput.CommentInput; import com.google.gerrit.extensions.client.Comment; import com.google.gerrit.extensions.config.FactoryModule; +import com.google.gerrit.extensions.restapi.RestApiException; import com.google.gerrit.server.notedb.Sequences; import com.google.inject.Scopes; import com.googlesource.gerrit.plugins.replication.ReplicationConfig; @@ -38,11 +40,13 @@ import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData; import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject; import java.io.IOException; +import java.util.List; import java.util.Optional; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.storage.file.FileBasedConfig; import org.junit.Before; import org.junit.Test; @@ -53,9 +57,12 @@ public class RevisionReaderIT extends LightweightPluginDaemonTest { RevisionReader objectUnderTest; + ReplicationFileBasedConfig replicationConfig; + @Before public void setup() { objectUnderTest = plugin.getSysInjector().getInstance(RevisionReader.class); + replicationConfig = plugin.getSysInjector().getInstance(ReplicationFileBasedConfig.class); } @Test @@ -64,7 +71,7 @@ String refName = RefNames.changeMetaRef(pushResult.getChange().getId()); Optional<RevisionData> revisionDataOption = - refObjectId(refName).flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId)); + refObjectId(refName).flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId, 0)); assertThat(revisionDataOption.isPresent()).isTrue(); RevisionData revisionData = revisionDataOption.get(); @@ -80,9 +87,65 @@ assertThat(revisionData.getBlobs()).isEmpty(); } - private Optional<RevisionData> readRevisionFromObjectUnderTest(String refName, ObjectId objId) { + @Test + public void shouldReadRefMetaObjectWithMaxNumberOfParents() throws Exception { + int numberOfParents = 3; + setReplicationConfig(numberOfParents); + Result pushResult = createChange(); + Id changeId = pushResult.getChange().getId(); + String refName = RefNames.changeMetaRef(pushResult.getChange().getId()); + + addMultipleComments(numberOfParents, changeId); + + Optional<RevisionData> revisionDataOption = + refObjectId(refName) + .flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId, numberOfParents)); + + assertThat(revisionDataOption.isPresent()).isTrue(); + List<ObjectId> parentObjectIds = revisionDataOption.get().getParentObjetIds(); + assertThat(parentObjectIds).hasSize(numberOfParents); + } + + @Test + public void shouldReadRefMetaObjectLimitedToMaxNumberOfParents() throws Exception { + int numberOfParents = 3; + setReplicationConfig(numberOfParents); + Result pushResult = createChange(); + Id changeId = pushResult.getChange().getId(); + String refName = RefNames.changeMetaRef(pushResult.getChange().getId()); + + addMultipleComments(numberOfParents + 1, changeId); + + Optional<RevisionData> revisionDataOption = + refObjectId(refName) + .flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId, numberOfParents)); + + assertThat(revisionDataOption.isPresent()).isTrue(); + List<ObjectId> parentObjectIds = revisionDataOption.get().getParentObjetIds(); + assertThat(parentObjectIds).hasSize(numberOfParents); + } + + private void addMultipleComments(int numberOfParents, Id changeId) throws RestApiException { + for (int i = 0; i < numberOfParents; i++) { + addComment(changeId); + } + } + + private void setReplicationConfig(int numberOfParents) throws IOException { + FileBasedConfig config = (FileBasedConfig) replicationConfig.getConfig(); + config.setInt( + "replication", null, RevisionReader.CONFIG_MAX_API_HISTORY_DEPTH, numberOfParents); + config.save(); + } + + private void addComment(Id changeId) throws RestApiException { + gApi.changes().id(changeId.get()).current().review(new ReviewInput().message("foo")); + } + + private Optional<RevisionData> readRevisionFromObjectUnderTest( + String refName, ObjectId objId, int maxParentsDepth) { try { - return objectUnderTest.read(project, objId, refName); + return objectUnderTest.read(project, objId, refName, maxParentsDepth); } catch (Exception e) { throw new IllegalStateException(e); } @@ -107,7 +170,7 @@ gApi.changes().id(changeId.get()).current().review(reviewInput); Optional<RevisionData> revisionDataOption = - refObjectId(refName).flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId)); + refObjectId(refName).flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId, 0)); assertThat(revisionDataOption.isPresent()).isTrue(); RevisionData revisionData = revisionDataOption.get(); @@ -131,7 +194,7 @@ createChange().assertOkStatus(); String refName = RefNames.REFS_SEQUENCES + Sequences.NAME_CHANGES; Optional<RevisionData> revisionDataOption = - refObjectId(refName).flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId)); + refObjectId(refName).flatMap(objId -> readRevisionFromObjectUnderTest(refName, objId, 0)); Truth8.assertThat(revisionDataOption).isEmpty(); }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ActionITBase.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ActionITBase.java index cef1051..1019b86 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ActionITBase.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ActionITBase.java
@@ -149,7 +149,8 @@ protected Optional<RevisionData> createRevisionData(NameKey projectName, String refName) throws Exception { try (Repository repository = repoManager.openRepository(projectName)) { - return revisionReader.read(projectName, repository.exactRef(refName).getObjectId(), refName); + return revisionReader.read( + projectName, repository.exactRef(refName).getObjectId(), refName, 0); } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java index 3678e82..814ba76 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java
@@ -37,6 +37,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Collections; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.junit.Before; @@ -55,8 +56,14 @@ String location = "http://gerrit-host/a/config/server/tasks/08d173e9"; int taskId = 1234; + private String sampleCommitObjectId = "9f8d52853089a3cf00c02ff7bd0817bd4353a95a"; + private String sampleTreeObjectId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; + private String sampleBlobObjectId = "b5d7bcf1d1c5b0f0726d10a16c8315f06f900bfb"; + private String sampleCommitContent = - "tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n" + "tree " + + sampleTreeObjectId + + "\n" + "parent 20eb48d28be86dc88fb4bef747f08de0fbefe12d\n" + "author Gerrit User 1000000 <1000000@69ec38f0-350e-4d9c-96d4-bc956f2faaac> 1610471611 +0100\n" + "committer Gerrit Code Review <root@maczech-XPS-15> 1610471611 +0100\n" @@ -102,7 +109,8 @@ new RevisionInput( label, refMetaName, - createSampleRevisionDataBlob(new RevisionObjectData(Constants.OBJ_BLOB, blobData))); + createSampleRevisionDataBlob( + new RevisionObjectData(sampleBlobObjectId, Constants.OBJ_BLOB, blobData))); Response<?> response = applyObjectAction.apply(projectResource, inputParams); @@ -148,8 +156,10 @@ @Test(expected = BadRequestException.class) public void shouldThrowBadRequestExceptionWhenMissingCommitObjectData() throws Exception { - RevisionObjectData commitData = new RevisionObjectData(Constants.OBJ_COMMIT, null); - RevisionObjectData treeData = new RevisionObjectData(Constants.OBJ_TREE, new byte[] {}); + RevisionObjectData commitData = + new RevisionObjectData(sampleCommitObjectId, Constants.OBJ_COMMIT, null); + RevisionObjectData treeData = + new RevisionObjectData(sampleTreeObjectId, Constants.OBJ_TREE, new byte[] {}); RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData(commitData, treeData)); @@ -159,7 +169,8 @@ @Test(expected = BadRequestException.class) public void shouldThrowBadRequestExceptionWhenMissingTreeObject() throws Exception { RevisionObjectData commitData = - new RevisionObjectData(Constants.OBJ_COMMIT, sampleCommitContent.getBytes()); + new RevisionObjectData( + sampleCommitObjectId, Constants.OBJ_COMMIT, sampleCommitContent.getBytes()); RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData(commitData, null)); @@ -192,17 +203,19 @@ private RevisionData createSampleRevisionData() { RevisionObjectData commitData = - new RevisionObjectData(Constants.OBJ_COMMIT, sampleCommitContent.getBytes()); - RevisionObjectData treeData = new RevisionObjectData(Constants.OBJ_TREE, new byte[] {}); + new RevisionObjectData( + sampleCommitObjectId, Constants.OBJ_COMMIT, sampleCommitContent.getBytes()); + RevisionObjectData treeData = + new RevisionObjectData(sampleTreeObjectId, Constants.OBJ_TREE, new byte[] {}); return createSampleRevisionData(commitData, treeData); } private RevisionData createSampleRevisionData( RevisionObjectData commitData, RevisionObjectData treeData) { - return new RevisionData(commitData, treeData, Lists.newArrayList()); + return new RevisionData(Collections.emptyList(), commitData, treeData, Lists.newArrayList()); } private RevisionData createSampleRevisionDataBlob(RevisionObjectData blob) { - return new RevisionData(null, null, Arrays.asList(blob)); + return new RevisionData(Collections.emptyList(), null, null, Arrays.asList(blob)); } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommandTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommandTest.java index 51051c0..d73a6e7 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommandTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommandTest.java
@@ -38,6 +38,7 @@ import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject; import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState; import java.io.IOException; +import java.util.Collections; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.RefUpdate; import org.junit.Before; @@ -55,6 +56,10 @@ private static final NameKey TEST_PROJECT_NAME = Project.nameKey("test-project"); private static final String TEST_REMOTE_NAME = "test-remote-name"; + private String sampleCommitObjectId = "9f8d52853089a3cf00c02ff7bd0817bd4353a95a"; + private String sampleTreeObjectId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"; + private String sampleBlobObjectId = "b5d7bcf1d1c5b0f0726d10a16c8315f06f900bfb"; + @Mock private PullReplicationStateLogger fetchStateLog; @Mock private ApplyObject applyObject; @Mock private ApplyObjectMetrics metrics; @@ -93,8 +98,10 @@ } private RevisionData createSampleRevisionData() { - RevisionObjectData commitData = new RevisionObjectData(Constants.OBJ_COMMIT, new byte[] {}); - RevisionObjectData treeData = new RevisionObjectData(Constants.OBJ_TREE, new byte[] {}); - return new RevisionData(commitData, treeData, Lists.newArrayList()); + RevisionObjectData commitData = + new RevisionObjectData(sampleCommitObjectId, Constants.OBJ_COMMIT, new byte[] {}); + RevisionObjectData treeData = + new RevisionObjectData(sampleTreeObjectId, Constants.OBJ_TREE, new byte[] {}); + return new RevisionData(Collections.emptyList(), commitData, treeData, Lists.newArrayList()); } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java index daf2001..4415a4b 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/DeleteRefCommandTest.java
@@ -16,6 +16,7 @@ import static com.google.common.truth.Truth.assertThat; import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -24,12 +25,22 @@ import com.google.gerrit.extensions.registration.DynamicItem; import com.google.gerrit.server.events.Event; import com.google.gerrit.server.events.EventDispatcher; +import com.google.gerrit.server.git.LocalDiskRepositoryManager; +import com.google.gerrit.server.permissions.PermissionBackend; +import com.google.gerrit.server.permissions.PermissionBackend.ForProject; +import com.google.gerrit.server.permissions.PermissionBackend.ForRef; +import com.google.gerrit.server.permissions.PermissionBackend.WithUser; import com.google.gerrit.server.project.ProjectCache; import com.google.gerrit.server.project.ProjectState; -import com.google.gerrit.server.restapi.project.DeleteRef; import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent; import com.googlesource.gerrit.plugins.replication.pull.PullReplicationStateLogger; +import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject; import java.util.Optional; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.RefDatabase; +import org.eclipse.jgit.lib.RefUpdate; +import org.eclipse.jgit.lib.RefUpdate.Result; +import org.eclipse.jgit.lib.Repository; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -48,19 +59,42 @@ @Mock private DynamicItem<EventDispatcher> eventDispatcherDataItem; @Mock private EventDispatcher eventDispatcher; @Mock private ProjectCache projectCache; - @Mock private DeleteRef deleteRef; + @Mock private ApplyObject applyObject; @Mock private ProjectState projectState; + @Mock private PermissionBackend permissionBackend; + @Mock private WithUser currentUser; + @Mock private ForProject forProject; + @Mock private ForRef forRef; + @Mock private LocalDiskRepositoryManager gitManager; + @Mock private RefUpdate refUpdate; + @Mock private Repository repository; + @Mock private Ref currentRef; + @Mock private RefDatabase refDb; @Captor ArgumentCaptor<Event> eventCaptor; private DeleteRefCommand objectUnderTest; @Before - public void setup() { + public void setup() throws Exception { when(eventDispatcherDataItem.get()).thenReturn(eventDispatcher); when(projectCache.get(any())).thenReturn(Optional.of(projectState)); + when(permissionBackend.currentUser()).thenReturn(currentUser); + when(currentUser.project(any())).thenReturn(forProject); + when(forProject.ref(any())).thenReturn(forRef); + when(gitManager.openRepository(any())).thenReturn(repository); + when(repository.updateRef(any())).thenReturn(refUpdate); + when(repository.getRefDatabase()).thenReturn(refDb); + when(refDb.exactRef(anyString())).thenReturn(currentRef); + when(refUpdate.delete()).thenReturn(Result.FORCED); objectUnderTest = - new DeleteRefCommand(fetchStateLog, projectCache, deleteRef, eventDispatcherDataItem); + new DeleteRefCommand( + fetchStateLog, + projectCache, + applyObject, + permissionBackend, + eventDispatcherDataItem, + gitManager); } @Test
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java index e1ad565..9af2d10 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java
@@ -35,6 +35,7 @@ import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection; import com.googlesource.gerrit.plugins.replication.pull.api.exception.RemoteConfigurationMissingException; import java.net.URISyntaxException; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -56,6 +57,7 @@ @Mock Source source; @Mock SourcesCollection sources; @Mock DynamicItem<EventDispatcher> eventDispatcher; + @Mock PullReplicationApiRequestMetrics apiRequestMetrics; @SuppressWarnings("rawtypes") @Mock @@ -76,7 +78,7 @@ when(fetchReplicationStateFactory.create(any())).thenReturn(state); when(source.getRemoteConfigName()).thenReturn(label); when(sources.getAll()).thenReturn(Lists.newArrayList(source)); - when(source.schedule(eq(projectName), eq(REF_NAME_TO_FETCH), eq(state), any())) + when(source.schedule(eq(projectName), eq(REF_NAME_TO_FETCH), eq(state), any(), any())) .thenReturn(CompletableFuture.completedFuture(null)); objectUnderTest = new FetchCommand(fetchReplicationStateFactory, fetchStateLog, sources, eventDispatcher); @@ -88,16 +90,18 @@ TimeoutException { objectUnderTest.fetchSync(projectName, label, REF_NAME_TO_FETCH); - verify(source, times(1)).schedule(projectName, REF_NAME_TO_FETCH, state, SYNC); + verify(source, times(1)) + .schedule(projectName, REF_NAME_TO_FETCH, state, SYNC, Optional.empty()); } @Test public void shouldScheduleRefFetchWithDelay() throws InterruptedException, ExecutionException, RemoteConfigurationMissingException, TimeoutException { - objectUnderTest.fetchAsync(projectName, label, REF_NAME_TO_FETCH); + objectUnderTest.fetchAsync(projectName, label, REF_NAME_TO_FETCH, apiRequestMetrics); - verify(source, times(1)).schedule(projectName, REF_NAME_TO_FETCH, state, ASYNC); + verify(source, times(1)) + .schedule(projectName, REF_NAME_TO_FETCH, state, ASYNC, Optional.of(apiRequestMetrics)); } @Test @@ -106,7 +110,8 @@ TimeoutException { objectUnderTest.fetchSync(projectName, label, REF_NAME_TO_FETCH); - verify(source, times(1)).schedule(projectName, REF_NAME_TO_FETCH, state, SYNC); + verify(source, times(1)) + .schedule(projectName, REF_NAME_TO_FETCH, state, SYNC, Optional.empty()); verify(state, times(1)).markAllFetchTasksScheduled(); } @@ -123,7 +128,8 @@ public void shouldUpdateStateWhenInterruptedException() throws InterruptedException, ExecutionException, TimeoutException { when(future.get(anyLong(), eq(TimeUnit.SECONDS))).thenThrow(new InterruptedException()); - when(source.schedule(projectName, REF_NAME_TO_FETCH, state, SYNC)).thenReturn(future); + when(source.schedule(projectName, REF_NAME_TO_FETCH, state, SYNC, Optional.empty())) + .thenReturn(future); InterruptedException e = assertThrows( @@ -138,7 +144,8 @@ throws InterruptedException, ExecutionException, TimeoutException { when(future.get(anyLong(), eq(TimeUnit.SECONDS))) .thenThrow(new ExecutionException(new Exception())); - when(source.schedule(projectName, REF_NAME_TO_FETCH, state, SYNC)).thenReturn(future); + when(source.schedule(projectName, REF_NAME_TO_FETCH, state, SYNC, Optional.empty())) + .thenReturn(future); ExecutionException e = assertThrows( @@ -152,7 +159,8 @@ public void shouldUpdateStateWhenTimeoutException() throws InterruptedException, ExecutionException, TimeoutException { when(future.get(anyLong(), eq(TimeUnit.SECONDS))).thenThrow(new TimeoutException()); - when(source.schedule(projectName, REF_NAME_TO_FETCH, state, SYNC)).thenReturn(future); + when(source.schedule(projectName, REF_NAME_TO_FETCH, state, SYNC, Optional.empty())) + .thenReturn(future); TimeoutException e = assertThrows(
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java index a1c2172..c45e7be 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java
@@ -38,6 +38,7 @@ import java.io.InputStreamReader; import java.net.URISyntaxException; import java.nio.ByteBuffer; +import java.util.Collections; import java.util.Optional; import org.apache.http.Header; import org.apache.http.client.ClientProtocolException; @@ -89,10 +90,22 @@ Header expectedHeader = new BasicHeader("Content-Type", "application/json"); SyncRefsFilter syncRefsFilter; + String commitObjectId = "9f8d52853089a3cf00c02ff7bd0817bd4353a95a"; + String treeObjectId = "77814d216a6cab2ddb9f2877fbbd0febdc0fa608"; + String blobObjectId = "bb383f5249c68a4cc8c82bdd1228b4a8883ff6e8"; + String expectedSendObjectPayload = - "{\"label\":\"Replication\",\"ref_name\":\"refs/heads/master\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"dHJlZSA3NzgxNGQyMTZhNmNhYjJkZGI5ZjI4NzdmYmJkMGZlYmRjMGZhNjA4CnBhcmVudCA5ODNmZjFhM2NmNzQ3MjVhNTNhNWRlYzhkMGMwNjEyMjEyOGY1YThkCmF1dGhvciBHZXJyaXQgVXNlciAxMDAwMDAwIDwxMDAwMDAwQDY5ZWMzOGYwLTM1MGUtNGQ5Yy05NmQ0LWJjOTU2ZjJmYWFhYz4gMTYxMDU3ODY0OCArMDEwMApjb21taXR0ZXIgR2Vycml0IENvZGUgUmV2aWV3IDxyb290QG1hY3plY2gtWFBTLTE1PiAxNjEwNTc4NjQ4ICswMTAwCgpVcGRhdGUgcGF0Y2ggc2V0IDEKClBhdGNoIFNldCAxOgoKKDEgY29tbWVudCkKClBhdGNoLXNldDogMQo\\u003d\"},\"tree_object\":{\"type\":2,\"content\":\"MTAwNjQ0IGJsb2IgYmIzODNmNTI0OWM2OGE0Y2M4YzgyYmRkMTIyOGI0YTg4ODNmZjZlOCAgICBmNzVhNjkwMDRhOTNiNGNjYzhjZTIxNWMxMjgwODYzNmMyYjc1Njc1\"},\"blobs\":[{\"type\":3,\"content\":\"ewogICJjb21tZW50cyI6IFsKICAgIHsKICAgICAgImtleSI6IHsKICAgICAgICAidXVpZCI6ICI5MGI1YWJmZl80ZjY3NTI2YSIsCiAgICAgICAgImZpbGVuYW1lIjogIi9DT01NSVRfTVNHIiwKICAgICAgICAicGF0Y2hTZXRJZCI6IDEKICAgICAgfSwKICAgICAgImxpbmVOYnIiOiA5LAogICAgICAiYXV0aG9yIjogewogICAgICAgICJpZCI6IDEwMDAwMDAKICAgICAgfSwKICAgICAgIndyaXR0ZW5PbiI6ICIyMDIxLTAxLTEzVDIyOjU3OjI4WiIsCiAgICAgICJzaWRlIjogMSwKICAgICAgIm1lc3NhZ2UiOiAidGVzdCBjb21tZW50IiwKICAgICAgInJhbmdlIjogewogICAgICAgICJzdGFydExpbmUiOiA5LAogICAgICAgICJzdGFydENoYXIiOiAyMSwKICAgICAgICAiZW5kTGluZSI6IDksCiAgICAgICAgImVuZENoYXIiOiAzNAogICAgICB9LAogICAgICAicmV2SWQiOiAiZjc1YTY5MDA0YTkzYjRjY2M4Y2UyMTVjMTI4MDg2MzZjMmI3NTY3NSIsCiAgICAgICJzZXJ2ZXJJZCI6ICI2OWVjMzhmMC0zNTBlLTRkOWMtOTZkNC1iYzk1NmYyZmFhYWMiLAogICAgICAidW5yZXNvbHZlZCI6IHRydWUKICAgIH0KICBdCn0\\u003d\"}]}}"; + "{\"label\":\"Replication\",\"ref_name\":\"refs/heads/master\",\"revision_data\":{\"commit_object\":{\"sha1\":\"" + + commitObjectId + + "\",\"type\":1,\"content\":\"dHJlZSA3NzgxNGQyMTZhNmNhYjJkZGI5ZjI4NzdmYmJkMGZlYmRjMGZhNjA4CnBhcmVudCA5ODNmZjFhM2NmNzQ3MjVhNTNhNWRlYzhkMGMwNjEyMjEyOGY1YThkCmF1dGhvciBHZXJyaXQgVXNlciAxMDAwMDAwIDwxMDAwMDAwQDY5ZWMzOGYwLTM1MGUtNGQ5Yy05NmQ0LWJjOTU2ZjJmYWFhYz4gMTYxMDU3ODY0OCArMDEwMApjb21taXR0ZXIgR2Vycml0IENvZGUgUmV2aWV3IDxyb290QG1hY3plY2gtWFBTLTE1PiAxNjEwNTc4NjQ4ICswMTAwCgpVcGRhdGUgcGF0Y2ggc2V0IDEKClBhdGNoIFNldCAxOgoKKDEgY29tbWVudCkKClBhdGNoLXNldDogMQo\\u003d\"},\"tree_object\":{\"sha1\":\"" + + treeObjectId + + "\",\"type\":2,\"content\":\"MTAwNjQ0IGJsb2IgYmIzODNmNTI0OWM2OGE0Y2M4YzgyYmRkMTIyOGI0YTg4ODNmZjZlOCAgICBmNzVhNjkwMDRhOTNiNGNjYzhjZTIxNWMxMjgwODYzNmMyYjc1Njc1\"},\"blobs\":[{\"sha1\":\"" + + blobObjectId + + "\",\"type\":3,\"content\":\"ewogICJjb21tZW50cyI6IFsKICAgIHsKICAgICAgImtleSI6IHsKICAgICAgICAidXVpZCI6ICI5MGI1YWJmZl80ZjY3NTI2YSIsCiAgICAgICAgImZpbGVuYW1lIjogIi9DT01NSVRfTVNHIiwKICAgICAgICAicGF0Y2hTZXRJZCI6IDEKICAgICAgfSwKICAgICAgImxpbmVOYnIiOiA5LAogICAgICAiYXV0aG9yIjogewogICAgICAgICJpZCI6IDEwMDAwMDAKICAgICAgfSwKICAgICAgIndyaXR0ZW5PbiI6ICIyMDIxLTAxLTEzVDIyOjU3OjI4WiIsCiAgICAgICJzaWRlIjogMSwKICAgICAgIm1lc3NhZ2UiOiAidGVzdCBjb21tZW50IiwKICAgICAgInJhbmdlIjogewogICAgICAgICJzdGFydExpbmUiOiA5LAogICAgICAgICJzdGFydENoYXIiOiAyMSwKICAgICAgICAiZW5kTGluZSI6IDksCiAgICAgICAgImVuZENoYXIiOiAzNAogICAgICB9LAogICAgICAicmV2SWQiOiAiZjc1YTY5MDA0YTkzYjRjY2M4Y2UyMTVjMTI4MDg2MzZjMmI3NTY3NSIsCiAgICAgICJzZXJ2ZXJJZCI6ICI2OWVjMzhmMC0zNTBlLTRkOWMtOTZkNC1iYzk1NmYyZmFhYWMiLAogICAgICAidW5yZXNvbHZlZCI6IHRydWUKICAgIH0KICBdCn0\\u003d\"}]}}"; String commitObject = - "tree 77814d216a6cab2ddb9f2877fbbd0febdc0fa608\n" + "tree " + + treeObjectId + + "\n" + "parent 983ff1a3cf74725a53a5dec8d0c06122128f5a8d\n" + "author Gerrit User 1000000 <1000000@69ec38f0-350e-4d9c-96d4-bc956f2faaac> 1610578648 +0100\n" + "committer Gerrit Code Review <root@maczech-XPS-15> 1610578648 +0100\n" @@ -105,7 +118,7 @@ + "\n" + "Patch-set: 1\n"; String treeObject = - "100644 blob bb383f5249c68a4cc8c82bdd1228b4a8883ff6e8 f75a69004a93b4ccc8ce215c12808636c2b75675"; + "100644 blob " + blobObjectId + " f75a69004a93b4ccc8ce215c12808636c2b75675"; String blobObject = "{\n" + " \"comments\": [\n" @@ -416,9 +429,12 @@ private RevisionData createSampleRevisionData() { RevisionObjectData commitData = - new RevisionObjectData(Constants.OBJ_COMMIT, commitObject.getBytes()); - RevisionObjectData treeData = new RevisionObjectData(Constants.OBJ_TREE, treeObject.getBytes()); - RevisionObjectData blobData = new RevisionObjectData(Constants.OBJ_BLOB, blobObject.getBytes()); - return new RevisionData(commitData, treeData, Lists.newArrayList(blobData)); + new RevisionObjectData(commitObjectId, Constants.OBJ_COMMIT, commitObject.getBytes()); + RevisionObjectData treeData = + new RevisionObjectData(treeObjectId, Constants.OBJ_TREE, treeObject.getBytes()); + RevisionObjectData blobData = + new RevisionObjectData(blobObjectId, Constants.OBJ_BLOB, blobObject.getBytes()); + return new RevisionData( + Collections.emptyList(), commitData, treeData, Lists.newArrayList(blobData)); } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultTest.java new file mode 100644 index 0000000..d82f3a5 --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResultTest.java
@@ -0,0 +1,55 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.client; + +import static com.google.common.truth.Truth.assertThat; + +import java.util.Arrays; +import java.util.Optional; +import javax.servlet.http.HttpServletResponse; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class HttpResultTest { + + @Parameterized.Parameters(name = "HTTP Status = {0} is successful: {1}") + public static Iterable<Object[]> data() { + return Arrays.asList( + new Object[][] { + {HttpServletResponse.SC_OK, true}, + {HttpServletResponse.SC_CREATED, true}, + {HttpServletResponse.SC_ACCEPTED, true}, + {HttpServletResponse.SC_NO_CONTENT, true}, + {HttpServletResponse.SC_BAD_REQUEST, false}, + {HttpServletResponse.SC_CONFLICT, false} + }); + } + + private Integer httpStatus; + private boolean isSuccessful; + + public HttpResultTest(Integer httpStatus, Boolean isSuccessful) { + this.httpStatus = httpStatus; + this.isSuccessful = isSuccessful; + } + + @Test + public void httpResultIsSuccessful() { + HttpResult httpResult = new HttpResult(httpStatus, Optional.empty()); + assertThat(httpResult.isSuccessful()).isEqualTo(isSuccessful); + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObjectIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObjectIT.java index 0f063a4..161830b 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObjectIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObjectIT.java
@@ -78,14 +78,15 @@ String refName = RefNames.changeMetaRef(pushResult.getChange().getId()); Optional<RevisionData> revisionData = - reader.read(Project.nameKey(testRepoProjectName), refName); + reader.read( + Project.nameKey(testRepoProjectName), pushResult.getCommit().toObjectId(), refName, 0); RefSpec refSpec = new RefSpec(refName); - objectUnderTest.apply(project, refSpec, revisionData.get()); + objectUnderTest.apply(project, refSpec, toArray(revisionData)); try (Repository repo = repoManager.openRepository(project); TestRepository<Repository> testRepo = new TestRepository<>(repo); ) { Optional<RevisionData> newRevisionData = - reader.read(project, repo.exactRef(refName).getObjectId(), refName); + reader.read(project, repo.exactRef(refName).getObjectId(), refName, 0); compareObjects(revisionData.get(), newRevisionData); testRepo.fsck(); } @@ -99,15 +100,15 @@ createChange(); String seqChangesRef = RefNames.REFS_SEQUENCES + "changes"; - Optional<RevisionData> revisionData = reader.read(allProjects, seqChangesRef); + Optional<RevisionData> revisionData = reader.read(allProjects, seqChangesRef, 0); RefSpec refSpec = new RefSpec(seqChangesRef); - objectUnderTest.apply(project, refSpec, revisionData.get()); + objectUnderTest.apply(project, refSpec, toArray(revisionData)); try (Repository repo = repoManager.openRepository(project); TestRepository<Repository> testRepo = new TestRepository<>(repo); ) { Optional<RevisionData> newRevisionData = - reader.read(project, repo.exactRef(seqChangesRef).getObjectId(), seqChangesRef); + reader.read(project, repo.exactRef(seqChangesRef).getObjectId(), seqChangesRef, 0); compareObjects(revisionData.get(), newRevisionData); testRepo.fsck(); } @@ -126,8 +127,8 @@ NameKey testRepoKey = Project.nameKey(testRepoProjectName); try (Repository repo = repoManager.openRepository(testRepoKey)) { Optional<RevisionData> revisionData = - reader.read(testRepoKey, repo.exactRef(refName).getObjectId(), refName); - objectUnderTest.apply(project, refSpec, revisionData.get()); + reader.read(testRepoKey, repo.exactRef(refName).getObjectId(), refName, 0); + objectUnderTest.apply(project, refSpec, toArray(revisionData)); } ReviewInput reviewInput = new ReviewInput(); @@ -138,12 +139,12 @@ try (Repository repo = repoManager.openRepository(project); TestRepository<Repository> testRepo = new TestRepository<>(repo)) { Optional<RevisionData> revisionDataWithComment = - reader.read(testRepoKey, repo.exactRef(refName).getObjectId(), refName); + reader.read(testRepoKey, repo.exactRef(refName).getObjectId(), refName, 0); - objectUnderTest.apply(project, refSpec, revisionDataWithComment.get()); + objectUnderTest.apply(project, refSpec, toArray(revisionDataWithComment)); Optional<RevisionData> newRevisionData = - reader.read(project, repo.exactRef(refName).getObjectId(), refName); + reader.read(project, repo.exactRef(refName).getObjectId(), refName, 0); compareObjects(revisionDataWithComment.get(), newRevisionData); @@ -168,12 +169,12 @@ gApi.changes().id(changeId.get()).current().review(reviewInput); Optional<RevisionData> revisionData = - reader.read(createTestProject, repo.exactRef(refName).getObjectId(), refName); + reader.read(createTestProject, repo.exactRef(refName).getObjectId(), refName, 0); RefSpec refSpec = new RefSpec(refName); assertThrows( MissingParentObjectException.class, - () -> objectUnderTest.apply(project, refSpec, revisionData.get())); + () -> objectUnderTest.apply(project, refSpec, toArray(revisionData))); } } @@ -229,4 +230,10 @@ bind(ApplyObject.class); } } + + private RevisionData[] toArray(Optional<RevisionData> optional) { + ImmutableList.Builder<RevisionData> listBuilder = ImmutableList.builder(); + optional.ifPresent(listBuilder::add); + return listBuilder.build().toArray(new RevisionData[1]); + } }