Merge branch 'stable-3.2' into stable-3.3 * stable-3.2: Propagate FetchRefReplicatedEvent Send small refs as a fetch call payload Send ref content as a fetch call payload Index change upon propagation Propagate FetchRefReplicatedEvent upon a Ref fetch Change-Id: I849d9244c623b9fa584f9ac75e773c2c3c3cb462
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectMetrics.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectMetrics.java new file mode 100644 index 0000000..78b6ddb --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ApplyObjectMetrics.java
@@ -0,0 +1,59 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull; + +import com.google.gerrit.extensions.annotations.PluginName; +import com.google.gerrit.metrics.Description; +import com.google.gerrit.metrics.Field; +import com.google.gerrit.metrics.MetricMaker; +import com.google.gerrit.metrics.Timer1; +import com.google.gerrit.server.logging.PluginMetadata; +import com.google.inject.Inject; +import com.google.inject.Singleton; + +@Singleton +public class ApplyObjectMetrics { + private final Timer1<String> executionTime; + + @Inject + ApplyObjectMetrics(@PluginName String pluginName, MetricMaker metricMaker) { + Field<String> SOURCE_FIELD = + Field.ofString( + "source", + (metadataBuilder, fieldValue) -> + metadataBuilder + .pluginName(pluginName) + .addPluginMetadata(PluginMetadata.create("source", fieldValue))) + .build(); + + executionTime = + metricMaker.newTimer( + "apply_object_latency", + new Description("Time spent applying object from remote source.") + .setCumulative() + .setUnit(Description.Units.MILLISECONDS), + SOURCE_FIELD); + } + + /** + * Start the replication latency timer from a source. + * + * @param name the source name. + * @return the timer context. + */ + public Timer1.Context<String> start(String name) { + return executionTime.start(name); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchRefReplicatedEvent.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchRefReplicatedEvent.java index eb825b8..cad6dfb 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchRefReplicatedEvent.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchRefReplicatedEvent.java
@@ -42,6 +42,10 @@ this.refUpdateResult = refUpdateResult; } + public String getStatus() { + return status; + } + @Override public int hashCode() { return Objects.hash(project, ref, status, refUpdateResult);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchResultProcessing.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchResultProcessing.java index cffc1f6..2d51239 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchResultProcessing.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchResultProcessing.java
@@ -18,6 +18,7 @@ import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.server.events.RefEvent; import com.google.gerrit.server.permissions.PermissionBackendException; +import com.google.inject.Inject; import java.lang.ref.WeakReference; import java.util.concurrent.atomic.AtomicBoolean; import org.eclipse.jgit.lib.RefUpdate; @@ -69,11 +70,15 @@ } public static class CommandProcessing extends FetchResultProcessing { + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private WeakReference<Command> sshCommand; private AtomicBoolean hasError = new AtomicBoolean(); + private final EventDispatcher dispatcher; - public CommandProcessing(Command sshCommand) { + @Inject + public CommandProcessing(Command sshCommand, EventDispatcher dispatcher) { this.sshCommand = new WeakReference<>(sshCommand); + this.dispatcher = dispatcher; } @Override @@ -112,6 +117,14 @@ sb.append(")"); } writeStdOut(sb.toString()); + try { + dispatcher.postEvent( + new FetchRefReplicatedEvent( + project, ref, resolveNodeName(uri), status, refUpdateResult)); + } catch (PermissionBackendException e) { + logger.atSevere().withCause(e).log( + "Cannot post event for ref '%s', project %s", ref, project); + } } @Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java index 9add07c..50ab913 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java
@@ -44,6 +44,8 @@ import com.googlesource.gerrit.plugins.replication.pull.client.FetchRestApiClient; import com.googlesource.gerrit.plugins.replication.pull.client.HttpClient; import com.googlesource.gerrit.plugins.replication.pull.client.SourceHttpClient; +import com.googlesource.gerrit.plugins.replication.pull.event.FetchRefReplicatedEventModule; +import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -65,8 +67,13 @@ @Override protected void configure() { + bind(RevisionReader.class).in(Scopes.SINGLETON); + bind(ApplyObject.class); + install(new PullReplicationApiModule()); + install(new FetchRefReplicatedEventModule()); + install( new FactoryModuleBuilder() .implement(HttpClient.class, SourceHttpClient.class)
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java index fc5ed7d..6c8ada0 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
@@ -17,6 +17,7 @@ import com.google.auto.value.AutoValue; import com.google.common.collect.Queues; import com.google.gerrit.entities.Project; +import com.google.gerrit.entities.Project.NameKey; import com.google.gerrit.extensions.events.GitReferenceUpdatedListener; import com.google.gerrit.extensions.events.LifecycleListener; import com.google.gerrit.extensions.registration.DynamicItem; @@ -26,16 +27,22 @@ import com.google.inject.Provider; import com.googlesource.gerrit.plugins.replication.ObservableQueue; import com.googlesource.gerrit.plugins.replication.pull.FetchResultProcessing.GitUpdateProcessing; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; +import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException; +import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException; import com.googlesource.gerrit.plugins.replication.pull.client.FetchRestApiClient; import com.googlesource.gerrit.plugins.replication.pull.client.HttpResult; +import java.io.IOException; import java.net.URISyntaxException; import java.util.HashSet; +import java.util.Optional; import java.util.Queue; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.transport.URIish; import org.slf4j.Logger; @@ -59,6 +66,7 @@ private FetchRestApiClient.Factory fetchClientFactory; private Integer fetchCallsTimeout; private RefsFilter refsFilter; + private RevisionReader revisionReader; @Inject ReplicationQueue( @@ -67,7 +75,8 @@ DynamicItem<EventDispatcher> dis, ReplicationStateListeners sl, FetchRestApiClient.Factory fetchClientFactory, - RefsFilter refsFilter) { + RefsFilter refsFilter, + RevisionReader revReader) { workQueue = wq; dispatcher = dis; sources = rd; @@ -75,6 +84,7 @@ beforeStartupEventsQueue = Queues.newConcurrentLinkedQueue(); this.fetchClientFactory = fetchClientFactory; this.refsFilter = refsFilter; + this.revisionReader = revReader; } @Override @@ -141,17 +151,10 @@ ForkJoinPool fetchCallsPool = null; try { fetchCallsPool = new ForkJoinPool(sources.get().getAll().size()); + + final Consumer<Source> callFunction = callFunction(project, refName, state); fetchCallsPool - .submit( - () -> - sources - .get() - .getAll() - .parallelStream() - .forEach( - source -> { - callFetch(source, project, refName, state); - })) + .submit(() -> sources.get().getAll().parallelStream().forEach(callFunction)) .get(fetchCallsTimeout, TimeUnit.MILLISECONDS); } catch (InterruptedException | ExecutionException | TimeoutException e) { stateLog.error( @@ -167,6 +170,74 @@ } } + private Consumer<Source> callFunction(NameKey project, String refName, ReplicationState state) { + CallFunction call = getCallFunction(project, refName, state); + + return (source) -> { + try { + call.call(source); + } catch (MissingParentObjectException e) { + callFetch(source, project, refName, state); + } + }; + } + + private CallFunction getCallFunction(NameKey project, String refName, ReplicationState state) { + try { + Optional<RevisionData> revisionData = revisionReader.read(project, refName); + if (revisionData.isPresent()) { + return ((source) -> callSendObject(source, project, refName, revisionData.get(), state)); + } + } catch (IOException | RefUpdateException e) { + stateLog.error( + String.format( + "Exception during reading ref: %s, project:%s, message: %s", + refName, project.get(), e.getMessage()), + e, + state); + } + + return (source) -> callFetch(source, project, refName, state); + } + + private void callSendObject( + Source source, + Project.NameKey project, + String refName, + RevisionData revision, + ReplicationState state) + throws MissingParentObjectException { + if (source.wouldFetchProject(project) && source.wouldFetchRef(refName)) { + for (String apiUrl : source.getApis()) { + try { + URIish uri = new URIish(apiUrl); + FetchRestApiClient fetchClient = fetchClientFactory.create(source); + + HttpResult result = fetchClient.callSendObject(project, refName, revision, uri); + if (!result.isSuccessful()) { + repLog.warn( + String.format( + "Pull replication rest api apply object call failed. Endpoint url: %s, reason:%s", + apiUrl, result.getMessage().orElse("unknown"))); + if (result.isParentObjectMissing()) { + throw new MissingParentObjectException( + project, refName, source.getRemoteConfigName()); + } + } + } catch (URISyntaxException e) { + stateLog.error(String.format("Cannot parse pull replication api url:%s", apiUrl), state); + } catch (IOException e) { + stateLog.error( + String.format( + "Exception during the pull replication fetch rest api call. Endpoint url:%s, message:%s", + apiUrl, e.getMessage()), + e, + state); + } + } + } + } + private void callFetch( Source source, Project.NameKey project, String refName, ReplicationState state) { if (source.wouldFetchProject(project) && source.wouldFetchRef(refName)) { @@ -176,7 +247,6 @@ FetchRestApiClient fetchClient = fetchClientFactory.create(source); HttpResult result = fetchClient.callFetch(project, refName, uri); - if (!result.isSuccessful()) { stateLog.warn( String.format( @@ -185,7 +255,7 @@ state); } } catch (URISyntaxException e) { - stateLog.warn(String.format("Cannot parse pull replication api url:%s", apiUrl), state); + stateLog.error(String.format("Cannot parse pull replication api url:%s", apiUrl), state); } catch (Exception e) { stateLog.error( String.format( @@ -227,4 +297,9 @@ public abstract ObjectId objectId(); } + + @FunctionalInterface + private interface CallFunction { + void call(Source source) throws MissingParentObjectException; + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java new file mode 100644 index 0000000..d88c5f5 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java
@@ -0,0 +1,162 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull; + +import static com.googlesource.gerrit.plugins.replication.pull.PullReplicationLogger.repLog; + +import com.google.common.collect.Lists; +import com.google.gerrit.entities.Project; +import com.google.gerrit.server.git.GitRepositoryManager; +import com.google.inject.Inject; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData; +import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException; +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import org.eclipse.jgit.diff.DiffEntry; +import org.eclipse.jgit.diff.DiffEntry.ChangeType; +import org.eclipse.jgit.errors.CorruptObjectException; +import org.eclipse.jgit.errors.IncorrectObjectTypeException; +import org.eclipse.jgit.errors.LargeObjectException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.errors.RepositoryNotFoundException; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevTree; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.TreeFilter; + +public class RevisionReader { + private static final String CONFIG_MAX_API_PAYLOAD_SIZE = "maxApiPayloadSize"; + private static final Long DEFAULT_MAX_PAYLOAD_SIZE_IN_BYTES = 10000L; + private GitRepositoryManager gitRepositoryManager; + private Long maxRefSize; + + @Inject + public RevisionReader(GitRepositoryManager gitRepositoryManager, Config cfg) { + this.gitRepositoryManager = gitRepositoryManager; + this.maxRefSize = + cfg.getLong("replication", CONFIG_MAX_API_PAYLOAD_SIZE, DEFAULT_MAX_PAYLOAD_SIZE_IN_BYTES); + } + + public Optional<RevisionData> read(Project.NameKey project, String refName) + throws MissingObjectException, IncorrectObjectTypeException, CorruptObjectException, + RepositoryNotFoundException, RefUpdateException, IOException { + try (Repository git = gitRepositoryManager.openRepository(project)) { + Ref head = git.exactRef(refName); + if (head == null) { + throw new RefUpdateException( + String.format("Cannot find ref %s in project %s", refName, project.get())); + } + + Long totalRefSize = 0l; + + ObjectLoader commitLoader = git.open(head.getObjectId()); + totalRefSize += commitLoader.getSize(); + verifySize(totalRefSize, commitLoader); + + RevCommit commit = RevCommit.parse(commitLoader.getCachedBytes()); + RevisionObjectData commitRev = + new RevisionObjectData(commit.getType(), commitLoader.getCachedBytes()); + + RevTree tree = commit.getTree(); + ObjectLoader treeLoader = git.open(commit.getTree().toObjectId()); + totalRefSize += treeLoader.getSize(); + verifySize(totalRefSize, treeLoader); + + RevisionObjectData treeRev = + new RevisionObjectData(tree.getType(), treeLoader.getCachedBytes()); + + List<RevisionObjectData> blobs = Lists.newLinkedList(); + try (TreeWalk walk = new TreeWalk(git)) { + if (commit.getParentCount() > 0) { + List<DiffEntry> diffEntries = readDiffs(git, commit, tree, walk); + blobs = readBlobs(git, totalRefSize, diffEntries); + } else { + walk.setRecursive(true); + walk.addTree(tree); + blobs = readBlobs(git, totalRefSize, walk); + } + } + return Optional.of(new RevisionData(commitRev, treeRev, blobs)); + } catch (LargeObjectException e) { + repLog.trace( + "Ref %s size for project %s is greater than configured '%s'", + refName, project, CONFIG_MAX_API_PAYLOAD_SIZE); + return Optional.empty(); + } + } + + private List<DiffEntry> readDiffs(Repository git, RevCommit commit, RevTree tree, TreeWalk walk) + throws MissingObjectException, IncorrectObjectTypeException, CorruptObjectException, + IOException { + walk.setFilter(TreeFilter.ANY_DIFF); + walk.reset(getParentTree(git, commit), tree); + return DiffEntry.scan(walk, true); + } + + private List<RevisionObjectData> readBlobs(Repository git, Long totalRefSize, TreeWalk walk) + throws MissingObjectException, IncorrectObjectTypeException, CorruptObjectException, + IOException { + List<RevisionObjectData> blobs = Lists.newLinkedList(); + while (walk.next()) { + ObjectId objectId = walk.getObjectId(0); + ObjectLoader objectLoader = git.open(objectId); + totalRefSize += objectLoader.getSize(); + verifySize(totalRefSize, objectLoader); + + RevisionObjectData rev = + new RevisionObjectData(objectLoader.getType(), objectLoader.getCachedBytes()); + blobs.add(rev); + } + return blobs; + } + + private List<RevisionObjectData> readBlobs( + Repository git, Long totalRefSize, List<DiffEntry> diffEntries) + throws MissingObjectException, IOException { + List<RevisionObjectData> blobs = Lists.newLinkedList(); + for (DiffEntry diffEntry : diffEntries) { + if (!ChangeType.DELETE.equals(diffEntry.getChangeType())) { + ObjectLoader objectLoader = git.open(diffEntry.getNewId().toObjectId()); + totalRefSize += objectLoader.getSize(); + verifySize(totalRefSize, objectLoader); + RevisionObjectData rev = + new RevisionObjectData(objectLoader.getType(), objectLoader.getCachedBytes()); + blobs.add(rev); + } + } + return blobs; + } + + private RevTree getParentTree(Repository git, RevCommit commit) + throws MissingObjectException, IOException { + RevCommit parent = commit.getParent(0); + ObjectLoader parentLoader = git.open(parent.getId()); + RevCommit parentCommit = RevCommit.parse(parentLoader.getCachedBytes()); + return parentCommit.getTree(); + } + + private void verifySize(Long totalRefSize, ObjectLoader loader) throws LargeObjectException { + if (loader.isLarge() || totalRefSize > maxRefSize) { + throw new LargeObjectException(); + } + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/StartFetchCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/StartFetchCommand.java index a3497e6..6ababe9 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/StartFetchCommand.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/StartFetchCommand.java
@@ -15,6 +15,8 @@ package com.googlesource.gerrit.plugins.replication.pull; import com.google.gerrit.extensions.annotations.RequiresCapability; +import com.google.gerrit.extensions.registration.DynamicItem; +import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.sshd.CommandMetaData; import com.google.gerrit.sshd.SshCommand; import com.google.inject.Inject; @@ -54,6 +56,8 @@ @Inject private ReplicationState.Factory fetchReplicationStateFactory; + @Inject private DynamicItem<EventDispatcher> eventDispatcher; + @Override protected void run() throws Failure { if (all && projectPatterns.size() > 0) { @@ -61,7 +65,8 @@ } ReplicationState state = - fetchReplicationStateFactory.create(new FetchResultProcessing.CommandProcessing(this)); + fetchReplicationStateFactory.create( + new FetchResultProcessing.CommandProcessing(this, eventDispatcher.get())); Future<?> future = null; ReplicationFilter projectFilter;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java new file mode 100644 index 0000000..db4edf5 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java
@@ -0,0 +1,151 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api; + +import static com.google.common.base.Preconditions.checkState; + +import com.google.common.base.Strings; +import com.google.common.flogger.FluentLogger; +import com.google.gerrit.entities.Project; +import com.google.gerrit.extensions.registration.DynamicItem; +import com.google.gerrit.extensions.restapi.AuthException; +import com.google.gerrit.extensions.restapi.BadRequestException; +import com.google.gerrit.extensions.restapi.ResourceConflictException; +import com.google.gerrit.extensions.restapi.Response; +import com.google.gerrit.extensions.restapi.RestApiException; +import com.google.gerrit.extensions.restapi.RestModifyView; +import com.google.gerrit.extensions.restapi.UnprocessableEntityException; +import com.google.gerrit.server.config.UrlFormatter; +import com.google.gerrit.server.git.WorkQueue; +import com.google.gerrit.server.ioutil.HexFormat; +import com.google.gerrit.server.project.ProjectResource; +import com.google.inject.Inject; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput; +import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException; +import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException; +import java.io.IOException; +import java.util.Objects; +import java.util.Optional; + +public class ApplyObjectAction implements RestModifyView<ProjectResource, RevisionInput> { + + private final ApplyObjectCommand command; + private final WorkQueue workQueue; + private final DynamicItem<UrlFormatter> urlFormatter; + private final FetchPreconditions preConditions; + + @Inject + public ApplyObjectAction( + ApplyObjectCommand command, + WorkQueue workQueue, + DynamicItem<UrlFormatter> urlFormatter, + FetchPreconditions preConditions) { + this.command = command; + this.workQueue = workQueue; + this.urlFormatter = urlFormatter; + this.preConditions = preConditions; + } + + @Override + public Response<?> apply(ProjectResource resource, RevisionInput input) throws RestApiException { + + if (!preConditions.canCallFetchApi()) { + throw new AuthException("not allowed to call fetch command"); + } + try { + if (Strings.isNullOrEmpty(input.getLabel())) { + throw new BadRequestException("Source label cannot be null or empty"); + } + if (Strings.isNullOrEmpty(input.getRefName())) { + throw new BadRequestException("Ref-update refname cannot be null or empty"); + } + + if (Objects.isNull(input.getRevisionData())) { + throw new BadRequestException("Ref-update revision data cannot be null or empty"); + } + + if (Objects.isNull(input.getRevisionData().getCommitObject()) + || Objects.isNull(input.getRevisionData().getCommitObject().getContent()) + || input.getRevisionData().getCommitObject().getContent().length == 0 + || Objects.isNull(input.getRevisionData().getCommitObject().getType())) { + throw new BadRequestException("Ref-update commit object cannot be null or empty"); + } + + if (Objects.isNull(input.getRevisionData().getTreeObject()) + || Objects.isNull(input.getRevisionData().getTreeObject().getContent()) + || Objects.isNull(input.getRevisionData().getTreeObject().getType())) { + throw new BadRequestException("Ref-update tree object cannot be null"); + } + + if (input.isAsync()) { + return applyAsync(resource.getNameKey(), input); + } + return applySync(resource.getNameKey(), input); + } catch (MissingParentObjectException e) { + throw new ResourceConflictException(e.getMessage(), e); + } catch (NumberFormatException | IOException e) { + throw RestApiException.wrap(e.getMessage(), e); + } catch (RefUpdateException e) { + throw new UnprocessableEntityException(e.getMessage()); + } + } + + private Response<?> applySync(Project.NameKey project, RevisionInput input) + throws NumberFormatException, IOException, RefUpdateException, MissingParentObjectException { + command.applyObject(project, input.getRefName(), input.getRevisionData(), input.getLabel()); + return Response.created(input); + } + + private Response.Accepted applyAsync(Project.NameKey project, RevisionInput input) { + @SuppressWarnings("unchecked") + WorkQueue.Task<Void> task = + (WorkQueue.Task<Void>) + workQueue.getDefaultQueue().submit(new ApplyObjectJob(command, project, input)); + Optional<String> url = + urlFormatter + .get() + .getRestUrl("a/config/server/tasks/" + HexFormat.fromInt(task.getTaskId())); + // We're in a HTTP handler, so must be present. + checkState(url.isPresent()); + return Response.accepted(url.get()); + } + + private static class ApplyObjectJob implements Runnable { + private static final FluentLogger log = FluentLogger.forEnclosingClass(); + + private ApplyObjectCommand command; + private Project.NameKey project; + private RevisionInput input; + + public ApplyObjectJob( + ApplyObjectCommand command, Project.NameKey project, RevisionInput input) { + this.command = command; + this.project = project; + this.input = input; + } + + @Override + public void run() { + try { + command.applyObject(project, input.getRefName(), input.getRevisionData(), input.getLabel()); + } catch (IOException | RefUpdateException | MissingParentObjectException e) { + log.atSevere().withCause(e).log( + "Exception during the applyObject call for project {} and ref name {}", + project.get(), + input.getRefName()); + } + } + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java new file mode 100644 index 0000000..ac7e79b --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java
@@ -0,0 +1,120 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api; + +import static com.googlesource.gerrit.plugins.replication.pull.PullReplicationLogger.repLog; +import static java.util.concurrent.TimeUnit.NANOSECONDS; + +import com.google.common.collect.ImmutableSet; +import com.google.common.flogger.FluentLogger; +import com.google.gerrit.entities.Project; +import com.google.gerrit.extensions.registration.DynamicItem; +import com.google.gerrit.metrics.Timer1; +import com.google.gerrit.server.events.EventDispatcher; +import com.google.gerrit.server.permissions.PermissionBackendException; +import com.google.inject.Inject; +import com.googlesource.gerrit.plugins.replication.pull.ApplyObjectMetrics; +import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent; +import com.googlesource.gerrit.plugins.replication.pull.PullReplicationStateLogger; +import com.googlesource.gerrit.plugins.replication.pull.ReplicationState; +import com.googlesource.gerrit.plugins.replication.pull.ReplicationState.RefFetchResult; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; +import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException; +import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException; +import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject; +import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState; +import java.io.IOException; +import java.util.Set; +import org.eclipse.jgit.lib.RefUpdate; +import org.eclipse.jgit.transport.RefSpec; + +public class ApplyObjectCommand { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + private static final Set<RefUpdate.Result> SUCCESSFUL_RESULTS = + ImmutableSet.of( + RefUpdate.Result.NEW, + RefUpdate.Result.FORCED, + RefUpdate.Result.NO_CHANGE, + RefUpdate.Result.FAST_FORWARD); + + private final PullReplicationStateLogger fetchStateLog; + private final ApplyObject applyObject; + private final ApplyObjectMetrics metrics; + private final DynamicItem<EventDispatcher> eventDispatcher; + + @Inject + public ApplyObjectCommand( + PullReplicationStateLogger fetchStateLog, + ApplyObject applyObject, + ApplyObjectMetrics metrics, + DynamicItem<EventDispatcher> eventDispatcher) { + this.fetchStateLog = fetchStateLog; + this.applyObject = applyObject; + this.metrics = metrics; + this.eventDispatcher = eventDispatcher; + } + + public void applyObject( + Project.NameKey name, String refName, RevisionData revisionData, String sourceLabel) + throws IOException, RefUpdateException, MissingParentObjectException { + + repLog.info("Apply object from {} for project {}, ref name {}", sourceLabel, name, refName); + Timer1.Context<String> context = metrics.start(sourceLabel); + RefUpdateState refUpdateState = applyObject.apply(name, new RefSpec(refName), revisionData); + long elapsed = NANOSECONDS.toMillis(context.stop()); + + try { + eventDispatcher + .get() + .postEvent( + new FetchRefReplicatedEvent( + name.get(), + refName, + sourceLabel, + getStatus(refUpdateState), + refUpdateState.getResult())); + } catch (PermissionBackendException e) { + logger.atSevere().withCause(e).log( + "Cannot post event for ref '%s', project %s", refName, name); + } + + if (!isSuccessful(refUpdateState.getResult())) { + String message = + String.format( + "RefUpdate failed with result %s for: sourceLcabel=%s, project=%s, refName=%s", + refUpdateState.getResult().name(), sourceLabel, name, refName); + fetchStateLog.error(message); + throw new RefUpdateException(message); + } + repLog.info( + "Apply object from {} for project {}, ref name {} completed in {}ms", + sourceLabel, + name, + refName, + elapsed); + } + + private RefFetchResult getStatus(RefUpdateState refUpdateState) { + return isSuccessful(refUpdateState.getResult()) + ? ReplicationState.RefFetchResult.SUCCEEDED + : ReplicationState.RefFetchResult.FAILED; + } + + private Boolean isSuccessful(RefUpdate.Result result) { + return SUCCESSFUL_RESULTS.contains(result); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java index 7470c2c..1088d9b 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java
@@ -15,6 +15,8 @@ package com.googlesource.gerrit.plugins.replication.pull.api; import com.google.gerrit.entities.Project; +import com.google.gerrit.extensions.registration.DynamicItem; +import com.google.gerrit.server.events.EventDispatcher; import com.google.inject.Inject; import com.googlesource.gerrit.plugins.replication.pull.Command; import com.googlesource.gerrit.plugins.replication.pull.FetchResultProcessing; @@ -34,22 +36,26 @@ private ReplicationState.Factory fetchReplicationStateFactory; private PullReplicationStateLogger fetchStateLog; private SourcesCollection sources; + private final DynamicItem<EventDispatcher> eventDispatcher; @Inject public FetchCommand( ReplicationState.Factory fetchReplicationStateFactory, PullReplicationStateLogger fetchStateLog, - SourcesCollection sources) { + SourcesCollection sources, + DynamicItem<EventDispatcher> eventDispatcher) { this.fetchReplicationStateFactory = fetchReplicationStateFactory; this.fetchStateLog = fetchStateLog; this.sources = sources; + this.eventDispatcher = eventDispatcher; } public void fetch(Project.NameKey name, String label, String refName) throws InterruptedException, ExecutionException, RemoteConfigurationMissingException, TimeoutException { ReplicationState state = - fetchReplicationStateFactory.create(new FetchResultProcessing.CommandProcessing(this)); + fetchReplicationStateFactory.create( + new FetchResultProcessing.CommandProcessing(this, eventDispatcher.get())); Optional<Source> source = sources.getAll().stream().filter(s -> s.getRemoteConfigName().equals(label)).findFirst(); if (!source.isPresent()) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiModule.java index 1663ad2..f94f49b 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/PullReplicationApiModule.java
@@ -26,7 +26,9 @@ @Override protected void configure() { bind(FetchAction.class).in(Scopes.SINGLETON); + bind(ApplyObjectAction.class).in(Scopes.SINGLETON); post(PROJECT_KIND, "fetch").to(FetchAction.class); + post(PROJECT_KIND, "apply-object").to(ApplyObjectAction.class); bind(FetchPreconditions.class).in(Scopes.SINGLETON); bind(CapabilityDefinition.class)
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionData.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionData.java new file mode 100644 index 0000000..bcd4e05 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionData.java
@@ -0,0 +1,46 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api.data; + +import java.util.List; + +public class RevisionData { + private RevisionObjectData commitObject; + + private RevisionObjectData treeObject; + + private List<RevisionObjectData> blobs; + + public RevisionData( + RevisionObjectData commitObject, + RevisionObjectData treeObject, + List<RevisionObjectData> blobs) { + this.commitObject = commitObject; + this.treeObject = treeObject; + this.blobs = blobs; + } + + public RevisionObjectData getCommitObject() { + return commitObject; + } + + public RevisionObjectData getTreeObject() { + return treeObject; + } + + public List<RevisionObjectData> getBlobs() { + return blobs; + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java new file mode 100644 index 0000000..160a720 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java
@@ -0,0 +1,52 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api.data; + +public class RevisionInput { + private String label; + + private String refName; + + private RevisionData revisionData; + + private boolean async; + + public RevisionInput(String label, String refName, RevisionData revisionData) { + this(label, refName, revisionData, false); + } + + public RevisionInput(String label, String refName, RevisionData revisionData, boolean async) { + this.label = label; + this.refName = refName; + this.revisionData = revisionData; + this.async = async; + } + + public String getLabel() { + return label; + } + + public String getRefName() { + return refName; + } + + public RevisionData getRevisionData() { + return revisionData; + } + + public boolean isAsync() { + return async; + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionObjectData.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionObjectData.java new file mode 100644 index 0000000..02ba06c --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionObjectData.java
@@ -0,0 +1,35 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api.data; + +import java.util.Base64; + +public class RevisionObjectData { + private final Integer type; + private final String content; + + public RevisionObjectData(int type, byte[] content) { + this.type = type; + this.content = content == null ? "" : Base64.getEncoder().encodeToString(content); + } + + public Integer getType() { + return type; + } + + public byte[] getContent() { + return Base64.getDecoder().decode(content); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/exception/MissingParentObjectException.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/exception/MissingParentObjectException.java new file mode 100644 index 0000000..a96c1ea --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/exception/MissingParentObjectException.java
@@ -0,0 +1,37 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api.exception; + +import com.google.gerrit.entities.Project; +import org.eclipse.jgit.lib.ObjectId; + +public class MissingParentObjectException extends Exception { + private static final long serialVersionUID = 1L; + + public MissingParentObjectException( + Project.NameKey project, String refName, ObjectId parentObjectId) { + super( + String.format( + "Missing parent object %s for project %s ref name: %s", + parentObjectId.getName(), project.get(), refName)); + } + + public MissingParentObjectException(Project.NameKey project, String refName, String targetName) { + super( + String.format( + "Missing parent object on %s for project %s ref name: %s", + targetName, project.get(), refName)); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/exception/RefUpdateException.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/exception/RefUpdateException.java new file mode 100644 index 0000000..f02f9c5 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/exception/RefUpdateException.java
@@ -0,0 +1,24 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api.exception; + +public class RefUpdateException extends Exception { + + private static final long serialVersionUID = 1L; + + public RefUpdateException(String msg) { + super(msg); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java index bdfc1c3..eed1afb 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
@@ -14,17 +14,24 @@ package com.googlesource.gerrit.plugins.replication.pull.client; +import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES; import static java.util.Objects.requireNonNull; import com.google.common.base.Strings; import com.google.common.flogger.FluentLogger; +import com.google.common.net.MediaType; import com.google.gerrit.entities.Project; +import com.google.gerrit.extensions.annotations.PluginName; import com.google.gerrit.extensions.restapi.Url; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; import com.googlesource.gerrit.plugins.replication.CredentialsFactory; import com.googlesource.gerrit.plugins.replication.ReplicationConfig; import com.googlesource.gerrit.plugins.replication.pull.Source; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Optional; @@ -48,6 +55,9 @@ private static final FluentLogger logger = FluentLogger.forEnclosingClass(); static String GERRIT_ADMIN_PROTOCOL_PREFIX = "gerrit+"; + private static final Gson GSON = + new GsonBuilder().setFieldNamingPolicy(LOWER_CASE_WITH_UNDERSCORES).create(); + public interface Factory { FetchRestApiClient create(Source source); } @@ -56,16 +66,19 @@ private final SourceHttpClient.Factory httpClientFactory; private final Source source; private final String instanceLabel; + private final String pluginName; @Inject FetchRestApiClient( CredentialsFactory credentials, SourceHttpClient.Factory httpClientFactory, ReplicationConfig replicationConfig, + @PluginName String pluginName, @Assisted Source source) { this.credentials = credentials; this.httpClientFactory = httpClientFactory; this.source = source; + this.pluginName = pluginName; this.instanceLabel = Strings.nullToEmpty( replicationConfig.getConfig().getString("replication", null, "instanceLabel")) @@ -90,6 +103,23 @@ return httpClientFactory.create(source).execute(post, this, getContext(targetUri)); } + public HttpResult callSendObject( + Project.NameKey project, String refName, RevisionData revisionData, URIish targetUri) + throws ClientProtocolException, IOException { + + RevisionInput input = new RevisionInput(instanceLabel, refName, revisionData); + + String url = + String.format( + "%s/a/projects/%s/%s~apply-object", + targetUri.toString(), Url.encode(project.get()), pluginName); + + HttpPost post = new HttpPost(url); + post.setEntity(new StringEntity(GSON.toJson(input))); + post.addHeader(new BasicHeader("Content-Type", MediaType.JSON_UTF_8.toString())); + return httpClientFactory.create(source).execute(post, this, getContext(targetUri)); + } + @Override public HttpResult handleResponse(HttpResponse response) { Optional<String> responseBody = Optional.empty();
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResult.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResult.java index 8a2b66d..dc01295 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResult.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/HttpResult.java
@@ -14,6 +14,7 @@ package com.googlesource.gerrit.plugins.replication.pull.client; +import static javax.servlet.http.HttpServletResponse.SC_CONFLICT; import static javax.servlet.http.HttpServletResponse.SC_CREATED; import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT; import static javax.servlet.http.HttpServletResponse.SC_OK; @@ -36,4 +37,8 @@ public boolean isSuccessful() { return responseCode == SC_CREATED || responseCode == SC_NO_CONTENT || responseCode == SC_OK; } + + public boolean isParentObjectMissing() { + return responseCode == SC_CONFLICT; + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandler.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandler.java new file mode 100644 index 0000000..fc96394 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandler.java
@@ -0,0 +1,58 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.event; + +import com.google.common.flogger.FluentLogger; +import com.google.gerrit.entities.Change; +import com.google.gerrit.entities.Project; +import com.google.gerrit.server.events.Event; +import com.google.gerrit.server.events.EventListener; +import com.google.gerrit.server.index.change.ChangeIndexer; +import com.google.inject.Inject; +import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent; +import com.googlesource.gerrit.plugins.replication.pull.ReplicationState; + +public class FetchRefReplicatedEventHandler implements EventListener { + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private ChangeIndexer changeIndexer; + + @Inject + FetchRefReplicatedEventHandler(ChangeIndexer changeIndexer) { + this.changeIndexer = changeIndexer; + } + + @Override + public void onEvent(Event event) { + if (event instanceof FetchRefReplicatedEvent) { + FetchRefReplicatedEvent fetchRefReplicatedEvent = (FetchRefReplicatedEvent) event; + Project.NameKey projectNameKey = fetchRefReplicatedEvent.getProjectNameKey(); + logger.atFine().log( + "Indexing ref '%s' for project %s", + fetchRefReplicatedEvent.getRefName(), projectNameKey.get()); + + Change.Id changeId = Change.Id.fromRef(fetchRefReplicatedEvent.getRefName()); + if (changeId != null + && fetchRefReplicatedEvent + .getStatus() + .equals(ReplicationState.RefFetchResult.SUCCEEDED.toString())) { + changeIndexer.index(projectNameKey, changeId); + } else { + logger.atWarning().log( + "Couldn't get changeId from refName. Skipping indexing of change %s for project %s", + fetchRefReplicatedEvent.getRefName(), projectNameKey.get()); + } + } + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventModule.java new file mode 100644 index 0000000..675563a --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventModule.java
@@ -0,0 +1,27 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.event; + +import com.google.gerrit.extensions.registration.DynamicSet; +import com.google.gerrit.lifecycle.LifecycleModule; +import com.google.gerrit.server.events.EventListener; + +public class FetchRefReplicatedEventModule extends LifecycleModule { + + @Override + protected void configure() { + DynamicSet.bind(binder(), EventListener.class).to(FetchRefReplicatedEventHandler.class); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java new file mode 100644 index 0000000..da63168 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java
@@ -0,0 +1,72 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.fetch; + +import com.google.gerrit.entities.Project; +import com.google.gerrit.server.git.GitRepositoryManager; +import com.google.gerrit.server.git.LocalDiskRepositoryManager; +import com.google.inject.Inject; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData; +import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException; +import java.io.IOException; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.RefUpdate; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.transport.RefSpec; + +public class ApplyObject { + + private final GitRepositoryManager gitManager; + + @Inject + public ApplyObject(LocalDiskRepositoryManager gitManager) { + this.gitManager = gitManager; + } + + public RefUpdateState apply(Project.NameKey name, RefSpec refSpec, RevisionData revisionData) + throws MissingParentObjectException, IOException { + try (Repository git = gitManager.openRepository(name)) { + + ObjectId newObjectID = null; + try (ObjectInserter oi = git.newObjectInserter()) { + RevisionObjectData commitObject = revisionData.getCommitObject(); + RevCommit commit = RevCommit.parse(commitObject.getContent()); + for (RevCommit parent : commit.getParents()) { + if (!git.getObjectDatabase().has(parent.getId())) { + throw new MissingParentObjectException(name, refSpec.getSource(), parent.getId()); + } + } + newObjectID = oi.insert(commitObject.getType(), commitObject.getContent()); + + RevisionObjectData treeObject = revisionData.getTreeObject(); + oi.insert(treeObject.getType(), treeObject.getContent()); + + for (RevisionObjectData rev : revisionData.getBlobs()) { + oi.insert(rev.getType(), rev.getContent()); + } + + oi.flush(); + } + RefUpdate ru = git.updateRef(refSpec.getSource()); + ru.setNewObjectId(newObjectID); + RefUpdate.Result result = ru.update(); + + return new RefUpdateState(refSpec.getSource(), result); + } + } +}
diff --git a/src/main/resources/Documentation/about.md b/src/main/resources/Documentation/about.md index d921e0b..22b52e7 100644 --- a/src/main/resources/Documentation/about.md +++ b/src/main/resources/Documentation/about.md
@@ -15,3 +15,8 @@ To be allowed to trigger pull replication a user must be a member of a group that is granted the 'Pull Replication' capability (provided by this plugin) or the 'Administrate Server' capability. + +Change Indexing +-------- + +Changes will be automatically indexed upon replication.
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index ae2c2fd..c3e5ad4 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -181,6 +181,13 @@ By default, all other refs are included. +replication.maxApiPayloadSize +: Maximum size in bytes of the ref to be sent as a REST Api call + payload. For refs larger than threshold git fetch operation + will be used. + + Default: 10000 + remote.NAME.url : Address of the remote server to fetch from. Single URL can be specified within a single remote block. A remote node can request
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchGitUpdateProcessingTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchGitUpdateProcessingTest.java index f0f1c54..77dc947 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchGitUpdateProcessingTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/FetchGitUpdateProcessingTest.java
@@ -21,6 +21,7 @@ import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.server.permissions.PermissionBackendException; +import com.googlesource.gerrit.plugins.replication.pull.FetchResultProcessing.CommandProcessing; import com.googlesource.gerrit.plugins.replication.pull.FetchResultProcessing.GitUpdateProcessing; import com.googlesource.gerrit.plugins.replication.pull.ReplicationState.RefFetchResult; import java.net.URISyntaxException; @@ -32,15 +33,20 @@ public class FetchGitUpdateProcessingTest { private EventDispatcher dispatcherMock; private GitUpdateProcessing gitUpdateProcessing; + private CommandProcessing commandProcessing; + private Command sshCommandMock; @Before public void setUp() throws Exception { dispatcherMock = mock(EventDispatcher.class); gitUpdateProcessing = new GitUpdateProcessing(dispatcherMock); + sshCommandMock = mock(Command.class); + commandProcessing = new CommandProcessing(sshCommandMock, dispatcherMock); } @Test - public void headRefReplicated() throws URISyntaxException, PermissionBackendException { + public void headRefReplicatedInGitUpdateProcessing() + throws URISyntaxException, PermissionBackendException { FetchRefReplicatedEvent expectedEvent = new FetchRefReplicatedEvent( "someProject", @@ -59,6 +65,26 @@ } @Test + public void headRefReplicatedInCommandProcessing() + throws URISyntaxException, PermissionBackendException { + FetchRefReplicatedEvent expectedEvent = + new FetchRefReplicatedEvent( + "someProject", + "refs/heads/master", + "someHost", + RefFetchResult.SUCCEEDED, + RefUpdate.Result.NEW); + + commandProcessing.onOneProjectReplicationDone( + "someProject", + "refs/heads/master", + new URIish("git://someHost/someProject.git"), + RefFetchResult.SUCCEEDED, + RefUpdate.Result.NEW); + verify(dispatcherMock, times(1)).postEvent(eq(expectedEvent)); + } + + @Test public void changeRefReplicated() throws URISyntaxException, PermissionBackendException { FetchRefReplicatedEvent expectedEvent = new FetchRefReplicatedEvent(
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java index 61520f4..44b69db 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
@@ -15,7 +15,14 @@ package com.googlesource.gerrit.plugins.replication.pull; import static java.nio.file.Files.createTempDirectory; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import com.google.gerrit.extensions.api.changes.NotifyHandling; import com.google.gerrit.extensions.common.AccountInfo; import com.google.gerrit.extensions.events.GitReferenceUpdatedListener; @@ -27,41 +34,129 @@ import com.google.inject.Provider; import com.googlesource.gerrit.plugins.replication.ReplicationConfig; import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; +import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException; import com.googlesource.gerrit.plugins.replication.pull.client.FetchRestApiClient; +import com.googlesource.gerrit.plugins.replication.pull.client.HttpResult; import java.io.IOException; import java.nio.file.Path; +import java.util.Optional; +import org.apache.http.client.ClientProtocolException; +import org.eclipse.jgit.errors.LargeObjectException; +import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.util.FS; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) public class ReplicationQueueTest { + private static int CONNECTION_TIMEOUT = 1000000; + @Mock private WorkQueue wq; + @Mock private Source source; + @Mock private SourcesCollection sourceCollection; @Mock private Provider<SourcesCollection> rd; @Mock private DynamicItem<EventDispatcher> dis; @Mock ReplicationStateListeners sl; + @Mock FetchRestApiClient fetchRestApiClient; @Mock FetchRestApiClient.Factory fetchClientFactory; @Mock AccountInfo accountInfo; + @Mock RevisionReader revReader; + @Mock RevisionData revisionData; + @Mock HttpResult httpResult; - RefsFilter refsFilter; - + private RefsFilter refsFilter; private ReplicationQueue objectUnderTest; private SitePaths sitePaths; private Path pluginDataPath; @Before - public void setup() throws IOException { + public void setup() throws IOException, LargeObjectException, RefUpdateException { Path sitePath = createTempPath("site"); sitePaths = new SitePaths(sitePath); Path pluginDataPath = createTempPath("data"); ReplicationConfig replicationConfig = new ReplicationFileBasedConfig(sitePaths, pluginDataPath); refsFilter = new RefsFilter(replicationConfig); - objectUnderTest = new ReplicationQueue(wq, rd, dis, sl, fetchClientFactory, refsFilter); + when(source.getConnectionTimeout()).thenReturn(CONNECTION_TIMEOUT); + when(source.wouldFetchProject(any())).thenReturn(true); + when(source.wouldFetchRef(anyString())).thenReturn(true); + ImmutableList<String> apis = ImmutableList.of("http://localhost:18080"); + when(source.getApis()).thenReturn(apis); + when(sourceCollection.getAll()).thenReturn(Lists.newArrayList(source)); + when(rd.get()).thenReturn(sourceCollection); + when(revReader.read(any(), anyString())).thenReturn(Optional.of(revisionData)); + when(fetchClientFactory.create(any())).thenReturn(fetchRestApiClient); + when(fetchRestApiClient.callSendObject(any(), anyString(), any(), any())) + .thenReturn(httpResult); + when(fetchRestApiClient.callFetch(any(), anyString(), any())).thenReturn(httpResult); + when(httpResult.isSuccessful()).thenReturn(true); + + objectUnderTest = + new ReplicationQueue(wq, rd, dis, sl, fetchClientFactory, refsFilter, revReader); + } + + @Test + public void shouldCallSendObjectWhenMetaRef() throws ClientProtocolException, IOException { + Event event = new TestEvent("refs/changes/01/1/meta"); + objectUnderTest.start(); + objectUnderTest.onGitReferenceUpdated(event); + + verify(fetchRestApiClient).callSendObject(any(), anyString(), any(), any()); + } + + @Test + public void shouldCallSendObjectWhenPatchSetRef() throws ClientProtocolException, IOException { + Event event = new TestEvent("refs/changes/01/1/1"); + objectUnderTest.start(); + objectUnderTest.onGitReferenceUpdated(event); + + verify(fetchRestApiClient).callSendObject(any(), anyString(), any(), any()); + } + + @Test + public void shouldFallbackToCallFetchWhenIOException() + throws ClientProtocolException, IOException, LargeObjectException, RefUpdateException { + Event event = new TestEvent("refs/changes/01/1/meta"); + objectUnderTest.start(); + + when(revReader.read(any(), anyString())).thenThrow(IOException.class); + + objectUnderTest.onGitReferenceUpdated(event); + + verify(fetchRestApiClient).callFetch(any(), anyString(), any()); + } + + @Test + public void shouldFallbackToCallFetchWhenLargeRef() + throws ClientProtocolException, IOException, LargeObjectException, RefUpdateException { + Event event = new TestEvent("refs/changes/01/1/1"); + objectUnderTest.start(); + + when(revReader.read(any(), anyString())).thenReturn(Optional.empty()); + + objectUnderTest.onGitReferenceUpdated(event); + + verify(fetchRestApiClient).callFetch(any(), anyString(), any()); + } + + @Test + public void shouldFallbackToCallFetchWhenParentObjectIsMissing() + throws ClientProtocolException, IOException { + Event event = new TestEvent("refs/changes/01/1/meta"); + objectUnderTest.start(); + + when(httpResult.isSuccessful()).thenReturn(false); + when(httpResult.isParentObjectMissing()).thenReturn(true); + when(fetchRestApiClient.callSendObject(any(), anyString(), any(), any())) + .thenReturn(httpResult); + + objectUnderTest.onGitReferenceUpdated(event); + + verify(fetchRestApiClient).callFetch(any(), anyString(), any()); } @Test @@ -69,7 +164,7 @@ Event event = new TestEvent("refs/users/00/1000000"); objectUnderTest.onGitReferenceUpdated(event); - Mockito.verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); + verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); } @Test @@ -77,7 +172,7 @@ Event event = new TestEvent("refs/groups/a1/a16d5b33cc789d60b682c654f03f9cc2feb12975"); objectUnderTest.onGitReferenceUpdated(event); - Mockito.verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); + verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); } @Test @@ -85,7 +180,7 @@ Event event = new TestEvent("refs/meta/group-names"); objectUnderTest.onGitReferenceUpdated(event); - Mockito.verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); + verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); } @Test @@ -93,7 +188,7 @@ Event event = new TestEvent("refs/sequences/changes"); objectUnderTest.onGitReferenceUpdated(event); - Mockito.verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); + verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); } @Test @@ -104,11 +199,13 @@ fileConfig.save(); ReplicationConfig replicationConfig = new ReplicationFileBasedConfig(sitePaths, pluginDataPath); refsFilter = new RefsFilter(replicationConfig); - objectUnderTest = new ReplicationQueue(wq, rd, dis, sl, fetchClientFactory, refsFilter); + + objectUnderTest = + new ReplicationQueue(wq, rd, dis, sl, fetchClientFactory, refsFilter, revReader); Event event = new TestEvent("refs/multi-site/version"); objectUnderTest.onGitReferenceUpdated(event); - Mockito.verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); + verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); } @Test @@ -116,7 +213,7 @@ Event event = new TestEvent("refs/starred-changes/41/2941/1000000"); objectUnderTest.onGitReferenceUpdated(event); - Mockito.verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); + verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); } @Test @@ -124,7 +221,7 @@ Event event = new TestEvent("refs/meta/config"); objectUnderTest.onGitReferenceUpdated(event); - Mockito.verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); + verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); } @Test @@ -132,7 +229,7 @@ Event event = new TestEvent("refs/meta/external-ids"); objectUnderTest.onGitReferenceUpdated(event); - Mockito.verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); + verifyZeroInteractions(wq, rd, dis, sl, fetchClientFactory, accountInfo); } protected static Path createTempPath(String prefix) throws IOException { @@ -141,9 +238,17 @@ private class TestEvent implements GitReferenceUpdatedListener.Event { private String refName; + private String projectName; + private ObjectId newObjectId; public TestEvent(String refName) { + this(refName, "defaultProject", ObjectId.zeroId()); + } + + public TestEvent(String refName, String projectName, ObjectId newObjectId) { this.refName = refName; + this.projectName = projectName; + this.newObjectId = newObjectId; } @Override @@ -153,7 +258,7 @@ @Override public String getProjectName() { - return null; + return projectName; } @Override @@ -168,7 +273,7 @@ @Override public String getNewObjectId() { - return null; + return newObjectId.getName(); } @Override
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java new file mode 100644 index 0000000..ca410d5 --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java
@@ -0,0 +1,121 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.gerrit.acceptance.LightweightPluginDaemonTest; +import com.google.gerrit.acceptance.PushOneCommit.Result; +import com.google.gerrit.acceptance.TestPlugin; +import com.google.gerrit.acceptance.UseLocalDisk; +import com.google.gerrit.entities.Change; +import com.google.gerrit.entities.Patch; +import com.google.gerrit.entities.RefNames; +import com.google.gerrit.extensions.api.changes.ReviewInput; +import com.google.gerrit.extensions.api.changes.ReviewInput.CommentInput; +import com.google.gerrit.extensions.client.Comment; +import com.google.gerrit.extensions.config.FactoryModule; +import com.google.inject.Inject; +import com.google.inject.Scopes; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData; +import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject; +import java.util.Optional; +import org.eclipse.jgit.lib.Constants; +import org.junit.Test; + +@UseLocalDisk +@TestPlugin( + name = "pull-replication", + sysModule = "com.googlesource.gerrit.plugins.replication.pull.RevisionReaderIT$TestModule") +public class RevisionReaderIT extends LightweightPluginDaemonTest { + @Inject RevisionReader objectUnderTest; + + @Test + public void shouldReadRefMetaObject() throws Exception { + Result pushResult = createChange(); + String refName = RefNames.changeMetaRef(pushResult.getChange().getId()); + + Optional<RevisionData> revisionDataOption = objectUnderTest.read(project, refName); + + assertThat(revisionDataOption.isPresent()).isTrue(); + RevisionData revisionData = revisionDataOption.get(); + assertThat(revisionData).isNotNull(); + assertThat(revisionData.getCommitObject()).isNotNull(); + assertThat(revisionData.getCommitObject().getType()).isEqualTo(Constants.OBJ_COMMIT); + assertThat(revisionData.getCommitObject().getContent()).isNotEmpty(); + + assertThat(revisionData.getTreeObject()).isNotNull(); + assertThat(revisionData.getTreeObject().getType()).isEqualTo(Constants.OBJ_TREE); + assertThat(revisionData.getTreeObject().getContent()).isEmpty(); + + assertThat(revisionData.getBlobs()).isEmpty(); + } + + @Test + public void shouldReadRefMetaObjectWithComments() throws Exception { + Result pushResult = createChange(); + Change.Id changeId = pushResult.getChange().getId(); + String refName = RefNames.changeMetaRef(changeId); + + CommentInput comment = createCommentInput(1, 0, 1, 1, "Test comment"); + + ReviewInput reviewInput = new ReviewInput(); + reviewInput.comments = ImmutableMap.of(Patch.COMMIT_MSG, ImmutableList.of(comment)); + gApi.changes().id(changeId.get()).current().review(reviewInput); + + Optional<RevisionData> revisionDataOption = objectUnderTest.read(project, refName); + + assertThat(revisionDataOption.isPresent()).isTrue(); + RevisionData revisionData = revisionDataOption.get(); + assertThat(revisionData).isNotNull(); + assertThat(revisionData.getCommitObject()).isNotNull(); + assertThat(revisionData.getCommitObject().getType()).isEqualTo(Constants.OBJ_COMMIT); + assertThat(revisionData.getCommitObject().getContent()).isNotEmpty(); + + assertThat(revisionData.getTreeObject()).isNotNull(); + assertThat(revisionData.getTreeObject().getType()).isEqualTo(Constants.OBJ_TREE); + assertThat(revisionData.getTreeObject().getContent()).isNotEmpty(); + + assertThat(revisionData.getBlobs()).hasSize(1); + RevisionObjectData blobObject = revisionData.getBlobs().get(0); + assertThat(blobObject.getType()).isEqualTo(Constants.OBJ_BLOB); + assertThat(blobObject.getContent()).isNotEmpty(); + } + + private CommentInput createCommentInput( + int startLine, int startCharacter, int endLine, int endCharacter, String message) { + CommentInput comment = new CommentInput(); + comment.range = new Comment.Range(); + comment.range.startLine = startLine; + comment.range.startCharacter = startCharacter; + comment.range.endLine = endLine; + comment.range.endCharacter = endCharacter; + comment.message = message; + comment.path = Patch.COMMIT_MSG; + return comment; + } + + @SuppressWarnings("unused") + private static class TestModule extends FactoryModule { + @Override + protected void configure() { + bind(RevisionReader.class).in(Scopes.SINGLETON); + bind(ApplyObject.class); + } + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java new file mode 100644 index 0000000..55fdee8 --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java
@@ -0,0 +1,239 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.http.HttpStatus.SC_ACCEPTED; +import static org.apache.http.HttpStatus.SC_CREATED; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.when; + +import com.google.common.collect.Lists; +import com.google.gerrit.entities.Project; +import com.google.gerrit.extensions.registration.DynamicItem; +import com.google.gerrit.extensions.restapi.AuthException; +import com.google.gerrit.extensions.restapi.BadRequestException; +import com.google.gerrit.extensions.restapi.ResourceConflictException; +import com.google.gerrit.extensions.restapi.Response; +import com.google.gerrit.extensions.restapi.RestApiException; +import com.google.gerrit.server.config.UrlFormatter; +import com.google.gerrit.server.git.WorkQueue; +import com.google.gerrit.server.git.WorkQueue.Task; +import com.google.gerrit.server.project.ProjectResource; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData; +import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException; +import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException; +import java.io.IOException; +import java.util.Optional; +import java.util.concurrent.ScheduledExecutorService; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +@RunWith(MockitoJUnitRunner.class) +public class ApplyObjectActionTest { + ApplyObjectAction applyObjectAction; + String label = "instance-2-label"; + String url = "file:///gerrit-host/instance-1/git/${name}.git"; + String refName = "refs/heads/master"; + String location = "http://gerrit-host/a/config/server/tasks/08d173e9"; + int taskId = 1234; + + private String sampleCommitContent = + "tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n" + + "parent 20eb48d28be86dc88fb4bef747f08de0fbefe12d\n" + + "author Gerrit User 1000000 <1000000@69ec38f0-350e-4d9c-96d4-bc956f2faaac> 1610471611 +0100\n" + + "committer Gerrit Code Review <root@maczech-XPS-15> 1610471611 +0100\n" + + "\n" + + "Update patch set 1\n" + + "\n" + + "Change has been successfully merged by Administrator\n" + + "\n" + + "Patch-set: 1\n" + + "Status: merged\n" + + "Tag: autogenerated:gerrit:merged\n" + + "Reviewer: Gerrit User 1000000 <1000000@69ec38f0-350e-4d9c-96d4-bc956f2faaac>\n" + + "Label: SUBM=+1\n" + + "Submission-id: 1904-1610471611558-783c0a2f\n" + + "Submitted-with: OK\n" + + "Submitted-with: OK: Code-Review: Gerrit User 1000000 <1000000@69ec38f0-350e-4d9c-96d4-bc956f2faaac>"; + + @Mock ApplyObjectCommand applyObjectCommand; + @Mock ProjectResource projectResource; + @Mock WorkQueue workQueue; + @Mock ScheduledExecutorService exceutorService; + @Mock DynamicItem<UrlFormatter> urlFormatterDynamicItem; + @Mock UrlFormatter urlFormatter; + @Mock WorkQueue.Task<Void> task; + @Mock FetchPreconditions preConditions; + + @Before + public void setup() { + when(workQueue.getDefaultQueue()).thenReturn(exceutorService); + when(urlFormatter.getRestUrl(anyString())).thenReturn(Optional.of(location)); + when(exceutorService.submit(any(Runnable.class))) + .thenAnswer( + new Answer<WorkQueue.Task<Void>>() { + @Override + public Task<Void> answer(InvocationOnMock invocation) throws Throwable { + return task; + } + }); + when(urlFormatterDynamicItem.get()).thenReturn(urlFormatter); + when(task.getTaskId()).thenReturn(taskId); + when(preConditions.canCallFetchApi()).thenReturn(true); + + applyObjectAction = + new ApplyObjectAction( + applyObjectCommand, workQueue, urlFormatterDynamicItem, preConditions); + } + + @Test + public void shouldReturnCreatedResponseCode() throws RestApiException { + RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData()); + + Response<?> response = applyObjectAction.apply(projectResource, inputParams); + + assertThat(response.statusCode()).isEqualTo(SC_CREATED); + } + + @SuppressWarnings("cast") + @Test + public void shouldReturnSourceUrlAndrefNameAsAResponseBody() throws Exception { + RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData()); + Response<?> response = applyObjectAction.apply(projectResource, inputParams); + + assertThat((RevisionInput) response.value()).isEqualTo(inputParams); + } + + @Test(expected = BadRequestException.class) + public void shouldThrowBadRequestExceptionWhenMissingLabel() throws Exception { + RevisionInput inputParams = new RevisionInput(null, refName, createSampleRevisionData()); + + applyObjectAction.apply(projectResource, inputParams); + } + + @Test(expected = BadRequestException.class) + public void shouldThrowBadRequestExceptionWhenEmptyLabel() throws Exception { + RevisionInput inputParams = new RevisionInput("", refName, createSampleRevisionData()); + + applyObjectAction.apply(projectResource, inputParams); + } + + @Test(expected = BadRequestException.class) + public void shouldThrowBadRequestExceptionWhenMissingRefName() throws Exception { + RevisionInput inputParams = new RevisionInput(label, null, createSampleRevisionData()); + + applyObjectAction.apply(projectResource, inputParams); + } + + @Test(expected = BadRequestException.class) + public void shouldThrowBadRequestExceptionWhenEmptyRefName() throws Exception { + RevisionInput inputParams = new RevisionInput(label, "", createSampleRevisionData()); + + applyObjectAction.apply(projectResource, inputParams); + } + + @Test(expected = BadRequestException.class) + public void shouldThrowBadRequestExceptionWhenMissingRevisionData() throws Exception { + RevisionInput inputParams = new RevisionInput(label, refName, null); + + applyObjectAction.apply(projectResource, inputParams); + } + + @Test(expected = BadRequestException.class) + public void shouldThrowBadRequestExceptionWhenMissingCommitObjectData() throws Exception { + RevisionObjectData commitData = new RevisionObjectData(Constants.OBJ_COMMIT, null); + RevisionObjectData treeData = new RevisionObjectData(Constants.OBJ_TREE, new byte[] {}); + RevisionInput inputParams = + new RevisionInput(label, refName, createSampleRevisionData(commitData, treeData)); + + applyObjectAction.apply(projectResource, inputParams); + } + + @Test(expected = BadRequestException.class) + public void shouldThrowBadRequestExceptionWhenMissingTreeObject() throws Exception { + RevisionObjectData commitData = + new RevisionObjectData(Constants.OBJ_COMMIT, sampleCommitContent.getBytes()); + RevisionInput inputParams = + new RevisionInput(label, refName, createSampleRevisionData(commitData, null)); + + applyObjectAction.apply(projectResource, inputParams); + } + + @Test(expected = AuthException.class) + public void shouldThrowAuthExceptionWhenCallFetchActionCapabilityNotAssigned() + throws RestApiException { + RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData()); + + when(preConditions.canCallFetchApi()).thenReturn(false); + + applyObjectAction.apply(projectResource, inputParams); + } + + @Test(expected = ResourceConflictException.class) + public void shouldThrowResourceConflictExceptionWhenMissingParentObject() + throws RestApiException, IOException, RefUpdateException, MissingParentObjectException { + RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData()); + + doThrow( + new MissingParentObjectException( + Project.nameKey("test_projects"), refName, ObjectId.zeroId())) + .when(applyObjectCommand) + .applyObject(any(), anyString(), any(), anyString()); + + applyObjectAction.apply(projectResource, inputParams); + } + + @Test + public void shouldReturnScheduledTaskForAsyncCall() throws RestApiException { + RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData(), true); + + Response<?> response = applyObjectAction.apply(projectResource, inputParams); + assertThat(response.statusCode()).isEqualTo(SC_ACCEPTED); + } + + @Test + public void shouldLocationHeaderForAsyncCall() throws Exception { + RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData(), true); + + Response<?> response = applyObjectAction.apply(projectResource, inputParams); + assertThat(response).isInstanceOf(Response.Accepted.class); + Response.Accepted acceptResponse = (Response.Accepted) response; + assertThat(acceptResponse.location()).isEqualTo(location); + } + + private RevisionData createSampleRevisionData() { + RevisionObjectData commitData = + new RevisionObjectData(Constants.OBJ_COMMIT, sampleCommitContent.getBytes()); + RevisionObjectData treeData = new RevisionObjectData(Constants.OBJ_TREE, new byte[] {}); + return createSampleRevisionData(commitData, treeData); + } + + private RevisionData createSampleRevisionData( + RevisionObjectData commitData, RevisionObjectData treeData) { + return new RevisionData(commitData, treeData, Lists.newArrayList()); + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommandTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommandTest.java new file mode 100644 index 0000000..51051c0 --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommandTest.java
@@ -0,0 +1,100 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.common.collect.Lists; +import com.google.gerrit.entities.Project; +import com.google.gerrit.entities.Project.NameKey; +import com.google.gerrit.extensions.registration.DynamicItem; +import com.google.gerrit.metrics.Timer1; +import com.google.gerrit.server.events.Event; +import com.google.gerrit.server.events.EventDispatcher; +import com.google.gerrit.server.permissions.PermissionBackendException; +import com.googlesource.gerrit.plugins.replication.pull.ApplyObjectMetrics; +import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent; +import com.googlesource.gerrit.plugins.replication.pull.PullReplicationStateLogger; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData; +import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException; +import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException; +import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject; +import com.googlesource.gerrit.plugins.replication.pull.fetch.RefUpdateState; +import java.io.IOException; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.RefUpdate; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class ApplyObjectCommandTest { + private static final String TEST_SOURCE_LABEL = "test-source-label"; + private static final String TEST_REF_NAME = "refs/changes/01/1/1"; + private static final NameKey TEST_PROJECT_NAME = Project.nameKey("test-project"); + private static final String TEST_REMOTE_NAME = "test-remote-name"; + + @Mock private PullReplicationStateLogger fetchStateLog; + @Mock private ApplyObject applyObject; + @Mock private ApplyObjectMetrics metrics; + @Mock private DynamicItem<EventDispatcher> eventDispatcherDataItem; + @Mock private EventDispatcher eventDispatcher; + @Mock private Timer1.Context<String> timetContext; + @Captor ArgumentCaptor<Event> eventCaptor; + + private ApplyObjectCommand objectUnderTest; + + @Before + public void setup() throws MissingParentObjectException, IOException { + RefUpdateState state = new RefUpdateState(TEST_REMOTE_NAME, RefUpdate.Result.NEW); + when(eventDispatcherDataItem.get()).thenReturn(eventDispatcher); + when(metrics.start(anyString())).thenReturn(timetContext); + when(timetContext.stop()).thenReturn(100L); + when(applyObject.apply(any(), any(), any())).thenReturn(state); + + objectUnderTest = + new ApplyObjectCommand(fetchStateLog, applyObject, metrics, eventDispatcherDataItem); + } + + @Test + public void shouldSendEventWhenApplyObject() + throws PermissionBackendException, IOException, RefUpdateException, + MissingParentObjectException { + objectUnderTest.applyObject( + TEST_PROJECT_NAME, TEST_REF_NAME, createSampleRevisionData(), TEST_SOURCE_LABEL); + + verify(eventDispatcher).postEvent(eventCaptor.capture()); + Event sentEvent = eventCaptor.getValue(); + assertThat(sentEvent).isInstanceOf(FetchRefReplicatedEvent.class); + FetchRefReplicatedEvent fetchEvent = (FetchRefReplicatedEvent) sentEvent; + assertThat(fetchEvent.getProjectNameKey()).isEqualTo(TEST_PROJECT_NAME); + assertThat(fetchEvent.getRefName()).isEqualTo(TEST_REF_NAME); + } + + private RevisionData createSampleRevisionData() { + RevisionObjectData commitData = new RevisionObjectData(Constants.OBJ_COMMIT, new byte[] {}); + RevisionObjectData treeData = new RevisionObjectData(Constants.OBJ_TREE, new byte[] {}); + return new RevisionData(commitData, treeData, Lists.newArrayList()); + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java index e625e0f..4cdad48 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java
@@ -25,6 +25,8 @@ import com.google.common.collect.Lists; import com.google.gerrit.entities.Project; +import com.google.gerrit.extensions.registration.DynamicItem; +import com.google.gerrit.server.events.EventDispatcher; import com.googlesource.gerrit.plugins.replication.pull.PullReplicationStateLogger; import com.googlesource.gerrit.plugins.replication.pull.ReplicationState; import com.googlesource.gerrit.plugins.replication.pull.Source; @@ -51,6 +53,7 @@ @Mock PullReplicationStateLogger fetchStateLog; @Mock Source source; @Mock SourcesCollection sources; + @Mock DynamicItem<EventDispatcher> eventDispatcher; @SuppressWarnings("rawtypes") @Mock @@ -73,7 +76,8 @@ when(sources.getAll()).thenReturn(Lists.newArrayList(source)); when(source.schedule(projectName, REF_NAME_TO_FETCH, state, true)) .thenReturn(CompletableFuture.completedFuture(null)); - objectUnderTest = new FetchCommand(fetchReplicationStateFactory, fetchStateLog, sources); + objectUnderTest = + new FetchCommand(fetchReplicationStateFactory, fetchStateLog, sources, eventDispatcher); } @Test
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java index c62ddab..771acff 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java
@@ -23,11 +23,14 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.common.collect.Lists; import com.google.gerrit.entities.Project; import com.google.gerrit.entities.RefNames; import com.googlesource.gerrit.plugins.replication.CredentialsFactory; import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig; import com.googlesource.gerrit.plugins.replication.pull.Source; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData; import java.io.IOException; import java.net.URISyntaxException; import java.nio.ByteBuffer; @@ -36,6 +39,7 @@ import org.apache.http.client.ClientProtocolException; import org.apache.http.client.methods.HttpPost; import org.apache.http.message.BasicHeader; +import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.transport.CredentialItem; import org.eclipse.jgit.transport.CredentialsProvider; @@ -64,12 +68,59 @@ @Captor ArgumentCaptor<HttpPost> httpPostCaptor; String api = "http://gerrit-host"; + String pluginName = "pull-replication"; String label = "Replication"; String refName = RefNames.REFS_HEADS + "master"; String expectedPayload = "{\"label\":\"Replication\", \"ref_name\": \"" + refName + "\"}"; Header expectedHeader = new BasicHeader("Content-Type", "application/json"); + String expectedSendObjectPayload = + "{\"label\":\"Replication\",\"ref_name\":\"refs/heads/master\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"dHJlZSA3NzgxNGQyMTZhNmNhYjJkZGI5ZjI4NzdmYmJkMGZlYmRjMGZhNjA4CnBhcmVudCA5ODNmZjFhM2NmNzQ3MjVhNTNhNWRlYzhkMGMwNjEyMjEyOGY1YThkCmF1dGhvciBHZXJyaXQgVXNlciAxMDAwMDAwIDwxMDAwMDAwQDY5ZWMzOGYwLTM1MGUtNGQ5Yy05NmQ0LWJjOTU2ZjJmYWFhYz4gMTYxMDU3ODY0OCArMDEwMApjb21taXR0ZXIgR2Vycml0IENvZGUgUmV2aWV3IDxyb290QG1hY3plY2gtWFBTLTE1PiAxNjEwNTc4NjQ4ICswMTAwCgpVcGRhdGUgcGF0Y2ggc2V0IDEKClBhdGNoIFNldCAxOgoKKDEgY29tbWVudCkKClBhdGNoLXNldDogMQo\\u003d\"},\"tree_object\":{\"type\":2,\"content\":\"MTAwNjQ0IGJsb2IgYmIzODNmNTI0OWM2OGE0Y2M4YzgyYmRkMTIyOGI0YTg4ODNmZjZlOCAgICBmNzVhNjkwMDRhOTNiNGNjYzhjZTIxNWMxMjgwODYzNmMyYjc1Njc1\"},\"blobs\":[{\"type\":3,\"content\":\"ewogICJjb21tZW50cyI6IFsKICAgIHsKICAgICAgImtleSI6IHsKICAgICAgICAidXVpZCI6ICI5MGI1YWJmZl80ZjY3NTI2YSIsCiAgICAgICAgImZpbGVuYW1lIjogIi9DT01NSVRfTVNHIiwKICAgICAgICAicGF0Y2hTZXRJZCI6IDEKICAgICAgfSwKICAgICAgImxpbmVOYnIiOiA5LAogICAgICAiYXV0aG9yIjogewogICAgICAgICJpZCI6IDEwMDAwMDAKICAgICAgfSwKICAgICAgIndyaXR0ZW5PbiI6ICIyMDIxLTAxLTEzVDIyOjU3OjI4WiIsCiAgICAgICJzaWRlIjogMSwKICAgICAgIm1lc3NhZ2UiOiAidGVzdCBjb21tZW50IiwKICAgICAgInJhbmdlIjogewogICAgICAgICJzdGFydExpbmUiOiA5LAogICAgICAgICJzdGFydENoYXIiOiAyMSwKICAgICAgICAiZW5kTGluZSI6IDksCiAgICAgICAgImVuZENoYXIiOiAzNAogICAgICB9LAogICAgICAicmV2SWQiOiAiZjc1YTY5MDA0YTkzYjRjY2M4Y2UyMTVjMTI4MDg2MzZjMmI3NTY3NSIsCiAgICAgICJzZXJ2ZXJJZCI6ICI2OWVjMzhmMC0zNTBlLTRkOWMtOTZkNC1iYzk1NmYyZmFhYWMiLAogICAgICAidW5yZXNvbHZlZCI6IHRydWUKICAgIH0KICBdCn0\\u003d\"}]},\"async\":false}"; + String commitObject = + "tree 77814d216a6cab2ddb9f2877fbbd0febdc0fa608\n" + + "parent 983ff1a3cf74725a53a5dec8d0c06122128f5a8d\n" + + "author Gerrit User 1000000 <1000000@69ec38f0-350e-4d9c-96d4-bc956f2faaac> 1610578648 +0100\n" + + "committer Gerrit Code Review <root@maczech-XPS-15> 1610578648 +0100\n" + + "\n" + + "Update patch set 1\n" + + "\n" + + "Patch Set 1:\n" + + "\n" + + "(1 comment)\n" + + "\n" + + "Patch-set: 1\n"; + String treeObject = + "100644 blob bb383f5249c68a4cc8c82bdd1228b4a8883ff6e8 f75a69004a93b4ccc8ce215c12808636c2b75675"; + String blobObject = + "{\n" + + " \"comments\": [\n" + + " {\n" + + " \"key\": {\n" + + " \"uuid\": \"90b5abff_4f67526a\",\n" + + " \"filename\": \"/COMMIT_MSG\",\n" + + " \"patchSetId\": 1\n" + + " },\n" + + " \"lineNbr\": 9,\n" + + " \"author\": {\n" + + " \"id\": 1000000\n" + + " },\n" + + " \"writtenOn\": \"2021-01-13T22:57:28Z\",\n" + + " \"side\": 1,\n" + + " \"message\": \"test comment\",\n" + + " \"range\": {\n" + + " \"startLine\": 9,\n" + + " \"startChar\": 21,\n" + + " \"endLine\": 9,\n" + + " \"endChar\": 34\n" + + " },\n" + + " \"revId\": \"f75a69004a93b4ccc8ce215c12808636c2b75675\",\n" + + " \"serverId\": \"69ec38f0-350e-4d9c-96d4-bc956f2faaac\",\n" + + " \"unresolved\": true\n" + + " }\n" + + " ]\n" + + "}"; + FetchRestApiClient objectUnderTest; @Before @@ -99,7 +150,8 @@ when(httpClient.execute(any(HttpPost.class), any(), any())).thenReturn(httpResult); when(httpClientFactory.create(any())).thenReturn(httpClient); objectUnderTest = - new FetchRestApiClient(credentials, httpClientFactory, replicationConfig, source); + new FetchRestApiClient( + credentials, httpClientFactory, replicationConfig, pluginName, source); } @Test @@ -142,11 +194,54 @@ } @Test + public void shouldCallSendObjectEndpoint() + throws ClientProtocolException, IOException, URISyntaxException { + + objectUnderTest.callSendObject( + Project.nameKey("test_repo"), refName, createSampleRevisionData(), new URIish(api)); + + verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any(), any()); + + HttpPost httpPost = httpPostCaptor.getValue(); + assertThat(httpPost.getURI().getHost()).isEqualTo("gerrit-host"); + assertThat(httpPost.getURI().getPath()) + .isEqualTo("/a/projects/test_repo/pull-replication~apply-object"); + } + + @Test + public void shouldCallSendObjectEndpointWithPayload() + throws ClientProtocolException, IOException, URISyntaxException { + + objectUnderTest.callSendObject( + Project.nameKey("test_repo"), refName, createSampleRevisionData(), new URIish(api)); + + verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any(), any()); + + HttpPost httpPost = httpPostCaptor.getValue(); + assertThat(readPayload(httpPost)).isEqualTo(expectedSendObjectPayload); + } + + @Test + public void shouldSetContentTypeHeaderForSendObjectCall() + throws ClientProtocolException, IOException, URISyntaxException { + + objectUnderTest.callFetch(Project.nameKey("test_repo"), refName, new URIish(api)); + + verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any(), any()); + + HttpPost httpPost = httpPostCaptor.getValue(); + assertThat(httpPost.getLastHeader("Content-Type").getValue()) + .isEqualTo(expectedHeader.getValue()); + } + + @Test public void shouldThrowExceptionWhenInstanceLabelIsNull() { when(config.getString("replication", null, "instanceLabel")).thenReturn(null); assertThrows( NullPointerException.class, - () -> new FetchRestApiClient(credentials, httpClientFactory, replicationConfig, source)); + () -> + new FetchRestApiClient( + credentials, httpClientFactory, replicationConfig, pluginName, source)); } @Test @@ -154,7 +249,9 @@ when(config.getString("replication", null, "instanceLabel")).thenReturn(" "); assertThrows( NullPointerException.class, - () -> new FetchRestApiClient(credentials, httpClientFactory, replicationConfig, source)); + () -> + new FetchRestApiClient( + credentials, httpClientFactory, replicationConfig, pluginName, source)); } @Test @@ -162,11 +259,21 @@ when(config.getString("replication", null, "instanceLabel")).thenReturn(""); assertThrows( NullPointerException.class, - () -> new FetchRestApiClient(credentials, httpClientFactory, replicationConfig, source)); + () -> + new FetchRestApiClient( + credentials, httpClientFactory, replicationConfig, pluginName, source)); } public String readPayload(HttpPost entity) throws UnsupportedOperationException, IOException { ByteBuffer buf = IO.readWholeStream(entity.getEntity().getContent(), 1024); return RawParseUtils.decode(buf.array(), buf.arrayOffset(), buf.limit()).trim(); } + + private RevisionData createSampleRevisionData() { + RevisionObjectData commitData = + new RevisionObjectData(Constants.OBJ_COMMIT, commitObject.getBytes()); + RevisionObjectData treeData = new RevisionObjectData(Constants.OBJ_TREE, treeObject.getBytes()); + RevisionObjectData blobData = new RevisionObjectData(Constants.OBJ_BLOB, blobObject.getBytes()); + return new RevisionData(commitData, treeData, Lists.newArrayList(blobData)); + } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandlerTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandlerTest.java new file mode 100644 index 0000000..1d87195 --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandlerTest.java
@@ -0,0 +1,97 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.event; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import com.google.gerrit.entities.Change; +import com.google.gerrit.entities.Project; +import com.google.gerrit.server.index.change.ChangeIndexer; +import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent; +import com.googlesource.gerrit.plugins.replication.pull.ReplicationState; +import org.eclipse.jgit.lib.RefUpdate; +import org.junit.Before; +import org.junit.Test; + +public class FetchRefReplicatedEventHandlerTest { + private ChangeIndexer changeIndexerMock; + private FetchRefReplicatedEventHandler fetchRefReplicatedEventHandler; + + @Before + public void setUp() throws Exception { + changeIndexerMock = mock(ChangeIndexer.class); + fetchRefReplicatedEventHandler = new FetchRefReplicatedEventHandler(changeIndexerMock); + } + + @Test + public void onEventShouldIndexExistingChange() { + Project.NameKey projectNameKey = Project.nameKey("testProject"); + String ref = "refs/changes/41/41/meta"; + Change.Id changeId = Change.Id.fromRef(ref); + fetchRefReplicatedEventHandler.onEvent( + new FetchRefReplicatedEvent( + projectNameKey.get(), + ref, + "aSourceNode", + ReplicationState.RefFetchResult.SUCCEEDED, + RefUpdate.Result.FAST_FORWARD)); + verify(changeIndexerMock, times(1)).index(eq(projectNameKey), eq(changeId)); + } + + @Test + public void onEventShouldNotIndexMissingChange() { + fetchRefReplicatedEventHandler.onEvent( + new FetchRefReplicatedEvent( + Project.nameKey("testProject").get(), + "invalidRef", + "aSourceNode", + ReplicationState.RefFetchResult.SUCCEEDED, + RefUpdate.Result.FAST_FORWARD)); + verify(changeIndexerMock, never()).index(any(), any()); + } + + @Test + public void onEventShouldNotIndexFailingChange() { + Project.NameKey projectNameKey = Project.nameKey("testProject"); + String ref = "refs/changes/41/41/meta"; + fetchRefReplicatedEventHandler.onEvent( + new FetchRefReplicatedEvent( + projectNameKey.get(), + ref, + "aSourceNode", + ReplicationState.RefFetchResult.FAILED, + RefUpdate.Result.FAST_FORWARD)); + verify(changeIndexerMock, never()).index(any(), any()); + } + + @Test + public void onEventShouldNotIndexNotAttemptedChange() { + Project.NameKey projectNameKey = Project.nameKey("testProject"); + String ref = "refs/changes/41/41/meta"; + fetchRefReplicatedEventHandler.onEvent( + new FetchRefReplicatedEvent( + projectNameKey.get(), + ref, + "aSourceNode", + ReplicationState.RefFetchResult.NOT_ATTEMPTED, + RefUpdate.Result.FAST_FORWARD)); + verify(changeIndexerMock, never()).index(any(), any()); + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObjectIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObjectIT.java new file mode 100644 index 0000000..5d80d94 --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObjectIT.java
@@ -0,0 +1,191 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.fetch; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.gerrit.testing.GerritJUnit.assertThrows; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.primitives.Bytes; +import com.google.gerrit.acceptance.LightweightPluginDaemonTest; +import com.google.gerrit.acceptance.PushOneCommit.Result; +import com.google.gerrit.acceptance.SkipProjectClone; +import com.google.gerrit.acceptance.TestPlugin; +import com.google.gerrit.acceptance.UseLocalDisk; +import com.google.gerrit.acceptance.testsuite.project.ProjectOperations; +import com.google.gerrit.entities.Change; +import com.google.gerrit.entities.Patch; +import com.google.gerrit.entities.Project; +import com.google.gerrit.entities.RefNames; +import com.google.gerrit.extensions.api.changes.ReviewInput; +import com.google.gerrit.extensions.api.changes.ReviewInput.CommentInput; +import com.google.gerrit.extensions.client.Comment; +import com.google.gerrit.extensions.config.FactoryModule; +import com.google.inject.Inject; +import com.google.inject.Scopes; +import com.googlesource.gerrit.plugins.replication.pull.RevisionReader; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData; +import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.transport.RefSpec; +import org.junit.Test; + +@SkipProjectClone +@UseLocalDisk +@TestPlugin( + name = "pull-replication", + sysModule = "com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObjectIT$TestModule") +public class ApplyObjectIT extends LightweightPluginDaemonTest { + private static final String TEST_REPLICATION_SUFFIX = "suffix1"; + + @Inject private ProjectOperations projectOperations; + @Inject RevisionReader reader; + @Inject ApplyObject objectUnderTest; + + @Test + public void shouldApplyRefMetaObject() throws Exception { + String testRepoProjectName = project + TEST_REPLICATION_SUFFIX; + testRepo = cloneProject(createTestProject(testRepoProjectName)); + + Result pushResult = createChange(); + String refName = RefNames.changeMetaRef(pushResult.getChange().getId()); + + Optional<RevisionData> revisionData = + reader.read(Project.nameKey(testRepoProjectName), refName); + + RefSpec refSpec = new RefSpec(refName); + objectUnderTest.apply(project, refSpec, revisionData.get()); + Optional<RevisionData> newRevisionData = reader.read(project, refName); + + compareObjects(revisionData.get(), newRevisionData); + + try (Repository repo = repoManager.openRepository(project); + TestRepository<Repository> testRepo = new TestRepository<>(repo); ) { + testRepo.fsck(); + } + } + + @Test + public void shouldApplyRefMetaObjectWithComments() throws Exception { + String testRepoProjectName = project + TEST_REPLICATION_SUFFIX; + testRepo = cloneProject(createTestProject(testRepoProjectName)); + + Result pushResult = createChange(); + Change.Id changeId = pushResult.getChange().getId(); + String refName = RefNames.changeMetaRef(changeId); + RefSpec refSpec = new RefSpec(refName); + + Optional<RevisionData> revisionData = + reader.read(Project.nameKey(testRepoProjectName), refName); + objectUnderTest.apply(project, refSpec, revisionData.get()); + + ReviewInput reviewInput = new ReviewInput(); + CommentInput comment = createCommentInput(1, 0, 1, 1, "Test comment"); + reviewInput.comments = ImmutableMap.of(Patch.COMMIT_MSG, ImmutableList.of(comment)); + gApi.changes().id(changeId.get()).current().review(reviewInput); + + Optional<RevisionData> revisionDataWithComment = + reader.read(Project.nameKey(testRepoProjectName), refName); + + objectUnderTest.apply(project, refSpec, revisionDataWithComment.get()); + + Optional<RevisionData> newRevisionData = reader.read(project, refName); + + compareObjects(revisionDataWithComment.get(), newRevisionData); + + try (Repository repo = repoManager.openRepository(project); + TestRepository<Repository> testRepo = new TestRepository<>(repo)) { + testRepo.fsck(); + } + } + + @Test + public void shouldThrowExceptionWhenParentCommitObjectIsMissing() throws Exception { + String testRepoProjectName = project + TEST_REPLICATION_SUFFIX; + testRepo = cloneProject(createTestProject(testRepoProjectName)); + + Result pushResult = createChange(); + Change.Id changeId = pushResult.getChange().getId(); + String refName = RefNames.changeMetaRef(changeId); + + CommentInput comment = createCommentInput(1, 0, 1, 1, "Test comment"); + ReviewInput reviewInput = new ReviewInput(); + reviewInput.comments = ImmutableMap.of(Patch.COMMIT_MSG, ImmutableList.of(comment)); + gApi.changes().id(changeId.get()).current().review(reviewInput); + + Optional<RevisionData> revisionData = + reader.read(Project.nameKey(testRepoProjectName), refName); + + RefSpec refSpec = new RefSpec(refName); + assertThrows( + MissingParentObjectException.class, + () -> objectUnderTest.apply(project, refSpec, revisionData.get())); + } + + private void compareObjects(RevisionData expected, Optional<RevisionData> actualOption) { + assertThat(actualOption.isPresent()).isTrue(); + RevisionData actual = actualOption.get(); + compareContent(expected.getCommitObject(), actual.getCommitObject()); + compareContent(expected.getTreeObject(), expected.getTreeObject()); + List<List<Byte>> actualBlobs = + actual.getBlobs().stream() + .map(revision -> Bytes.asList(revision.getContent())) + .collect(Collectors.toList()); + List<List<Byte>> expectedBlobs = + expected.getBlobs().stream() + .map(revision -> Bytes.asList(revision.getContent())) + .collect(Collectors.toList()); + assertThat(actualBlobs).containsExactlyElementsIn(expectedBlobs); + } + + private void compareContent(RevisionObjectData expected, RevisionObjectData actual) { + assertThat(actual.getType()).isEqualTo(expected.getType()); + assertThat(Bytes.asList(actual.getContent())) + .containsExactlyElementsIn(Bytes.asList(expected.getContent())) + .inOrder(); + } + + private CommentInput createCommentInput( + int startLine, int startCharacter, int endLine, int endCharacter, String message) { + CommentInput comment = new CommentInput(); + comment.range = new Comment.Range(); + comment.range.startLine = startLine; + comment.range.startCharacter = startCharacter; + comment.range.endLine = endLine; + comment.range.endCharacter = endCharacter; + comment.message = message; + comment.path = Patch.COMMIT_MSG; + return comment; + } + + private Project.NameKey createTestProject(String name) throws Exception { + return projectOperations.newProject().name(name).create(); + } + + @SuppressWarnings("unused") + private static class TestModule extends FactoryModule { + @Override + protected void configure() { + bind(RevisionReader.class).in(Scopes.SINGLETON); + bind(ApplyObject.class); + } + } +}