Merge changes from topic "asynchronous_fetch" into stable-3.1 * changes: Add replication delay for asynchronous fetch calls Allow asynchronous fetch calls
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchAll.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchAll.java index baffff7..bcd86d7 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchAll.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchAll.java
@@ -29,7 +29,11 @@ private final ReplicationStateListener stateLog; public interface Factory { - FetchAll create(String urlMatch, ReplicationFilter filter, ReplicationState state, boolean now); + FetchAll create( + String urlMatch, + ReplicationFilter filter, + ReplicationState state, + ReplicationType replicationType); } private final WorkQueue workQueue; @@ -37,7 +41,7 @@ private final String urlMatch; private final ReplicationFilter filter; private final ReplicationState state; - private final boolean now; + private final ReplicationType replicationType; private final SourcesCollection sources; @Inject @@ -49,7 +53,7 @@ @Assisted @Nullable String urlMatch, @Assisted ReplicationFilter filter, @Assisted ReplicationState state, - @Assisted boolean now) { + @Assisted ReplicationType replicationType) { this.workQueue = wq; this.projectCache = projectCache; this.stateLog = stateLog; @@ -57,7 +61,7 @@ this.urlMatch = urlMatch; this.filter = filter; this.state = state; - this.now = now; + this.replicationType = replicationType; } Future<?> schedule(long delay, TimeUnit unit) { @@ -69,7 +73,7 @@ try { for (Project.NameKey nameKey : projectCache.all()) { if (filter.matches(nameKey)) { - scheduleFullSync(nameKey, urlMatch, state, now); + scheduleFullSync(nameKey, urlMatch, state, replicationType); } } } catch (Exception e) { @@ -79,12 +83,15 @@ } private void scheduleFullSync( - Project.NameKey project, String urlMatch, ReplicationState state, boolean now) { + Project.NameKey project, + String urlMatch, + ReplicationState state, + ReplicationType replicationType) { for (Source cfg : sources.getAll()) { if (cfg.wouldFetchProject(project)) { for (URIish uri : cfg.getURIs(project, urlMatch)) { - cfg.schedule(project, FetchOne.ALL_REFS, uri, state, now); + cfg.schedule(project, FetchOne.ALL_REFS, uri, state, replicationType); } } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/OnStartStop.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/OnStartStop.java index d8c4a8d..5cf8bb6 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/OnStartStop.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/OnStartStop.java
@@ -14,6 +14,8 @@ package com.googlesource.gerrit.plugins.replication.pull; +import static com.googlesource.gerrit.plugins.replication.pull.ReplicationType.ASYNC; + import com.google.common.util.concurrent.Atomics; import com.google.gerrit.extensions.events.LifecycleListener; import com.google.gerrit.extensions.registration.DynamicItem; @@ -65,7 +67,7 @@ new FetchResultProcessing.GitUpdateProcessing(eventDispatcher.get())); fetchAllFuture.set( fetchAll - .create(null, ReplicationFilter.all(), state, false) + .create(null, ReplicationFilter.all(), state, ASYNC) .schedule(30, TimeUnit.SECONDS)); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java index 6c8ada0..cdba012 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
@@ -32,6 +32,7 @@ import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException; import com.googlesource.gerrit.plugins.replication.pull.client.FetchRestApiClient; import com.googlesource.gerrit.plugins.replication.pull.client.HttpResult; +import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter; import java.io.IOException; import java.net.URISyntaxException; import java.util.HashSet; @@ -65,7 +66,7 @@ private final Queue<ReferenceUpdatedEvent> beforeStartupEventsQueue; private FetchRestApiClient.Factory fetchClientFactory; private Integer fetchCallsTimeout; - private RefsFilter refsFilter; + private ExcludedRefsFilter refsFilter; private RevisionReader revisionReader; @Inject @@ -75,7 +76,7 @@ DynamicItem<EventDispatcher> dis, ReplicationStateListeners sl, FetchRestApiClient.Factory fetchClientFactory, - RefsFilter refsFilter, + ExcludedRefsFilter refsFilter, RevisionReader revReader) { workQueue = wq; dispatcher = dis;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationType.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationType.java new file mode 100644 index 0000000..308a90b --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationType.java
@@ -0,0 +1,20 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull; + +public enum ReplicationType { + SYNC, + ASYNC +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java index 0b429de..ea2ce32 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
@@ -16,6 +16,7 @@ import static com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig.replaceName; import static com.googlesource.gerrit.plugins.replication.pull.FetchResultProcessing.resolveNodeName; +import static com.googlesource.gerrit.plugins.replication.pull.ReplicationType.SYNC; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; @@ -383,13 +384,20 @@ } public Future<?> schedule( - Project.NameKey project, String ref, ReplicationState state, boolean now) { + Project.NameKey project, + String ref, + ReplicationState state, + ReplicationType replicationType) { URIish uri = getURI(project); - return schedule(project, ref, uri, state, now); + return schedule(project, ref, uri, state, replicationType); } public Future<?> schedule( - Project.NameKey project, String ref, URIish uri, ReplicationState state, boolean now) { + Project.NameKey project, + String ref, + URIish uri, + ReplicationState state, + ReplicationType replicationType) { repLog.info("scheduling replication {}:{} => {}", uri, ref, project); if (!shouldReplicate(project, ref, state)) { @@ -429,7 +437,7 @@ addRef(e, ref); e.addState(ref, state); pending.put(uri, e); - f = pool.schedule(e, now ? 0 : config.getDelay(), TimeUnit.SECONDS); + f = pool.schedule(e, isSyncCall(replicationType) ? 0 : config.getDelay(), TimeUnit.SECONDS); } else if (!e.getRefs().contains(ref)) { addRef(e, ref); e.addState(ref, state); @@ -452,6 +460,10 @@ postReplicationScheduledEvent(e, ref); } + private boolean isSyncCall(ReplicationType replicationType) { + return SYNC.equals(replicationType); + } + /** * It schedules again a FetchOp instance. *
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java index e50461d..8046d49 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java
@@ -23,7 +23,7 @@ import org.eclipse.jgit.transport.RemoteConfig; public class SourceConfiguration implements RemoteConfiguration { - static final int DEFAULT_REPLICATION_DELAY = 0; + static final int DEFAULT_REPLICATION_DELAY = 4; static final int DEFAULT_RESCHEDULE_DELAY = 3; static final int DEFAULT_SLOW_LATENCY_THRESHOLD_SECS = 900; static final int DEFAULT_MAX_CONNECTION_INACTIVITY_MS = 10000;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/StartFetchCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/StartFetchCommand.java index 6ababe9..97f8e9e 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/StartFetchCommand.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/StartFetchCommand.java
@@ -14,6 +14,9 @@ package com.googlesource.gerrit.plugins.replication.pull; +import static com.googlesource.gerrit.plugins.replication.pull.ReplicationType.ASYNC; +import static com.googlesource.gerrit.plugins.replication.pull.ReplicationType.SYNC; + import com.google.gerrit.extensions.annotations.RequiresCapability; import com.google.gerrit.extensions.registration.DynamicItem; import com.google.gerrit.server.events.EventDispatcher; @@ -77,7 +80,10 @@ projectFilter = new ReplicationFilter(projectPatterns); } - future = fetchFactory.create(urlMatch, projectFilter, state, now).schedule(0, TimeUnit.SECONDS); + future = + fetchFactory + .create(urlMatch, projectFilter, state, replicationType(now)) + .schedule(0, TimeUnit.SECONDS); if (wait) { if (future != null) { @@ -105,6 +111,10 @@ } } + private ReplicationType replicationType(Boolean now) { + return now ? SYNC : ASYNC; + } + @Override public void writeStdOutSync(String message) { if (wait) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java index 678ff73..77c54fa 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java
@@ -14,12 +14,7 @@ package com.googlesource.gerrit.plugins.replication.pull.api; -import static com.google.common.base.Preconditions.checkState; - import com.google.common.base.Strings; -import com.google.common.flogger.FluentLogger; -import com.google.gerrit.entities.Project; -import com.google.gerrit.extensions.registration.DynamicItem; import com.google.gerrit.extensions.restapi.AuthException; import com.google.gerrit.extensions.restapi.BadRequestException; import com.google.gerrit.extensions.restapi.ResourceConflictException; @@ -27,9 +22,6 @@ import com.google.gerrit.extensions.restapi.RestApiException; import com.google.gerrit.extensions.restapi.RestModifyView; import com.google.gerrit.extensions.restapi.UnprocessableEntityException; -import com.google.gerrit.server.config.UrlFormatter; -import com.google.gerrit.server.git.WorkQueue; -import com.google.gerrit.server.ioutil.HexFormat; import com.google.gerrit.server.project.ProjectResource; import com.google.inject.Inject; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput; @@ -37,24 +29,15 @@ import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException; import java.io.IOException; import java.util.Objects; -import java.util.Optional; public class ApplyObjectAction implements RestModifyView<ProjectResource, RevisionInput> { private final ApplyObjectCommand command; - private final WorkQueue workQueue; - private final DynamicItem<UrlFormatter> urlFormatter; private final FetchPreconditions preConditions; @Inject - public ApplyObjectAction( - ApplyObjectCommand command, - WorkQueue workQueue, - DynamicItem<UrlFormatter> urlFormatter, - FetchPreconditions preConditions) { + public ApplyObjectAction(ApplyObjectCommand command, FetchPreconditions preConditions) { this.command = command; - this.workQueue = workQueue; - this.urlFormatter = urlFormatter; this.preConditions = preConditions; } @@ -89,10 +72,9 @@ throw new BadRequestException("Ref-update tree object cannot be null"); } - if (input.isAsync()) { - return applyAsync(resource.getNameKey(), input); - } - return applySync(resource.getNameKey(), input); + command.applyObject( + resource.getNameKey(), input.getRefName(), input.getRevisionData(), input.getLabel()); + return Response.created(input); } catch (MissingParentObjectException e) { throw new ResourceConflictException(e.getMessage(), e); } catch (NumberFormatException | IOException e) { @@ -101,51 +83,4 @@ throw new UnprocessableEntityException(e.getMessage()); } } - - private Response<?> applySync(Project.NameKey project, RevisionInput input) - throws NumberFormatException, IOException, RefUpdateException, MissingParentObjectException { - command.applyObject(project, input.getRefName(), input.getRevisionData(), input.getLabel()); - return Response.created(input); - } - - private Response.Accepted applyAsync(Project.NameKey project, RevisionInput input) { - @SuppressWarnings("unchecked") - WorkQueue.Task<Void> task = - (WorkQueue.Task<Void>) - workQueue.getDefaultQueue().submit(new ApplyObjectJob(command, project, input)); - Optional<String> url = - urlFormatter - .get() - .getRestUrl("a/config/server/tasks/" + HexFormat.fromInt(task.getTaskId())); - // We're in a HTTP handler, so must be present. - checkState(url.isPresent()); - return Response.accepted(url.get()); - } - - private static class ApplyObjectJob implements Runnable { - private static final FluentLogger log = FluentLogger.forEnclosingClass(); - - private ApplyObjectCommand command; - private Project.NameKey project; - private RevisionInput input; - - public ApplyObjectJob( - ApplyObjectCommand command, Project.NameKey project, RevisionInput input) { - this.command = command; - this.project = project; - this.input = input; - } - - @Override - public void run() { - try { - command.applyObject(project, input.getRefName(), input.getRevisionData(), input.getLabel()); - } catch (IOException | RefUpdateException | MissingParentObjectException e) { - log.atSevere().withCause(e).log( - "Exception during the applyObject call for project {} and ref name {}", - project.get(), - input.getRefName()); - } - } - } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java index 6970299..92cc180 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java
@@ -93,7 +93,7 @@ private Response<?> applySync(Project.NameKey project, Input input) throws InterruptedException, ExecutionException, RemoteConfigurationMissingException, TimeoutException { - command.fetch(project, input.label, input.refName); + command.fetchSync(project, input.label, input.refName); return Response.created(input); } @@ -127,7 +127,7 @@ @Override public void run() { try { - command.fetch(project, input.label, input.refName); + command.fetchAsync(project, input.label, input.refName); } catch (InterruptedException | ExecutionException | RemoteConfigurationMissingException
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java index 1088d9b..e1ac9ac 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java
@@ -14,6 +14,9 @@ package com.googlesource.gerrit.plugins.replication.pull.api; +import static com.googlesource.gerrit.plugins.replication.pull.ReplicationType.ASYNC; +import static com.googlesource.gerrit.plugins.replication.pull.ReplicationType.SYNC; + import com.google.gerrit.entities.Project; import com.google.gerrit.extensions.registration.DynamicItem; import com.google.gerrit.server.events.EventDispatcher; @@ -22,6 +25,7 @@ import com.googlesource.gerrit.plugins.replication.pull.FetchResultProcessing; import com.googlesource.gerrit.plugins.replication.pull.PullReplicationStateLogger; import com.googlesource.gerrit.plugins.replication.pull.ReplicationState; +import com.googlesource.gerrit.plugins.replication.pull.ReplicationType; import com.googlesource.gerrit.plugins.replication.pull.Source; import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection; import com.googlesource.gerrit.plugins.replication.pull.api.exception.RemoteConfigurationMissingException; @@ -50,7 +54,19 @@ this.eventDispatcher = eventDispatcher; } - public void fetch(Project.NameKey name, String label, String refName) + public void fetchAsync(Project.NameKey name, String label, String refName) + throws InterruptedException, ExecutionException, RemoteConfigurationMissingException, + TimeoutException { + fetch(name, label, refName, ASYNC); + } + + public void fetchSync(Project.NameKey name, String label, String refName) + throws InterruptedException, ExecutionException, RemoteConfigurationMissingException, + TimeoutException { + fetch(name, label, refName, SYNC); + } + + private void fetch(Project.NameKey name, String label, String refName, ReplicationType fetchType) throws InterruptedException, ExecutionException, RemoteConfigurationMissingException, TimeoutException { ReplicationState state = @@ -66,7 +82,7 @@ try { state.markAllFetchTasksScheduled(); - Future<?> future = source.get().schedule(name, refName, state, true); + Future<?> future = source.get().schedule(name, refName, state, fetchType); future.get(source.get().getTimeout(), TimeUnit.SECONDS); } catch (ExecutionException | IllegalStateException
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java index 160a720..bc3e218 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java
@@ -21,17 +21,10 @@ private RevisionData revisionData; - private boolean async; - public RevisionInput(String label, String refName, RevisionData revisionData) { - this(label, refName, revisionData, false); - } - - public RevisionInput(String label, String refName, RevisionData revisionData, boolean async) { this.label = label; this.refName = refName; this.revisionData = revisionData; - this.async = async; } public String getLabel() { @@ -45,8 +38,4 @@ public RevisionData getRevisionData() { return revisionData; } - - public boolean isAsync() { - return async; - } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java index eed1afb..7b876df 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
@@ -32,6 +32,7 @@ import com.googlesource.gerrit.plugins.replication.pull.Source; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput; +import com.googlesource.gerrit.plugins.replication.pull.filter.SyncRefsFilter; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Optional; @@ -67,18 +68,21 @@ private final Source source; private final String instanceLabel; private final String pluginName; + private final SyncRefsFilter syncRefsFilter; @Inject FetchRestApiClient( CredentialsFactory credentials, SourceHttpClient.Factory httpClientFactory, ReplicationConfig replicationConfig, + SyncRefsFilter syncRefsFilter, @PluginName String pluginName, @Assisted Source source) { this.credentials = credentials; this.httpClientFactory = httpClientFactory; this.source = source; this.pluginName = pluginName; + this.syncRefsFilter = syncRefsFilter; this.instanceLabel = Strings.nullToEmpty( replicationConfig.getConfig().getString("replication", null, "instanceLabel")) @@ -93,11 +97,13 @@ String.format( "%s/a/projects/%s/pull-replication~fetch", targetUri.toString(), Url.encode(project.get())); - + Boolean callAsync = !syncRefsFilter.match(refName); HttpPost post = new HttpPost(url); post.setEntity( new StringEntity( - String.format("{\"label\":\"%s\", \"ref_name\": \"%s\"}", instanceLabel, refName), + String.format( + "{\"label\":\"%s\", \"ref_name\": \"%s\", \"async\":%s}", + instanceLabel, refName, callAsync), StandardCharsets.UTF_8)); post.addHeader(new BasicHeader("Content-Type", "application/json")); return httpClientFactory.create(source).execute(post, this, getContext(targetUri));
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/ExcludedRefsFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/ExcludedRefsFilter.java new file mode 100644 index 0000000..f2e4ab3 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/ExcludedRefsFilter.java
@@ -0,0 +1,51 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.filter; + +import com.google.common.collect.ImmutableList; +import com.google.gerrit.entities.RefNames; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.googlesource.gerrit.plugins.replication.ReplicationConfig; +import java.util.List; +import org.eclipse.jgit.lib.Config; + +@Singleton +public class ExcludedRefsFilter extends RefsFilter { + @Inject + public ExcludedRefsFilter(ReplicationConfig replicationConfig) { + super(replicationConfig); + } + + @Override + protected List<String> getRefNamePatterns(Config cfg) { + return ImmutableList.<String>builder() + .addAll(getDefaultExcludeRefPatterns()) + .addAll(ImmutableList.copyOf(cfg.getStringList("replication", null, "excludeRefs"))) + .build(); + } + + private List<String> getDefaultExcludeRefPatterns() { + return ImmutableList.of( + RefNames.REFS_USERS + "*", + RefNames.REFS_CONFIG, + RefNames.REFS_SEQUENCES + "*", + RefNames.REFS_EXTERNAL_IDS, + RefNames.REFS_GROUPS + "*", + RefNames.REFS_GROUPNAMES, + RefNames.REFS_CACHE_AUTOMERGE + "*", + RefNames.REFS_STARRED_CHANGES + "*"); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RefsFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/RefsFilter.java similarity index 70% rename from src/main/java/com/googlesource/gerrit/plugins/replication/pull/RefsFilter.java rename to src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/RefsFilter.java index de6c97d..0bd7564 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RefsFilter.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/RefsFilter.java
@@ -12,20 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package com.googlesource.gerrit.plugins.replication.pull; +package com.googlesource.gerrit.plugins.replication.pull.filter; import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; import com.google.gerrit.common.data.AccessSection; -import com.google.gerrit.entities.RefNames; -import com.google.inject.Inject; -import com.google.inject.Singleton; import com.googlesource.gerrit.plugins.replication.ReplicationConfig; import java.util.List; import org.eclipse.jgit.lib.Config; -@Singleton -public class RefsFilter { +public abstract class RefsFilter { public enum PatternType { REGEX, WILDCARD, @@ -44,7 +39,6 @@ private final List<String> refsPatterns; - @Inject public RefsFilter(ReplicationConfig replicationConfig) { refsPatterns = getRefNamePatterns(replicationConfig.getConfig()); } @@ -66,12 +60,7 @@ return false; } - private List<String> getRefNamePatterns(Config cfg) { - return ImmutableList.<String>builder() - .addAll(getDefaultExcludeRefPatterns()) - .addAll(ImmutableList.copyOf(cfg.getStringList("replication", null, "excludeRefs"))) - .build(); - } + protected abstract List<String> getRefNamePatterns(Config cfg); private boolean matchesPattern(String refName, String pattern) { boolean match = false; @@ -87,16 +76,4 @@ } return match; } - - private List<String> getDefaultExcludeRefPatterns() { - return ImmutableList.of( - RefNames.REFS_USERS + "*", - RefNames.REFS_CONFIG, - RefNames.REFS_SEQUENCES + "*", - RefNames.REFS_EXTERNAL_IDS, - RefNames.REFS_GROUPS + "*", - RefNames.REFS_GROUPNAMES, - RefNames.REFS_CACHE_AUTOMERGE + "*", - RefNames.REFS_STARRED_CHANGES + "*"); - } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/SyncRefsFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/SyncRefsFilter.java new file mode 100644 index 0000000..a069935 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/SyncRefsFilter.java
@@ -0,0 +1,35 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.filter; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.googlesource.gerrit.plugins.replication.ReplicationConfig; +import java.util.List; +import org.eclipse.jgit.lib.Config; + +@Singleton +public class SyncRefsFilter extends RefsFilter { + @Inject + public SyncRefsFilter(ReplicationConfig replicationConfig) { + super(replicationConfig); + } + + @Override + protected List<String> getRefNamePatterns(Config cfg) { + return ImmutableList.copyOf(cfg.getStringList("replication", null, "syncRefs")); + } +}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index c3e5ad4..126cd1b 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -181,6 +181,30 @@ By default, all other refs are included. +replication.syncRefs +: Specify for which refs git fetch calls should be executed synchronously. + It can be provided more than once, and supports three formats: regular expressions, + wildcard matching, and single ref matching. All three formats match are case-sensitive. + + Values starting with a caret `^` are treated as regular + expressions. For the regular expressions details please follow + official [java documentation](https://docs.oracle.com/javase/tutorial/essential/regex/). + + Please note that regular expressions could also be used + with inverse match. + + Values that are not regular expressions and end in `*` are + treated as wildcard matches. Wildcards match refs whose + name agrees from the beginning until the trailing `*`. So + `foo/b*` would match the refs `foo/b`, `foo/bar`, and + `foo/baz`, but neither `foobar`, nor `bar/foo/baz`. + + Values that are neither regular expressions nor wildcards are + treated as single ref matches. So `foo/bar` matches only + the ref `foo/bar`, but no other refs. + + By default, set to '*' (all refs are replicated synchronously). + replication.maxApiPayloadSize : Maximum size in bytes of the ref to be sent as a REST Api call payload. For refs larger than threshold git fetch operation @@ -268,13 +292,14 @@ Defaults to 0 seconds, wait indefinitely. remote.NAME.replicationDelay -: Time to wait before scheduling a remote fetch operation. Setting - the delay to 0 effectively disables the delay, causing the fetch - to start as soon as possible. +: Time to wait before scheduling an asynchronous remote fetch + operation. Setting the delay to 0 effectively disables the delay, + causing the fetch to start as soon as possible. This is a Gerrit specific extension to the Git remote block. - By default, 0 seconds. + By default for asynchronous fetch, 4 seconds. For a synchronous fetch + replicationDelay is zero. remote.NAME.rescheduleDelay : Delay when rescheduling a fetch operation due to an in-flight fetch
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java index 44b69db..327c6ba 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
@@ -38,6 +38,7 @@ import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException; import com.googlesource.gerrit.plugins.replication.pull.client.FetchRestApiClient; import com.googlesource.gerrit.plugins.replication.pull.client.HttpResult; +import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter; import java.io.IOException; import java.nio.file.Path; import java.util.Optional; @@ -69,7 +70,7 @@ @Mock RevisionData revisionData; @Mock HttpResult httpResult; - private RefsFilter refsFilter; + private ExcludedRefsFilter refsFilter; private ReplicationQueue objectUnderTest; private SitePaths sitePaths; private Path pluginDataPath; @@ -80,7 +81,7 @@ sitePaths = new SitePaths(sitePath); Path pluginDataPath = createTempPath("data"); ReplicationConfig replicationConfig = new ReplicationFileBasedConfig(sitePaths, pluginDataPath); - refsFilter = new RefsFilter(replicationConfig); + refsFilter = new ExcludedRefsFilter(replicationConfig); when(source.getConnectionTimeout()).thenReturn(CONNECTION_TIMEOUT); when(source.wouldFetchProject(any())).thenReturn(true); when(source.wouldFetchRef(anyString())).thenReturn(true); @@ -198,7 +199,7 @@ fileConfig.setString("replication", null, "excludeRefs", "refs/multi-site/version"); fileConfig.save(); ReplicationConfig replicationConfig = new ReplicationFileBasedConfig(sitePaths, pluginDataPath); - refsFilter = new RefsFilter(replicationConfig); + refsFilter = new ExcludedRefsFilter(replicationConfig); objectUnderTest = new ReplicationQueue(wq, rd, dis, sl, fetchClientFactory, refsFilter, revReader);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionIT.java new file mode 100644 index 0000000..75c31de --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionIT.java
@@ -0,0 +1,272 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull.api; + +import static com.google.common.truth.Truth.assertThat; +import static java.util.stream.Collectors.toList; + +import com.google.gerrit.acceptance.LightweightPluginDaemonTest; +import com.google.gerrit.acceptance.PushOneCommit.Result; +import com.google.gerrit.acceptance.SkipProjectClone; +import com.google.gerrit.acceptance.TestPlugin; +import com.google.gerrit.acceptance.UseLocalDisk; +import com.google.gerrit.acceptance.testsuite.project.ProjectOperations; +import com.google.gerrit.entities.Project; +import com.google.gerrit.entities.RefNames; +import com.google.gerrit.extensions.restapi.Url; +import com.google.gerrit.server.config.SitePaths; +import com.google.inject.Inject; +import com.googlesource.gerrit.plugins.replication.CredentialsFactory; +import com.googlesource.gerrit.plugins.replication.pull.RevisionReader; +import com.googlesource.gerrit.plugins.replication.pull.Source; +import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection; +import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; +import com.googlesource.gerrit.plugins.replication.pull.client.SourceHttpClient; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import java.util.Optional; +import org.apache.http.HttpResponse; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.ResponseHandler; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.message.BasicHeader; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.util.FS; +import org.junit.Test; + +@SkipProjectClone +@UseLocalDisk +@TestPlugin( + name = "pull-replication", + sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule") +public class ApplyObjectActionIT extends LightweightPluginDaemonTest { + private static final Optional<String> ALL_PROJECTS = Optional.empty(); + private static final int TEST_REPLICATION_DELAY = 60; + private static final String TEST_REPLICATION_SUFFIX = "suffix1"; + private static final String TEST_REPLICATION_REMOTE = "remote1"; + + private Path gitPath; + private FileBasedConfig config; + private FileBasedConfig secureConfig; + private RevisionReader revisionReader; + + @Inject private SitePaths sitePaths; + @Inject private ProjectOperations projectOperations; + CredentialsFactory credentials; + Source source; + SourceHttpClient.Factory httpClientFactory; + String url; + + @Override + public void setUpTestPlugin() throws Exception { + gitPath = sitePaths.site_path.resolve("git"); + + config = + new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED); + setReplicationSource( + TEST_REPLICATION_REMOTE, + TEST_REPLICATION_SUFFIX, + ALL_PROJECTS); // Simulates a full replication.config initialization + config.save(); + + secureConfig = + new FileBasedConfig(sitePaths.etc_dir.resolve("secure.config").toFile(), FS.DETECTED); + setReplicationCredentials(TEST_REPLICATION_REMOTE, admin.username(), admin.httpPassword()); + secureConfig.save(); + + super.setUpTestPlugin(); + + httpClientFactory = plugin.getSysInjector().getInstance(SourceHttpClient.Factory.class); + credentials = plugin.getSysInjector().getInstance(CredentialsFactory.class); + revisionReader = plugin.getSysInjector().getInstance(RevisionReader.class); + source = plugin.getSysInjector().getInstance(SourcesCollection.class).getAll().get(0); + + url = + String.format( + "%s/a/projects/%s/pull-replication~apply-object", + adminRestSession.url(), Url.encode(project.get())); + } + + @Test + public void shouldAcceptPayloadWithAsyncField() throws Exception { + String payloadWithAsyncFieldTemplate = + "{\"label\":\"" + + TEST_REPLICATION_REMOTE + + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}, \"async\":true}"; + + String refName = createRef(); + Optional<RevisionData> revisionDataOption = createRevisionData(refName); + assertThat(revisionDataOption.isPresent()).isTrue(); + + RevisionData revisionData = revisionDataOption.get(); + String sendObjectPayload = createPayload(payloadWithAsyncFieldTemplate, refName, revisionData); + + HttpPost post = createRequest(sendObjectPayload); + httpClientFactory.create(source).execute(post, assertHttpResponseCode(201), getContext()); + } + + @Test + public void shouldAcceptPayloadWithoutAsyncField() throws Exception { + String payloadWithoutAsyncFieldTemplate = + "{\"label\":\"" + + TEST_REPLICATION_REMOTE + + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}}"; + + String refName = createRef(); + Optional<RevisionData> revisionDataOption = createRevisionData(refName); + assertThat(revisionDataOption.isPresent()).isTrue(); + + RevisionData revisionData = revisionDataOption.get(); + String sendObjectPayload = + createPayload(payloadWithoutAsyncFieldTemplate, refName, revisionData); + + HttpPost post = createRequest(sendObjectPayload); + httpClientFactory.create(source).execute(post, assertHttpResponseCode(201), getContext()); + } + + @Test + public void shouldReturnBadRequestCodeWhenMandatoryFieldLabelIsMissing() throws Exception { + String payloadWithoutLabelFieldTemplate = + "{\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}, \"async\":true}"; + + String refName = createRef(); + Optional<RevisionData> revisionDataOption = createRevisionData(refName); + assertThat(revisionDataOption.isPresent()).isTrue(); + + RevisionData revisionData = revisionDataOption.get(); + String sendObjectPayload = + createPayload(payloadWithoutLabelFieldTemplate, refName, revisionData); + + HttpPost post = createRequest(sendObjectPayload); + httpClientFactory.create(source).execute(post, assertHttpResponseCode(400), getContext()); + } + + @Test + public void shouldReturnBadRequestCodeWhenPayloadIsNotAProperJSON() throws Exception { + String wrongPayloadTemplate = + "{\"label\":\"" + + TEST_REPLICATION_REMOTE + + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}, \"async\":true,}"; + + String refName = createRef(); + Optional<RevisionData> revisionDataOption = createRevisionData(refName); + assertThat(revisionDataOption.isPresent()).isTrue(); + + RevisionData revisionData = revisionDataOption.get(); + String sendObjectPayload = createPayload(wrongPayloadTemplate, refName, revisionData); + + HttpPost post = createRequest(sendObjectPayload); + httpClientFactory.create(source).execute(post, assertHttpResponseCode(400), getContext()); + } + + private String createPayload( + String wrongPayloadTemplate, String refName, RevisionData revisionData) { + String sendObjectPayload = + String.format( + wrongPayloadTemplate, + refName, + encode(revisionData.getCommitObject().getContent()), + encode(revisionData.getTreeObject().getContent())); + return sendObjectPayload; + } + + private HttpPost createRequest(String sendObjectPayload) { + HttpPost post = new HttpPost(url); + post.setEntity(new StringEntity(sendObjectPayload, StandardCharsets.UTF_8)); + post.addHeader(new BasicHeader("Content-Type", "application/json")); + return post; + } + + private String createRef() throws Exception { + testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX)); + + Result pushResult = createChange("topic", "test.txt", "test_content"); + return RefNames.changeMetaRef(pushResult.getChange().getId()); + } + + private Optional<RevisionData> createRevisionData(String refName) throws Exception { + return revisionReader.read(Project.nameKey(project + TEST_REPLICATION_SUFFIX), refName); + } + + private Object encode(byte[] content) { + return Base64.getEncoder().encodeToString(content); + } + + public ResponseHandler<Object> assertHttpResponseCode(int responseCode) { + return new ResponseHandler<Object>() { + + @Override + public Object handleResponse(HttpResponse response) + throws ClientProtocolException, IOException { + assertThat(response.getStatusLine().getStatusCode()).isEqualTo(responseCode); + return null; + } + }; + } + + private HttpClientContext getContext() { + HttpClientContext ctx = HttpClientContext.create(); + CredentialsProvider adapted = new BasicCredentialsProvider(); + adapted.setCredentials( + AuthScope.ANY, new UsernamePasswordCredentials(admin.username(), admin.httpPassword())); + ctx.setCredentialsProvider(adapted); + return ctx; + } + + private Project.NameKey createTestProject(String name) throws Exception { + return projectOperations.newProject().name(name).parent(project).create(); + } + + private void setReplicationSource( + String remoteName, String replicaSuffix, Optional<String> project) throws IOException { + setReplicationSource(remoteName, Arrays.asList(replicaSuffix), project); + } + + private void setReplicationSource( + String remoteName, List<String> replicaSuffixes, Optional<String> project) + throws IOException { + + List<String> replicaUrls = + replicaSuffixes.stream() + .map(suffix -> gitPath.resolve("${name}" + suffix + ".git").toString()) + .collect(toList()); + config.setString("replication", null, "instanceLabel", remoteName); + config.setStringList("remote", remoteName, "url", replicaUrls); + config.setString("remote", remoteName, "apiUrl", adminRestSession.url()); + config.setString("remote", remoteName, "fetch", "+refs/tags/*:refs/tags/*"); + config.setInt("remote", remoteName, "timeout", 600); + config.setInt("remote", remoteName, "replicationDelay", TEST_REPLICATION_DELAY); + project.ifPresent(prj -> config.setString("remote", remoteName, "projects", prj)); + config.setBoolean("gerrit", null, "autoReload", true); + config.save(); + } + + private void setReplicationCredentials(String remoteName, String username, String password) + throws IOException { + secureConfig.setString("remote", remoteName, "username", username); + secureConfig.setString("remote", remoteName, "password", password); + secureConfig.save(); + } +}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java index 55fdee8..8738046 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java
@@ -15,7 +15,6 @@ package com.googlesource.gerrit.plugins.replication.pull.api; import static com.google.common.truth.Truth.assertThat; -import static org.apache.http.HttpStatus.SC_ACCEPTED; import static org.apache.http.HttpStatus.SC_CREATED; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; @@ -24,15 +23,11 @@ import com.google.common.collect.Lists; import com.google.gerrit.entities.Project; -import com.google.gerrit.extensions.registration.DynamicItem; import com.google.gerrit.extensions.restapi.AuthException; import com.google.gerrit.extensions.restapi.BadRequestException; import com.google.gerrit.extensions.restapi.ResourceConflictException; import com.google.gerrit.extensions.restapi.Response; import com.google.gerrit.extensions.restapi.RestApiException; -import com.google.gerrit.server.config.UrlFormatter; -import com.google.gerrit.server.git.WorkQueue; -import com.google.gerrit.server.git.WorkQueue.Task; import com.google.gerrit.server.project.ProjectResource; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput; @@ -40,17 +35,13 @@ import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException; import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException; import java.io.IOException; -import java.util.Optional; -import java.util.concurrent.ScheduledExecutorService; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; -import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; -import org.mockito.stubbing.Answer; @RunWith(MockitoJUnitRunner.class) public class ApplyObjectActionTest { @@ -82,32 +73,13 @@ @Mock ApplyObjectCommand applyObjectCommand; @Mock ProjectResource projectResource; - @Mock WorkQueue workQueue; - @Mock ScheduledExecutorService exceutorService; - @Mock DynamicItem<UrlFormatter> urlFormatterDynamicItem; - @Mock UrlFormatter urlFormatter; - @Mock WorkQueue.Task<Void> task; @Mock FetchPreconditions preConditions; @Before public void setup() { - when(workQueue.getDefaultQueue()).thenReturn(exceutorService); - when(urlFormatter.getRestUrl(anyString())).thenReturn(Optional.of(location)); - when(exceutorService.submit(any(Runnable.class))) - .thenAnswer( - new Answer<WorkQueue.Task<Void>>() { - @Override - public Task<Void> answer(InvocationOnMock invocation) throws Throwable { - return task; - } - }); - when(urlFormatterDynamicItem.get()).thenReturn(urlFormatter); - when(task.getTaskId()).thenReturn(taskId); when(preConditions.canCallFetchApi()).thenReturn(true); - applyObjectAction = - new ApplyObjectAction( - applyObjectCommand, workQueue, urlFormatterDynamicItem, preConditions); + applyObjectAction = new ApplyObjectAction(applyObjectCommand, preConditions); } @Test @@ -207,24 +179,6 @@ applyObjectAction.apply(projectResource, inputParams); } - @Test - public void shouldReturnScheduledTaskForAsyncCall() throws RestApiException { - RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData(), true); - - Response<?> response = applyObjectAction.apply(projectResource, inputParams); - assertThat(response.statusCode()).isEqualTo(SC_ACCEPTED); - } - - @Test - public void shouldLocationHeaderForAsyncCall() throws Exception { - RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData(), true); - - Response<?> response = applyObjectAction.apply(projectResource, inputParams); - assertThat(response).isInstanceOf(Response.Accepted.class); - Response.Accepted acceptResponse = (Response.Accepted) response; - assertThat(acceptResponse.location()).isEqualTo(location); - } - private RevisionData createSampleRevisionData() { RevisionObjectData commitData = new RevisionObjectData(Constants.OBJ_COMMIT, sampleCommitContent.getBytes());
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionTest.java index 2c888c8..fb7f3d1 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionTest.java
@@ -147,7 +147,7 @@ inputParams.label = label; inputParams.refName = refName; - doThrow(new InterruptedException()).when(fetchCommand).fetch(any(), any(), any()); + doThrow(new InterruptedException()).when(fetchCommand).fetchSync(any(), any(), any()); fetchAction.apply(projectResource, inputParams); } @@ -162,7 +162,7 @@ doThrow(new RemoteConfigurationMissingException("")) .when(fetchCommand) - .fetch(any(), any(), any()); + .fetchSync(any(), any(), any()); fetchAction.apply(projectResource, inputParams); } @@ -177,7 +177,7 @@ doThrow(new ExecutionException(new RuntimeException())) .when(fetchCommand) - .fetch(any(), any(), any()); + .fetchSync(any(), any(), any()); fetchAction.apply(projectResource, inputParams); } @@ -190,7 +190,7 @@ inputParams.label = label; inputParams.refName = refName; - doThrow(new IllegalStateException()).when(fetchCommand).fetch(any(), any(), any()); + doThrow(new IllegalStateException()).when(fetchCommand).fetchSync(any(), any(), any()); fetchAction.apply(projectResource, inputParams); } @@ -203,7 +203,7 @@ inputParams.label = label; inputParams.refName = refName; - doThrow(new TimeoutException()).when(fetchCommand).fetch(any(), any(), any()); + doThrow(new TimeoutException()).when(fetchCommand).fetchSync(any(), any(), any()); fetchAction.apply(projectResource, inputParams); }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java index 4cdad48..e1ad565 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java
@@ -15,6 +15,8 @@ package com.googlesource.gerrit.plugins.replication.pull.api; import static com.google.gerrit.testing.GerritJUnit.assertThrows; +import static com.googlesource.gerrit.plugins.replication.pull.ReplicationType.ASYNC; +import static com.googlesource.gerrit.plugins.replication.pull.ReplicationType.SYNC; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyLong; import static org.mockito.Mockito.anyString; @@ -74,7 +76,7 @@ when(fetchReplicationStateFactory.create(any())).thenReturn(state); when(source.getRemoteConfigName()).thenReturn(label); when(sources.getAll()).thenReturn(Lists.newArrayList(source)); - when(source.schedule(projectName, REF_NAME_TO_FETCH, state, true)) + when(source.schedule(eq(projectName), eq(REF_NAME_TO_FETCH), eq(state), any())) .thenReturn(CompletableFuture.completedFuture(null)); objectUnderTest = new FetchCommand(fetchReplicationStateFactory, fetchStateLog, sources, eventDispatcher); @@ -84,18 +86,27 @@ public void shouldScheduleRefFetch() throws InterruptedException, ExecutionException, RemoteConfigurationMissingException, TimeoutException { - objectUnderTest.fetch(projectName, label, REF_NAME_TO_FETCH); + objectUnderTest.fetchSync(projectName, label, REF_NAME_TO_FETCH); - verify(source, times(1)).schedule(projectName, REF_NAME_TO_FETCH, state, true); + verify(source, times(1)).schedule(projectName, REF_NAME_TO_FETCH, state, SYNC); + } + + @Test + public void shouldScheduleRefFetchWithDelay() + throws InterruptedException, ExecutionException, RemoteConfigurationMissingException, + TimeoutException { + objectUnderTest.fetchAsync(projectName, label, REF_NAME_TO_FETCH); + + verify(source, times(1)).schedule(projectName, REF_NAME_TO_FETCH, state, ASYNC); } @Test public void shouldMarkAllFetchTasksScheduled() throws InterruptedException, ExecutionException, RemoteConfigurationMissingException, TimeoutException { - objectUnderTest.fetch(projectName, label, REF_NAME_TO_FETCH); + objectUnderTest.fetchSync(projectName, label, REF_NAME_TO_FETCH); - verify(source, times(1)).schedule(projectName, REF_NAME_TO_FETCH, state, true); + verify(source, times(1)).schedule(projectName, REF_NAME_TO_FETCH, state, SYNC); verify(state, times(1)).markAllFetchTasksScheduled(); } @@ -103,7 +114,7 @@ public void shouldUpdateStateWhenRemoteConfigNameIsMissing() { assertThrows( RemoteConfigurationMissingException.class, - () -> objectUnderTest.fetch(projectName, "unknownLabel", REF_NAME_TO_FETCH)); + () -> objectUnderTest.fetchSync(projectName, "unknownLabel", REF_NAME_TO_FETCH)); verify(fetchStateLog, times(1)).error(anyString(), eq(state)); } @@ -112,12 +123,12 @@ public void shouldUpdateStateWhenInterruptedException() throws InterruptedException, ExecutionException, TimeoutException { when(future.get(anyLong(), eq(TimeUnit.SECONDS))).thenThrow(new InterruptedException()); - when(source.schedule(projectName, REF_NAME_TO_FETCH, state, true)).thenReturn(future); + when(source.schedule(projectName, REF_NAME_TO_FETCH, state, SYNC)).thenReturn(future); InterruptedException e = assertThrows( InterruptedException.class, - () -> objectUnderTest.fetch(projectName, label, REF_NAME_TO_FETCH)); + () -> objectUnderTest.fetchSync(projectName, label, REF_NAME_TO_FETCH)); verify(fetchStateLog, times(1)).error(anyString(), eq(e), eq(state)); } @@ -127,12 +138,12 @@ throws InterruptedException, ExecutionException, TimeoutException { when(future.get(anyLong(), eq(TimeUnit.SECONDS))) .thenThrow(new ExecutionException(new Exception())); - when(source.schedule(projectName, REF_NAME_TO_FETCH, state, true)).thenReturn(future); + when(source.schedule(projectName, REF_NAME_TO_FETCH, state, SYNC)).thenReturn(future); ExecutionException e = assertThrows( ExecutionException.class, - () -> objectUnderTest.fetch(projectName, label, REF_NAME_TO_FETCH)); + () -> objectUnderTest.fetchSync(projectName, label, REF_NAME_TO_FETCH)); verify(fetchStateLog, times(1)).error(anyString(), eq(e), eq(state)); } @@ -141,12 +152,12 @@ public void shouldUpdateStateWhenTimeoutException() throws InterruptedException, ExecutionException, TimeoutException { when(future.get(anyLong(), eq(TimeUnit.SECONDS))).thenThrow(new TimeoutException()); - when(source.schedule(projectName, REF_NAME_TO_FETCH, state, true)).thenReturn(future); + when(source.schedule(projectName, REF_NAME_TO_FETCH, state, SYNC)).thenReturn(future); TimeoutException e = assertThrows( TimeoutException.class, - () -> objectUnderTest.fetch(projectName, label, REF_NAME_TO_FETCH)); + () -> objectUnderTest.fetchSync(projectName, label, REF_NAME_TO_FETCH)); verify(fetchStateLog, times(1)).error(anyString(), eq(e), eq(state)); } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java index 771acff..a3b0b02 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java
@@ -31,6 +31,7 @@ import com.googlesource.gerrit.plugins.replication.pull.Source; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData; import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData; +import com.googlesource.gerrit.plugins.replication.pull.filter.SyncRefsFilter; import java.io.IOException; import java.net.URISyntaxException; import java.nio.ByteBuffer; @@ -72,11 +73,15 @@ String label = "Replication"; String refName = RefNames.REFS_HEADS + "master"; - String expectedPayload = "{\"label\":\"Replication\", \"ref_name\": \"" + refName + "\"}"; + String expectedPayload = + "{\"label\":\"Replication\", \"ref_name\": \"" + refName + "\", \"async\":false}"; + String expectedAsyncPayload = + "{\"label\":\"Replication\", \"ref_name\": \"" + refName + "\", \"async\":true}"; Header expectedHeader = new BasicHeader("Content-Type", "application/json"); + SyncRefsFilter syncRefsFilter; String expectedSendObjectPayload = - "{\"label\":\"Replication\",\"ref_name\":\"refs/heads/master\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"dHJlZSA3NzgxNGQyMTZhNmNhYjJkZGI5ZjI4NzdmYmJkMGZlYmRjMGZhNjA4CnBhcmVudCA5ODNmZjFhM2NmNzQ3MjVhNTNhNWRlYzhkMGMwNjEyMjEyOGY1YThkCmF1dGhvciBHZXJyaXQgVXNlciAxMDAwMDAwIDwxMDAwMDAwQDY5ZWMzOGYwLTM1MGUtNGQ5Yy05NmQ0LWJjOTU2ZjJmYWFhYz4gMTYxMDU3ODY0OCArMDEwMApjb21taXR0ZXIgR2Vycml0IENvZGUgUmV2aWV3IDxyb290QG1hY3plY2gtWFBTLTE1PiAxNjEwNTc4NjQ4ICswMTAwCgpVcGRhdGUgcGF0Y2ggc2V0IDEKClBhdGNoIFNldCAxOgoKKDEgY29tbWVudCkKClBhdGNoLXNldDogMQo\\u003d\"},\"tree_object\":{\"type\":2,\"content\":\"MTAwNjQ0IGJsb2IgYmIzODNmNTI0OWM2OGE0Y2M4YzgyYmRkMTIyOGI0YTg4ODNmZjZlOCAgICBmNzVhNjkwMDRhOTNiNGNjYzhjZTIxNWMxMjgwODYzNmMyYjc1Njc1\"},\"blobs\":[{\"type\":3,\"content\":\"ewogICJjb21tZW50cyI6IFsKICAgIHsKICAgICAgImtleSI6IHsKICAgICAgICAidXVpZCI6ICI5MGI1YWJmZl80ZjY3NTI2YSIsCiAgICAgICAgImZpbGVuYW1lIjogIi9DT01NSVRfTVNHIiwKICAgICAgICAicGF0Y2hTZXRJZCI6IDEKICAgICAgfSwKICAgICAgImxpbmVOYnIiOiA5LAogICAgICAiYXV0aG9yIjogewogICAgICAgICJpZCI6IDEwMDAwMDAKICAgICAgfSwKICAgICAgIndyaXR0ZW5PbiI6ICIyMDIxLTAxLTEzVDIyOjU3OjI4WiIsCiAgICAgICJzaWRlIjogMSwKICAgICAgIm1lc3NhZ2UiOiAidGVzdCBjb21tZW50IiwKICAgICAgInJhbmdlIjogewogICAgICAgICJzdGFydExpbmUiOiA5LAogICAgICAgICJzdGFydENoYXIiOiAyMSwKICAgICAgICAiZW5kTGluZSI6IDksCiAgICAgICAgImVuZENoYXIiOiAzNAogICAgICB9LAogICAgICAicmV2SWQiOiAiZjc1YTY5MDA0YTkzYjRjY2M4Y2UyMTVjMTI4MDg2MzZjMmI3NTY3NSIsCiAgICAgICJzZXJ2ZXJJZCI6ICI2OWVjMzhmMC0zNTBlLTRkOWMtOTZkNC1iYzk1NmYyZmFhYWMiLAogICAgICAidW5yZXNvbHZlZCI6IHRydWUKICAgIH0KICBdCn0\\u003d\"}]},\"async\":false}"; + "{\"label\":\"Replication\",\"ref_name\":\"refs/heads/master\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"dHJlZSA3NzgxNGQyMTZhNmNhYjJkZGI5ZjI4NzdmYmJkMGZlYmRjMGZhNjA4CnBhcmVudCA5ODNmZjFhM2NmNzQ3MjVhNTNhNWRlYzhkMGMwNjEyMjEyOGY1YThkCmF1dGhvciBHZXJyaXQgVXNlciAxMDAwMDAwIDwxMDAwMDAwQDY5ZWMzOGYwLTM1MGUtNGQ5Yy05NmQ0LWJjOTU2ZjJmYWFhYz4gMTYxMDU3ODY0OCArMDEwMApjb21taXR0ZXIgR2Vycml0IENvZGUgUmV2aWV3IDxyb290QG1hY3plY2gtWFBTLTE1PiAxNjEwNTc4NjQ4ICswMTAwCgpVcGRhdGUgcGF0Y2ggc2V0IDEKClBhdGNoIFNldCAxOgoKKDEgY29tbWVudCkKClBhdGNoLXNldDogMQo\\u003d\"},\"tree_object\":{\"type\":2,\"content\":\"MTAwNjQ0IGJsb2IgYmIzODNmNTI0OWM2OGE0Y2M4YzgyYmRkMTIyOGI0YTg4ODNmZjZlOCAgICBmNzVhNjkwMDRhOTNiNGNjYzhjZTIxNWMxMjgwODYzNmMyYjc1Njc1\"},\"blobs\":[{\"type\":3,\"content\":\"ewogICJjb21tZW50cyI6IFsKICAgIHsKICAgICAgImtleSI6IHsKICAgICAgICAidXVpZCI6ICI5MGI1YWJmZl80ZjY3NTI2YSIsCiAgICAgICAgImZpbGVuYW1lIjogIi9DT01NSVRfTVNHIiwKICAgICAgICAicGF0Y2hTZXRJZCI6IDEKICAgICAgfSwKICAgICAgImxpbmVOYnIiOiA5LAogICAgICAiYXV0aG9yIjogewogICAgICAgICJpZCI6IDEwMDAwMDAKICAgICAgfSwKICAgICAgIndyaXR0ZW5PbiI6ICIyMDIxLTAxLTEzVDIyOjU3OjI4WiIsCiAgICAgICJzaWRlIjogMSwKICAgICAgIm1lc3NhZ2UiOiAidGVzdCBjb21tZW50IiwKICAgICAgInJhbmdlIjogewogICAgICAgICJzdGFydExpbmUiOiA5LAogICAgICAgICJzdGFydENoYXIiOiAyMSwKICAgICAgICAiZW5kTGluZSI6IDksCiAgICAgICAgImVuZENoYXIiOiAzNAogICAgICB9LAogICAgICAicmV2SWQiOiAiZjc1YTY5MDA0YTkzYjRjY2M4Y2UyMTVjMTI4MDg2MzZjMmI3NTY3NSIsCiAgICAgICJzZXJ2ZXJJZCI6ICI2OWVjMzhmMC0zNTBlLTRkOWMtOTZkNC1iYzk1NmYyZmFhYWMiLAogICAgICAidW5yZXNvbHZlZCI6IHRydWUKICAgIH0KICBdCn0\\u003d\"}]}}"; String commitObject = "tree 77814d216a6cab2ddb9f2877fbbd0febdc0fa608\n" + "parent 983ff1a3cf74725a53a5dec8d0c06122128f5a8d\n" @@ -143,15 +148,17 @@ when(credentialProvider.get(any(), any(CredentialItem.class))).thenReturn(true); when(credentials.create(anyString())).thenReturn(credentialProvider); when(replicationConfig.getConfig()).thenReturn(config); + when(config.getStringList("replication", null, "syncRefs")).thenReturn(new String[0]); when(source.getRemoteConfigName()).thenReturn("Replication"); when(config.getString("replication", null, "instanceLabel")).thenReturn(label); HttpResult httpResult = new HttpResult(SC_CREATED, Optional.of("result message")); when(httpClient.execute(any(HttpPost.class), any(), any())).thenReturn(httpResult); when(httpClientFactory.create(any())).thenReturn(httpClient); + syncRefsFilter = new SyncRefsFilter(replicationConfig); objectUnderTest = new FetchRestApiClient( - credentials, httpClientFactory, replicationConfig, pluginName, source); + credentials, httpClientFactory, replicationConfig, syncRefsFilter, pluginName, source); } @Test @@ -169,6 +176,67 @@ } @Test + public void shouldByDefaultCallSyncFetchForAllRefs() + throws ClientProtocolException, IOException, URISyntaxException { + + syncRefsFilter = new SyncRefsFilter(replicationConfig); + objectUnderTest = + new FetchRestApiClient( + credentials, httpClientFactory, replicationConfig, syncRefsFilter, pluginName, source); + + objectUnderTest.callFetch(Project.nameKey("test_repo"), refName, new URIish(api)); + + verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any(), any()); + + HttpPost httpPost = httpPostCaptor.getValue(); + assertThat(readPayload(httpPost)).isEqualTo(expectedPayload); + } + + @Test + public void shouldCallAsyncFetchForAllRefs() + throws ClientProtocolException, IOException, URISyntaxException { + + when(config.getStringList("replication", null, "syncRefs")) + .thenReturn(new String[] {"NO_SYNC_REFS"}); + syncRefsFilter = new SyncRefsFilter(replicationConfig); + objectUnderTest = + new FetchRestApiClient( + credentials, httpClientFactory, replicationConfig, syncRefsFilter, pluginName, source); + + objectUnderTest.callFetch(Project.nameKey("test_repo"), refName, new URIish(api)); + + verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any(), any()); + + HttpPost httpPost = httpPostCaptor.getValue(); + assertThat(readPayload(httpPost)).isEqualTo(expectedAsyncPayload); + } + + @Test + public void shouldCallSyncFetchOnlyForMetaRef() + throws ClientProtocolException, IOException, URISyntaxException { + String metaRefName = "refs/changes/01/101/meta"; + String expectedMetaRefPayload = + "{\"label\":\"Replication\", \"ref_name\": \"" + metaRefName + "\", \"async\":false}"; + + when(config.getStringList("replication", null, "syncRefs")) + .thenReturn(new String[] {"^refs\\/changes\\/.*\\/meta"}); + syncRefsFilter = new SyncRefsFilter(replicationConfig); + objectUnderTest = + new FetchRestApiClient( + credentials, httpClientFactory, replicationConfig, syncRefsFilter, pluginName, source); + + objectUnderTest.callFetch(Project.nameKey("test_repo"), refName, new URIish(api)); + verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any(), any()); + HttpPost httpPost = httpPostCaptor.getValue(); + assertThat(readPayload(httpPost)).isEqualTo(expectedAsyncPayload); + + objectUnderTest.callFetch(Project.nameKey("test_repo"), metaRefName, new URIish(api)); + verify(httpClient, times(2)).execute(httpPostCaptor.capture(), any(), any()); + httpPost = httpPostCaptor.getValue(); + assertThat(readPayload(httpPost)).isEqualTo(expectedMetaRefPayload); + } + + @Test public void shouldCallFetchEndpointWithPayload() throws ClientProtocolException, IOException, URISyntaxException { @@ -241,7 +309,12 @@ NullPointerException.class, () -> new FetchRestApiClient( - credentials, httpClientFactory, replicationConfig, pluginName, source)); + credentials, + httpClientFactory, + replicationConfig, + syncRefsFilter, + pluginName, + source)); } @Test @@ -251,7 +324,12 @@ NullPointerException.class, () -> new FetchRestApiClient( - credentials, httpClientFactory, replicationConfig, pluginName, source)); + credentials, + httpClientFactory, + replicationConfig, + syncRefsFilter, + pluginName, + source)); } @Test @@ -261,7 +339,12 @@ NullPointerException.class, () -> new FetchRestApiClient( - credentials, httpClientFactory, replicationConfig, pluginName, source)); + credentials, + httpClientFactory, + replicationConfig, + syncRefsFilter, + pluginName, + source)); } public String readPayload(HttpPost entity) throws UnsupportedOperationException, IOException {