Merge branch 'stable-3.2' into stable-3.3
* stable-3.2:
FetchAction: Fix FloggerFormatString pattern flagged by error prone
Add replication delay for asynchronous fetch calls
Trigger indexing only for local ref updates
Index change only for meta ref propagation
Allow asynchronous fetch calls
Comment the LocalDiskRepositoryManager in ApplyObject
Fix issue with reading maxApiPayloadSize property
Change-Id: Id9a78f577699bfa67b1c8a2666f2ab6e08de4f43
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Context.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Context.java
new file mode 100644
index 0000000..34bf3d2
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Context.java
@@ -0,0 +1,39 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull;
+
+/**
+ * Allows to tag event as local to avoid consuming remote events.
+ *
+ * <p>TODO: Gerrit v3.1 doesn't have concept of the instanceId so ThreadLocal must be used. From
+ * Gerrit v3.2 replace ThreadLocal with instanceId.
+ */
+public class Context {
+ private static final ThreadLocal<Boolean> localEvent = ThreadLocal.withInitial(() -> false);
+
+ private Context() {}
+
+ public static Boolean isLocalEvent() {
+ return localEvent.get();
+ }
+
+ public static void setLocalEvent(Boolean b) {
+ localEvent.set(b);
+ }
+
+ public static void unsetLocalEvent() {
+ localEvent.remove();
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchAll.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchAll.java
index baffff7..bcd86d7 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchAll.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchAll.java
@@ -29,7 +29,11 @@
private final ReplicationStateListener stateLog;
public interface Factory {
- FetchAll create(String urlMatch, ReplicationFilter filter, ReplicationState state, boolean now);
+ FetchAll create(
+ String urlMatch,
+ ReplicationFilter filter,
+ ReplicationState state,
+ ReplicationType replicationType);
}
private final WorkQueue workQueue;
@@ -37,7 +41,7 @@
private final String urlMatch;
private final ReplicationFilter filter;
private final ReplicationState state;
- private final boolean now;
+ private final ReplicationType replicationType;
private final SourcesCollection sources;
@Inject
@@ -49,7 +53,7 @@
@Assisted @Nullable String urlMatch,
@Assisted ReplicationFilter filter,
@Assisted ReplicationState state,
- @Assisted boolean now) {
+ @Assisted ReplicationType replicationType) {
this.workQueue = wq;
this.projectCache = projectCache;
this.stateLog = stateLog;
@@ -57,7 +61,7 @@
this.urlMatch = urlMatch;
this.filter = filter;
this.state = state;
- this.now = now;
+ this.replicationType = replicationType;
}
Future<?> schedule(long delay, TimeUnit unit) {
@@ -69,7 +73,7 @@
try {
for (Project.NameKey nameKey : projectCache.all()) {
if (filter.matches(nameKey)) {
- scheduleFullSync(nameKey, urlMatch, state, now);
+ scheduleFullSync(nameKey, urlMatch, state, replicationType);
}
}
} catch (Exception e) {
@@ -79,12 +83,15 @@
}
private void scheduleFullSync(
- Project.NameKey project, String urlMatch, ReplicationState state, boolean now) {
+ Project.NameKey project,
+ String urlMatch,
+ ReplicationState state,
+ ReplicationType replicationType) {
for (Source cfg : sources.getAll()) {
if (cfg.wouldFetchProject(project)) {
for (URIish uri : cfg.getURIs(project, urlMatch)) {
- cfg.schedule(project, FetchOne.ALL_REFS, uri, state, now);
+ cfg.schedule(project, FetchOne.ALL_REFS, uri, state, replicationType);
}
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchResultProcessing.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchResultProcessing.java
index 2d51239..ab16318 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchResultProcessing.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/FetchResultProcessing.java
@@ -118,12 +118,15 @@
}
writeStdOut(sb.toString());
try {
+ Context.setLocalEvent(true);
dispatcher.postEvent(
new FetchRefReplicatedEvent(
project, ref, resolveNodeName(uri), status, refUpdateResult));
} catch (PermissionBackendException e) {
logger.atSevere().withCause(e).log(
"Cannot post event for ref '%s', project %s", ref, project);
+ } finally {
+ Context.unsetLocalEvent();
}
}
@@ -200,9 +203,12 @@
private void postEvent(RefEvent event) {
try {
+ Context.setLocalEvent(true);
dispatcher.postEvent(event);
} catch (PermissionBackendException e) {
logger.atSevere().withCause(e).log("Cannot post event");
+ } finally {
+ Context.unsetLocalEvent();
}
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/OnStartStop.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/OnStartStop.java
index d8c4a8d..5cf8bb6 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/OnStartStop.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/OnStartStop.java
@@ -14,6 +14,8 @@
package com.googlesource.gerrit.plugins.replication.pull;
+import static com.googlesource.gerrit.plugins.replication.pull.ReplicationType.ASYNC;
+
import com.google.common.util.concurrent.Atomics;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.extensions.registration.DynamicItem;
@@ -65,7 +67,7 @@
new FetchResultProcessing.GitUpdateProcessing(eventDispatcher.get()));
fetchAllFuture.set(
fetchAll
- .create(null, ReplicationFilter.all(), state, false)
+ .create(null, ReplicationFilter.all(), state, ASYNC)
.schedule(30, TimeUnit.SECONDS));
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
index 6c8ada0..cdba012 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueue.java
@@ -32,6 +32,7 @@
import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException;
import com.googlesource.gerrit.plugins.replication.pull.client.FetchRestApiClient;
import com.googlesource.gerrit.plugins.replication.pull.client.HttpResult;
+import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.HashSet;
@@ -65,7 +66,7 @@
private final Queue<ReferenceUpdatedEvent> beforeStartupEventsQueue;
private FetchRestApiClient.Factory fetchClientFactory;
private Integer fetchCallsTimeout;
- private RefsFilter refsFilter;
+ private ExcludedRefsFilter refsFilter;
private RevisionReader revisionReader;
@Inject
@@ -75,7 +76,7 @@
DynamicItem<EventDispatcher> dis,
ReplicationStateListeners sl,
FetchRestApiClient.Factory fetchClientFactory,
- RefsFilter refsFilter,
+ ExcludedRefsFilter refsFilter,
RevisionReader revReader) {
workQueue = wq;
dispatcher = dis;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationType.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationType.java
new file mode 100644
index 0000000..308a90b
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationType.java
@@ -0,0 +1,20 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull;
+
+public enum ReplicationType {
+ SYNC,
+ ASYNC
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java
index d88c5f5..5718249 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReader.java
@@ -20,6 +20,7 @@
import com.google.gerrit.entities.Project;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData;
import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException;
@@ -33,7 +34,6 @@
import org.eclipse.jgit.errors.LargeObjectException;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.errors.RepositoryNotFoundException;
-import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.Ref;
@@ -50,10 +50,11 @@
private Long maxRefSize;
@Inject
- public RevisionReader(GitRepositoryManager gitRepositoryManager, Config cfg) {
+ public RevisionReader(GitRepositoryManager gitRepositoryManager, ReplicationConfig cfg) {
this.gitRepositoryManager = gitRepositoryManager;
this.maxRefSize =
- cfg.getLong("replication", CONFIG_MAX_API_PAYLOAD_SIZE, DEFAULT_MAX_PAYLOAD_SIZE_IN_BYTES);
+ cfg.getConfig()
+ .getLong("replication", CONFIG_MAX_API_PAYLOAD_SIZE, DEFAULT_MAX_PAYLOAD_SIZE_IN_BYTES);
}
public Optional<RevisionData> read(Project.NameKey project, String refName)
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
index ec61a5d..2d8bce4 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/Source.java
@@ -16,6 +16,7 @@
import static com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig.replaceName;
import static com.googlesource.gerrit.plugins.replication.pull.FetchResultProcessing.resolveNodeName;
+import static com.googlesource.gerrit.plugins.replication.pull.ReplicationType.SYNC;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
@@ -380,13 +381,20 @@
}
public Future<?> schedule(
- Project.NameKey project, String ref, ReplicationState state, boolean now) {
+ Project.NameKey project,
+ String ref,
+ ReplicationState state,
+ ReplicationType replicationType) {
URIish uri = getURI(project);
- return schedule(project, ref, uri, state, now);
+ return schedule(project, ref, uri, state, replicationType);
}
public Future<?> schedule(
- Project.NameKey project, String ref, URIish uri, ReplicationState state, boolean now) {
+ Project.NameKey project,
+ String ref,
+ URIish uri,
+ ReplicationState state,
+ ReplicationType replicationType) {
repLog.info("scheduling replication {}:{} => {}", uri, ref, project);
if (!shouldReplicate(project, ref, state)) {
@@ -426,7 +434,7 @@
addRef(e, ref);
e.addState(ref, state);
pending.put(uri, e);
- f = pool.schedule(e, now ? 0 : config.getDelay(), TimeUnit.SECONDS);
+ f = pool.schedule(e, isSyncCall(replicationType) ? 0 : config.getDelay(), TimeUnit.SECONDS);
} else if (!e.getRefs().contains(ref)) {
addRef(e, ref);
e.addState(ref, state);
@@ -449,6 +457,10 @@
postReplicationScheduledEvent(e, ref);
}
+ private boolean isSyncCall(ReplicationType replicationType) {
+ return SYNC.equals(replicationType);
+ }
+
/**
* It schedules again a FetchOp instance.
*
@@ -750,15 +762,20 @@
private void postReplicationFailedEvent(FetchOne fetchOp, RefUpdate.Result result) {
Project.NameKey project = fetchOp.getProjectNameKey();
String sourceNode = resolveNodeName(fetchOp.getURI());
- for (String ref : fetchOp.getRefs()) {
- FetchRefReplicatedEvent event =
- new FetchRefReplicatedEvent(
- project.get(), ref, sourceNode, ReplicationState.RefFetchResult.FAILED, result);
- try {
- eventDispatcher.get().postEvent(BranchNameKey.create(project, ref), event);
- } catch (PermissionBackendException e) {
- repLog.error("error posting event", e);
+ try {
+ Context.setLocalEvent(true);
+ for (String ref : fetchOp.getRefs()) {
+ FetchRefReplicatedEvent event =
+ new FetchRefReplicatedEvent(
+ project.get(), ref, sourceNode, ReplicationState.RefFetchResult.FAILED, result);
+ try {
+ eventDispatcher.get().postEvent(BranchNameKey.create(project, ref), event);
+ } catch (PermissionBackendException e) {
+ repLog.error("error posting event", e);
+ }
}
+ } finally {
+ Context.unsetLocalEvent();
}
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java
index e50461d..8046d49 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourceConfiguration.java
@@ -23,7 +23,7 @@
import org.eclipse.jgit.transport.RemoteConfig;
public class SourceConfiguration implements RemoteConfiguration {
- static final int DEFAULT_REPLICATION_DELAY = 0;
+ static final int DEFAULT_REPLICATION_DELAY = 4;
static final int DEFAULT_RESCHEDULE_DELAY = 3;
static final int DEFAULT_SLOW_LATENCY_THRESHOLD_SECS = 900;
static final int DEFAULT_MAX_CONNECTION_INACTIVITY_MS = 10000;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/StartFetchCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/StartFetchCommand.java
index 6ababe9..97f8e9e 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/StartFetchCommand.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/StartFetchCommand.java
@@ -14,6 +14,9 @@
package com.googlesource.gerrit.plugins.replication.pull;
+import static com.googlesource.gerrit.plugins.replication.pull.ReplicationType.ASYNC;
+import static com.googlesource.gerrit.plugins.replication.pull.ReplicationType.SYNC;
+
import com.google.gerrit.extensions.annotations.RequiresCapability;
import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.server.events.EventDispatcher;
@@ -77,7 +80,10 @@
projectFilter = new ReplicationFilter(projectPatterns);
}
- future = fetchFactory.create(urlMatch, projectFilter, state, now).schedule(0, TimeUnit.SECONDS);
+ future =
+ fetchFactory
+ .create(urlMatch, projectFilter, state, replicationType(now))
+ .schedule(0, TimeUnit.SECONDS);
if (wait) {
if (future != null) {
@@ -105,6 +111,10 @@
}
}
+ private ReplicationType replicationType(Boolean now) {
+ return now ? SYNC : ASYNC;
+ }
+
@Override
public void writeStdOutSync(String message) {
if (wait) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java
index db4edf5..f029834 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectAction.java
@@ -14,12 +14,7 @@
package com.googlesource.gerrit.plugins.replication.pull.api;
-import static com.google.common.base.Preconditions.checkState;
-
import com.google.common.base.Strings;
-import com.google.common.flogger.FluentLogger;
-import com.google.gerrit.entities.Project;
-import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.extensions.restapi.AuthException;
import com.google.gerrit.extensions.restapi.BadRequestException;
import com.google.gerrit.extensions.restapi.ResourceConflictException;
@@ -27,9 +22,6 @@
import com.google.gerrit.extensions.restapi.RestApiException;
import com.google.gerrit.extensions.restapi.RestModifyView;
import com.google.gerrit.extensions.restapi.UnprocessableEntityException;
-import com.google.gerrit.server.config.UrlFormatter;
-import com.google.gerrit.server.git.WorkQueue;
-import com.google.gerrit.server.ioutil.HexFormat;
import com.google.gerrit.server.project.ProjectResource;
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput;
@@ -37,24 +29,15 @@
import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException;
import java.io.IOException;
import java.util.Objects;
-import java.util.Optional;
public class ApplyObjectAction implements RestModifyView<ProjectResource, RevisionInput> {
private final ApplyObjectCommand command;
- private final WorkQueue workQueue;
- private final DynamicItem<UrlFormatter> urlFormatter;
private final FetchPreconditions preConditions;
@Inject
- public ApplyObjectAction(
- ApplyObjectCommand command,
- WorkQueue workQueue,
- DynamicItem<UrlFormatter> urlFormatter,
- FetchPreconditions preConditions) {
+ public ApplyObjectAction(ApplyObjectCommand command, FetchPreconditions preConditions) {
this.command = command;
- this.workQueue = workQueue;
- this.urlFormatter = urlFormatter;
this.preConditions = preConditions;
}
@@ -89,10 +72,9 @@
throw new BadRequestException("Ref-update tree object cannot be null");
}
- if (input.isAsync()) {
- return applyAsync(resource.getNameKey(), input);
- }
- return applySync(resource.getNameKey(), input);
+ command.applyObject(
+ resource.getNameKey(), input.getRefName(), input.getRevisionData(), input.getLabel());
+ return Response.created(input);
} catch (MissingParentObjectException e) {
throw new ResourceConflictException(e.getMessage(), e);
} catch (NumberFormatException | IOException e) {
@@ -101,51 +83,4 @@
throw new UnprocessableEntityException(e.getMessage());
}
}
-
- private Response<?> applySync(Project.NameKey project, RevisionInput input)
- throws NumberFormatException, IOException, RefUpdateException, MissingParentObjectException {
- command.applyObject(project, input.getRefName(), input.getRevisionData(), input.getLabel());
- return Response.created(input);
- }
-
- private Response.Accepted applyAsync(Project.NameKey project, RevisionInput input) {
- @SuppressWarnings("unchecked")
- WorkQueue.Task<Void> task =
- (WorkQueue.Task<Void>)
- workQueue.getDefaultQueue().submit(new ApplyObjectJob(command, project, input));
- Optional<String> url =
- urlFormatter
- .get()
- .getRestUrl("a/config/server/tasks/" + HexFormat.fromInt(task.getTaskId()));
- // We're in a HTTP handler, so must be present.
- checkState(url.isPresent());
- return Response.accepted(url.get());
- }
-
- private static class ApplyObjectJob implements Runnable {
- private static final FluentLogger log = FluentLogger.forEnclosingClass();
-
- private ApplyObjectCommand command;
- private Project.NameKey project;
- private RevisionInput input;
-
- public ApplyObjectJob(
- ApplyObjectCommand command, Project.NameKey project, RevisionInput input) {
- this.command = command;
- this.project = project;
- this.input = input;
- }
-
- @Override
- public void run() {
- try {
- command.applyObject(project, input.getRefName(), input.getRevisionData(), input.getLabel());
- } catch (IOException | RefUpdateException | MissingParentObjectException e) {
- log.atSevere().withCause(e).log(
- "Exception during the applyObject call for project {} and ref name {}",
- project.get(),
- input.getRefName());
- }
- }
- }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java
index ac7e79b..276aa16 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectCommand.java
@@ -26,6 +26,7 @@
import com.google.gerrit.server.permissions.PermissionBackendException;
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.replication.pull.ApplyObjectMetrics;
+import com.googlesource.gerrit.plugins.replication.pull.Context;
import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent;
import com.googlesource.gerrit.plugins.replication.pull.PullReplicationStateLogger;
import com.googlesource.gerrit.plugins.replication.pull.ReplicationState;
@@ -78,6 +79,7 @@
long elapsed = NANOSECONDS.toMillis(context.stop());
try {
+ Context.setLocalEvent(true);
eventDispatcher
.get()
.postEvent(
@@ -90,6 +92,8 @@
} catch (PermissionBackendException e) {
logger.atSevere().withCause(e).log(
"Cannot post event for ref '%s', project %s", refName, name);
+ } finally {
+ Context.unsetLocalEvent();
}
if (!isSuccessful(refUpdateState.getResult())) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java
index c44bcbc..2767f1d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchAction.java
@@ -93,7 +93,7 @@
private Response<?> applySync(Project.NameKey project, Input input)
throws InterruptedException, ExecutionException, RemoteConfigurationMissingException,
TimeoutException {
- command.fetch(project, input.label, input.refName);
+ command.fetchSync(project, input.label, input.refName);
return Response.created(input);
}
@@ -127,16 +127,14 @@
@Override
public void run() {
try {
- command.fetch(project, input.label, input.refName);
+ command.fetchAsync(project, input.label, input.refName);
} catch (InterruptedException
| ExecutionException
| RemoteConfigurationMissingException
| TimeoutException e) {
log.atSevere().withCause(e).log(
- "Exception during the async fetch call for project {}, label {} and ref name {}",
- project.get(),
- input.label,
- input.refName);
+ "Exception during the async fetch call for project %s, label %s and ref name %s",
+ project.get(), input.label, input.refName);
}
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java
index 1088d9b..e1ac9ac 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommand.java
@@ -14,6 +14,9 @@
package com.googlesource.gerrit.plugins.replication.pull.api;
+import static com.googlesource.gerrit.plugins.replication.pull.ReplicationType.ASYNC;
+import static com.googlesource.gerrit.plugins.replication.pull.ReplicationType.SYNC;
+
import com.google.gerrit.entities.Project;
import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.server.events.EventDispatcher;
@@ -22,6 +25,7 @@
import com.googlesource.gerrit.plugins.replication.pull.FetchResultProcessing;
import com.googlesource.gerrit.plugins.replication.pull.PullReplicationStateLogger;
import com.googlesource.gerrit.plugins.replication.pull.ReplicationState;
+import com.googlesource.gerrit.plugins.replication.pull.ReplicationType;
import com.googlesource.gerrit.plugins.replication.pull.Source;
import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection;
import com.googlesource.gerrit.plugins.replication.pull.api.exception.RemoteConfigurationMissingException;
@@ -50,7 +54,19 @@
this.eventDispatcher = eventDispatcher;
}
- public void fetch(Project.NameKey name, String label, String refName)
+ public void fetchAsync(Project.NameKey name, String label, String refName)
+ throws InterruptedException, ExecutionException, RemoteConfigurationMissingException,
+ TimeoutException {
+ fetch(name, label, refName, ASYNC);
+ }
+
+ public void fetchSync(Project.NameKey name, String label, String refName)
+ throws InterruptedException, ExecutionException, RemoteConfigurationMissingException,
+ TimeoutException {
+ fetch(name, label, refName, SYNC);
+ }
+
+ private void fetch(Project.NameKey name, String label, String refName, ReplicationType fetchType)
throws InterruptedException, ExecutionException, RemoteConfigurationMissingException,
TimeoutException {
ReplicationState state =
@@ -66,7 +82,7 @@
try {
state.markAllFetchTasksScheduled();
- Future<?> future = source.get().schedule(name, refName, state, true);
+ Future<?> future = source.get().schedule(name, refName, state, fetchType);
future.get(source.get().getTimeout(), TimeUnit.SECONDS);
} catch (ExecutionException
| IllegalStateException
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java
index 160a720..bc3e218 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/api/data/RevisionInput.java
@@ -21,17 +21,10 @@
private RevisionData revisionData;
- private boolean async;
-
public RevisionInput(String label, String refName, RevisionData revisionData) {
- this(label, refName, revisionData, false);
- }
-
- public RevisionInput(String label, String refName, RevisionData revisionData, boolean async) {
this.label = label;
this.refName = refName;
this.revisionData = revisionData;
- this.async = async;
}
public String getLabel() {
@@ -45,8 +38,4 @@
public RevisionData getRevisionData() {
return revisionData;
}
-
- public boolean isAsync() {
- return async;
- }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
index eed1afb..7b876df 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
@@ -32,6 +32,7 @@
import com.googlesource.gerrit.plugins.replication.pull.Source;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput;
+import com.googlesource.gerrit.plugins.replication.pull.filter.SyncRefsFilter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
@@ -67,18 +68,21 @@
private final Source source;
private final String instanceLabel;
private final String pluginName;
+ private final SyncRefsFilter syncRefsFilter;
@Inject
FetchRestApiClient(
CredentialsFactory credentials,
SourceHttpClient.Factory httpClientFactory,
ReplicationConfig replicationConfig,
+ SyncRefsFilter syncRefsFilter,
@PluginName String pluginName,
@Assisted Source source) {
this.credentials = credentials;
this.httpClientFactory = httpClientFactory;
this.source = source;
this.pluginName = pluginName;
+ this.syncRefsFilter = syncRefsFilter;
this.instanceLabel =
Strings.nullToEmpty(
replicationConfig.getConfig().getString("replication", null, "instanceLabel"))
@@ -93,11 +97,13 @@
String.format(
"%s/a/projects/%s/pull-replication~fetch",
targetUri.toString(), Url.encode(project.get()));
-
+ Boolean callAsync = !syncRefsFilter.match(refName);
HttpPost post = new HttpPost(url);
post.setEntity(
new StringEntity(
- String.format("{\"label\":\"%s\", \"ref_name\": \"%s\"}", instanceLabel, refName),
+ String.format(
+ "{\"label\":\"%s\", \"ref_name\": \"%s\", \"async\":%s}",
+ instanceLabel, refName, callAsync),
StandardCharsets.UTF_8));
post.addHeader(new BasicHeader("Content-Type", "application/json"));
return httpClientFactory.create(source).execute(post, this, getContext(targetUri));
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandler.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandler.java
index fc96394..a618e16 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandler.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandler.java
@@ -17,10 +17,12 @@
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.entities.Change;
import com.google.gerrit.entities.Project;
+import com.google.gerrit.entities.RefNames;
import com.google.gerrit.server.events.Event;
import com.google.gerrit.server.events.EventListener;
import com.google.gerrit.server.index.change.ChangeIndexer;
import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.replication.pull.Context;
import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent;
import com.googlesource.gerrit.plugins.replication.pull.ReplicationState;
@@ -35,18 +37,21 @@
@Override
public void onEvent(Event event) {
- if (event instanceof FetchRefReplicatedEvent) {
+ if (event instanceof FetchRefReplicatedEvent && isLocalEvent()) {
FetchRefReplicatedEvent fetchRefReplicatedEvent = (FetchRefReplicatedEvent) event;
+ if (!RefNames.isNoteDbMetaRef(fetchRefReplicatedEvent.getRefName())
+ || !fetchRefReplicatedEvent
+ .getStatus()
+ .equals(ReplicationState.RefFetchResult.SUCCEEDED.toString())) {
+ return;
+ }
+
Project.NameKey projectNameKey = fetchRefReplicatedEvent.getProjectNameKey();
logger.atFine().log(
"Indexing ref '%s' for project %s",
fetchRefReplicatedEvent.getRefName(), projectNameKey.get());
-
Change.Id changeId = Change.Id.fromRef(fetchRefReplicatedEvent.getRefName());
- if (changeId != null
- && fetchRefReplicatedEvent
- .getStatus()
- .equals(ReplicationState.RefFetchResult.SUCCEEDED.toString())) {
+ if (changeId != null) {
changeIndexer.index(projectNameKey, changeId);
} else {
logger.atWarning().log(
@@ -55,4 +60,8 @@
}
}
}
+
+ private boolean isLocalEvent() {
+ return Context.isLocalEvent();
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java
index da63168..03362bf 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObject.java
@@ -15,7 +15,6 @@
package com.googlesource.gerrit.plugins.replication.pull.fetch;
import com.google.gerrit.entities.Project;
-import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.LocalDiskRepositoryManager;
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData;
@@ -31,8 +30,12 @@
public class ApplyObject {
- private final GitRepositoryManager gitManager;
+ private final LocalDiskRepositoryManager gitManager;
+ // NOTE: We do need specifically the LocalDiskRepositoryManager to make sure
+ // to be able to write onto the directly physical repository without any wrapper.
+ // Using for instance the multi-site wrapper injected by Guice would result
+ // in a split-brain because of the misalignment of local vs. global refs values.
@Inject
public ApplyObject(LocalDiskRepositoryManager gitManager) {
this.gitManager = gitManager;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/ExcludedRefsFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/ExcludedRefsFilter.java
new file mode 100644
index 0000000..f2e4ab3
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/ExcludedRefsFilter.java
@@ -0,0 +1,51 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.filter;
+
+import com.google.common.collect.ImmutableList;
+import com.google.gerrit.entities.RefNames;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig;
+import java.util.List;
+import org.eclipse.jgit.lib.Config;
+
+@Singleton
+public class ExcludedRefsFilter extends RefsFilter {
+ @Inject
+ public ExcludedRefsFilter(ReplicationConfig replicationConfig) {
+ super(replicationConfig);
+ }
+
+ @Override
+ protected List<String> getRefNamePatterns(Config cfg) {
+ return ImmutableList.<String>builder()
+ .addAll(getDefaultExcludeRefPatterns())
+ .addAll(ImmutableList.copyOf(cfg.getStringList("replication", null, "excludeRefs")))
+ .build();
+ }
+
+ private List<String> getDefaultExcludeRefPatterns() {
+ return ImmutableList.of(
+ RefNames.REFS_USERS + "*",
+ RefNames.REFS_CONFIG,
+ RefNames.REFS_SEQUENCES + "*",
+ RefNames.REFS_EXTERNAL_IDS,
+ RefNames.REFS_GROUPS + "*",
+ RefNames.REFS_GROUPNAMES,
+ RefNames.REFS_CACHE_AUTOMERGE + "*",
+ RefNames.REFS_STARRED_CHANGES + "*");
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RefsFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/RefsFilter.java
similarity index 70%
rename from src/main/java/com/googlesource/gerrit/plugins/replication/pull/RefsFilter.java
rename to src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/RefsFilter.java
index 0b9a691..7ec19ea 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/RefsFilter.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/RefsFilter.java
@@ -12,20 +12,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package com.googlesource.gerrit.plugins.replication.pull;
+package com.googlesource.gerrit.plugins.replication.pull.filter;
import com.google.common.base.Strings;
-import com.google.common.collect.ImmutableList;
import com.google.gerrit.entities.AccessSection;
-import com.google.gerrit.entities.RefNames;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
import com.googlesource.gerrit.plugins.replication.ReplicationConfig;
import java.util.List;
import org.eclipse.jgit.lib.Config;
-@Singleton
-public class RefsFilter {
+public abstract class RefsFilter {
public enum PatternType {
REGEX,
WILDCARD,
@@ -44,7 +39,6 @@
private final List<String> refsPatterns;
- @Inject
public RefsFilter(ReplicationConfig replicationConfig) {
refsPatterns = getRefNamePatterns(replicationConfig.getConfig());
}
@@ -66,12 +60,7 @@
return false;
}
- private List<String> getRefNamePatterns(Config cfg) {
- return ImmutableList.<String>builder()
- .addAll(getDefaultExcludeRefPatterns())
- .addAll(ImmutableList.copyOf(cfg.getStringList("replication", null, "excludeRefs")))
- .build();
- }
+ protected abstract List<String> getRefNamePatterns(Config cfg);
private boolean matchesPattern(String refName, String pattern) {
boolean match = false;
@@ -87,16 +76,4 @@
}
return match;
}
-
- private List<String> getDefaultExcludeRefPatterns() {
- return ImmutableList.of(
- RefNames.REFS_USERS + "*",
- RefNames.REFS_CONFIG,
- RefNames.REFS_SEQUENCES + "*",
- RefNames.REFS_EXTERNAL_IDS,
- RefNames.REFS_GROUPS + "*",
- RefNames.REFS_GROUPNAMES,
- RefNames.REFS_CACHE_AUTOMERGE + "*",
- RefNames.REFS_STARRED_CHANGES + "*");
- }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/SyncRefsFilter.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/SyncRefsFilter.java
new file mode 100644
index 0000000..a069935
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/filter/SyncRefsFilter.java
@@ -0,0 +1,35 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.filter;
+
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig;
+import java.util.List;
+import org.eclipse.jgit.lib.Config;
+
+@Singleton
+public class SyncRefsFilter extends RefsFilter {
+ @Inject
+ public SyncRefsFilter(ReplicationConfig replicationConfig) {
+ super(replicationConfig);
+ }
+
+ @Override
+ protected List<String> getRefNamePatterns(Config cfg) {
+ return ImmutableList.copyOf(cfg.getStringList("replication", null, "syncRefs"));
+ }
+}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index c3e5ad4..126cd1b 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -181,6 +181,30 @@
By default, all other refs are included.
+replication.syncRefs
+: Specify for which refs git fetch calls should be executed synchronously.
+ It can be provided more than once, and supports three formats: regular expressions,
+ wildcard matching, and single ref matching. All three formats match are case-sensitive.
+
+ Values starting with a caret `^` are treated as regular
+ expressions. For the regular expressions details please follow
+ official [java documentation](https://docs.oracle.com/javase/tutorial/essential/regex/).
+
+ Please note that regular expressions could also be used
+ with inverse match.
+
+ Values that are not regular expressions and end in `*` are
+ treated as wildcard matches. Wildcards match refs whose
+ name agrees from the beginning until the trailing `*`. So
+ `foo/b*` would match the refs `foo/b`, `foo/bar`, and
+ `foo/baz`, but neither `foobar`, nor `bar/foo/baz`.
+
+ Values that are neither regular expressions nor wildcards are
+ treated as single ref matches. So `foo/bar` matches only
+ the ref `foo/bar`, but no other refs.
+
+ By default, set to '*' (all refs are replicated synchronously).
+
replication.maxApiPayloadSize
: Maximum size in bytes of the ref to be sent as a REST Api call
payload. For refs larger than threshold git fetch operation
@@ -268,13 +292,14 @@
Defaults to 0 seconds, wait indefinitely.
remote.NAME.replicationDelay
-: Time to wait before scheduling a remote fetch operation. Setting
- the delay to 0 effectively disables the delay, causing the fetch
- to start as soon as possible.
+: Time to wait before scheduling an asynchronous remote fetch
+ operation. Setting the delay to 0 effectively disables the delay,
+ causing the fetch to start as soon as possible.
This is a Gerrit specific extension to the Git remote block.
- By default, 0 seconds.
+ By default for asynchronous fetch, 4 seconds. For a synchronous fetch
+ replicationDelay is zero.
remote.NAME.rescheduleDelay
: Delay when rescheduling a fetch operation due to an in-flight fetch
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
index 44b69db..327c6ba 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/ReplicationQueueTest.java
@@ -38,6 +38,7 @@
import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException;
import com.googlesource.gerrit.plugins.replication.pull.client.FetchRestApiClient;
import com.googlesource.gerrit.plugins.replication.pull.client.HttpResult;
+import com.googlesource.gerrit.plugins.replication.pull.filter.ExcludedRefsFilter;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Optional;
@@ -69,7 +70,7 @@
@Mock RevisionData revisionData;
@Mock HttpResult httpResult;
- private RefsFilter refsFilter;
+ private ExcludedRefsFilter refsFilter;
private ReplicationQueue objectUnderTest;
private SitePaths sitePaths;
private Path pluginDataPath;
@@ -80,7 +81,7 @@
sitePaths = new SitePaths(sitePath);
Path pluginDataPath = createTempPath("data");
ReplicationConfig replicationConfig = new ReplicationFileBasedConfig(sitePaths, pluginDataPath);
- refsFilter = new RefsFilter(replicationConfig);
+ refsFilter = new ExcludedRefsFilter(replicationConfig);
when(source.getConnectionTimeout()).thenReturn(CONNECTION_TIMEOUT);
when(source.wouldFetchProject(any())).thenReturn(true);
when(source.wouldFetchRef(anyString())).thenReturn(true);
@@ -198,7 +199,7 @@
fileConfig.setString("replication", null, "excludeRefs", "refs/multi-site/version");
fileConfig.save();
ReplicationConfig replicationConfig = new ReplicationFileBasedConfig(sitePaths, pluginDataPath);
- refsFilter = new RefsFilter(replicationConfig);
+ refsFilter = new ExcludedRefsFilter(replicationConfig);
objectUnderTest =
new ReplicationQueue(wq, rd, dis, sl, fetchClientFactory, refsFilter, revReader);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java
index ca410d5..d1a4c85 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/RevisionReaderIT.java
@@ -29,13 +29,15 @@
import com.google.gerrit.extensions.api.changes.ReviewInput.CommentInput;
import com.google.gerrit.extensions.client.Comment;
import com.google.gerrit.extensions.config.FactoryModule;
-import com.google.inject.Inject;
import com.google.inject.Scopes;
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig;
+import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData;
import com.googlesource.gerrit.plugins.replication.pull.fetch.ApplyObject;
import java.util.Optional;
import org.eclipse.jgit.lib.Constants;
+import org.junit.Before;
import org.junit.Test;
@UseLocalDisk
@@ -43,7 +45,12 @@
name = "pull-replication",
sysModule = "com.googlesource.gerrit.plugins.replication.pull.RevisionReaderIT$TestModule")
public class RevisionReaderIT extends LightweightPluginDaemonTest {
- @Inject RevisionReader objectUnderTest;
+ RevisionReader objectUnderTest;
+
+ @Before
+ public void setup() {
+ objectUnderTest = plugin.getSysInjector().getInstance(RevisionReader.class);
+ }
@Test
public void shouldReadRefMetaObject() throws Exception {
@@ -114,6 +121,7 @@
private static class TestModule extends FactoryModule {
@Override
protected void configure() {
+ bind(ReplicationConfig.class).to(ReplicationFileBasedConfig.class);
bind(RevisionReader.class).in(Scopes.SINGLETON);
bind(ApplyObject.class);
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionIT.java
new file mode 100644
index 0000000..75c31de
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionIT.java
@@ -0,0 +1,272 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication.pull.api;
+
+import static com.google.common.truth.Truth.assertThat;
+import static java.util.stream.Collectors.toList;
+
+import com.google.gerrit.acceptance.LightweightPluginDaemonTest;
+import com.google.gerrit.acceptance.PushOneCommit.Result;
+import com.google.gerrit.acceptance.SkipProjectClone;
+import com.google.gerrit.acceptance.TestPlugin;
+import com.google.gerrit.acceptance.UseLocalDisk;
+import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
+import com.google.gerrit.entities.Project;
+import com.google.gerrit.entities.RefNames;
+import com.google.gerrit.extensions.restapi.Url;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.inject.Inject;
+import com.googlesource.gerrit.plugins.replication.CredentialsFactory;
+import com.googlesource.gerrit.plugins.replication.pull.RevisionReader;
+import com.googlesource.gerrit.plugins.replication.pull.Source;
+import com.googlesource.gerrit.plugins.replication.pull.SourcesCollection;
+import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData;
+import com.googlesource.gerrit.plugins.replication.pull.client.SourceHttpClient;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.List;
+import java.util.Optional;
+import org.apache.http.HttpResponse;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.ResponseHandler;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.message.BasicHeader;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
+import org.junit.Test;
+
+@SkipProjectClone
+@UseLocalDisk
+@TestPlugin(
+ name = "pull-replication",
+ sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule")
+public class ApplyObjectActionIT extends LightweightPluginDaemonTest {
+ private static final Optional<String> ALL_PROJECTS = Optional.empty();
+ private static final int TEST_REPLICATION_DELAY = 60;
+ private static final String TEST_REPLICATION_SUFFIX = "suffix1";
+ private static final String TEST_REPLICATION_REMOTE = "remote1";
+
+ private Path gitPath;
+ private FileBasedConfig config;
+ private FileBasedConfig secureConfig;
+ private RevisionReader revisionReader;
+
+ @Inject private SitePaths sitePaths;
+ @Inject private ProjectOperations projectOperations;
+ CredentialsFactory credentials;
+ Source source;
+ SourceHttpClient.Factory httpClientFactory;
+ String url;
+
+ @Override
+ public void setUpTestPlugin() throws Exception {
+ gitPath = sitePaths.site_path.resolve("git");
+
+ config =
+ new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED);
+ setReplicationSource(
+ TEST_REPLICATION_REMOTE,
+ TEST_REPLICATION_SUFFIX,
+ ALL_PROJECTS); // Simulates a full replication.config initialization
+ config.save();
+
+ secureConfig =
+ new FileBasedConfig(sitePaths.etc_dir.resolve("secure.config").toFile(), FS.DETECTED);
+ setReplicationCredentials(TEST_REPLICATION_REMOTE, admin.username(), admin.httpPassword());
+ secureConfig.save();
+
+ super.setUpTestPlugin();
+
+ httpClientFactory = plugin.getSysInjector().getInstance(SourceHttpClient.Factory.class);
+ credentials = plugin.getSysInjector().getInstance(CredentialsFactory.class);
+ revisionReader = plugin.getSysInjector().getInstance(RevisionReader.class);
+ source = plugin.getSysInjector().getInstance(SourcesCollection.class).getAll().get(0);
+
+ url =
+ String.format(
+ "%s/a/projects/%s/pull-replication~apply-object",
+ adminRestSession.url(), Url.encode(project.get()));
+ }
+
+ @Test
+ public void shouldAcceptPayloadWithAsyncField() throws Exception {
+ String payloadWithAsyncFieldTemplate =
+ "{\"label\":\""
+ + TEST_REPLICATION_REMOTE
+ + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}, \"async\":true}";
+
+ String refName = createRef();
+ Optional<RevisionData> revisionDataOption = createRevisionData(refName);
+ assertThat(revisionDataOption.isPresent()).isTrue();
+
+ RevisionData revisionData = revisionDataOption.get();
+ String sendObjectPayload = createPayload(payloadWithAsyncFieldTemplate, refName, revisionData);
+
+ HttpPost post = createRequest(sendObjectPayload);
+ httpClientFactory.create(source).execute(post, assertHttpResponseCode(201), getContext());
+ }
+
+ @Test
+ public void shouldAcceptPayloadWithoutAsyncField() throws Exception {
+ String payloadWithoutAsyncFieldTemplate =
+ "{\"label\":\""
+ + TEST_REPLICATION_REMOTE
+ + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}}";
+
+ String refName = createRef();
+ Optional<RevisionData> revisionDataOption = createRevisionData(refName);
+ assertThat(revisionDataOption.isPresent()).isTrue();
+
+ RevisionData revisionData = revisionDataOption.get();
+ String sendObjectPayload =
+ createPayload(payloadWithoutAsyncFieldTemplate, refName, revisionData);
+
+ HttpPost post = createRequest(sendObjectPayload);
+ httpClientFactory.create(source).execute(post, assertHttpResponseCode(201), getContext());
+ }
+
+ @Test
+ public void shouldReturnBadRequestCodeWhenMandatoryFieldLabelIsMissing() throws Exception {
+ String payloadWithoutLabelFieldTemplate =
+ "{\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}, \"async\":true}";
+
+ String refName = createRef();
+ Optional<RevisionData> revisionDataOption = createRevisionData(refName);
+ assertThat(revisionDataOption.isPresent()).isTrue();
+
+ RevisionData revisionData = revisionDataOption.get();
+ String sendObjectPayload =
+ createPayload(payloadWithoutLabelFieldTemplate, refName, revisionData);
+
+ HttpPost post = createRequest(sendObjectPayload);
+ httpClientFactory.create(source).execute(post, assertHttpResponseCode(400), getContext());
+ }
+
+ @Test
+ public void shouldReturnBadRequestCodeWhenPayloadIsNotAProperJSON() throws Exception {
+ String wrongPayloadTemplate =
+ "{\"label\":\""
+ + TEST_REPLICATION_REMOTE
+ + "\",\"ref_name\":\"%s\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"%s\"},\"tree_object\":{\"type\":2,\"content\":\"%s\"},\"blobs\":[]}, \"async\":true,}";
+
+ String refName = createRef();
+ Optional<RevisionData> revisionDataOption = createRevisionData(refName);
+ assertThat(revisionDataOption.isPresent()).isTrue();
+
+ RevisionData revisionData = revisionDataOption.get();
+ String sendObjectPayload = createPayload(wrongPayloadTemplate, refName, revisionData);
+
+ HttpPost post = createRequest(sendObjectPayload);
+ httpClientFactory.create(source).execute(post, assertHttpResponseCode(400), getContext());
+ }
+
+ private String createPayload(
+ String wrongPayloadTemplate, String refName, RevisionData revisionData) {
+ String sendObjectPayload =
+ String.format(
+ wrongPayloadTemplate,
+ refName,
+ encode(revisionData.getCommitObject().getContent()),
+ encode(revisionData.getTreeObject().getContent()));
+ return sendObjectPayload;
+ }
+
+ private HttpPost createRequest(String sendObjectPayload) {
+ HttpPost post = new HttpPost(url);
+ post.setEntity(new StringEntity(sendObjectPayload, StandardCharsets.UTF_8));
+ post.addHeader(new BasicHeader("Content-Type", "application/json"));
+ return post;
+ }
+
+ private String createRef() throws Exception {
+ testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX));
+
+ Result pushResult = createChange("topic", "test.txt", "test_content");
+ return RefNames.changeMetaRef(pushResult.getChange().getId());
+ }
+
+ private Optional<RevisionData> createRevisionData(String refName) throws Exception {
+ return revisionReader.read(Project.nameKey(project + TEST_REPLICATION_SUFFIX), refName);
+ }
+
+ private Object encode(byte[] content) {
+ return Base64.getEncoder().encodeToString(content);
+ }
+
+ public ResponseHandler<Object> assertHttpResponseCode(int responseCode) {
+ return new ResponseHandler<Object>() {
+
+ @Override
+ public Object handleResponse(HttpResponse response)
+ throws ClientProtocolException, IOException {
+ assertThat(response.getStatusLine().getStatusCode()).isEqualTo(responseCode);
+ return null;
+ }
+ };
+ }
+
+ private HttpClientContext getContext() {
+ HttpClientContext ctx = HttpClientContext.create();
+ CredentialsProvider adapted = new BasicCredentialsProvider();
+ adapted.setCredentials(
+ AuthScope.ANY, new UsernamePasswordCredentials(admin.username(), admin.httpPassword()));
+ ctx.setCredentialsProvider(adapted);
+ return ctx;
+ }
+
+ private Project.NameKey createTestProject(String name) throws Exception {
+ return projectOperations.newProject().name(name).parent(project).create();
+ }
+
+ private void setReplicationSource(
+ String remoteName, String replicaSuffix, Optional<String> project) throws IOException {
+ setReplicationSource(remoteName, Arrays.asList(replicaSuffix), project);
+ }
+
+ private void setReplicationSource(
+ String remoteName, List<String> replicaSuffixes, Optional<String> project)
+ throws IOException {
+
+ List<String> replicaUrls =
+ replicaSuffixes.stream()
+ .map(suffix -> gitPath.resolve("${name}" + suffix + ".git").toString())
+ .collect(toList());
+ config.setString("replication", null, "instanceLabel", remoteName);
+ config.setStringList("remote", remoteName, "url", replicaUrls);
+ config.setString("remote", remoteName, "apiUrl", adminRestSession.url());
+ config.setString("remote", remoteName, "fetch", "+refs/tags/*:refs/tags/*");
+ config.setInt("remote", remoteName, "timeout", 600);
+ config.setInt("remote", remoteName, "replicationDelay", TEST_REPLICATION_DELAY);
+ project.ifPresent(prj -> config.setString("remote", remoteName, "projects", prj));
+ config.setBoolean("gerrit", null, "autoReload", true);
+ config.save();
+ }
+
+ private void setReplicationCredentials(String remoteName, String username, String password)
+ throws IOException {
+ secureConfig.setString("remote", remoteName, "username", username);
+ secureConfig.setString("remote", remoteName, "password", password);
+ secureConfig.save();
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java
index 55fdee8..8738046 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/ApplyObjectActionTest.java
@@ -15,7 +15,6 @@
package com.googlesource.gerrit.plugins.replication.pull.api;
import static com.google.common.truth.Truth.assertThat;
-import static org.apache.http.HttpStatus.SC_ACCEPTED;
import static org.apache.http.HttpStatus.SC_CREATED;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
@@ -24,15 +23,11 @@
import com.google.common.collect.Lists;
import com.google.gerrit.entities.Project;
-import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.extensions.restapi.AuthException;
import com.google.gerrit.extensions.restapi.BadRequestException;
import com.google.gerrit.extensions.restapi.ResourceConflictException;
import com.google.gerrit.extensions.restapi.Response;
import com.google.gerrit.extensions.restapi.RestApiException;
-import com.google.gerrit.server.config.UrlFormatter;
-import com.google.gerrit.server.git.WorkQueue;
-import com.google.gerrit.server.git.WorkQueue.Task;
import com.google.gerrit.server.project.ProjectResource;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionInput;
@@ -40,17 +35,13 @@
import com.googlesource.gerrit.plugins.replication.pull.api.exception.MissingParentObjectException;
import com.googlesource.gerrit.plugins.replication.pull.api.exception.RefUpdateException;
import java.io.IOException;
-import java.util.Optional;
-import java.util.concurrent.ScheduledExecutorService;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
-import org.mockito.invocation.InvocationOnMock;
import org.mockito.junit.MockitoJUnitRunner;
-import org.mockito.stubbing.Answer;
@RunWith(MockitoJUnitRunner.class)
public class ApplyObjectActionTest {
@@ -82,32 +73,13 @@
@Mock ApplyObjectCommand applyObjectCommand;
@Mock ProjectResource projectResource;
- @Mock WorkQueue workQueue;
- @Mock ScheduledExecutorService exceutorService;
- @Mock DynamicItem<UrlFormatter> urlFormatterDynamicItem;
- @Mock UrlFormatter urlFormatter;
- @Mock WorkQueue.Task<Void> task;
@Mock FetchPreconditions preConditions;
@Before
public void setup() {
- when(workQueue.getDefaultQueue()).thenReturn(exceutorService);
- when(urlFormatter.getRestUrl(anyString())).thenReturn(Optional.of(location));
- when(exceutorService.submit(any(Runnable.class)))
- .thenAnswer(
- new Answer<WorkQueue.Task<Void>>() {
- @Override
- public Task<Void> answer(InvocationOnMock invocation) throws Throwable {
- return task;
- }
- });
- when(urlFormatterDynamicItem.get()).thenReturn(urlFormatter);
- when(task.getTaskId()).thenReturn(taskId);
when(preConditions.canCallFetchApi()).thenReturn(true);
- applyObjectAction =
- new ApplyObjectAction(
- applyObjectCommand, workQueue, urlFormatterDynamicItem, preConditions);
+ applyObjectAction = new ApplyObjectAction(applyObjectCommand, preConditions);
}
@Test
@@ -207,24 +179,6 @@
applyObjectAction.apply(projectResource, inputParams);
}
- @Test
- public void shouldReturnScheduledTaskForAsyncCall() throws RestApiException {
- RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData(), true);
-
- Response<?> response = applyObjectAction.apply(projectResource, inputParams);
- assertThat(response.statusCode()).isEqualTo(SC_ACCEPTED);
- }
-
- @Test
- public void shouldLocationHeaderForAsyncCall() throws Exception {
- RevisionInput inputParams = new RevisionInput(label, refName, createSampleRevisionData(), true);
-
- Response<?> response = applyObjectAction.apply(projectResource, inputParams);
- assertThat(response).isInstanceOf(Response.Accepted.class);
- Response.Accepted acceptResponse = (Response.Accepted) response;
- assertThat(acceptResponse.location()).isEqualTo(location);
- }
-
private RevisionData createSampleRevisionData() {
RevisionObjectData commitData =
new RevisionObjectData(Constants.OBJ_COMMIT, sampleCommitContent.getBytes());
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionTest.java
index 2c888c8..fb7f3d1 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchActionTest.java
@@ -147,7 +147,7 @@
inputParams.label = label;
inputParams.refName = refName;
- doThrow(new InterruptedException()).when(fetchCommand).fetch(any(), any(), any());
+ doThrow(new InterruptedException()).when(fetchCommand).fetchSync(any(), any(), any());
fetchAction.apply(projectResource, inputParams);
}
@@ -162,7 +162,7 @@
doThrow(new RemoteConfigurationMissingException(""))
.when(fetchCommand)
- .fetch(any(), any(), any());
+ .fetchSync(any(), any(), any());
fetchAction.apply(projectResource, inputParams);
}
@@ -177,7 +177,7 @@
doThrow(new ExecutionException(new RuntimeException()))
.when(fetchCommand)
- .fetch(any(), any(), any());
+ .fetchSync(any(), any(), any());
fetchAction.apply(projectResource, inputParams);
}
@@ -190,7 +190,7 @@
inputParams.label = label;
inputParams.refName = refName;
- doThrow(new IllegalStateException()).when(fetchCommand).fetch(any(), any(), any());
+ doThrow(new IllegalStateException()).when(fetchCommand).fetchSync(any(), any(), any());
fetchAction.apply(projectResource, inputParams);
}
@@ -203,7 +203,7 @@
inputParams.label = label;
inputParams.refName = refName;
- doThrow(new TimeoutException()).when(fetchCommand).fetch(any(), any(), any());
+ doThrow(new TimeoutException()).when(fetchCommand).fetchSync(any(), any(), any());
fetchAction.apply(projectResource, inputParams);
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java
index 4cdad48..e1ad565 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/api/FetchCommandTest.java
@@ -15,6 +15,8 @@
package com.googlesource.gerrit.plugins.replication.pull.api;
import static com.google.gerrit.testing.GerritJUnit.assertThrows;
+import static com.googlesource.gerrit.plugins.replication.pull.ReplicationType.ASYNC;
+import static com.googlesource.gerrit.plugins.replication.pull.ReplicationType.SYNC;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyLong;
import static org.mockito.Mockito.anyString;
@@ -74,7 +76,7 @@
when(fetchReplicationStateFactory.create(any())).thenReturn(state);
when(source.getRemoteConfigName()).thenReturn(label);
when(sources.getAll()).thenReturn(Lists.newArrayList(source));
- when(source.schedule(projectName, REF_NAME_TO_FETCH, state, true))
+ when(source.schedule(eq(projectName), eq(REF_NAME_TO_FETCH), eq(state), any()))
.thenReturn(CompletableFuture.completedFuture(null));
objectUnderTest =
new FetchCommand(fetchReplicationStateFactory, fetchStateLog, sources, eventDispatcher);
@@ -84,18 +86,27 @@
public void shouldScheduleRefFetch()
throws InterruptedException, ExecutionException, RemoteConfigurationMissingException,
TimeoutException {
- objectUnderTest.fetch(projectName, label, REF_NAME_TO_FETCH);
+ objectUnderTest.fetchSync(projectName, label, REF_NAME_TO_FETCH);
- verify(source, times(1)).schedule(projectName, REF_NAME_TO_FETCH, state, true);
+ verify(source, times(1)).schedule(projectName, REF_NAME_TO_FETCH, state, SYNC);
+ }
+
+ @Test
+ public void shouldScheduleRefFetchWithDelay()
+ throws InterruptedException, ExecutionException, RemoteConfigurationMissingException,
+ TimeoutException {
+ objectUnderTest.fetchAsync(projectName, label, REF_NAME_TO_FETCH);
+
+ verify(source, times(1)).schedule(projectName, REF_NAME_TO_FETCH, state, ASYNC);
}
@Test
public void shouldMarkAllFetchTasksScheduled()
throws InterruptedException, ExecutionException, RemoteConfigurationMissingException,
TimeoutException {
- objectUnderTest.fetch(projectName, label, REF_NAME_TO_FETCH);
+ objectUnderTest.fetchSync(projectName, label, REF_NAME_TO_FETCH);
- verify(source, times(1)).schedule(projectName, REF_NAME_TO_FETCH, state, true);
+ verify(source, times(1)).schedule(projectName, REF_NAME_TO_FETCH, state, SYNC);
verify(state, times(1)).markAllFetchTasksScheduled();
}
@@ -103,7 +114,7 @@
public void shouldUpdateStateWhenRemoteConfigNameIsMissing() {
assertThrows(
RemoteConfigurationMissingException.class,
- () -> objectUnderTest.fetch(projectName, "unknownLabel", REF_NAME_TO_FETCH));
+ () -> objectUnderTest.fetchSync(projectName, "unknownLabel", REF_NAME_TO_FETCH));
verify(fetchStateLog, times(1)).error(anyString(), eq(state));
}
@@ -112,12 +123,12 @@
public void shouldUpdateStateWhenInterruptedException()
throws InterruptedException, ExecutionException, TimeoutException {
when(future.get(anyLong(), eq(TimeUnit.SECONDS))).thenThrow(new InterruptedException());
- when(source.schedule(projectName, REF_NAME_TO_FETCH, state, true)).thenReturn(future);
+ when(source.schedule(projectName, REF_NAME_TO_FETCH, state, SYNC)).thenReturn(future);
InterruptedException e =
assertThrows(
InterruptedException.class,
- () -> objectUnderTest.fetch(projectName, label, REF_NAME_TO_FETCH));
+ () -> objectUnderTest.fetchSync(projectName, label, REF_NAME_TO_FETCH));
verify(fetchStateLog, times(1)).error(anyString(), eq(e), eq(state));
}
@@ -127,12 +138,12 @@
throws InterruptedException, ExecutionException, TimeoutException {
when(future.get(anyLong(), eq(TimeUnit.SECONDS)))
.thenThrow(new ExecutionException(new Exception()));
- when(source.schedule(projectName, REF_NAME_TO_FETCH, state, true)).thenReturn(future);
+ when(source.schedule(projectName, REF_NAME_TO_FETCH, state, SYNC)).thenReturn(future);
ExecutionException e =
assertThrows(
ExecutionException.class,
- () -> objectUnderTest.fetch(projectName, label, REF_NAME_TO_FETCH));
+ () -> objectUnderTest.fetchSync(projectName, label, REF_NAME_TO_FETCH));
verify(fetchStateLog, times(1)).error(anyString(), eq(e), eq(state));
}
@@ -141,12 +152,12 @@
public void shouldUpdateStateWhenTimeoutException()
throws InterruptedException, ExecutionException, TimeoutException {
when(future.get(anyLong(), eq(TimeUnit.SECONDS))).thenThrow(new TimeoutException());
- when(source.schedule(projectName, REF_NAME_TO_FETCH, state, true)).thenReturn(future);
+ when(source.schedule(projectName, REF_NAME_TO_FETCH, state, SYNC)).thenReturn(future);
TimeoutException e =
assertThrows(
TimeoutException.class,
- () -> objectUnderTest.fetch(projectName, label, REF_NAME_TO_FETCH));
+ () -> objectUnderTest.fetchSync(projectName, label, REF_NAME_TO_FETCH));
verify(fetchStateLog, times(1)).error(anyString(), eq(e), eq(state));
}
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java
index 771acff..a3b0b02 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClientTest.java
@@ -31,6 +31,7 @@
import com.googlesource.gerrit.plugins.replication.pull.Source;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData;
+import com.googlesource.gerrit.plugins.replication.pull.filter.SyncRefsFilter;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
@@ -72,11 +73,15 @@
String label = "Replication";
String refName = RefNames.REFS_HEADS + "master";
- String expectedPayload = "{\"label\":\"Replication\", \"ref_name\": \"" + refName + "\"}";
+ String expectedPayload =
+ "{\"label\":\"Replication\", \"ref_name\": \"" + refName + "\", \"async\":false}";
+ String expectedAsyncPayload =
+ "{\"label\":\"Replication\", \"ref_name\": \"" + refName + "\", \"async\":true}";
Header expectedHeader = new BasicHeader("Content-Type", "application/json");
+ SyncRefsFilter syncRefsFilter;
String expectedSendObjectPayload =
- "{\"label\":\"Replication\",\"ref_name\":\"refs/heads/master\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"dHJlZSA3NzgxNGQyMTZhNmNhYjJkZGI5ZjI4NzdmYmJkMGZlYmRjMGZhNjA4CnBhcmVudCA5ODNmZjFhM2NmNzQ3MjVhNTNhNWRlYzhkMGMwNjEyMjEyOGY1YThkCmF1dGhvciBHZXJyaXQgVXNlciAxMDAwMDAwIDwxMDAwMDAwQDY5ZWMzOGYwLTM1MGUtNGQ5Yy05NmQ0LWJjOTU2ZjJmYWFhYz4gMTYxMDU3ODY0OCArMDEwMApjb21taXR0ZXIgR2Vycml0IENvZGUgUmV2aWV3IDxyb290QG1hY3plY2gtWFBTLTE1PiAxNjEwNTc4NjQ4ICswMTAwCgpVcGRhdGUgcGF0Y2ggc2V0IDEKClBhdGNoIFNldCAxOgoKKDEgY29tbWVudCkKClBhdGNoLXNldDogMQo\\u003d\"},\"tree_object\":{\"type\":2,\"content\":\"MTAwNjQ0IGJsb2IgYmIzODNmNTI0OWM2OGE0Y2M4YzgyYmRkMTIyOGI0YTg4ODNmZjZlOCAgICBmNzVhNjkwMDRhOTNiNGNjYzhjZTIxNWMxMjgwODYzNmMyYjc1Njc1\"},\"blobs\":[{\"type\":3,\"content\":\"ewogICJjb21tZW50cyI6IFsKICAgIHsKICAgICAgImtleSI6IHsKICAgICAgICAidXVpZCI6ICI5MGI1YWJmZl80ZjY3NTI2YSIsCiAgICAgICAgImZpbGVuYW1lIjogIi9DT01NSVRfTVNHIiwKICAgICAgICAicGF0Y2hTZXRJZCI6IDEKICAgICAgfSwKICAgICAgImxpbmVOYnIiOiA5LAogICAgICAiYXV0aG9yIjogewogICAgICAgICJpZCI6IDEwMDAwMDAKICAgICAgfSwKICAgICAgIndyaXR0ZW5PbiI6ICIyMDIxLTAxLTEzVDIyOjU3OjI4WiIsCiAgICAgICJzaWRlIjogMSwKICAgICAgIm1lc3NhZ2UiOiAidGVzdCBjb21tZW50IiwKICAgICAgInJhbmdlIjogewogICAgICAgICJzdGFydExpbmUiOiA5LAogICAgICAgICJzdGFydENoYXIiOiAyMSwKICAgICAgICAiZW5kTGluZSI6IDksCiAgICAgICAgImVuZENoYXIiOiAzNAogICAgICB9LAogICAgICAicmV2SWQiOiAiZjc1YTY5MDA0YTkzYjRjY2M4Y2UyMTVjMTI4MDg2MzZjMmI3NTY3NSIsCiAgICAgICJzZXJ2ZXJJZCI6ICI2OWVjMzhmMC0zNTBlLTRkOWMtOTZkNC1iYzk1NmYyZmFhYWMiLAogICAgICAidW5yZXNvbHZlZCI6IHRydWUKICAgIH0KICBdCn0\\u003d\"}]},\"async\":false}";
+ "{\"label\":\"Replication\",\"ref_name\":\"refs/heads/master\",\"revision_data\":{\"commit_object\":{\"type\":1,\"content\":\"dHJlZSA3NzgxNGQyMTZhNmNhYjJkZGI5ZjI4NzdmYmJkMGZlYmRjMGZhNjA4CnBhcmVudCA5ODNmZjFhM2NmNzQ3MjVhNTNhNWRlYzhkMGMwNjEyMjEyOGY1YThkCmF1dGhvciBHZXJyaXQgVXNlciAxMDAwMDAwIDwxMDAwMDAwQDY5ZWMzOGYwLTM1MGUtNGQ5Yy05NmQ0LWJjOTU2ZjJmYWFhYz4gMTYxMDU3ODY0OCArMDEwMApjb21taXR0ZXIgR2Vycml0IENvZGUgUmV2aWV3IDxyb290QG1hY3plY2gtWFBTLTE1PiAxNjEwNTc4NjQ4ICswMTAwCgpVcGRhdGUgcGF0Y2ggc2V0IDEKClBhdGNoIFNldCAxOgoKKDEgY29tbWVudCkKClBhdGNoLXNldDogMQo\\u003d\"},\"tree_object\":{\"type\":2,\"content\":\"MTAwNjQ0IGJsb2IgYmIzODNmNTI0OWM2OGE0Y2M4YzgyYmRkMTIyOGI0YTg4ODNmZjZlOCAgICBmNzVhNjkwMDRhOTNiNGNjYzhjZTIxNWMxMjgwODYzNmMyYjc1Njc1\"},\"blobs\":[{\"type\":3,\"content\":\"ewogICJjb21tZW50cyI6IFsKICAgIHsKICAgICAgImtleSI6IHsKICAgICAgICAidXVpZCI6ICI5MGI1YWJmZl80ZjY3NTI2YSIsCiAgICAgICAgImZpbGVuYW1lIjogIi9DT01NSVRfTVNHIiwKICAgICAgICAicGF0Y2hTZXRJZCI6IDEKICAgICAgfSwKICAgICAgImxpbmVOYnIiOiA5LAogICAgICAiYXV0aG9yIjogewogICAgICAgICJpZCI6IDEwMDAwMDAKICAgICAgfSwKICAgICAgIndyaXR0ZW5PbiI6ICIyMDIxLTAxLTEzVDIyOjU3OjI4WiIsCiAgICAgICJzaWRlIjogMSwKICAgICAgIm1lc3NhZ2UiOiAidGVzdCBjb21tZW50IiwKICAgICAgInJhbmdlIjogewogICAgICAgICJzdGFydExpbmUiOiA5LAogICAgICAgICJzdGFydENoYXIiOiAyMSwKICAgICAgICAiZW5kTGluZSI6IDksCiAgICAgICAgImVuZENoYXIiOiAzNAogICAgICB9LAogICAgICAicmV2SWQiOiAiZjc1YTY5MDA0YTkzYjRjY2M4Y2UyMTVjMTI4MDg2MzZjMmI3NTY3NSIsCiAgICAgICJzZXJ2ZXJJZCI6ICI2OWVjMzhmMC0zNTBlLTRkOWMtOTZkNC1iYzk1NmYyZmFhYWMiLAogICAgICAidW5yZXNvbHZlZCI6IHRydWUKICAgIH0KICBdCn0\\u003d\"}]}}";
String commitObject =
"tree 77814d216a6cab2ddb9f2877fbbd0febdc0fa608\n"
+ "parent 983ff1a3cf74725a53a5dec8d0c06122128f5a8d\n"
@@ -143,15 +148,17 @@
when(credentialProvider.get(any(), any(CredentialItem.class))).thenReturn(true);
when(credentials.create(anyString())).thenReturn(credentialProvider);
when(replicationConfig.getConfig()).thenReturn(config);
+ when(config.getStringList("replication", null, "syncRefs")).thenReturn(new String[0]);
when(source.getRemoteConfigName()).thenReturn("Replication");
when(config.getString("replication", null, "instanceLabel")).thenReturn(label);
HttpResult httpResult = new HttpResult(SC_CREATED, Optional.of("result message"));
when(httpClient.execute(any(HttpPost.class), any(), any())).thenReturn(httpResult);
when(httpClientFactory.create(any())).thenReturn(httpClient);
+ syncRefsFilter = new SyncRefsFilter(replicationConfig);
objectUnderTest =
new FetchRestApiClient(
- credentials, httpClientFactory, replicationConfig, pluginName, source);
+ credentials, httpClientFactory, replicationConfig, syncRefsFilter, pluginName, source);
}
@Test
@@ -169,6 +176,67 @@
}
@Test
+ public void shouldByDefaultCallSyncFetchForAllRefs()
+ throws ClientProtocolException, IOException, URISyntaxException {
+
+ syncRefsFilter = new SyncRefsFilter(replicationConfig);
+ objectUnderTest =
+ new FetchRestApiClient(
+ credentials, httpClientFactory, replicationConfig, syncRefsFilter, pluginName, source);
+
+ objectUnderTest.callFetch(Project.nameKey("test_repo"), refName, new URIish(api));
+
+ verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any(), any());
+
+ HttpPost httpPost = httpPostCaptor.getValue();
+ assertThat(readPayload(httpPost)).isEqualTo(expectedPayload);
+ }
+
+ @Test
+ public void shouldCallAsyncFetchForAllRefs()
+ throws ClientProtocolException, IOException, URISyntaxException {
+
+ when(config.getStringList("replication", null, "syncRefs"))
+ .thenReturn(new String[] {"NO_SYNC_REFS"});
+ syncRefsFilter = new SyncRefsFilter(replicationConfig);
+ objectUnderTest =
+ new FetchRestApiClient(
+ credentials, httpClientFactory, replicationConfig, syncRefsFilter, pluginName, source);
+
+ objectUnderTest.callFetch(Project.nameKey("test_repo"), refName, new URIish(api));
+
+ verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any(), any());
+
+ HttpPost httpPost = httpPostCaptor.getValue();
+ assertThat(readPayload(httpPost)).isEqualTo(expectedAsyncPayload);
+ }
+
+ @Test
+ public void shouldCallSyncFetchOnlyForMetaRef()
+ throws ClientProtocolException, IOException, URISyntaxException {
+ String metaRefName = "refs/changes/01/101/meta";
+ String expectedMetaRefPayload =
+ "{\"label\":\"Replication\", \"ref_name\": \"" + metaRefName + "\", \"async\":false}";
+
+ when(config.getStringList("replication", null, "syncRefs"))
+ .thenReturn(new String[] {"^refs\\/changes\\/.*\\/meta"});
+ syncRefsFilter = new SyncRefsFilter(replicationConfig);
+ objectUnderTest =
+ new FetchRestApiClient(
+ credentials, httpClientFactory, replicationConfig, syncRefsFilter, pluginName, source);
+
+ objectUnderTest.callFetch(Project.nameKey("test_repo"), refName, new URIish(api));
+ verify(httpClient, times(1)).execute(httpPostCaptor.capture(), any(), any());
+ HttpPost httpPost = httpPostCaptor.getValue();
+ assertThat(readPayload(httpPost)).isEqualTo(expectedAsyncPayload);
+
+ objectUnderTest.callFetch(Project.nameKey("test_repo"), metaRefName, new URIish(api));
+ verify(httpClient, times(2)).execute(httpPostCaptor.capture(), any(), any());
+ httpPost = httpPostCaptor.getValue();
+ assertThat(readPayload(httpPost)).isEqualTo(expectedMetaRefPayload);
+ }
+
+ @Test
public void shouldCallFetchEndpointWithPayload()
throws ClientProtocolException, IOException, URISyntaxException {
@@ -241,7 +309,12 @@
NullPointerException.class,
() ->
new FetchRestApiClient(
- credentials, httpClientFactory, replicationConfig, pluginName, source));
+ credentials,
+ httpClientFactory,
+ replicationConfig,
+ syncRefsFilter,
+ pluginName,
+ source));
}
@Test
@@ -251,7 +324,12 @@
NullPointerException.class,
() ->
new FetchRestApiClient(
- credentials, httpClientFactory, replicationConfig, pluginName, source));
+ credentials,
+ httpClientFactory,
+ replicationConfig,
+ syncRefsFilter,
+ pluginName,
+ source));
}
@Test
@@ -261,7 +339,12 @@
NullPointerException.class,
() ->
new FetchRestApiClient(
- credentials, httpClientFactory, replicationConfig, pluginName, source));
+ credentials,
+ httpClientFactory,
+ replicationConfig,
+ syncRefsFilter,
+ pluginName,
+ source));
}
public String readPayload(HttpPost entity) throws UnsupportedOperationException, IOException {
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandlerTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandlerTest.java
index 1d87195..e528eca 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandlerTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/event/FetchRefReplicatedEventHandlerTest.java
@@ -24,6 +24,7 @@
import com.google.gerrit.entities.Change;
import com.google.gerrit.entities.Project;
import com.google.gerrit.server.index.change.ChangeIndexer;
+import com.googlesource.gerrit.plugins.replication.pull.Context;
import com.googlesource.gerrit.plugins.replication.pull.FetchRefReplicatedEvent;
import com.googlesource.gerrit.plugins.replication.pull.ReplicationState;
import org.eclipse.jgit.lib.RefUpdate;
@@ -45,6 +46,26 @@
Project.NameKey projectNameKey = Project.nameKey("testProject");
String ref = "refs/changes/41/41/meta";
Change.Id changeId = Change.Id.fromRef(ref);
+ try {
+ Context.setLocalEvent(true);
+ fetchRefReplicatedEventHandler.onEvent(
+ new FetchRefReplicatedEvent(
+ projectNameKey.get(),
+ ref,
+ "aSourceNode",
+ ReplicationState.RefFetchResult.SUCCEEDED,
+ RefUpdate.Result.FAST_FORWARD));
+ verify(changeIndexerMock, times(1)).index(eq(projectNameKey), eq(changeId));
+ } finally {
+ Context.unsetLocalEvent();
+ }
+ }
+
+ @Test
+ public void onEventShouldNotIndexIfNotLocalEvent() {
+ Project.NameKey projectNameKey = Project.nameKey("testProject");
+ String ref = "refs/changes/41/41/meta";
+ Change.Id changeId = Change.Id.fromRef(ref);
fetchRefReplicatedEventHandler.onEvent(
new FetchRefReplicatedEvent(
projectNameKey.get(),
@@ -52,7 +73,22 @@
"aSourceNode",
ReplicationState.RefFetchResult.SUCCEEDED,
RefUpdate.Result.FAST_FORWARD));
- verify(changeIndexerMock, times(1)).index(eq(projectNameKey), eq(changeId));
+ verify(changeIndexerMock, never()).index(eq(projectNameKey), eq(changeId));
+ }
+
+ @Test
+ public void onEventShouldIndexOnlyMetaRef() {
+ Project.NameKey projectNameKey = Project.nameKey("testProject");
+ String ref = "refs/changes/41/41/1";
+ Change.Id changeId = Change.Id.fromRef(ref);
+ fetchRefReplicatedEventHandler.onEvent(
+ new FetchRefReplicatedEvent(
+ projectNameKey.get(),
+ ref,
+ "aSourceNode",
+ ReplicationState.RefFetchResult.SUCCEEDED,
+ RefUpdate.Result.FAST_FORWARD));
+ verify(changeIndexerMock, never()).index(eq(projectNameKey), eq(changeId));
}
@Test
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObjectIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObjectIT.java
index 5d80d94..f6cd342 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObjectIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/fetch/ApplyObjectIT.java
@@ -36,6 +36,8 @@
import com.google.gerrit.extensions.config.FactoryModule;
import com.google.inject.Inject;
import com.google.inject.Scopes;
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig;
+import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig;
import com.googlesource.gerrit.plugins.replication.pull.RevisionReader;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionData;
import com.googlesource.gerrit.plugins.replication.pull.api.data.RevisionObjectData;
@@ -46,6 +48,7 @@
import org.eclipse.jgit.junit.TestRepository;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.transport.RefSpec;
+import org.junit.Before;
import org.junit.Test;
@SkipProjectClone
@@ -57,8 +60,13 @@
private static final String TEST_REPLICATION_SUFFIX = "suffix1";
@Inject private ProjectOperations projectOperations;
- @Inject RevisionReader reader;
@Inject ApplyObject objectUnderTest;
+ RevisionReader reader;
+
+ @Before
+ public void setup() {
+ reader = plugin.getSysInjector().getInstance(RevisionReader.class);
+ }
@Test
public void shouldApplyRefMetaObject() throws Exception {
@@ -184,6 +192,7 @@
private static class TestModule extends FactoryModule {
@Override
protected void configure() {
+ bind(ReplicationConfig.class).to(ReplicationFileBasedConfig.class);
bind(RevisionReader.class).in(Scopes.SINGLETON);
bind(ApplyObject.class);
}
