Merge branch 'stable-2.16' into stable-3.0
* stable-2.16:
Fix log statement formatting
Change-Id: I57cee891b7610fdaf7ef37b2fe1cf11b3010157a
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java
index de6e91e..30e8245 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java
@@ -33,10 +33,12 @@
@Singleton
static class DefaultAdminApiFactory implements AdminApiFactory {
protected final SshHelper sshHelper;
+ private final GerritRestApi.Factory gerritRestApiFactory;
@Inject
- public DefaultAdminApiFactory(SshHelper sshHelper) {
+ public DefaultAdminApiFactory(SshHelper sshHelper, GerritRestApi.Factory gerritRestApiFactory) {
this.sshHelper = sshHelper;
+ this.gerritRestApiFactory = gerritRestApiFactory;
}
@Override
@@ -47,6 +49,8 @@
return Optional.of(new LocalFS(uri));
} else if (isSSH(uri)) {
return Optional.of(new RemoteSsh(sshHelper, uri));
+ } else if (isGerritHttp(uri)) {
+ return Optional.of(gerritRestApiFactory.create(uri));
}
return Optional.empty();
}
@@ -70,4 +74,9 @@
}
return false;
}
+
+ public static boolean isGerritHttp(URIish uri) {
+ String scheme = uri.getScheme();
+ return scheme != null && scheme.toLowerCase().contains("gerrit+http");
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
index 02daa6d..945f869 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
@@ -14,9 +14,12 @@
package com.googlesource.gerrit.plugins.replication;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Multimap;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.common.FileUtil;
import com.google.gerrit.extensions.annotations.PluginData;
+import com.google.gerrit.extensions.annotations.PluginName;
+import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.git.WorkQueue;
import com.google.inject.Inject;
@@ -25,11 +28,18 @@
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.transport.URIish;
@Singleton
public class AutoReloadConfigDecorator implements ReplicationConfig {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+ private static final long RELOAD_DELAY = 120;
+ private static final long RELOAD_INTERVAL = 60;
private volatile ReplicationFileBasedConfig currentConfig;
private long currentConfigTs;
@@ -41,6 +51,8 @@
// Use Provider<> instead of injecting the ReplicationQueue because of circular dependency with
// ReplicationConfig
private final Provider<ReplicationQueue> replicationQueue;
+ private final ScheduledExecutorService autoReloadExecutor;
+ private ScheduledFuture<?> autoReloadRunnable;
private volatile boolean shuttingDown;
@@ -49,7 +61,9 @@
SitePaths site,
Destination.Factory destinationFactory,
Provider<ReplicationQueue> replicationQueue,
- @PluginData Path pluginDataDir)
+ @PluginData Path pluginDataDir,
+ @PluginName String pluginName,
+ WorkQueue workQueue)
throws ConfigInvalidException, IOException {
this.site = site;
this.destinationFactory = destinationFactory;
@@ -57,6 +71,7 @@
this.currentConfig = loadConfig();
this.currentConfigTs = getLastModified(currentConfig);
this.replicationQueue = replicationQueue;
+ this.autoReloadExecutor = workQueue.createQueue(1, pluginName + "_auto-reload-config");
}
private static long getLastModified(ReplicationFileBasedConfig cfg) {
@@ -73,11 +88,16 @@
@Override
public synchronized List<Destination> getDestinations(FilterType filterType) {
- reloadIfNeeded();
return currentConfig.getDestinations(filterType);
}
- private void reloadIfNeeded() {
+ @Override
+ public synchronized Multimap<Destination, URIish> getURIs(
+ Optional<String> remoteName, Project.NameKey projectName, FilterType filterType) {
+ return currentConfig.getURIs(remoteName, projectName, filterType);
+ }
+
+ private synchronized void reloadIfNeeded() {
reload(false);
}
@@ -128,6 +148,11 @@
}
@Override
+ public synchronized int getMaxRefsToLog() {
+ return currentConfig.getMaxRefsToLog();
+ }
+
+ @Override
public synchronized boolean isEmpty() {
return currentConfig.isEmpty();
}
@@ -153,6 +178,10 @@
@Override
public int shutdown() {
this.shuttingDown = true;
+ if (autoReloadRunnable != null) {
+ autoReloadRunnable.cancel(false);
+ autoReloadRunnable = null;
+ }
return currentConfig.shutdown();
}
@@ -160,6 +189,9 @@
public synchronized void startup(WorkQueue workQueue) {
shuttingDown = false;
currentConfig.startup(workQueue);
+ autoReloadRunnable =
+ autoReloadExecutor.scheduleAtFixedRate(
+ this::reloadIfNeeded, RELOAD_DELAY, RELOAD_INTERVAL, TimeUnit.SECONDS);
}
@Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
index 29a7ee6..98f364d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
@@ -23,6 +23,7 @@
import java.nio.file.Files;
import java.util.concurrent.atomic.AtomicReference;
import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.transport.CredentialsProvider;
public class AutoReloadSecureCredentialsFactoryDecorator implements CredentialsFactory {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -50,7 +51,7 @@
}
@Override
- public SecureCredentialsProvider create(String remoteName) {
+ public CredentialsProvider create(String remoteName) {
try {
if (needsReload()) {
secureCredentialsFactory.compareAndSet(
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java
new file mode 100644
index 0000000..a8dede3
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java
@@ -0,0 +1,70 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
+import com.google.gerrit.extensions.registration.DynamicItem;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
+import java.util.Optional;
+import org.eclipse.jgit.transport.RemoteConfig;
+import org.eclipse.jgit.transport.URIish;
+
+public class CreateProjectTask {
+ interface Factory {
+ CreateProjectTask create(Project.NameKey project, String head);
+ }
+
+ private final RemoteConfig config;
+ private final ReplicationConfig replicationConfig;
+ private final DynamicItem<AdminApiFactory> adminApiFactory;
+ private final Project.NameKey project;
+ private final String head;
+
+ @Inject
+ CreateProjectTask(
+ RemoteConfig config,
+ ReplicationConfig replicationConfig,
+ DynamicItem<AdminApiFactory> adminApiFactory,
+ @Assisted Project.NameKey project,
+ @Assisted String head) {
+ this.config = config;
+ this.replicationConfig = replicationConfig;
+ this.adminApiFactory = adminApiFactory;
+ this.project = project;
+ this.head = head;
+ }
+
+ public boolean create() {
+ return replicationConfig
+ .getURIs(Optional.of(config.getName()), project, FilterType.PROJECT_CREATION).values()
+ .stream()
+ .map(u -> createProject(u, project, head))
+ .reduce(true, (a, b) -> a && b);
+ }
+
+ private boolean createProject(URIish replicateURI, Project.NameKey projectName, String head) {
+ Optional<AdminApi> adminApi = adminApiFactory.get().create(replicateURI);
+ if (adminApi.isPresent() && adminApi.get().createProject(projectName, head)) {
+ return true;
+ }
+
+ repLog.warn("Cannot create new project {} on remote site {}.", projectName, replicateURI);
+ return false;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java
index 10719c1..3bb64ab 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java
@@ -13,7 +13,9 @@
// limitations under the License.
package com.googlesource.gerrit.plugins.replication;
+import org.eclipse.jgit.transport.CredentialsProvider;
+
public interface CredentialsFactory {
- SecureCredentialsProvider create(String remoteName);
+ CredentialsProvider create(String remoteName);
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java
new file mode 100644
index 0000000..f9b2ad7
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java
@@ -0,0 +1,66 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
+import com.google.gerrit.extensions.registration.DynamicItem;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.server.ioutil.HexFormat;
+import com.google.gerrit.server.util.IdGenerator;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import java.util.Optional;
+import org.eclipse.jgit.transport.URIish;
+
+public class DeleteProjectTask implements Runnable {
+ interface Factory {
+ DeleteProjectTask create(URIish replicateURI, Project.NameKey project);
+ }
+
+ private final DynamicItem<AdminApiFactory> adminApiFactory;
+ private final int id;
+ private final URIish replicateURI;
+ private final Project.NameKey project;
+
+ @Inject
+ DeleteProjectTask(
+ DynamicItem<AdminApiFactory> adminApiFactory,
+ IdGenerator ig,
+ @Assisted URIish replicateURI,
+ @Assisted Project.NameKey project) {
+ this.adminApiFactory = adminApiFactory;
+ this.id = ig.next();
+ this.replicateURI = replicateURI;
+ this.project = project;
+ }
+
+ @Override
+ public void run() {
+ Optional<AdminApi> adminApi = adminApiFactory.get().create(replicateURI);
+ if (adminApi.isPresent()) {
+ adminApi.get().deleteProject(project);
+ return;
+ }
+
+ repLog.warn("Cannot delete project {} on remote site {}.", project, replicateURI);
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "[%s] delete-project %s at %s", HexFormat.fromInt(id), project.get(), replicateURI);
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
index 966248a..b206dbe 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
@@ -15,6 +15,7 @@
package com.googlesource.gerrit.plugins.replication;
import static com.googlesource.gerrit.plugins.replication.PushResultProcessing.resolveNodeName;
+import static com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig.replaceName;
import static org.eclipse.jgit.transport.RemoteRefUpdate.Status.NON_EXISTING;
import static org.eclipse.jgit.transport.RemoteRefUpdate.Status.REJECTED_OTHER_REASON;
@@ -32,14 +33,12 @@
import com.google.gerrit.reviewdb.client.Branch;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.client.RefNames;
-import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.PluginUser;
import com.google.gerrit.server.account.GroupBackend;
import com.google.gerrit.server.account.GroupBackends;
import com.google.gerrit.server.account.GroupIncludeCache;
import com.google.gerrit.server.account.ListGroupMembership;
-import com.google.gerrit.server.config.RequestScopedReviewDbProvider;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.PerThreadRequestScope;
@@ -56,6 +55,7 @@
import com.google.inject.Injector;
import com.google.inject.Provider;
import com.google.inject.Provides;
+import com.google.inject.Scopes;
import com.google.inject.assistedinject.Assisted;
import com.google.inject.assistedinject.FactoryModuleBuilder;
import com.google.inject.servlet.RequestScoped;
@@ -74,6 +74,7 @@
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import org.apache.commons.io.FilenameUtils;
+import org.apache.http.impl.client.CloseableHttpClient;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
@@ -97,6 +98,8 @@
private final Map<URIish, PushOne> pending = new HashMap<>();
private final Map<URIish, PushOne> inFlight = new HashMap<>();
private final PushOne.Factory opFactory;
+ private final DeleteProjectTask.Factory deleteProjectFactory;
+ private final UpdateHeadTask.Factory updateHeadFactory;
private final GitRepositoryManager gitManager;
private final PermissionBackend permissionBackend;
private final Provider<CurrentUser> userProvider;
@@ -173,24 +176,24 @@
bind(Destination.class).toInstance(Destination.this);
bind(RemoteConfig.class).toInstance(config.getRemoteConfig());
install(new FactoryModuleBuilder().build(PushOne.Factory.class));
+ install(new FactoryModuleBuilder().build(CreateProjectTask.Factory.class));
+ install(new FactoryModuleBuilder().build(DeleteProjectTask.Factory.class));
+ install(new FactoryModuleBuilder().build(UpdateHeadTask.Factory.class));
+
+ DynamicItem.itemOf(binder(), AdminApiFactory.class);
+ DynamicItem.bind(binder(), AdminApiFactory.class)
+ .to(AdminApiFactory.DefaultAdminApiFactory.class);
+
+ install(new FactoryModuleBuilder().build(GerritRestApi.Factory.class));
+ bind(CloseableHttpClient.class)
+ .toProvider(HttpClientProvider.class)
+ .in(Scopes.SINGLETON);
}
@Provides
public PerThreadRequestScope.Scoper provideScoper(
- final PerThreadRequestScope.Propagator propagator,
- final Provider<RequestScopedReviewDbProvider> dbProvider) {
- final RequestContext requestContext =
- new RequestContext() {
- @Override
- public CurrentUser getUser() {
- return remoteUser;
- }
-
- @Override
- public Provider<ReviewDb> getReviewDbProvider() {
- return dbProvider.get();
- }
- };
+ final PerThreadRequestScope.Propagator propagator) {
+ final RequestContext requestContext = () -> remoteUser;
return new PerThreadRequestScope.Scoper() {
@Override
public <T> Callable<T> scope(Callable<T> callable) {
@@ -201,6 +204,8 @@
});
opFactory = child.getInstance(PushOne.Factory.class);
+ deleteProjectFactory = child.getInstance(DeleteProjectTask.Factory.class);
+ updateHeadFactory = child.getInstance(UpdateHeadTask.Factory.class);
threadScoper = child.getInstance(PerThreadRequestScope.Scoper.class);
}
@@ -231,24 +236,35 @@
public int shutdown() {
int cnt = 0;
if (pool != null) {
- repLog.warn("Cancelling replication events");
+ synchronized (stateLock) {
+ int numPending = pending.size();
+ int numInFlight = inFlight.size();
- foreachPushOp(
- pending,
- push -> {
- push.cancel();
- return null;
- });
- pending.clear();
- foreachPushOp(
- inFlight,
- push -> {
- push.setCanceledWhileRunning();
- return null;
- });
- inFlight.clear();
- cnt = pool.shutdownNow().size();
- pool = null;
+ if (numPending > 0 || numInFlight > 0) {
+ repLog.warn(
+ "Cancelling replication events (pending={}, inFlight={}) for destination {}",
+ numPending,
+ numInFlight,
+ getRemoteConfigName());
+
+ foreachPushOp(
+ pending,
+ push -> {
+ push.cancel();
+ return null;
+ });
+ pending.clear();
+ foreachPushOp(
+ inFlight,
+ push -> {
+ push.setCanceledWhileRunning();
+ return null;
+ });
+ inFlight.clear();
+ }
+ cnt = pool.shutdownNow().size();
+ pool = null;
+ }
}
return cnt;
}
@@ -291,47 +307,44 @@
try {
return threadScoper
.scope(
- new Callable<Boolean>() {
- @Override
- public Boolean call() throws NoSuchProjectException, PermissionBackendException {
- ProjectState projectState;
- try {
- projectState = projectCache.checkedGet(project);
- } catch (IOException e) {
- repLog.warn("Error reading project {} from cache", project, e);
- return false;
- }
- if (projectState == null) {
- repLog.debug("Project {} does not exist", project);
- throw new NoSuchProjectException(project);
- }
- if (!projectState.statePermitsRead()) {
- repLog.debug("Project {} does not permit read", project);
- return false;
- }
- if (!shouldReplicate(projectState, userProvider.get())) {
- repLog.debug("Project {} should not be replicated", project);
- return false;
- }
- if (PushOne.ALL_REFS.equals(ref)) {
- return true;
- }
- try {
- permissionBackend
- .user(userProvider.get())
- .project(project)
- .ref(ref)
- .check(RefPermission.READ);
- } catch (AuthException e) {
- repLog.debug(
- "Ref {} on project {} is not visible to calling user {}",
- ref,
- project,
- userProvider.get().getUserName().orElse("unknown"));
- return false;
- }
+ () -> {
+ ProjectState projectState;
+ try {
+ projectState = projectCache.checkedGet(project);
+ } catch (IOException e) {
+ repLog.warn("Error reading project {} from cache", project, e);
+ return false;
+ }
+ if (projectState == null) {
+ repLog.debug("Project {} does not exist", project);
+ throw new NoSuchProjectException(project);
+ }
+ if (!projectState.statePermitsRead()) {
+ repLog.debug("Project {} does not permit read", project);
+ return false;
+ }
+ if (!shouldReplicate(projectState, userProvider.get())) {
+ repLog.debug("Project {} should not be replicated", project);
+ return false;
+ }
+ if (PushOne.ALL_REFS.equals(ref)) {
return true;
}
+ try {
+ permissionBackend
+ .user(userProvider.get())
+ .project(project)
+ .ref(ref)
+ .check(RefPermission.READ);
+ } catch (AuthException e) {
+ repLog.debug(
+ "Ref {} on project {} is not visible to calling user {}",
+ ref,
+ project,
+ userProvider.get().getUserName().orElse("unknown"));
+ return false;
+ }
+ return true;
})
.call();
} catch (NoSuchProjectException err) {
@@ -347,20 +360,17 @@
try {
return threadScoper
.scope(
- new Callable<Boolean>() {
- @Override
- public Boolean call() throws NoSuchProjectException, PermissionBackendException {
- ProjectState projectState;
- try {
- projectState = projectCache.checkedGet(project);
- } catch (IOException e) {
- return false;
- }
- if (projectState == null) {
- throw new NoSuchProjectException(project);
- }
- return shouldReplicate(projectState, userProvider.get());
+ () -> {
+ ProjectState projectState;
+ try {
+ projectState = projectCache.checkedGet(project);
+ } catch (IOException e) {
+ return false;
}
+ if (projectState == null) {
+ throw new NoSuchProjectException(project);
+ }
+ return shouldReplicate(projectState, userProvider.get());
})
.call();
} catch (NoSuchProjectException err) {
@@ -443,6 +453,18 @@
}
}
+ void scheduleDeleteProject(URIish uri, Project.NameKey project) {
+ @SuppressWarnings("unused")
+ ScheduledFuture<?> ignored =
+ pool.schedule(deleteProjectFactory.create(uri, project), 0, TimeUnit.SECONDS);
+ }
+
+ void scheduleUpdateHead(URIish uri, Project.NameKey project, String newHead) {
+ @SuppressWarnings("unused")
+ ScheduledFuture<?> ignored =
+ pool.schedule(updateHeadFactory.create(uri, project, newHead), 0, TimeUnit.SECONDS);
+ }
+
private void addRef(PushOne e, String ref) {
e.addRef(ref);
postReplicationScheduledEvent(e, ref);
@@ -677,8 +699,7 @@
} else if (!remoteNameStyle.equals("slash")) {
repLog.debug("Unknown remoteNameStyle: {}, falling back to slash", remoteNameStyle);
}
- String replacedPath =
- ReplicationQueue.replaceName(template.getPath(), name, isSingleProjectMatch());
+ String replacedPath = replaceName(template.getPath(), name, isSingleProjectMatch());
return (replacedPath != null) ? template.setPath(replacedPath) : template;
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java
new file mode 100644
index 0000000..eac56df
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java
@@ -0,0 +1,135 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.googlesource.gerrit.plugins.replication.GerritSshApi.GERRIT_ADMIN_PROTOCOL_PREFIX;
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.gerrit.extensions.restapi.Url;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import java.io.IOException;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicHeader;
+import org.eclipse.jgit.transport.CredentialItem;
+import org.eclipse.jgit.transport.RemoteConfig;
+import org.eclipse.jgit.transport.URIish;
+
+public class GerritRestApi implements AdminApi {
+
+ public interface Factory {
+ GerritRestApi create(URIish uri);
+ }
+
+ private final CredentialsFactory credentials;
+ private final CloseableHttpClient httpClient;
+ private final RemoteConfig remoteConfig;
+ private final URIish uri;
+
+ @Inject
+ GerritRestApi(
+ CredentialsFactory credentials,
+ CloseableHttpClient httpClient,
+ RemoteConfig remoteConfig,
+ @Assisted URIish uri) {
+ this.credentials = credentials;
+ this.httpClient = httpClient;
+ this.remoteConfig = remoteConfig;
+ this.uri = uri;
+ }
+
+ @Override
+ public boolean createProject(Project.NameKey project, String head) {
+ repLog.info("Creating project {} on {}", project, uri);
+ String url = String.format("%s/a/projects/%s", toHttpUri(uri), Url.encode(project.get()));
+ try {
+ return httpClient
+ .execute(new HttpPut(url), new HttpResponseHandler(), getContext())
+ .isSuccessful();
+ } catch (IOException e) {
+ repLog.error("Couldn't perform project creation on {}", uri, e);
+ return false;
+ }
+ }
+
+ @Override
+ public boolean deleteProject(Project.NameKey project) {
+ repLog.info("Deleting project {} on {}", project, uri);
+ String url = String.format("%s/a/projects/%s", toHttpUri(uri), Url.encode(project.get()));
+ try {
+ httpClient.execute(new HttpDelete(url), new HttpResponseHandler(), getContext());
+ return true;
+ } catch (IOException e) {
+ repLog.error("Couldn't perform project deletion on {}", uri, e);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean updateHead(Project.NameKey project, String newHead) {
+ repLog.info("Updating head of {} on {}", project, uri);
+ String url = String.format("%s/a/projects/%s/HEAD", toHttpUri(uri), Url.encode(project.get()));
+ try {
+ HttpPut req = new HttpPut(url);
+ req.setEntity(new StringEntity(String.format("{\"ref\": \"%s\"}", newHead), UTF_8.name()));
+ req.addHeader(new BasicHeader("Content-Type", "application/json"));
+ httpClient.execute(req, new HttpResponseHandler(), getContext());
+ return true;
+ } catch (IOException e) {
+ repLog.error("Couldn't perform update head on {}", uri, e);
+ }
+ return false;
+ }
+
+ private HttpClientContext getContext() {
+ HttpClientContext ctx = HttpClientContext.create();
+ ctx.setCredentialsProvider(adapt(credentials.create(remoteConfig.getName())));
+ return ctx;
+ }
+
+ private CredentialsProvider adapt(org.eclipse.jgit.transport.CredentialsProvider cp) {
+ CredentialItem.Username user = new CredentialItem.Username();
+ CredentialItem.Password pass = new CredentialItem.Password();
+ if (cp.supports(user, pass) && cp.get(uri, user, pass)) {
+ CredentialsProvider adapted = new BasicCredentialsProvider();
+ adapted.setCredentials(
+ AuthScope.ANY,
+ new UsernamePasswordCredentials(user.getValue(), new String(pass.getValue())));
+ return adapted;
+ }
+ return null;
+ }
+
+ private static String toHttpUri(URIish uri) {
+ String u = uri.toString();
+ if (u.startsWith(GERRIT_ADMIN_PROTOCOL_PREFIX)) {
+ u = u.substring(GERRIT_ADMIN_PROTOCOL_PREFIX.length());
+ }
+ if (u.endsWith("/")) {
+ return u.substring(0, u.length() - 1);
+ }
+ return u;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
index ccd8bf4..6dcc80e 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
@@ -28,7 +28,7 @@
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
static final int SSH_COMMAND_FAILED = -1;
- private static final String GERRIT_ADMIN_PROTOCOL_PREFIX = "gerrit+";
+ static final String GERRIT_ADMIN_PROTOCOL_PREFIX = "gerrit+";
protected final SshHelper sshHelper;
protected final URIish uri;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/HttpClientProvider.java b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpClientProvider.java
new file mode 100644
index 0000000..916059c
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpClientProvider.java
@@ -0,0 +1,108 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.ProvisionException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import javax.net.ssl.SSLContext;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.HttpClientConnectionManager;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.ssl.SSLContexts;
+import org.eclipse.jgit.lib.Config;
+
+/** Provides an HTTP client with SSL capabilities. */
+class HttpClientProvider implements Provider<CloseableHttpClient> {
+ private static final int CONNECTIONS_PER_ROUTE = 100;
+
+ // Up to 2 target instances with the max number of connections per host:
+ private static final int MAX_CONNECTIONS = 2 * CONNECTIONS_PER_ROUTE;
+
+ private static final int MAX_CONNECTION_INACTIVITY = 10000;
+ private static final int DEFAULT_TIMEOUT_MS = 5000;
+
+ private final Config cfg;
+ private final SitePaths site;
+
+ @Inject
+ HttpClientProvider(@GerritServerConfig Config cfg, SitePaths site) {
+ this.cfg = cfg;
+ this.site = site;
+ }
+
+ @Override
+ public CloseableHttpClient get() {
+ try {
+ return HttpClients.custom()
+ .setConnectionManager(customConnectionManager())
+ .setDefaultRequestConfig(customRequestConfig())
+ .build();
+ } catch (Exception e) {
+ throw new ProvisionException("Couldn't create CloseableHttpClient", e);
+ }
+ }
+
+ private RequestConfig customRequestConfig() {
+ return RequestConfig.custom()
+ .setConnectTimeout(DEFAULT_TIMEOUT_MS)
+ .setSocketTimeout(DEFAULT_TIMEOUT_MS)
+ .setConnectionRequestTimeout(DEFAULT_TIMEOUT_MS)
+ .build();
+ }
+
+ private HttpClientConnectionManager customConnectionManager() throws Exception {
+ Registry<ConnectionSocketFactory> socketFactoryRegistry =
+ RegistryBuilder.<ConnectionSocketFactory>create()
+ .register("https", buildSslSocketFactory())
+ .register("http", PlainConnectionSocketFactory.INSTANCE)
+ .build();
+ PoolingHttpClientConnectionManager connManager =
+ new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ connManager.setDefaultMaxPerRoute(CONNECTIONS_PER_ROUTE);
+ connManager.setMaxTotal(MAX_CONNECTIONS);
+ connManager.setValidateAfterInactivity(MAX_CONNECTION_INACTIVITY);
+ return connManager;
+ }
+
+ private SSLConnectionSocketFactory buildSslSocketFactory() throws Exception {
+ String keyStore = cfg.getString("httpd", null, "sslKeyStore");
+ if (keyStore == null) {
+ keyStore = "etc/keystore";
+ }
+ return new SSLConnectionSocketFactory(createSSLContext(site.resolve(keyStore)));
+ }
+
+ private SSLContext createSSLContext(Path keyStorePath) throws Exception {
+ SSLContext ctx;
+ if (Files.exists(keyStorePath)) {
+ ctx = SSLContexts.custom().loadTrustMaterial(keyStorePath.toFile()).build();
+ } else {
+ ctx = SSLContext.getDefault();
+ }
+ return ctx;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponse.java b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponse.java
new file mode 100644
index 0000000..595acc7
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponse.java
@@ -0,0 +1,68 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.base.Preconditions;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.ByteBuffer;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.eclipse.jgit.util.IO;
+import org.eclipse.jgit.util.RawParseUtils;
+
+public class HttpResponse implements AutoCloseable {
+
+ protected CloseableHttpResponse response;
+ protected Reader reader;
+
+ HttpResponse(CloseableHttpResponse response) {
+ this.response = response;
+ }
+
+ public Reader getReader() throws IllegalStateException, IOException {
+ if (reader == null && response.getEntity() != null) {
+ reader = new InputStreamReader(response.getEntity().getContent(), UTF_8);
+ }
+ return reader;
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ Reader reader = getReader();
+ if (reader != null) {
+ while (reader.read() != -1) {
+ // Empty
+ }
+ }
+ } finally {
+ response.close();
+ }
+ }
+
+ public int getStatusCode() {
+ return response.getStatusLine().getStatusCode();
+ }
+
+ public String getEntityContent() throws IOException {
+ Preconditions.checkNotNull(response, "Response is not initialized.");
+ Preconditions.checkNotNull(response.getEntity(), "Response.Entity is not initialized.");
+ ByteBuffer buf = IO.readWholeStream(response.getEntity().getContent(), 1024);
+ return RawParseUtils.decode(buf.array(), buf.arrayOffset(), buf.limit()).trim();
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponseHandler.java b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponseHandler.java
new file mode 100644
index 0000000..6a7c73e
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponseHandler.java
@@ -0,0 +1,71 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static javax.servlet.http.HttpServletResponse.SC_CREATED;
+import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+
+import com.google.common.flogger.FluentLogger;
+import com.googlesource.gerrit.plugins.replication.HttpResponseHandler.HttpResult;
+import java.io.IOException;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.ResponseHandler;
+import org.apache.http.util.EntityUtils;
+
+class HttpResponseHandler implements ResponseHandler<HttpResult> {
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+ static class HttpResult {
+ private final boolean successful;
+ private final String message;
+
+ HttpResult(boolean successful, String message) {
+ this.successful = successful;
+ this.message = message;
+ }
+
+ boolean isSuccessful() {
+ return successful;
+ }
+
+ String getMessage() {
+ return message;
+ }
+ }
+
+ @Override
+ public HttpResult handleResponse(HttpResponse response) {
+ return new HttpResult(isSuccessful(response), parseResponse(response));
+ }
+
+ private static boolean isSuccessful(HttpResponse response) {
+ int sc = response.getStatusLine().getStatusCode();
+ return sc == SC_CREATED || sc == SC_NO_CONTENT || sc == SC_OK;
+ }
+
+ private static String parseResponse(HttpResponse response) {
+ HttpEntity entity = response.getEntity();
+ if (entity != null) {
+ try {
+ return EntityUtils.toString(entity);
+ } catch (IOException e) {
+ logger.atSevere().withCause(e).log("Error parsing entity");
+ }
+ }
+ return "";
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
index 8ccd54a..b64b024 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
@@ -16,6 +16,7 @@
import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static java.util.stream.Collectors.toMap;
import com.google.common.base.Throwables;
import com.google.common.collect.LinkedListMultimap;
@@ -53,7 +54,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicBoolean;
import org.eclipse.jgit.errors.NoRemoteRepositoryException;
import org.eclipse.jgit.errors.NotSupportedException;
@@ -93,9 +93,9 @@
private final PermissionBackend permissionBackend;
private final Destination pool;
private final RemoteConfig config;
+ private final ReplicationConfig replConfig;
private final CredentialsProvider credentialsProvider;
private final PerThreadRequestScope.Scoper threadScoper;
- private final ReplicationQueue replicationQueue;
private final Project.NameKey projectName;
private final URIish uri;
@@ -114,6 +114,7 @@
private final long createdAt;
private final ReplicationMetrics metrics;
private final ProjectCache projectCache;
+ private final CreateProjectTask.Factory createProjectFactory;
private final AtomicBoolean canceledWhileRunning;
private final TransportFactory transportFactory;
private DynamicItem<ReplicationPushFilter> replicationPushFilter;
@@ -124,13 +125,14 @@
PermissionBackend permissionBackend,
Destination p,
RemoteConfig c,
+ ReplicationConfig rc,
CredentialsFactory cpFactory,
PerThreadRequestScope.Scoper ts,
- ReplicationQueue rq,
IdGenerator ig,
ReplicationStateListeners sl,
ReplicationMetrics m,
ProjectCache pc,
+ CreateProjectTask.Factory cpf,
TransportFactory tf,
@Assisted Project.NameKey d,
@Assisted URIish u) {
@@ -138,9 +140,9 @@
this.permissionBackend = permissionBackend;
pool = p;
config = c;
+ replConfig = rc;
credentialsProvider = cpFactory.create(c.getName());
threadScoper = ts;
- replicationQueue = rq;
projectName = d;
uri = u;
lockRetryCount = 0;
@@ -150,6 +152,7 @@
createdAt = System.nanoTime();
metrics = m;
projectCache = pc;
+ createProjectFactory = cpf;
canceledWhileRunning = new AtomicBoolean(false);
maxRetries = p.getMaxRetries();
transportFactory = tf;
@@ -293,12 +296,9 @@
try {
threadScoper
.scope(
- new Callable<Void>() {
- @Override
- public Void call() {
- runPushOperation();
- return null;
- }
+ () -> {
+ runPushOperation();
+ return null;
})
.call();
} catch (Exception e) {
@@ -417,8 +417,7 @@
if (pool.isCreateMissingRepos()) {
try {
Ref head = git.exactRef(Constants.HEAD);
- if (replicationQueue.createProject(
- config.getName(), projectName, head != null ? getName(head) : null)) {
+ if (createProject(projectName, head != null ? getName(head) : null)) {
repLog.warn("Missing repository created; retry replication to {}", uri);
pool.reschedule(this, Destination.RetryReason.REPOSITORY_MISSING);
} else {
@@ -435,6 +434,10 @@
}
}
+ private boolean createProject(Project.NameKey project, String head) {
+ return createProjectFactory.create(project, head).create();
+ }
+
private String getName(Ref ref) {
Ref target = ref;
while (target.isSymbolic()) {
@@ -464,7 +467,16 @@
return new PushResult();
}
- repLog.info("Push to {} references: {}", uri, todo);
+ if (replConfig.getMaxRefsToLog() == 0 || todo.size() <= replConfig.getMaxRefsToLog()) {
+ repLog.info("Push to {} references: {}", uri, todo);
+ } else {
+ repLog.info(
+ "Push to {} references (first {} of {} listed): {}",
+ uri,
+ replConfig.getMaxRefsToLog(),
+ todo.size(),
+ todo.subList(0, replConfig.getMaxRefsToLog()));
+ }
return tn.push(NullProgressMonitor.INSTANCE, todo);
}
@@ -476,7 +488,8 @@
return Collections.emptyList();
}
- Map<String, Ref> local = git.getAllRefs();
+ Map<String, Ref> local =
+ git.getRefDatabase().getRefs().stream().collect(toMap(Ref::getName, r -> r));
boolean filter;
PermissionBackend.ForProject forProject = permissionBackend.currentUser().project(projectName);
try {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java
index 8542034..92ba4be 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java
@@ -15,10 +15,10 @@
package com.googlesource.gerrit.plugins.replication;
import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.exceptions.StorageException;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.events.RefEvent;
import com.google.gerrit.server.permissions.PermissionBackendException;
-import com.google.gwtorm.server.OrmException;
import com.googlesource.gerrit.plugins.replication.ReplicationState.RefPushResult;
import java.lang.ref.WeakReference;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -202,7 +202,7 @@
private void postEvent(RefEvent event) {
try {
dispatcher.postEvent(event);
- } catch (OrmException | PermissionBackendException e) {
+ } catch (StorageException | PermissionBackendException e) {
logger.atSevere().withCause(e).log("Cannot post event");
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
index c9531e3..929c538 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
@@ -13,9 +13,13 @@
// limitations under the License.
package com.googlesource.gerrit.plugins.replication;
+import com.google.common.collect.Multimap;
+import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.git.WorkQueue;
import java.nio.file.Path;
import java.util.List;
+import java.util.Optional;
+import org.eclipse.jgit.transport.URIish;
public interface ReplicationConfig {
@@ -27,10 +31,15 @@
List<Destination> getDestinations(FilterType filterType);
+ Multimap<Destination, URIish> getURIs(
+ Optional<String> remoteName, Project.NameKey projectName, FilterType filterType);
+
boolean isReplicateAllOnPluginStart();
boolean isDefaultForceUpdate();
+ int getMaxRefsToLog();
+
boolean isEmpty();
Path getEventsDirectory();
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
index 683fb9b..1f0c40e 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
@@ -13,15 +13,24 @@
// limitations under the License.
package com.googlesource.gerrit.plugins.replication;
+import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isGerrit;
+import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isGerritHttp;
+import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isSSH;
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toList;
import com.google.common.base.Strings;
+import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.SetMultimap;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.extensions.annotations.PluginData;
+import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.config.ConfigUtil;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.git.WorkQueue;
@@ -33,6 +42,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
import org.eclipse.jgit.errors.ConfigInvalidException;
@@ -52,6 +62,7 @@
private Path cfgPath;
private boolean replicateAllOnPluginStart;
private boolean defaultForceUpdate;
+ private int maxRefsToLog;
private int sshCommandTimeout;
private int sshConnectionTimeout = DEFAULT_SSH_CONNECTION_TIMEOUT_MS;
private final FileBasedConfig config;
@@ -117,6 +128,8 @@
defaultForceUpdate = config.getBoolean("gerrit", "defaultForceUpdate", false);
+ maxRefsToLog = config.getInt("gerrit", "maxRefsToLog", 0);
+
sshCommandTimeout =
(int) ConfigUtil.getTimeUnit(config, "gerrit", null, "sshCommandTimeout", 0, SECONDS);
sshConnectionTimeout =
@@ -167,6 +180,77 @@
return dest.build();
}
+ @Override
+ public Multimap<Destination, URIish> getURIs(
+ Optional<String> remoteName, Project.NameKey projectName, FilterType filterType) {
+ if (getDestinations(filterType).isEmpty()) {
+ return ImmutableMultimap.of();
+ }
+
+ SetMultimap<Destination, URIish> uris = HashMultimap.create();
+ for (Destination config : getDestinations(filterType)) {
+ if (!config.wouldPushProject(projectName)) {
+ continue;
+ }
+
+ if (remoteName.isPresent() && !config.getRemoteConfigName().equals(remoteName.get())) {
+ continue;
+ }
+
+ boolean adminURLUsed = false;
+
+ for (String url : config.getAdminUrls()) {
+ if (Strings.isNullOrEmpty(url)) {
+ continue;
+ }
+
+ URIish uri;
+ try {
+ uri = new URIish(url);
+ } catch (URISyntaxException e) {
+ repLog.warn("adminURL '{}' is invalid: {}", url, e.getMessage());
+ continue;
+ }
+
+ if (!isGerrit(uri) && !isGerritHttp(uri)) {
+ String path =
+ replaceName(uri.getPath(), projectName.get(), config.isSingleProjectMatch());
+ if (path == null) {
+ repLog.warn("adminURL {} does not contain ${name}", uri);
+ continue;
+ }
+
+ uri = uri.setPath(path);
+ if (!isSSH(uri)) {
+ repLog.warn("adminURL '{}' is invalid: only SSH and HTTP are supported", uri);
+ continue;
+ }
+ }
+ uris.put(config, uri);
+ adminURLUsed = true;
+ }
+
+ if (!adminURLUsed) {
+ for (URIish uri : config.getURIs(projectName, "*")) {
+ uris.put(config, uri);
+ }
+ }
+ }
+ return uris;
+ }
+
+ static String replaceName(String in, String name, boolean keyIsOptional) {
+ String key = "${name}";
+ int n = in.indexOf(key);
+ if (0 <= n) {
+ return in.substring(0, n) + name + in.substring(n + key.length());
+ }
+ if (keyIsOptional) {
+ return in;
+ }
+ return null;
+ }
+
/* (non-Javadoc)
* @see com.googlesource.gerrit.plugins.replication.ReplicationConfig#isReplicateAllOnPluginStart()
*/
@@ -183,6 +267,11 @@
return defaultForceUpdate;
}
+ @Override
+ public int getMaxRefsToLog() {
+ return maxRefsToLog;
+ }
+
private static List<RemoteConfig> allRemotes(FileBasedConfig cfg) throws ConfigInvalidException {
Set<String> names = cfg.getSubsections("remote");
List<RemoteConfig> result = Lists.newArrayListWithCapacity(names.size());
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
index 5fdb375..835d068 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
@@ -22,7 +22,6 @@
import com.google.gerrit.extensions.events.HeadUpdatedListener;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.extensions.events.ProjectDeletedListener;
-import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.extensions.registration.DynamicSet;
import com.google.gerrit.server.events.EventTypes;
import com.google.inject.AbstractModule;
@@ -67,10 +66,6 @@
EventTypes.register(ReplicationScheduledEvent.TYPE, ReplicationScheduledEvent.class);
bind(SshSessionFactory.class).toProvider(ReplicationSshSessionFactoryProvider.class);
- DynamicItem.itemOf(binder(), AdminApiFactory.class);
- DynamicItem.bind(binder(), AdminApiFactory.class)
- .to(AdminApiFactory.DefaultAdminApiFactory.class);
-
bind(TransportFactory.class).to(TransportFactoryImpl.class).in(Scopes.SINGLETON);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
index e494c24..cad7623 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
@@ -14,29 +14,22 @@
package com.googlesource.gerrit.plugins.replication;
-import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isGerrit;
-import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isSSH;
-
+import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Objects;
-import com.google.common.base.Strings;
import com.google.common.collect.Queues;
-import com.google.gerrit.common.Nullable;
+import com.google.gerrit.common.UsedAt;
import com.google.gerrit.extensions.events.GitReferenceUpdatedListener;
import com.google.gerrit.extensions.events.HeadUpdatedListener;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.extensions.events.ProjectDeletedListener;
import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.reviewdb.client.Project;
-import com.google.gerrit.server.UsedAt;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.git.WorkQueue;
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdateProcessing;
import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
import com.googlesource.gerrit.plugins.replication.ReplicationTasksStorage.ReplicateRefUpdate;
-import java.net.URISyntaxException;
-import java.util.Collections;
import java.util.HashSet;
import java.util.Optional;
import java.util.Queue;
@@ -56,22 +49,9 @@
private final ReplicationStateListener stateLog;
- static String replaceName(String in, String name, boolean keyIsOptional) {
- String key = "${name}";
- int n = in.indexOf(key);
- if (0 <= n) {
- return in.substring(0, n) + name + in.substring(n + key.length());
- }
- if (keyIsOptional) {
- return in;
- }
- return null;
- }
-
private final WorkQueue workQueue;
private final DynamicItem<EventDispatcher> dispatcher;
private final ReplicationConfig config;
- private final DynamicItem<AdminApiFactory> adminApiFactory;
private final ReplicationTasksStorage replicationTasksStorage;
private volatile boolean running;
private volatile boolean replaying;
@@ -80,7 +60,6 @@
@Inject
ReplicationQueue(
WorkQueue wq,
- DynamicItem<AdminApiFactory> aaf,
ReplicationConfig rc,
DynamicItem<EventDispatcher> dis,
ReplicationStateListeners sl,
@@ -89,7 +68,6 @@
dispatcher = dis;
config = rc;
stateLog = sl;
- adminApiFactory = aaf;
replicationTasksStorage = rts;
beforeStartupEventsQueue = Queues.newConcurrentLinkedQueue();
}
@@ -156,7 +134,7 @@
stateLog.warn(
"Replication plugin did not finish startup before event, event replication is postponed",
state);
- beforeStartupEventsQueue.add(new ReferenceUpdatedEvent(projectName, refName));
+ beforeStartupEventsQueue.add(ReferenceUpdatedEvent.create(projectName, refName));
return;
}
@@ -195,6 +173,7 @@
}
private void firePendingEvents() {
+ replaying = true;
try {
Set<String> eventsReplayed = new HashSet<>();
replaying = true;
@@ -217,163 +196,39 @@
@Override
public void onProjectDeleted(ProjectDeletedListener.Event event) {
- Project.NameKey projectName = new Project.NameKey(event.getProjectName());
- for (URIish uri : getURIs(null, projectName, FilterType.PROJECT_DELETION)) {
- deleteProject(uri, projectName);
- }
+ Project.NameKey p = new Project.NameKey(event.getProjectName());
+ config.getURIs(Optional.empty(), p, FilterType.PROJECT_DELETION).entries().stream()
+ .forEach(e -> e.getKey().scheduleDeleteProject(e.getValue(), p));
}
@Override
public void onHeadUpdated(HeadUpdatedListener.Event event) {
- Project.NameKey project = new Project.NameKey(event.getProjectName());
- for (URIish uri : getURIs(null, project, FilterType.ALL)) {
- updateHead(uri, project, event.getNewHeadName());
- }
+ Project.NameKey p = new Project.NameKey(event.getProjectName());
+ config.getURIs(Optional.empty(), p, FilterType.ALL).entries().stream()
+ .forEach(e -> e.getKey().scheduleUpdateHead(e.getValue(), p, event.getNewHeadName()));
}
private void fireBeforeStartupEvents() {
Set<String> eventsReplayed = new HashSet<>();
for (ReferenceUpdatedEvent event : beforeStartupEventsQueue) {
- String eventKey = String.format("%s:%s", event.getProjectName(), event.getRefName());
+ String eventKey = String.format("%s:%s", event.projectName(), event.refName());
if (!eventsReplayed.contains(eventKey)) {
repLog.info("Firing pending task {}", event);
- onGitReferenceUpdated(event.getProjectName(), event.getRefName());
+ onGitReferenceUpdated(event.projectName(), event.refName());
eventsReplayed.add(eventKey);
}
}
}
- private Set<URIish> getURIs(
- @Nullable String remoteName, Project.NameKey projectName, FilterType filterType) {
- if (config.getDestinations(filterType).isEmpty()) {
- return Collections.emptySet();
- }
- if (!running) {
- repLog.error("Replication plugin did not finish startup before event");
- return Collections.emptySet();
+ @AutoValue
+ abstract static class ReferenceUpdatedEvent {
+
+ static ReferenceUpdatedEvent create(String projectName, String refName) {
+ return new AutoValue_ReplicationQueue_ReferenceUpdatedEvent(projectName, refName);
}
- Set<URIish> uris = new HashSet<>();
- for (Destination config : this.config.getDestinations(filterType)) {
- if (!config.wouldPushProject(projectName)) {
- continue;
- }
+ public abstract String projectName();
- if (remoteName != null && !config.getRemoteConfigName().equals(remoteName)) {
- continue;
- }
-
- boolean adminURLUsed = false;
-
- for (String url : config.getAdminUrls()) {
- if (Strings.isNullOrEmpty(url)) {
- continue;
- }
-
- URIish uri;
- try {
- uri = new URIish(url);
- } catch (URISyntaxException e) {
- repLog.warn("adminURL '{}' is invalid: {}", url, e.getMessage());
- continue;
- }
-
- if (!isGerrit(uri)) {
- String path =
- replaceName(uri.getPath(), projectName.get(), config.isSingleProjectMatch());
- if (path == null) {
- repLog.warn("adminURL {} does not contain ${name}", uri);
- continue;
- }
-
- uri = uri.setPath(path);
- if (!isSSH(uri)) {
- repLog.warn("adminURL '{}' is invalid: only SSH is supported", uri);
- continue;
- }
- }
- uris.add(uri);
- adminURLUsed = true;
- }
-
- if (!adminURLUsed) {
- for (URIish uri : config.getURIs(projectName, "*")) {
- uris.add(uri);
- }
- }
- }
- return uris;
- }
-
- public boolean createProject(String remoteName, Project.NameKey project, String head) {
- boolean success = true;
- for (URIish uri : getURIs(remoteName, project, FilterType.PROJECT_CREATION)) {
- success &= createProject(uri, project, head);
- }
- return success;
- }
-
- private boolean createProject(URIish replicateURI, Project.NameKey projectName, String head) {
- Optional<AdminApi> adminApi = adminApiFactory.get().create(replicateURI);
- if (adminApi.isPresent() && adminApi.get().createProject(projectName, head)) {
- return true;
- }
-
- warnCannotPerform("create new project", replicateURI);
- return false;
- }
-
- private void deleteProject(URIish replicateURI, Project.NameKey projectName) {
- Optional<AdminApi> adminApi = adminApiFactory.get().create(replicateURI);
- if (adminApi.isPresent()) {
- adminApi.get().deleteProject(projectName);
- return;
- }
-
- warnCannotPerform("delete project", replicateURI);
- }
-
- private void updateHead(URIish replicateURI, Project.NameKey projectName, String newHead) {
- Optional<AdminApi> adminApi = adminApiFactory.get().create(replicateURI);
- if (adminApi.isPresent()) {
- adminApi.get().updateHead(projectName, newHead);
- return;
- }
-
- warnCannotPerform("update HEAD of project", replicateURI);
- }
-
- private void warnCannotPerform(String op, URIish uri) {
- repLog.warn("Cannot {} on remote site {}.", op, uri);
- }
-
- private static class ReferenceUpdatedEvent {
- private String projectName;
- private String refName;
-
- public ReferenceUpdatedEvent(String projectName, String refName) {
- this.projectName = projectName;
- this.refName = refName;
- }
-
- public String getProjectName() {
- return projectName;
- }
-
- public String getRefName() {
- return refName;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(projectName, refName);
- }
-
- @Override
- public boolean equals(Object obj) {
- return (obj instanceof ReferenceUpdatedEvent)
- && Objects.equal(projectName, ((ReferenceUpdatedEvent) obj).projectName)
- && Objects.equal(refName, ((ReferenceUpdatedEvent) obj).refName);
- }
+ public abstract String refName();
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java
index c518091..18a4cc2 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java
@@ -21,6 +21,8 @@
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.transport.CredentialsProvider;
+import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.eclipse.jgit.util.FS;
/** Looks up a remote's password in secure.config. */
@@ -49,9 +51,9 @@
}
@Override
- public SecureCredentialsProvider create(String remoteName) {
+ public CredentialsProvider create(String remoteName) {
String user = Objects.toString(config.getString("remote", remoteName, "username"), "");
String pass = Objects.toString(config.getString("remote", remoteName, "password"), "");
- return new SecureCredentialsProvider(user, pass);
+ return new UsernamePasswordCredentialsProvider(user, pass);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java
deleted file mode 100644
index 62b4036..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java
+++ /dev/null
@@ -1,80 +0,0 @@
-// Copyright (C) 2011 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.replication;
-
-import org.eclipse.jgit.errors.UnsupportedCredentialItem;
-import org.eclipse.jgit.transport.CredentialItem;
-import org.eclipse.jgit.transport.CredentialsProvider;
-import org.eclipse.jgit.transport.URIish;
-
-/** Looks up a remote's password in secure.config. */
-public class SecureCredentialsProvider extends CredentialsProvider {
- private final String cfgUser;
- private final String cfgPass;
-
- SecureCredentialsProvider(String user, String pass) {
- cfgUser = user;
- cfgPass = pass;
- }
-
- @Override
- public boolean isInteractive() {
- return false;
- }
-
- @Override
- public boolean supports(CredentialItem... items) {
- for (CredentialItem i : items) {
- if (i instanceof CredentialItem.Username) {
- continue;
- } else if (i instanceof CredentialItem.Password) {
- continue;
- } else {
- return false;
- }
- }
- return true;
- }
-
- @Override
- public boolean get(URIish uri, CredentialItem... items) throws UnsupportedCredentialItem {
- String username = uri.getUser();
- if (username == null) {
- username = cfgUser;
- }
- if (username == null) {
- return false;
- }
-
- String password = uri.getPass();
- if (password == null) {
- password = cfgPass;
- }
- if (password == null) {
- return false;
- }
-
- for (CredentialItem i : items) {
- if (i instanceof CredentialItem.Username) {
- ((CredentialItem.Username) i).setValue(username);
- } else if (i instanceof CredentialItem.Password) {
- ((CredentialItem.Password) i).setValue(password.toCharArray());
- } else {
- throw new UnsupportedCredentialItem(uri, i.getPromptText());
- }
- }
- return true;
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java
new file mode 100644
index 0000000..70452b4
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java
@@ -0,0 +1,70 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
+import com.google.gerrit.extensions.registration.DynamicItem;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.server.ioutil.HexFormat;
+import com.google.gerrit.server.util.IdGenerator;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import java.util.Optional;
+import org.eclipse.jgit.transport.URIish;
+
+public class UpdateHeadTask implements Runnable {
+ private final DynamicItem<AdminApiFactory> adminApiFactory;
+ private final int id;
+ private final URIish replicateURI;
+ private final Project.NameKey project;
+ private final String newHead;
+
+ interface Factory {
+ UpdateHeadTask create(URIish uri, Project.NameKey project, String newHead);
+ }
+
+ @Inject
+ UpdateHeadTask(
+ DynamicItem<AdminApiFactory> adminApiFactory,
+ IdGenerator ig,
+ @Assisted URIish replicateURI,
+ @Assisted Project.NameKey project,
+ @Assisted String newHead) {
+ this.adminApiFactory = adminApiFactory;
+ this.id = ig.next();
+ this.replicateURI = replicateURI;
+ this.project = project;
+ this.newHead = newHead;
+ }
+
+ @Override
+ public void run() {
+ Optional<AdminApi> adminApi = adminApiFactory.get().create(replicateURI);
+ if (adminApi.isPresent()) {
+ adminApi.get().updateHead(project, newHead);
+ return;
+ }
+
+ repLog.warn("Cannot update HEAD of project {} on remote site {}.", project, replicateURI);
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "[%s] update-head of %s at %s to %s",
+ HexFormat.fromInt(id), project.get(), replicateURI, newHead);
+ }
+}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 1309aeb..194238c 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -82,6 +82,10 @@
: If true, the default push refspec will be set to use forced
update to the remote when no refspec is given. By default, false.
+gerrit.maxRefsToLog
+: Number of refs, that are pushed during replication, to be logged.
+ For printing all refs to the logs, use a value of 0. By default, 0.
+
gerrit.sshCommandTimeout
: Timeout for SSH command execution. If 0, there is no timeout and
the client waits indefinitely. By default, 0.
@@ -177,6 +181,21 @@
local environment. In that case, an alternative SSH url could
be specified to repository creation.
+ To enable replication to different Gerrit instance use
+ `gerrit+http://` or `gerrit+https://` as protocol name followed
+ by hostname of another Gerrit server eg.
+
+ `gerrit+http://replica2.my.org/`
+ <br>
+ `gerrit+https://replica3.my.org/`
+
+ In this case replication will use Gerrit's REST API
+ to create/remove projects and update repository HEAD references.
+
+ NOTE: In order to replicate project deletion, the
+ link:https://gerrit-review.googlesource.com/admin/projects/plugins/delete-project delete-project[delete-project]
+ plugin must be installed on the other Gerrit.
+
*Backward compatibility notice*
Before Gerrit v2.13 it was possible to enable replication to different
@@ -186,10 +205,7 @@
`gerrit+ssh://replica1.my.org/`
In that case replication would have used Gerrit's SSH API to
- create/remove projects and update repository HEAD references
- and, in order to replicate project deletion, the
- link:https://gerrit-review.googlesource.com/admin/projects/plugins/delete-project delete-project[delete-project]
- plugin was needed to be installed on the other Gerrit.
+ create/remove projects and update repository HEAD references.
The `gerrit+ssh` option is kept for backward compatibility, however
the use-case behind it is not valid anymore since the introduction of
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/AbstractConfigTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/AbstractConfigTest.java
new file mode 100644
index 0000000..77dc1cc
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/AbstractConfigTest.java
@@ -0,0 +1,116 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.google.common.truth.Truth.assertThat;
+import static java.nio.file.Files.createTempDirectory;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.getCurrentArguments;
+import static org.easymock.EasyMock.isA;
+import static org.easymock.EasyMock.replay;
+
+import com.google.gerrit.server.config.SitePaths;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.easymock.IAnswer;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
+import org.junit.Before;
+import org.junit.Ignore;
+
+@Ignore
+public abstract class AbstractConfigTest {
+ protected final Path sitePath;
+ protected final SitePaths sitePaths;
+ protected final Destination.Factory destinationFactoryMock;
+ protected final Path pluginDataPath;
+
+ static class FakeDestination extends Destination {
+ public final DestinationConfiguration config;
+
+ protected FakeDestination(DestinationConfiguration config) {
+ super(injectorMock(), null, null, null, null, null, null, null, null, null, null, config);
+ this.config = config;
+ }
+
+ private static Injector injectorMock() {
+ Injector injector = createNiceMock(Injector.class);
+ Injector childInjectorMock = createNiceMock(Injector.class);
+ expect(injector.createChildInjector((Module) anyObject())).andReturn(childInjectorMock);
+ replay(childInjectorMock);
+ replay(injector);
+ return injector;
+ }
+ }
+
+ AbstractConfigTest() throws IOException {
+ sitePath = createTempPath("site");
+ sitePaths = new SitePaths(sitePath);
+ pluginDataPath = createTempPath("data");
+ destinationFactoryMock = createMock(Destination.Factory.class);
+ }
+
+ @Before
+ public void setup() {
+ expect(destinationFactoryMock.create(isA(DestinationConfiguration.class)))
+ .andAnswer(
+ new IAnswer<Destination>() {
+ @Override
+ public Destination answer() throws Throwable {
+ return new FakeDestination((DestinationConfiguration) getCurrentArguments()[0]);
+ }
+ })
+ .anyTimes();
+ replay(destinationFactoryMock);
+ }
+
+ protected static Path createTempPath(String prefix) throws IOException {
+ return createTempDirectory(prefix);
+ }
+
+ protected FileBasedConfig newReplicationConfig() {
+ FileBasedConfig replicationConfig =
+ new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED);
+ return replicationConfig;
+ }
+
+ protected void assertThatIsDestination(
+ Destination destination, String remoteName, String... remoteUrls) {
+ DestinationConfiguration destinationConfig = ((FakeDestination) destination).config;
+ assertThat(destinationConfig.getRemoteConfig().getName()).isEqualTo(remoteName);
+ assertThat(destinationConfig.getUrls()).containsExactlyElementsIn(remoteUrls);
+ }
+
+ protected void assertThatContainsDestination(
+ List<Destination> destinations, String remoteName, String... remoteUrls) {
+ List<Destination> matchingDestinations =
+ destinations.stream()
+ .filter(
+ (Destination dst) ->
+ ((FakeDestination) dst).config.getRemoteConfig().getName().equals(remoteName))
+ .collect(Collectors.toList());
+
+ assertThat(matchingDestinations).isNotEmpty();
+
+ assertThatIsDestination(matchingDestinations.get(0), remoteName, remoteUrls);
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecoratorTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecoratorTest.java
new file mode 100644
index 0000000..211cafa
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecoratorTest.java
@@ -0,0 +1,252 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.easymock.EasyMock.anyInt;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+
+import com.google.gerrit.server.git.WorkQueue;
+import com.google.inject.util.Providers;
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.junit.Before;
+import org.junit.Test;
+
+public class AutoReloadConfigDecoratorTest extends AbstractConfigTest {
+ private AutoReloadConfigDecorator autoReloadConfig;
+ private ReplicationQueue replicationQueueMock;
+ private WorkQueue workQueueMock;
+ private FakeExecutorService executorService = new FakeExecutorService();
+
+ public class FakeExecutorService implements ScheduledExecutorService {
+ public Runnable refreshCommand;
+
+ @Override
+ public void shutdown() {}
+
+ @Override
+ public List<Runnable> shutdownNow() {
+ return null;
+ }
+
+ @Override
+ public boolean isShutdown() {
+ return false;
+ }
+
+ @Override
+ public boolean isTerminated() {
+ return false;
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+ return false;
+ }
+
+ @Override
+ public <T> Future<T> submit(Callable<T> task) {
+ return null;
+ }
+
+ @Override
+ public <T> Future<T> submit(Runnable task, T result) {
+ return null;
+ }
+
+ @Override
+ public Future<?> submit(Runnable task) {
+ return null;
+ }
+
+ @Override
+ public <T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks)
+ throws InterruptedException {
+ return null;
+ }
+
+ @Override
+ public <T> List<Future<T>> invokeAll(
+ Collection<? extends Callable<T>> tasks, long timeout, TimeUnit unit)
+ throws InterruptedException {
+ return null;
+ }
+
+ @Override
+ public <T> T invokeAny(Collection<? extends Callable<T>> tasks)
+ throws InterruptedException, ExecutionException {
+ return null;
+ }
+
+ @Override
+ public <T> T invokeAny(Collection<? extends Callable<T>> tasks, long timeout, TimeUnit unit)
+ throws InterruptedException, ExecutionException, TimeoutException {
+ return null;
+ }
+
+ @Override
+ public void execute(Runnable command) {}
+
+ @Override
+ public ScheduledFuture<?> schedule(Runnable command, long delay, TimeUnit unit) {
+ return null;
+ }
+
+ @Override
+ public <V> ScheduledFuture<V> schedule(Callable<V> callable, long delay, TimeUnit unit) {
+ return null;
+ }
+
+ @Override
+ public ScheduledFuture<?> scheduleAtFixedRate(
+ Runnable command, long initialDelay, long period, TimeUnit unit) {
+ refreshCommand = command;
+ return null;
+ }
+
+ @Override
+ public ScheduledFuture<?> scheduleWithFixedDelay(
+ Runnable command, long initialDelay, long delay, TimeUnit unit) {
+ return null;
+ }
+ }
+
+ public AutoReloadConfigDecoratorTest() throws IOException {
+ super();
+ }
+
+ @Override
+ @Before
+ public void setup() {
+ super.setup();
+
+ setupMocks();
+ }
+
+ private void setupMocks() {
+ replicationQueueMock = createNiceMock(ReplicationQueue.class);
+ expect(replicationQueueMock.isRunning()).andReturn(true);
+ replay(replicationQueueMock);
+
+ workQueueMock = createNiceMock(WorkQueue.class);
+ expect(workQueueMock.createQueue(anyInt(), anyObject(String.class))).andReturn(executorService);
+ replay(workQueueMock);
+ }
+
+ @Test
+ public void shouldLoadNotEmptyInitialReplicationConfig() throws Exception {
+ FileBasedConfig replicationConfig = newReplicationConfig();
+ String remoteName = "foo";
+ String remoteUrl = "ssh://git@git.somewhere.com/${name}";
+ replicationConfig.setString("remote", remoteName, "url", remoteUrl);
+ replicationConfig.save();
+
+ autoReloadConfig =
+ new AutoReloadConfigDecorator(
+ sitePaths,
+ destinationFactoryMock,
+ Providers.of(replicationQueueMock),
+ pluginDataPath,
+ "replication",
+ workQueueMock);
+
+ List<Destination> destinations = autoReloadConfig.getDestinations(FilterType.ALL);
+ assertThat(destinations).hasSize(1);
+ assertThatIsDestination(destinations.get(0), remoteName, remoteUrl);
+ }
+
+ @Test
+ public void shouldAutoReloadReplicationConfig() throws Exception {
+ FileBasedConfig replicationConfig = newReplicationConfig();
+ replicationConfig.setBoolean("gerrit", null, "autoReload", true);
+ String remoteName1 = "foo";
+ String remoteUrl1 = "ssh://git@git.foo.com/${name}";
+ replicationConfig.setString("remote", remoteName1, "url", remoteUrl1);
+ replicationConfig.save();
+
+ autoReloadConfig =
+ new AutoReloadConfigDecorator(
+ sitePaths,
+ destinationFactoryMock,
+ Providers.of(replicationQueueMock),
+ pluginDataPath,
+ "replication",
+ workQueueMock);
+ autoReloadConfig.startup(workQueueMock);
+
+ List<Destination> destinations = autoReloadConfig.getDestinations(FilterType.ALL);
+ assertThat(destinations).hasSize(1);
+ assertThatIsDestination(destinations.get(0), remoteName1, remoteUrl1);
+
+ TimeUnit.SECONDS.sleep(1); // Allow the filesystem to change the update TS
+
+ String remoteName2 = "bar";
+ String remoteUrl2 = "ssh://git@git.bar.com/${name}";
+ replicationConfig.setString("remote", remoteName2, "url", remoteUrl2);
+ replicationConfig.save();
+ executorService.refreshCommand.run();
+
+ destinations = autoReloadConfig.getDestinations(FilterType.ALL);
+ assertThat(destinations).hasSize(2);
+ assertThatContainsDestination(destinations, remoteName1, remoteUrl1);
+ assertThatContainsDestination(destinations, remoteName2, remoteUrl2);
+ }
+
+ @Test
+ public void shouldNotAutoReloadReplicationConfigIfDisabled() throws Exception {
+ String remoteName1 = "foo";
+ String remoteUrl1 = "ssh://git@git.foo.com/${name}";
+ FileBasedConfig replicationConfig = newReplicationConfig();
+ replicationConfig.setBoolean("gerrit", null, "autoReload", false);
+ replicationConfig.setString("remote", remoteName1, "url", remoteUrl1);
+ replicationConfig.save();
+
+ autoReloadConfig =
+ new AutoReloadConfigDecorator(
+ sitePaths,
+ destinationFactoryMock,
+ Providers.of(replicationQueueMock),
+ pluginDataPath,
+ "replication",
+ workQueueMock);
+ autoReloadConfig.startup(workQueueMock);
+
+ List<Destination> destinations = autoReloadConfig.getDestinations(FilterType.ALL);
+ assertThat(destinations).hasSize(1);
+ assertThatIsDestination(destinations.get(0), remoteName1, remoteUrl1);
+
+ TimeUnit.SECONDS.sleep(1); // Allow the filesystem to change the update TS
+
+ replicationConfig.setString("remote", "bar", "url", "ssh://git@git.bar.com/${name}");
+ replicationConfig.save();
+ executorService.refreshCommand.run();
+
+ assertThat(autoReloadConfig.getDestinations(FilterType.ALL)).isEqualTo(destinations);
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
index 5fa7b98..0f6d629 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
@@ -15,20 +15,13 @@
package com.googlesource.gerrit.plugins.replication;
import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.reset;
import static org.easymock.EasyMock.verify;
-import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.permissions.PermissionBackendException;
-import com.google.gwtorm.client.KeyUtil;
-import com.google.gwtorm.server.OrmException;
-import com.google.gwtorm.server.SchemaFactory;
-import com.google.gwtorm.server.StandardKeyEncoder;
import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdateProcessing;
import com.googlesource.gerrit.plugins.replication.ReplicationState.RefPushResult;
import java.net.URISyntaxException;
@@ -37,12 +30,7 @@
import org.junit.Before;
import org.junit.Test;
-@SuppressWarnings("unchecked")
public class GitUpdateProcessingTest {
- static {
- KeyUtil.setEncoderImpl(new StandardKeyEncoder());
- }
-
private EventDispatcher dispatcherMock;
private GitUpdateProcessing gitUpdateProcessing;
@@ -50,17 +38,11 @@
public void setUp() throws Exception {
dispatcherMock = createMock(EventDispatcher.class);
replay(dispatcherMock);
- ReviewDb reviewDbMock = createNiceMock(ReviewDb.class);
- replay(reviewDbMock);
- SchemaFactory<ReviewDb> schemaMock = createMock(SchemaFactory.class);
- expect(schemaMock.open()).andReturn(reviewDbMock).anyTimes();
- replay(schemaMock);
gitUpdateProcessing = new GitUpdateProcessing(dispatcherMock);
}
@Test
- public void headRefReplicated()
- throws URISyntaxException, OrmException, PermissionBackendException {
+ public void headRefReplicated() throws URISyntaxException, PermissionBackendException {
reset(dispatcherMock);
RefReplicatedEvent expectedEvent =
new RefReplicatedEvent(
@@ -83,8 +65,7 @@
}
@Test
- public void changeRefReplicated()
- throws URISyntaxException, OrmException, PermissionBackendException {
+ public void changeRefReplicated() throws URISyntaxException, PermissionBackendException {
reset(dispatcherMock);
RefReplicatedEvent expectedEvent =
new RefReplicatedEvent(
@@ -107,7 +88,7 @@
}
@Test
- public void onAllNodesReplicated() throws OrmException, PermissionBackendException {
+ public void onAllNodesReplicated() throws PermissionBackendException {
reset(dispatcherMock);
RefReplicationDoneEvent expectedDoneEvent =
new RefReplicationDoneEvent("someProject", "refs/heads/master", 5);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java
index af065b3..c010ddb 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java
@@ -53,6 +53,7 @@
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdRef;
import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.RefDatabase;
import org.eclipse.jgit.lib.RefUpdate;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.storage.file.FileBasedConfig;
@@ -82,7 +83,6 @@
private RefSpec refSpecMock;
private CredentialsFactory credentialsFactory;
private PerThreadRequestScope.Scoper threadRequestScoperMock;
- private ReplicationQueue replicationQueueMock;
private IdGenerator idGeneratorMock;
private ReplicationStateListeners replicationStateListenersMock;
private ReplicationMetrics replicationMetricsMock;
@@ -94,10 +94,13 @@
private PushConnection pushConnection;
private ProjectState projectStateMock;
private RefUpdate refUpdateMock;
+ private CreateProjectTask.Factory createProjectTaskFactoryMock;
+ private ReplicationConfig replicationConfigMock;
+ private RefDatabase refDatabaseMock;
private Project.NameKey projectNameKey;
private URIish urish;
- private Map<String, Ref> localRefs;
+ private List<Ref> localRefs;
private Map<String, Ref> remoteRefs;
private CountDownLatch isCallFinished;
@@ -112,8 +115,7 @@
new ObjectIdRef.Unpeeled(
NEW, "foo", ObjectId.fromString("0000000000000000000000000000000000000001"));
- localRefs = new HashMap<>();
- localRefs.put("fooProject", newLocalRef);
+ localRefs = Arrays.asList(newLocalRef);
Ref remoteRef = new ObjectIdRef.Unpeeled(NEW, "foo", ObjectId.zeroId());
remoteRefs = new HashMap<>();
@@ -147,7 +149,6 @@
setupFetchConnectionMock();
setupPushConnectionMock();
setupRequestScopeMock();
- replicationQueueMock = createNiceMock(ReplicationQueue.class);
idGeneratorMock = createNiceMock(IdGenerator.class);
replicationStateListenersMock = createNiceMock(ReplicationStateListeners.class);
@@ -158,6 +159,8 @@
setupProjectCacheMock();
+ replicationConfigMock = createNiceMock(ReplicationConfig.class);
+
replay(
gitRepositoryManagerMock,
refUpdateMock,
@@ -167,7 +170,6 @@
remoteConfigMock,
credentialsFactory,
threadRequestScoperMock,
- replicationQueueMock,
idGeneratorMock,
replicationStateListenersMock,
replicationMetricsMock,
@@ -179,7 +181,9 @@
forProjectMock,
fetchConnection,
pushConnection,
- refSpecMock);
+ refSpecMock,
+ refDatabaseMock,
+ replicationConfigMock);
}
@Test
@@ -260,13 +264,14 @@
permissionBackendMock,
destinationMock,
remoteConfigMock,
+ replicationConfigMock,
credentialsFactory,
threadRequestScoperMock,
- replicationQueueMock,
idGeneratorMock,
replicationStateListenersMock,
replicationMetricsMock,
projectCacheMock,
+ createProjectTaskFactoryMock,
transportFactoryMock,
projectNameKey,
urish);
@@ -358,11 +363,12 @@
expect(gitRepositoryManagerMock.openRepository(projectNameKey)).andReturn(repositoryMock);
}
- @SuppressWarnings("deprecation")
private void setupRepositoryMock(FileBasedConfig config) throws IOException {
repositoryMock = createNiceMock(Repository.class);
+ refDatabaseMock = createNiceMock(RefDatabase.class);
expect(repositoryMock.getConfig()).andReturn(config).anyTimes();
- expect(repositoryMock.getAllRefs()).andReturn(localRefs);
+ expect(repositoryMock.getRefDatabase()).andReturn(refDatabaseMock);
+ expect(refDatabaseMock.getRefs()).andReturn(localRefs);
expect(repositoryMock.updateRef("fooProject")).andReturn(refUpdateMock);
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfigTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfigTest.java
new file mode 100644
index 0000000..36cc209
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfigTest.java
@@ -0,0 +1,72 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
+import java.io.IOException;
+import java.util.List;
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.junit.Test;
+
+public class ReplicationFileBasedConfigTest extends AbstractConfigTest {
+
+ public ReplicationFileBasedConfigTest() throws IOException {
+ super();
+ }
+
+ @Test
+ public void shouldLoadOneDestination() throws Exception {
+ String remoteName = "foo";
+ String remoteUrl = "ssh://git@git.somewhere.com/${name}";
+ FileBasedConfig config = newReplicationConfig();
+ config.setString("remote", remoteName, "url", remoteUrl);
+ config.save();
+
+ ReplicationFileBasedConfig replicationConfig = newReplicationFileBasedConfig();
+ List<Destination> destinations = replicationConfig.getDestinations(FilterType.ALL);
+ assertThat(destinations).hasSize(1);
+
+ assertThatIsDestination(destinations.get(0), remoteName, remoteUrl);
+ }
+
+ @Test
+ public void shouldLoadTwoDestinations() throws Exception {
+ String remoteName1 = "foo";
+ String remoteUrl1 = "ssh://git@git.somewhere.com/${name}";
+ String remoteName2 = "bar";
+ String remoteUrl2 = "ssh://git@git.elsewhere.com/${name}";
+ FileBasedConfig config = newReplicationConfig();
+ config.setString("remote", remoteName1, "url", remoteUrl1);
+ config.setString("remote", remoteName2, "url", remoteUrl2);
+ config.save();
+
+ ReplicationFileBasedConfig replicationConfig = newReplicationFileBasedConfig();
+ List<Destination> destinations = replicationConfig.getDestinations(FilterType.ALL);
+ assertThat(destinations).hasSize(2);
+
+ assertThatIsDestination(destinations.get(0), remoteName1, remoteUrl1);
+ assertThatIsDestination(destinations.get(1), remoteName2, remoteUrl2);
+ }
+
+ private ReplicationFileBasedConfig newReplicationFileBasedConfig()
+ throws ConfigInvalidException, IOException {
+ ReplicationFileBasedConfig replicationConfig =
+ new ReplicationFileBasedConfig(sitePaths, destinationFactoryMock, pluginDataPath);
+ return replicationConfig;
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java
index cec57a7..62664b1 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java
@@ -24,6 +24,7 @@
import com.google.gerrit.acceptance.PushOneCommit.Result;
import com.google.gerrit.acceptance.TestPlugin;
import com.google.gerrit.acceptance.UseLocalDisk;
+import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
import com.google.gerrit.extensions.annotations.PluginData;
import com.google.gerrit.extensions.api.projects.BranchInput;
import com.google.gerrit.extensions.common.ProjectInfo;
@@ -61,6 +62,7 @@
private static final Duration TEST_TIMEOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 2);
@Inject private SitePaths sitePaths;
+ @Inject private ProjectOperations projectOperations;
private Path pluginDataDir;
private Path gitPath;
private Path storagePath;
@@ -105,7 +107,7 @@
@Test
public void shouldReplicateNewChangeRef() throws Exception {
- Project.NameKey targetProject = createTestProject("projectreplica");
+ Project.NameKey targetProject = createTestProject(project + "replica");
setReplicationDestination("foo", "replica", ALL_PROJECTS);
reloadConfig();
@@ -130,7 +132,7 @@
setReplicationDestination("foo", "replica", ALL_PROJECTS);
reloadConfig();
- Project.NameKey targetProject = createTestProject("projectreplica");
+ Project.NameKey targetProject = createTestProject(project + "replica");
String newBranch = "refs/heads/mybranch";
String master = "refs/heads/master";
BranchInput input = new BranchInput();
@@ -152,8 +154,8 @@
@Test
public void shouldReplicateNewBranchToTwoRemotes() throws Exception {
- Project.NameKey targetProject1 = createTestProject("projectreplica1");
- Project.NameKey targetProject2 = createTestProject("projectreplica2");
+ Project.NameKey targetProject1 = createTestProject(project + "replica1");
+ Project.NameKey targetProject2 = createTestProject(project + "replica2");
setReplicationDestination("foo1", "replica1", ALL_PROJECTS);
setReplicationDestination("foo2", "replica2", ALL_PROJECTS);
@@ -184,8 +186,8 @@
@Test
public void shouldCreateIndividualReplicationTasksForEveryRemoteUrlPair() throws Exception {
List<String> replicaSuffixes = Arrays.asList("replica1", "replica2");
- createTestProject("projectreplica1");
- createTestProject("projectreplica2");
+ createTestProject(project + "replica1");
+ createTestProject(project + "replica2");
setReplicationDestination("foo1", replicaSuffixes, ALL_PROJECTS);
setReplicationDestination("foo2", replicaSuffixes, ALL_PROJECTS);
@@ -203,7 +205,7 @@
@Test
public void shouldCreateOneReplicationTaskWhenSchedulingRepoFullSync() throws Exception {
- createTestProject("projectreplica");
+ createTestProject(project + "replica");
setReplicationDestination("foo", "replica", ALL_PROJECTS);
reloadConfig();
@@ -218,7 +220,7 @@
@Test
public void shouldMatchTemplatedURL() throws Exception {
- createTestProject("projectreplica");
+ createTestProject(project + "replica");
setReplicationDestination("foo", "replica", ALL_PROJECTS);
reloadConfig();
@@ -239,7 +241,7 @@
@Test
public void shouldMatchRealURL() throws Exception {
- createTestProject("projectreplica");
+ createTestProject(project + "replica");
setReplicationDestination("foo", "replica", ALL_PROJECTS);
reloadConfig();
@@ -264,7 +266,7 @@
setReplicationDestination("foo", "replica", ALL_PROJECTS);
reloadConfig();
- Project.NameKey targetProject = createTestProject("projectreplica");
+ Project.NameKey targetProject = createTestProject(project + "replica");
String newHead = "refs/heads/newhead";
String master = "refs/heads/master";
BranchInput input = new BranchInput();
@@ -284,7 +286,7 @@
@Test
public void shouldNotDrainTheQueueWhenReloading() throws Exception {
// Setup repo to replicate
- Project.NameKey targetProject = createTestProject("projectreplica");
+ Project.NameKey targetProject = createTestProject(project + "replica");
String remoteName = "doNotDrainQueue";
setReplicationDestination(remoteName, "replica", ALL_PROJECTS);
@@ -306,7 +308,7 @@
@Test
public void shouldDrainTheQueueWhenReloading() throws Exception {
// Setup repo to replicate
- Project.NameKey targetProject = createTestProject("projectreplica");
+ Project.NameKey targetProject = createTestProject(project + "replica");
String remoteName = "drainQueue";
setReplicationDestination(remoteName, "replica", ALL_PROJECTS);
@@ -328,8 +330,26 @@
}
}
- private Project.NameKey createTestProject(String name) throws Exception {
- return createProject(name);
+ @Test
+ public void shouldNotDropEventsWhenStarting() throws Exception {
+ Project.NameKey targetProject = createTestProject(project + "replica");
+
+ setReplicationDestination("foo", "replica", ALL_PROJECTS);
+ reloadConfig();
+
+ replicationQueueStop();
+ Result pushResult = createChange();
+ replicationQueueStart();
+
+ RevCommit sourceCommit = pushResult.getCommit();
+ String sourceRef = pushResult.getPatchSet().getRefName();
+
+ try (Repository repo = repoManager.openRepository(targetProject)) {
+ waitUntil(() -> checkedGetRef(repo, sourceRef) != null);
+ Ref targetBranchRef = getRef(repo, sourceRef);
+ assertThat(targetBranchRef).isNotNull();
+ assertThat(targetBranchRef.getObjectId()).isEqualTo(sourceCommit.getId());
+ }
}
private Ref getRef(Repository repo, String branchName) throws IOException {
@@ -369,11 +389,35 @@
}
private void reloadConfig() {
- plugin.getSysInjector().getInstance(AutoReloadConfigDecorator.class).forceReload();
+ getAutoReloadConfigDecoratorInstance().forceReload();
}
private void shutdownConfig() {
- plugin.getSysInjector().getInstance(AutoReloadConfigDecorator.class).shutdown();
+ getAutoReloadConfigDecoratorInstance().shutdown();
+ }
+
+ private void replicationQueueStart() {
+ getReplicationQueueInstance().start();
+ }
+
+ private void replicationQueueStop() {
+ getReplicationQueueInstance().stop();
+ }
+
+ private AutoReloadConfigDecorator getAutoReloadConfigDecoratorInstance() {
+ return getInstance(AutoReloadConfigDecorator.class);
+ }
+
+ private ReplicationQueue getReplicationQueueInstance() {
+ return getInstance(ReplicationQueue.class);
+ }
+
+ private <T> T getInstance(Class<T> classObj) {
+ return plugin.getSysInjector().getInstance(classObj);
+ }
+
+ private Project.NameKey createTestProject(String name) throws Exception {
+ return projectOperations.newProject().name(name).create();
}
private List<ReplicateRefUpdate> listReplicationTasks(String refRegex) {
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationQueueIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationQueueIT.java
deleted file mode 100644
index 1a5c6a1..0000000
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationQueueIT.java
+++ /dev/null
@@ -1,133 +0,0 @@
-// Copyright (C) 2019 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.replication;
-
-import static com.google.common.truth.Truth.assertThat;
-import static java.util.stream.Collectors.toList;
-
-import com.google.common.base.Stopwatch;
-import com.google.common.flogger.FluentLogger;
-import com.google.gerrit.acceptance.LightweightPluginDaemonTest;
-import com.google.gerrit.acceptance.PushOneCommit.Result;
-import com.google.gerrit.acceptance.TestPlugin;
-import com.google.gerrit.acceptance.UseLocalDisk;
-import com.google.gerrit.reviewdb.client.Project;
-import com.google.gerrit.server.config.SitePaths;
-import com.google.inject.Inject;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.time.Duration;
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Supplier;
-import org.eclipse.jgit.lib.Ref;
-import org.eclipse.jgit.lib.Repository;
-import org.eclipse.jgit.revwalk.RevCommit;
-import org.eclipse.jgit.storage.file.FileBasedConfig;
-import org.eclipse.jgit.util.FS;
-import org.junit.Test;
-
-@UseLocalDisk
-@TestPlugin(
- name = "replication",
- sysModule = "com.googlesource.gerrit.plugins.replication.ReplicationModule")
-public class ReplicationQueueIT extends LightweightPluginDaemonTest {
- private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
- private static final int TEST_REPLICATION_DELAY = 1;
- private static final Duration TEST_TIMEOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 2);
-
- @Inject private SitePaths sitePaths;
- private Path gitPath;
- private FileBasedConfig config;
-
- @Override
- public void setUpTestPlugin() throws Exception {
- gitPath = sitePaths.site_path.resolve("git");
- config =
- new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED);
-
- setReplicationDestination("foo", "replica");
- super.setUpTestPlugin();
- }
-
- @Test
- public void shouldNotDropEventsWhenStarting() throws Exception {
- Project.NameKey targetProject = createProject("projectreplica");
-
- replicationQueueStop();
- Result pushResult = createChange();
- replicationQueueStart();
-
- RevCommit sourceCommit = pushResult.getCommit();
- String sourceRef = pushResult.getPatchSet().getRefName();
-
- try (Repository repo = repoManager.openRepository(targetProject)) {
- waitUntil(() -> checkedGetRef(repo, sourceRef) != null);
- Ref targetBranchRef = getRef(repo, sourceRef);
- assertThat(targetBranchRef).isNotNull();
- assertThat(targetBranchRef.getObjectId()).isEqualTo(sourceCommit.getId());
- }
- }
-
- private Ref getRef(Repository repo, String branchName) throws IOException {
- return repo.getRefDatabase().exactRef(branchName);
- }
-
- private Ref checkedGetRef(Repository repo, String branchName) {
- try {
- return repo.getRefDatabase().exactRef(branchName);
- } catch (Exception e) {
- logger.atSevere().withCause(e).log("failed to get ref %s in repo %s", branchName, repo);
- return null;
- }
- }
-
- private void setReplicationDestination(String remoteName, String replicaSuffix)
- throws IOException {
- setReplicationDestination(remoteName, Arrays.asList(replicaSuffix));
- }
-
- private void setReplicationDestination(String remoteName, List<String> replicaSuffixes)
- throws IOException {
-
- List<String> replicaUrls =
- replicaSuffixes.stream()
- .map(suffix -> gitPath.resolve("${name}" + suffix + ".git").toString())
- .collect(toList());
- config.setStringList("remote", remoteName, "url", replicaUrls);
- config.setInt("remote", remoteName, "replicationDelay", TEST_REPLICATION_DELAY);
- config.save();
- }
-
- private void waitUntil(Supplier<Boolean> waitCondition) throws InterruptedException {
- Stopwatch stopwatch = Stopwatch.createStarted();
- while (!waitCondition.get()) {
- if (stopwatch.elapsed().compareTo(TEST_TIMEOUT) > 0) {
- throw new InterruptedException();
- }
- TimeUnit.MILLISECONDS.sleep(50);
- }
- }
-
- private void replicationQueueStart() {
- plugin.getSysInjector().getInstance(ReplicationQueue.class).start();
- }
-
- private void replicationQueueStop() {
- plugin.getSysInjector().getInstance(ReplicationQueue.class).stop();
- }
-}
