Merge branch 'stable-2.16' into stable-3.0
* stable-2.16:
Fix test build rule to include *IT tests
ReplicationIT: Do not use deprecated getRef()
First ReplicationIT test for new project and change
Revert "Delete event file only after replication completed for all destinations"
Adjust ReplicationIT to use ProjectOperations and compose the project name
accordingly.
Change-Id: I6cf94217e745da88b3343e2a8fd3d9a85884ca79
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java
index 528aff2..e5a1628 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java
@@ -23,10 +23,12 @@
public class AdminApiFactory {
private final SshHelper sshHelper;
+ private final GerritRestApi.Factory gerritRestApiFactory;
@Inject
- AdminApiFactory(SshHelper sshHelper) {
+ AdminApiFactory(SshHelper sshHelper, GerritRestApi.Factory gerritRestApiFactory) {
this.sshHelper = sshHelper;
+ this.gerritRestApiFactory = gerritRestApiFactory;
}
public Optional<AdminApi> create(URIish uri) {
@@ -36,6 +38,8 @@
return Optional.of(new LocalFS(uri));
} else if (isSSH(uri)) {
return Optional.of(new RemoteSsh(sshHelper, uri));
+ } else if (isGerritHttp(uri)) {
+ return Optional.of(gerritRestApiFactory.create(uri));
}
return Optional.empty();
}
@@ -58,4 +62,9 @@
}
return false;
}
+
+ public static boolean isGerritHttp(URIish uri) {
+ String scheme = uri.getScheme();
+ return scheme != null && scheme.toLowerCase().contains("gerrit+http");
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
index f30efc8..21c3960 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
@@ -14,9 +14,12 @@
package com.googlesource.gerrit.plugins.replication;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Multimap;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.common.FileUtil;
import com.google.gerrit.extensions.annotations.PluginData;
+import com.google.gerrit.extensions.annotations.PluginName;
+import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.git.WorkQueue;
import com.google.inject.Inject;
@@ -25,11 +28,18 @@
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.transport.URIish;
@Singleton
public class AutoReloadConfigDecorator implements ReplicationConfig {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+ private static final long RELOAD_DELAY = 120;
+ private static final long RELOAD_INTERVAL = 60;
private ReplicationFileBasedConfig currentConfig;
private long currentConfigTs;
@@ -41,13 +51,17 @@
// Use Provider<> instead of injecting the ReplicationQueue because of circular dependency with
// ReplicationConfig
private final Provider<ReplicationQueue> replicationQueue;
+ private final ScheduledExecutorService autoReloadExecutor;
+ private ScheduledFuture<?> autoReloadRunnable;
@Inject
public AutoReloadConfigDecorator(
SitePaths site,
Destination.Factory destinationFactory,
Provider<ReplicationQueue> replicationQueue,
- @PluginData Path pluginDataDir)
+ @PluginData Path pluginDataDir,
+ @PluginName String pluginName,
+ WorkQueue workQueue)
throws ConfigInvalidException, IOException {
this.site = site;
this.destinationFactory = destinationFactory;
@@ -55,6 +69,7 @@
this.currentConfig = loadConfig();
this.currentConfigTs = getLastModified(currentConfig);
this.replicationQueue = replicationQueue;
+ this.autoReloadExecutor = workQueue.createQueue(1, pluginName + "_auto-reload-config");
}
private static long getLastModified(ReplicationFileBasedConfig cfg) {
@@ -71,11 +86,16 @@
@Override
public synchronized List<Destination> getDestinations(FilterType filterType) {
- reloadIfNeeded();
return currentConfig.getDestinations(filterType);
}
- private void reloadIfNeeded() {
+ @Override
+ public synchronized Multimap<Destination, URIish> getURIs(
+ Optional<String> remoteName, Project.NameKey projectName, FilterType filterType) {
+ return currentConfig.getURIs(remoteName, projectName, filterType);
+ }
+
+ private synchronized void reloadIfNeeded() {
reload(false);
}
@@ -125,6 +145,11 @@
}
@Override
+ public synchronized int getMaxRefsToLog() {
+ return currentConfig.getMaxRefsToLog();
+ }
+
+ @Override
public synchronized boolean isEmpty() {
return currentConfig.isEmpty();
}
@@ -136,12 +161,19 @@
@Override
public synchronized int shutdown() {
+ if (autoReloadRunnable != null) {
+ autoReloadRunnable.cancel(false);
+ autoReloadRunnable = null;
+ }
return currentConfig.shutdown();
}
@Override
public synchronized void startup(WorkQueue workQueue) {
currentConfig.startup(workQueue);
+ autoReloadRunnable =
+ autoReloadExecutor.scheduleAtFixedRate(
+ this::reloadIfNeeded, RELOAD_DELAY, RELOAD_INTERVAL, TimeUnit.SECONDS);
}
@Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
index 29a7ee6..98f364d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
@@ -23,6 +23,7 @@
import java.nio.file.Files;
import java.util.concurrent.atomic.AtomicReference;
import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.transport.CredentialsProvider;
public class AutoReloadSecureCredentialsFactoryDecorator implements CredentialsFactory {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -50,7 +51,7 @@
}
@Override
- public SecureCredentialsProvider create(String remoteName) {
+ public CredentialsProvider create(String remoteName) {
try {
if (needsReload()) {
secureCredentialsFactory.compareAndSet(
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java
new file mode 100644
index 0000000..e3d57de
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java
@@ -0,0 +1,69 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
+import java.util.Optional;
+import org.eclipse.jgit.transport.RemoteConfig;
+import org.eclipse.jgit.transport.URIish;
+
+public class CreateProjectTask {
+ interface Factory {
+ CreateProjectTask create(Project.NameKey project, String head);
+ }
+
+ private final RemoteConfig config;
+ private final ReplicationConfig replicationConfig;
+ private final AdminApiFactory adminApiFactory;
+ private final Project.NameKey project;
+ private final String head;
+
+ @Inject
+ CreateProjectTask(
+ RemoteConfig config,
+ ReplicationConfig replicationConfig,
+ AdminApiFactory adminApiFactory,
+ @Assisted Project.NameKey project,
+ @Assisted String head) {
+ this.config = config;
+ this.replicationConfig = replicationConfig;
+ this.adminApiFactory = adminApiFactory;
+ this.project = project;
+ this.head = head;
+ }
+
+ public boolean create() {
+ return replicationConfig
+ .getURIs(Optional.of(config.getName()), project, FilterType.PROJECT_CREATION).values()
+ .stream()
+ .map(u -> createProject(u, project, head))
+ .reduce(true, (a, b) -> a && b);
+ }
+
+ private boolean createProject(URIish replicateURI, Project.NameKey projectName, String head) {
+ Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI);
+ if (adminApi.isPresent() && adminApi.get().createProject(projectName, head)) {
+ return true;
+ }
+
+ repLog.warn("Cannot create new project {} on remote site {}.", projectName, replicateURI);
+ return false;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java
index 10719c1..3bb64ab 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java
@@ -13,7 +13,9 @@
// limitations under the License.
package com.googlesource.gerrit.plugins.replication;
+import org.eclipse.jgit.transport.CredentialsProvider;
+
public interface CredentialsFactory {
- SecureCredentialsProvider create(String remoteName);
+ CredentialsProvider create(String remoteName);
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java
new file mode 100644
index 0000000..96d8c70
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java
@@ -0,0 +1,65 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.server.ioutil.HexFormat;
+import com.google.gerrit.server.util.IdGenerator;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import java.util.Optional;
+import org.eclipse.jgit.transport.URIish;
+
+public class DeleteProjectTask implements Runnable {
+ interface Factory {
+ DeleteProjectTask create(URIish replicateURI, Project.NameKey project);
+ }
+
+ private final AdminApiFactory adminApiFactory;
+ private final int id;
+ private final URIish replicateURI;
+ private final Project.NameKey project;
+
+ @Inject
+ DeleteProjectTask(
+ AdminApiFactory adminApiFactory,
+ IdGenerator ig,
+ @Assisted URIish replicateURI,
+ @Assisted Project.NameKey project) {
+ this.adminApiFactory = adminApiFactory;
+ this.id = ig.next();
+ this.replicateURI = replicateURI;
+ this.project = project;
+ }
+
+ @Override
+ public void run() {
+ Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI);
+ if (adminApi.isPresent()) {
+ adminApi.get().deleteProject(project);
+ return;
+ }
+
+ repLog.warn("Cannot delete project {} on remote site {}.", project, replicateURI);
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "[%s] delete-project %s at %s", HexFormat.fromInt(id), project.get(), replicateURI);
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
index 969f8e7..2677152 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
@@ -15,6 +15,7 @@
package com.googlesource.gerrit.plugins.replication;
import static com.googlesource.gerrit.plugins.replication.PushResultProcessing.resolveNodeName;
+import static com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig.replaceName;
import static org.eclipse.jgit.transport.RemoteRefUpdate.Status.NON_EXISTING;
import static org.eclipse.jgit.transport.RemoteRefUpdate.Status.REJECTED_OTHER_REASON;
@@ -32,14 +33,12 @@
import com.google.gerrit.reviewdb.client.Branch;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.client.RefNames;
-import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.PluginUser;
import com.google.gerrit.server.account.GroupBackend;
import com.google.gerrit.server.account.GroupBackends;
import com.google.gerrit.server.account.GroupIncludeCache;
import com.google.gerrit.server.account.ListGroupMembership;
-import com.google.gerrit.server.config.RequestScopedReviewDbProvider;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.PerThreadRequestScope;
@@ -56,6 +55,7 @@
import com.google.inject.Injector;
import com.google.inject.Provider;
import com.google.inject.Provides;
+import com.google.inject.Scopes;
import com.google.inject.assistedinject.Assisted;
import com.google.inject.assistedinject.FactoryModuleBuilder;
import com.google.inject.servlet.RequestScoped;
@@ -73,6 +73,7 @@
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import org.apache.commons.io.FilenameUtils;
+import org.apache.http.impl.client.CloseableHttpClient;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
@@ -94,6 +95,8 @@
private final Map<URIish, PushOne> pending = new HashMap<>();
private final Map<URIish, PushOne> inFlight = new HashMap<>();
private final PushOne.Factory opFactory;
+ private final DeleteProjectTask.Factory deleteProjectFactory;
+ private final UpdateHeadTask.Factory updateHeadFactory;
private final GitRepositoryManager gitManager;
private final PermissionBackend permissionBackend;
private final Provider<CurrentUser> userProvider;
@@ -167,24 +170,20 @@
bind(Destination.class).toInstance(Destination.this);
bind(RemoteConfig.class).toInstance(config.getRemoteConfig());
install(new FactoryModuleBuilder().build(PushOne.Factory.class));
+ install(new FactoryModuleBuilder().build(CreateProjectTask.Factory.class));
+ install(new FactoryModuleBuilder().build(DeleteProjectTask.Factory.class));
+ install(new FactoryModuleBuilder().build(UpdateHeadTask.Factory.class));
+ bind(AdminApiFactory.class);
+ install(new FactoryModuleBuilder().build(GerritRestApi.Factory.class));
+ bind(CloseableHttpClient.class)
+ .toProvider(HttpClientProvider.class)
+ .in(Scopes.SINGLETON);
}
@Provides
public PerThreadRequestScope.Scoper provideScoper(
- final PerThreadRequestScope.Propagator propagator,
- final Provider<RequestScopedReviewDbProvider> dbProvider) {
- final RequestContext requestContext =
- new RequestContext() {
- @Override
- public CurrentUser getUser() {
- return remoteUser;
- }
-
- @Override
- public Provider<ReviewDb> getReviewDbProvider() {
- return dbProvider.get();
- }
- };
+ final PerThreadRequestScope.Propagator propagator) {
+ final RequestContext requestContext = () -> remoteUser;
return new PerThreadRequestScope.Scoper() {
@Override
public <T> Callable<T> scope(Callable<T> callable) {
@@ -195,6 +194,8 @@
});
opFactory = child.getInstance(PushOne.Factory.class);
+ deleteProjectFactory = child.getInstance(DeleteProjectTask.Factory.class);
+ updateHeadFactory = child.getInstance(UpdateHeadTask.Factory.class);
threadScoper = child.getInstance(PerThreadRequestScope.Scoper.class);
}
@@ -225,24 +226,35 @@
public int shutdown() {
int cnt = 0;
if (pool != null) {
- repLog.warn("Cancelling replication events");
+ synchronized (stateLock) {
+ int numPending = pending.size();
+ int numInFlight = inFlight.size();
- foreachPushOp(
- pending,
- push -> {
- push.cancel();
- return null;
- });
- pending.clear();
- foreachPushOp(
- inFlight,
- push -> {
- push.setCanceledWhileRunning();
- return null;
- });
- inFlight.clear();
- cnt = pool.shutdownNow().size();
- pool = null;
+ if (numPending > 0 || numInFlight > 0) {
+ repLog.warn(
+ "Cancelling replication events (pending={}, inFlight={}) for destination {}",
+ numPending,
+ numInFlight,
+ getRemoteConfigName());
+
+ foreachPushOp(
+ pending,
+ push -> {
+ push.cancel();
+ return null;
+ });
+ pending.clear();
+ foreachPushOp(
+ inFlight,
+ push -> {
+ push.setCanceledWhileRunning();
+ return null;
+ });
+ inFlight.clear();
+ }
+ cnt = pool.shutdownNow().size();
+ pool = null;
+ }
}
return cnt;
}
@@ -279,38 +291,35 @@
try {
return threadScoper
.scope(
- new Callable<Boolean>() {
- @Override
- public Boolean call() throws NoSuchProjectException, PermissionBackendException {
- ProjectState projectState;
- try {
- projectState = projectCache.checkedGet(project);
- } catch (IOException e) {
- return false;
- }
- if (projectState == null) {
- throw new NoSuchProjectException(project);
- }
- if (!projectState.statePermitsRead()) {
- return false;
- }
- if (!shouldReplicate(projectState, userProvider.get())) {
- return false;
- }
- if (PushOne.ALL_REFS.equals(ref)) {
- return true;
- }
- try {
- permissionBackend
- .user(userProvider.get())
- .project(project)
- .ref(ref)
- .check(RefPermission.READ);
- } catch (AuthException e) {
- return false;
- }
+ () -> {
+ ProjectState projectState;
+ try {
+ projectState = projectCache.checkedGet(project);
+ } catch (IOException e) {
+ return false;
+ }
+ if (projectState == null) {
+ throw new NoSuchProjectException(project);
+ }
+ if (!projectState.statePermitsRead()) {
+ return false;
+ }
+ if (!shouldReplicate(projectState, userProvider.get())) {
+ return false;
+ }
+ if (PushOne.ALL_REFS.equals(ref)) {
return true;
}
+ try {
+ permissionBackend
+ .user(userProvider.get())
+ .project(project)
+ .ref(ref)
+ .check(RefPermission.READ);
+ } catch (AuthException e) {
+ return false;
+ }
+ return true;
})
.call();
} catch (NoSuchProjectException err) {
@@ -326,20 +335,17 @@
try {
return threadScoper
.scope(
- new Callable<Boolean>() {
- @Override
- public Boolean call() throws NoSuchProjectException, PermissionBackendException {
- ProjectState projectState;
- try {
- projectState = projectCache.checkedGet(project);
- } catch (IOException e) {
- return false;
- }
- if (projectState == null) {
- throw new NoSuchProjectException(project);
- }
- return shouldReplicate(projectState, userProvider.get());
+ () -> {
+ ProjectState projectState;
+ try {
+ projectState = projectCache.checkedGet(project);
+ } catch (IOException e) {
+ return false;
}
+ if (projectState == null) {
+ throw new NoSuchProjectException(project);
+ }
+ return shouldReplicate(projectState, userProvider.get());
})
.call();
} catch (NoSuchProjectException err) {
@@ -421,6 +427,18 @@
}
}
+ void scheduleDeleteProject(URIish uri, Project.NameKey project) {
+ @SuppressWarnings("unused")
+ ScheduledFuture<?> ignored =
+ pool.schedule(deleteProjectFactory.create(uri, project), 0, TimeUnit.SECONDS);
+ }
+
+ void scheduleUpdateHead(URIish uri, Project.NameKey project, String newHead) {
+ @SuppressWarnings("unused")
+ ScheduledFuture<?> ignored =
+ pool.schedule(updateHeadFactory.create(uri, project, newHead), 0, TimeUnit.SECONDS);
+ }
+
private void addRef(PushOne e, String ref) {
e.addRef(ref);
postReplicationScheduledEvent(e, ref);
@@ -621,8 +639,7 @@
} else if (!remoteNameStyle.equals("slash")) {
repLog.debug("Unknown remoteNameStyle: {}, falling back to slash", remoteNameStyle);
}
- String replacedPath =
- ReplicationQueue.replaceName(uri.getPath(), name, isSingleProjectMatch());
+ String replacedPath = replaceName(uri.getPath(), name, isSingleProjectMatch());
if (replacedPath != null) {
uri = uri.setPath(replacedPath);
r.add(uri);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java
new file mode 100644
index 0000000..55fbed1
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java
@@ -0,0 +1,132 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.googlesource.gerrit.plugins.replication.GerritSshApi.GERRIT_ADMIN_PROTOCOL_PREFIX;
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
+import com.google.common.base.Charsets;
+import com.google.gerrit.extensions.restapi.Url;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import java.io.IOException;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicHeader;
+import org.eclipse.jgit.transport.CredentialItem;
+import org.eclipse.jgit.transport.RemoteConfig;
+import org.eclipse.jgit.transport.URIish;
+
+public class GerritRestApi implements AdminApi {
+
+ public interface Factory {
+ GerritRestApi create(URIish uri);
+ }
+
+ private final CredentialsFactory credentials;
+ private final CloseableHttpClient httpClient;
+ private final RemoteConfig remoteConfig;
+ private final URIish uri;
+
+ @Inject
+ GerritRestApi(
+ CredentialsFactory credentials,
+ CloseableHttpClient httpClient,
+ RemoteConfig remoteConfig,
+ @Assisted URIish uri) {
+ this.credentials = credentials;
+ this.httpClient = httpClient;
+ this.remoteConfig = remoteConfig;
+ this.uri = uri;
+ }
+
+ @Override
+ public boolean createProject(Project.NameKey project, String head) {
+ repLog.info("Creating project {} on {}", project, uri);
+ String url = String.format("%s/a/projects/%s", toHttpUri(uri), Url.encode(project.get()));
+ try {
+ return httpClient
+ .execute(new HttpPut(url), new HttpResponseHandler(), getContext())
+ .isSuccessful();
+ } catch (IOException e) {
+ repLog.error("Couldn't perform project creation on {}", uri, e);
+ return false;
+ }
+ }
+
+ @Override
+ public void deleteProject(Project.NameKey project) {
+ repLog.info("Deleting project {} on {}", project, uri);
+ String url = String.format("%s/a/projects/%s", toHttpUri(uri), Url.encode(project.get()));
+ try {
+ httpClient.execute(new HttpDelete(url), new HttpResponseHandler(), getContext());
+ } catch (IOException e) {
+ repLog.error("Couldn't perform project deletion on {}", uri, e);
+ }
+ }
+
+ @Override
+ public void updateHead(Project.NameKey project, String newHead) {
+ repLog.info("Updating head of {} on {}", project, uri);
+ String url = String.format("%s/a/projects/%s/HEAD", toHttpUri(uri), Url.encode(project.get()));
+ try {
+ HttpPut req = new HttpPut(url);
+ req.setEntity(
+ new StringEntity(String.format("{\"ref\": \"%s\"}", newHead), Charsets.UTF_8.name()));
+ req.addHeader(new BasicHeader("Content-Type", "application/json"));
+ httpClient.execute(req, new HttpResponseHandler(), getContext());
+ } catch (IOException e) {
+ repLog.error("Couldn't perform update head on {}", uri, e);
+ }
+ }
+
+ private HttpClientContext getContext() {
+ HttpClientContext ctx = HttpClientContext.create();
+ ctx.setCredentialsProvider(adapt(credentials.create(remoteConfig.getName())));
+ return ctx;
+ }
+
+ private CredentialsProvider adapt(org.eclipse.jgit.transport.CredentialsProvider cp) {
+ CredentialItem.Username user = new CredentialItem.Username();
+ CredentialItem.Password pass = new CredentialItem.Password();
+ if (cp.supports(user, pass) && cp.get(uri, user, pass)) {
+ CredentialsProvider adapted = new BasicCredentialsProvider();
+ adapted.setCredentials(
+ AuthScope.ANY,
+ new UsernamePasswordCredentials(user.getValue(), new String(pass.getValue())));
+ return adapted;
+ }
+ return null;
+ }
+
+ private static String toHttpUri(URIish uri) {
+ String u = uri.toString();
+ if (u.startsWith(GERRIT_ADMIN_PROTOCOL_PREFIX)) {
+ u = u.substring(GERRIT_ADMIN_PROTOCOL_PREFIX.length());
+ }
+ if (u.endsWith("/")) {
+ return u.substring(0, u.length() - 1);
+ }
+ return u;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
index 0e8764a..9a92d4e 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
@@ -28,7 +28,7 @@
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
static int SSH_COMMAND_FAILED = -1;
- private static String GERRIT_ADMIN_PROTOCOL_PREFIX = "gerrit+";
+ static String GERRIT_ADMIN_PROTOCOL_PREFIX = "gerrit+";
protected final SshHelper sshHelper;
protected final URIish uri;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/HttpClientProvider.java b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpClientProvider.java
new file mode 100644
index 0000000..916059c
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpClientProvider.java
@@ -0,0 +1,108 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.ProvisionException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import javax.net.ssl.SSLContext;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.HttpClientConnectionManager;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.ssl.SSLContexts;
+import org.eclipse.jgit.lib.Config;
+
+/** Provides an HTTP client with SSL capabilities. */
+class HttpClientProvider implements Provider<CloseableHttpClient> {
+ private static final int CONNECTIONS_PER_ROUTE = 100;
+
+ // Up to 2 target instances with the max number of connections per host:
+ private static final int MAX_CONNECTIONS = 2 * CONNECTIONS_PER_ROUTE;
+
+ private static final int MAX_CONNECTION_INACTIVITY = 10000;
+ private static final int DEFAULT_TIMEOUT_MS = 5000;
+
+ private final Config cfg;
+ private final SitePaths site;
+
+ @Inject
+ HttpClientProvider(@GerritServerConfig Config cfg, SitePaths site) {
+ this.cfg = cfg;
+ this.site = site;
+ }
+
+ @Override
+ public CloseableHttpClient get() {
+ try {
+ return HttpClients.custom()
+ .setConnectionManager(customConnectionManager())
+ .setDefaultRequestConfig(customRequestConfig())
+ .build();
+ } catch (Exception e) {
+ throw new ProvisionException("Couldn't create CloseableHttpClient", e);
+ }
+ }
+
+ private RequestConfig customRequestConfig() {
+ return RequestConfig.custom()
+ .setConnectTimeout(DEFAULT_TIMEOUT_MS)
+ .setSocketTimeout(DEFAULT_TIMEOUT_MS)
+ .setConnectionRequestTimeout(DEFAULT_TIMEOUT_MS)
+ .build();
+ }
+
+ private HttpClientConnectionManager customConnectionManager() throws Exception {
+ Registry<ConnectionSocketFactory> socketFactoryRegistry =
+ RegistryBuilder.<ConnectionSocketFactory>create()
+ .register("https", buildSslSocketFactory())
+ .register("http", PlainConnectionSocketFactory.INSTANCE)
+ .build();
+ PoolingHttpClientConnectionManager connManager =
+ new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ connManager.setDefaultMaxPerRoute(CONNECTIONS_PER_ROUTE);
+ connManager.setMaxTotal(MAX_CONNECTIONS);
+ connManager.setValidateAfterInactivity(MAX_CONNECTION_INACTIVITY);
+ return connManager;
+ }
+
+ private SSLConnectionSocketFactory buildSslSocketFactory() throws Exception {
+ String keyStore = cfg.getString("httpd", null, "sslKeyStore");
+ if (keyStore == null) {
+ keyStore = "etc/keystore";
+ }
+ return new SSLConnectionSocketFactory(createSSLContext(site.resolve(keyStore)));
+ }
+
+ private SSLContext createSSLContext(Path keyStorePath) throws Exception {
+ SSLContext ctx;
+ if (Files.exists(keyStorePath)) {
+ ctx = SSLContexts.custom().loadTrustMaterial(keyStorePath.toFile()).build();
+ } else {
+ ctx = SSLContext.getDefault();
+ }
+ return ctx;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponse.java b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponse.java
new file mode 100644
index 0000000..595acc7
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponse.java
@@ -0,0 +1,68 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.base.Preconditions;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.ByteBuffer;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.eclipse.jgit.util.IO;
+import org.eclipse.jgit.util.RawParseUtils;
+
+public class HttpResponse implements AutoCloseable {
+
+ protected CloseableHttpResponse response;
+ protected Reader reader;
+
+ HttpResponse(CloseableHttpResponse response) {
+ this.response = response;
+ }
+
+ public Reader getReader() throws IllegalStateException, IOException {
+ if (reader == null && response.getEntity() != null) {
+ reader = new InputStreamReader(response.getEntity().getContent(), UTF_8);
+ }
+ return reader;
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ Reader reader = getReader();
+ if (reader != null) {
+ while (reader.read() != -1) {
+ // Empty
+ }
+ }
+ } finally {
+ response.close();
+ }
+ }
+
+ public int getStatusCode() {
+ return response.getStatusLine().getStatusCode();
+ }
+
+ public String getEntityContent() throws IOException {
+ Preconditions.checkNotNull(response, "Response is not initialized.");
+ Preconditions.checkNotNull(response.getEntity(), "Response.Entity is not initialized.");
+ ByteBuffer buf = IO.readWholeStream(response.getEntity().getContent(), 1024);
+ return RawParseUtils.decode(buf.array(), buf.arrayOffset(), buf.limit()).trim();
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponseHandler.java b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponseHandler.java
new file mode 100644
index 0000000..6a7c73e
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponseHandler.java
@@ -0,0 +1,71 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static javax.servlet.http.HttpServletResponse.SC_CREATED;
+import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+
+import com.google.common.flogger.FluentLogger;
+import com.googlesource.gerrit.plugins.replication.HttpResponseHandler.HttpResult;
+import java.io.IOException;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.ResponseHandler;
+import org.apache.http.util.EntityUtils;
+
+class HttpResponseHandler implements ResponseHandler<HttpResult> {
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+ static class HttpResult {
+ private final boolean successful;
+ private final String message;
+
+ HttpResult(boolean successful, String message) {
+ this.successful = successful;
+ this.message = message;
+ }
+
+ boolean isSuccessful() {
+ return successful;
+ }
+
+ String getMessage() {
+ return message;
+ }
+ }
+
+ @Override
+ public HttpResult handleResponse(HttpResponse response) {
+ return new HttpResult(isSuccessful(response), parseResponse(response));
+ }
+
+ private static boolean isSuccessful(HttpResponse response) {
+ int sc = response.getStatusLine().getStatusCode();
+ return sc == SC_CREATED || sc == SC_NO_CONTENT || sc == SC_OK;
+ }
+
+ private static String parseResponse(HttpResponse response) {
+ HttpEntity entity = response.getEntity();
+ if (entity != null) {
+ try {
+ return EntityUtils.toString(entity);
+ } catch (IOException e) {
+ logger.atSevere().withCause(e).log("Error parsing entity");
+ }
+ }
+ return "";
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
index 5794f6e..56cecfe 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
@@ -16,6 +16,7 @@
import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static java.util.stream.Collectors.toMap;
import com.google.common.base.Throwables;
import com.google.common.collect.LinkedListMultimap;
@@ -53,7 +54,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicBoolean;
import org.eclipse.jgit.errors.NoRemoteRepositoryException;
import org.eclipse.jgit.errors.NotSupportedException;
@@ -93,9 +93,9 @@
private final PermissionBackend permissionBackend;
private final Destination pool;
private final RemoteConfig config;
+ private final ReplicationConfig replConfig;
private final CredentialsProvider credentialsProvider;
private final PerThreadRequestScope.Scoper threadScoper;
- private final ReplicationQueue replicationQueue;
private final Project.NameKey projectName;
private final URIish uri;
@@ -113,6 +113,7 @@
private final long createdAt;
private final ReplicationMetrics metrics;
private final ProjectCache projectCache;
+ private final CreateProjectTask.Factory createProjectFactory;
private final AtomicBoolean canceledWhileRunning;
private final TransportFactory transportFactory;
private DynamicItem<ReplicationPushFilter> replicationPushFilter;
@@ -123,13 +124,14 @@
PermissionBackend permissionBackend,
Destination p,
RemoteConfig c,
+ ReplicationConfig rc,
CredentialsFactory cpFactory,
PerThreadRequestScope.Scoper ts,
- ReplicationQueue rq,
IdGenerator ig,
ReplicationStateListeners sl,
ReplicationMetrics m,
ProjectCache pc,
+ CreateProjectTask.Factory cpf,
TransportFactory tf,
@Assisted Project.NameKey d,
@Assisted URIish u) {
@@ -137,9 +139,9 @@
this.permissionBackend = permissionBackend;
pool = p;
config = c;
+ replConfig = rc;
credentialsProvider = cpFactory.create(c.getName());
threadScoper = ts;
- replicationQueue = rq;
projectName = d;
uri = u;
lockRetryCount = 0;
@@ -149,6 +151,7 @@
createdAt = System.nanoTime();
metrics = m;
projectCache = pc;
+ createProjectFactory = cpf;
canceledWhileRunning = new AtomicBoolean(false);
maxRetries = p.getMaxRetries();
transportFactory = tf;
@@ -292,12 +295,9 @@
try {
threadScoper
.scope(
- new Callable<Void>() {
- @Override
- public Void call() {
- runPushOperation();
- return null;
- }
+ () -> {
+ runPushOperation();
+ return null;
})
.call();
} catch (Exception e) {
@@ -419,8 +419,7 @@
if (pool.isCreateMissingRepos()) {
try {
Ref head = git.exactRef(Constants.HEAD);
- if (replicationQueue.createProject(
- config.getName(), projectName, head != null ? getName(head) : null)) {
+ if (createProject(projectName, head != null ? getName(head) : null)) {
repLog.warn("Missing repository created; retry replication to {}", uri);
pool.reschedule(this, Destination.RetryReason.REPOSITORY_MISSING);
} else {
@@ -437,6 +436,10 @@
}
}
+ private boolean createProject(Project.NameKey project, String head) {
+ return createProjectFactory.create(project, head).create();
+ }
+
private String getName(Ref ref) {
Ref target = ref;
while (target.isSymbolic()) {
@@ -467,7 +470,16 @@
return new PushResult();
}
- repLog.info("Push to {} references: {}", uri, todo);
+ if (replConfig.getMaxRefsToLog() == 0 || todo.size() <= replConfig.getMaxRefsToLog()) {
+ repLog.info("Push to {} references: {}", uri, todo);
+ } else {
+ repLog.info(
+ "Push to {} references (first {} of {} listed): {}",
+ uri,
+ replConfig.getMaxRefsToLog(),
+ todo.size(),
+ todo.subList(0, replConfig.getMaxRefsToLog()));
+ }
return tn.push(NullProgressMonitor.INSTANCE, todo);
}
@@ -479,7 +491,8 @@
return Collections.emptyList();
}
- Map<String, Ref> local = git.getAllRefs();
+ Map<String, Ref> local =
+ git.getRefDatabase().getRefs().stream().collect(toMap(Ref::getName, r -> r));
boolean filter;
PermissionBackend.ForProject forProject = permissionBackend.currentUser().project(projectName);
try {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java
index ae0662d..ad68d42 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java
@@ -15,10 +15,10 @@
package com.googlesource.gerrit.plugins.replication;
import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.exceptions.StorageException;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.events.RefEvent;
import com.google.gerrit.server.permissions.PermissionBackendException;
-import com.google.gwtorm.server.OrmException;
import com.googlesource.gerrit.plugins.replication.ReplicationState.RefPushResult;
import java.lang.ref.WeakReference;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -187,7 +187,7 @@
private void postEvent(RefEvent event) {
try {
dispatcher.postEvent(event);
- } catch (OrmException | PermissionBackendException e) {
+ } catch (StorageException | PermissionBackendException e) {
logger.atSevere().withCause(e).log("Cannot post event");
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
index c9531e3..929c538 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
@@ -13,9 +13,13 @@
// limitations under the License.
package com.googlesource.gerrit.plugins.replication;
+import com.google.common.collect.Multimap;
+import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.git.WorkQueue;
import java.nio.file.Path;
import java.util.List;
+import java.util.Optional;
+import org.eclipse.jgit.transport.URIish;
public interface ReplicationConfig {
@@ -27,10 +31,15 @@
List<Destination> getDestinations(FilterType filterType);
+ Multimap<Destination, URIish> getURIs(
+ Optional<String> remoteName, Project.NameKey projectName, FilterType filterType);
+
boolean isReplicateAllOnPluginStart();
boolean isDefaultForceUpdate();
+ int getMaxRefsToLog();
+
boolean isEmpty();
Path getEventsDirectory();
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
index d1be563..a075f12 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
@@ -13,15 +13,24 @@
// limitations under the License.
package com.googlesource.gerrit.plugins.replication;
+import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isGerrit;
+import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isGerritHttp;
+import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isSSH;
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toList;
import com.google.common.base.Strings;
+import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.SetMultimap;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.extensions.annotations.PluginData;
+import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.config.ConfigUtil;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.git.WorkQueue;
@@ -33,6 +42,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
import org.eclipse.jgit.errors.ConfigInvalidException;
@@ -52,6 +62,7 @@
private Path cfgPath;
private boolean replicateAllOnPluginStart;
private boolean defaultForceUpdate;
+ private int maxRefsToLog;
private int sshCommandTimeout;
private int sshConnectionTimeout = DEFAULT_SSH_CONNECTION_TIMEOUT_MS;
private final FileBasedConfig config;
@@ -117,6 +128,8 @@
defaultForceUpdate = config.getBoolean("gerrit", "defaultForceUpdate", false);
+ maxRefsToLog = config.getInt("gerrit", "maxRefsToLog", 0);
+
sshCommandTimeout =
(int) ConfigUtil.getTimeUnit(config, "gerrit", null, "sshCommandTimeout", 0, SECONDS);
sshConnectionTimeout =
@@ -167,6 +180,77 @@
return dest.build();
}
+ @Override
+ public Multimap<Destination, URIish> getURIs(
+ Optional<String> remoteName, Project.NameKey projectName, FilterType filterType) {
+ if (getDestinations(filterType).isEmpty()) {
+ return ImmutableMultimap.of();
+ }
+
+ SetMultimap<Destination, URIish> uris = HashMultimap.create();
+ for (Destination config : getDestinations(filterType)) {
+ if (!config.wouldPushProject(projectName)) {
+ continue;
+ }
+
+ if (remoteName.isPresent() && !config.getRemoteConfigName().equals(remoteName.get())) {
+ continue;
+ }
+
+ boolean adminURLUsed = false;
+
+ for (String url : config.getAdminUrls()) {
+ if (Strings.isNullOrEmpty(url)) {
+ continue;
+ }
+
+ URIish uri;
+ try {
+ uri = new URIish(url);
+ } catch (URISyntaxException e) {
+ repLog.warn("adminURL '{}' is invalid: {}", url, e.getMessage());
+ continue;
+ }
+
+ if (!isGerrit(uri) && !isGerritHttp(uri)) {
+ String path =
+ replaceName(uri.getPath(), projectName.get(), config.isSingleProjectMatch());
+ if (path == null) {
+ repLog.warn("adminURL {} does not contain ${name}", uri);
+ continue;
+ }
+
+ uri = uri.setPath(path);
+ if (!isSSH(uri)) {
+ repLog.warn("adminURL '{}' is invalid: only SSH and HTTP are supported", uri);
+ continue;
+ }
+ }
+ uris.put(config, uri);
+ adminURLUsed = true;
+ }
+
+ if (!adminURLUsed) {
+ for (URIish uri : config.getURIs(projectName, "*")) {
+ uris.put(config, uri);
+ }
+ }
+ }
+ return uris;
+ }
+
+ static String replaceName(String in, String name, boolean keyIsOptional) {
+ String key = "${name}";
+ int n = in.indexOf(key);
+ if (0 <= n) {
+ return in.substring(0, n) + name + in.substring(n + key.length());
+ }
+ if (keyIsOptional) {
+ return in;
+ }
+ return null;
+ }
+
/* (non-Javadoc)
* @see com.googlesource.gerrit.plugins.replication.ReplicationConfig#isReplicateAllOnPluginStart()
*/
@@ -183,6 +267,11 @@
return defaultForceUpdate;
}
+ @Override
+ public int getMaxRefsToLog() {
+ return maxRefsToLog;
+ }
+
private static List<RemoteConfig> allRemotes(FileBasedConfig cfg) throws ConfigInvalidException {
Set<String> names = cfg.getSubsections("remote");
List<RemoteConfig> result = Lists.newArrayListWithCapacity(names.size());
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
index 1898a4f..f9a2b2c 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
@@ -67,8 +67,6 @@
EventTypes.register(ReplicationScheduledEvent.TYPE, ReplicationScheduledEvent.class);
bind(SshSessionFactory.class).toProvider(ReplicationSshSessionFactoryProvider.class);
- bind(AdminApiFactory.class);
-
bind(TransportFactory.class).to(TransportFactoryImpl.class).in(Scopes.SINGLETON);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
index 203bc2f..6229686 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
@@ -14,11 +14,6 @@
package com.googlesource.gerrit.plugins.replication;
-import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isGerrit;
-import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isSSH;
-
-import com.google.common.base.Strings;
-import com.google.gerrit.common.Nullable;
import com.google.gerrit.extensions.events.GitReferenceUpdatedListener;
import com.google.gerrit.extensions.events.HeadUpdatedListener;
import com.google.gerrit.extensions.events.LifecycleListener;
@@ -30,11 +25,7 @@
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdateProcessing;
import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
-import java.net.URISyntaxException;
-import java.util.Collections;
-import java.util.HashSet;
import java.util.Optional;
-import java.util.Set;
import org.eclipse.jgit.transport.URIish;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,22 +41,9 @@
private final ReplicationStateListener stateLog;
- static String replaceName(String in, String name, boolean keyIsOptional) {
- String key = "${name}";
- int n = in.indexOf(key);
- if (0 <= n) {
- return in.substring(0, n) + name + in.substring(n + key.length());
- }
- if (keyIsOptional) {
- return in;
- }
- return null;
- }
-
private final WorkQueue workQueue;
private final DynamicItem<EventDispatcher> dispatcher;
private final ReplicationConfig config;
- private final AdminApiFactory adminApiFactory;
private final ReplicationState.Factory replicationStateFactory;
private final EventsStorage eventsStorage;
private volatile boolean running;
@@ -74,7 +52,6 @@
@Inject
ReplicationQueue(
WorkQueue wq,
- AdminApiFactory aaf,
ReplicationConfig rc,
DynamicItem<EventDispatcher> dis,
ReplicationStateListeners sl,
@@ -84,7 +61,6 @@
dispatcher = dis;
config = rc;
stateLog = sl;
- adminApiFactory = aaf;
replicationStateFactory = rsf;
eventsStorage = es;
}
@@ -175,121 +151,15 @@
@Override
public void onProjectDeleted(ProjectDeletedListener.Event event) {
- Project.NameKey projectName = new Project.NameKey(event.getProjectName());
- for (URIish uri : getURIs(null, projectName, FilterType.PROJECT_DELETION)) {
- deleteProject(uri, projectName);
- }
+ Project.NameKey p = new Project.NameKey(event.getProjectName());
+ config.getURIs(Optional.empty(), p, FilterType.PROJECT_DELETION).entries().stream()
+ .forEach(e -> e.getKey().scheduleDeleteProject(e.getValue(), p));
}
@Override
public void onHeadUpdated(HeadUpdatedListener.Event event) {
- Project.NameKey project = new Project.NameKey(event.getProjectName());
- for (URIish uri : getURIs(null, project, FilterType.ALL)) {
- updateHead(uri, project, event.getNewHeadName());
- }
- }
-
- private Set<URIish> getURIs(
- @Nullable String remoteName, Project.NameKey projectName, FilterType filterType) {
- if (config.getDestinations(filterType).isEmpty()) {
- return Collections.emptySet();
- }
- if (!running) {
- repLog.error("Replication plugin did not finish startup before event");
- return Collections.emptySet();
- }
-
- Set<URIish> uris = new HashSet<>();
- for (Destination config : this.config.getDestinations(filterType)) {
- if (!config.wouldPushProject(projectName)) {
- continue;
- }
-
- if (remoteName != null && !config.getRemoteConfigName().equals(remoteName)) {
- continue;
- }
-
- boolean adminURLUsed = false;
-
- for (String url : config.getAdminUrls()) {
- if (Strings.isNullOrEmpty(url)) {
- continue;
- }
-
- URIish uri;
- try {
- uri = new URIish(url);
- } catch (URISyntaxException e) {
- repLog.warn("adminURL '{}' is invalid: {}", url, e.getMessage());
- continue;
- }
-
- if (!isGerrit(uri)) {
- String path =
- replaceName(uri.getPath(), projectName.get(), config.isSingleProjectMatch());
- if (path == null) {
- repLog.warn("adminURL {} does not contain ${name}", uri);
- continue;
- }
-
- uri = uri.setPath(path);
- if (!isSSH(uri)) {
- repLog.warn("adminURL '{}' is invalid: only SSH is supported", uri);
- continue;
- }
- }
- uris.add(uri);
- adminURLUsed = true;
- }
-
- if (!adminURLUsed) {
- for (URIish uri : config.getURIs(projectName, "*")) {
- uris.add(uri);
- }
- }
- }
- return uris;
- }
-
- public boolean createProject(String remoteName, Project.NameKey project, String head) {
- boolean success = true;
- for (URIish uri : getURIs(remoteName, project, FilterType.PROJECT_CREATION)) {
- success &= createProject(uri, project, head);
- }
- return success;
- }
-
- private boolean createProject(URIish replicateURI, Project.NameKey projectName, String head) {
- Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI);
- if (adminApi.isPresent() && adminApi.get().createProject(projectName, head)) {
- return true;
- }
-
- warnCannotPerform("create new project", replicateURI);
- return false;
- }
-
- private void deleteProject(URIish replicateURI, Project.NameKey projectName) {
- Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI);
- if (adminApi.isPresent()) {
- adminApi.get().deleteProject(projectName);
- return;
- }
-
- warnCannotPerform("delete project", replicateURI);
- }
-
- private void updateHead(URIish replicateURI, Project.NameKey projectName, String newHead) {
- Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI);
- if (adminApi.isPresent()) {
- adminApi.get().updateHead(projectName, newHead);
- return;
- }
-
- warnCannotPerform("update HEAD of project", replicateURI);
- }
-
- private void warnCannotPerform(String op, URIish uri) {
- repLog.warn("Cannot {} on remote site {}.", op, uri);
+ Project.NameKey p = new Project.NameKey(event.getProjectName());
+ config.getURIs(Optional.empty(), p, FilterType.ALL).entries().stream()
+ .forEach(e -> e.getKey().scheduleUpdateHead(e.getValue(), p, event.getNewHeadName()));
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java
index c518091..18a4cc2 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java
@@ -21,6 +21,8 @@
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.transport.CredentialsProvider;
+import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.eclipse.jgit.util.FS;
/** Looks up a remote's password in secure.config. */
@@ -49,9 +51,9 @@
}
@Override
- public SecureCredentialsProvider create(String remoteName) {
+ public CredentialsProvider create(String remoteName) {
String user = Objects.toString(config.getString("remote", remoteName, "username"), "");
String pass = Objects.toString(config.getString("remote", remoteName, "password"), "");
- return new SecureCredentialsProvider(user, pass);
+ return new UsernamePasswordCredentialsProvider(user, pass);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java
deleted file mode 100644
index 62b4036..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java
+++ /dev/null
@@ -1,80 +0,0 @@
-// Copyright (C) 2011 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.replication;
-
-import org.eclipse.jgit.errors.UnsupportedCredentialItem;
-import org.eclipse.jgit.transport.CredentialItem;
-import org.eclipse.jgit.transport.CredentialsProvider;
-import org.eclipse.jgit.transport.URIish;
-
-/** Looks up a remote's password in secure.config. */
-public class SecureCredentialsProvider extends CredentialsProvider {
- private final String cfgUser;
- private final String cfgPass;
-
- SecureCredentialsProvider(String user, String pass) {
- cfgUser = user;
- cfgPass = pass;
- }
-
- @Override
- public boolean isInteractive() {
- return false;
- }
-
- @Override
- public boolean supports(CredentialItem... items) {
- for (CredentialItem i : items) {
- if (i instanceof CredentialItem.Username) {
- continue;
- } else if (i instanceof CredentialItem.Password) {
- continue;
- } else {
- return false;
- }
- }
- return true;
- }
-
- @Override
- public boolean get(URIish uri, CredentialItem... items) throws UnsupportedCredentialItem {
- String username = uri.getUser();
- if (username == null) {
- username = cfgUser;
- }
- if (username == null) {
- return false;
- }
-
- String password = uri.getPass();
- if (password == null) {
- password = cfgPass;
- }
- if (password == null) {
- return false;
- }
-
- for (CredentialItem i : items) {
- if (i instanceof CredentialItem.Username) {
- ((CredentialItem.Username) i).setValue(username);
- } else if (i instanceof CredentialItem.Password) {
- ((CredentialItem.Password) i).setValue(password.toCharArray());
- } else {
- throw new UnsupportedCredentialItem(uri, i.getPromptText());
- }
- }
- return true;
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java
new file mode 100644
index 0000000..db58f49
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java
@@ -0,0 +1,69 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.server.ioutil.HexFormat;
+import com.google.gerrit.server.util.IdGenerator;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import java.util.Optional;
+import org.eclipse.jgit.transport.URIish;
+
+public class UpdateHeadTask implements Runnable {
+ private final AdminApiFactory adminApiFactory;
+ private final int id;
+ private final URIish replicateURI;
+ private final Project.NameKey project;
+ private final String newHead;
+
+ interface Factory {
+ UpdateHeadTask create(URIish uri, Project.NameKey project, String newHead);
+ }
+
+ @Inject
+ UpdateHeadTask(
+ AdminApiFactory adminApiFactory,
+ IdGenerator ig,
+ @Assisted URIish replicateURI,
+ @Assisted Project.NameKey project,
+ @Assisted String newHead) {
+ this.adminApiFactory = adminApiFactory;
+ this.id = ig.next();
+ this.replicateURI = replicateURI;
+ this.project = project;
+ this.newHead = newHead;
+ }
+
+ @Override
+ public void run() {
+ Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI);
+ if (adminApi.isPresent()) {
+ adminApi.get().updateHead(project, newHead);
+ return;
+ }
+
+ repLog.warn("Cannot update HEAD of project {} on remote site {}.", project, replicateURI);
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "[%s] update-head of %s at %s to %s",
+ HexFormat.fromInt(id), project.get(), replicateURI, newHead);
+ }
+}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 417b2ad..f58845d 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -82,6 +82,10 @@
: If true, the default push refspec will be set to use forced
update to the remote when no refspec is given. By default, false.
+gerrit.maxRefsToLog
+: Number of refs, that are pushed during replication, to be logged.
+ For printing all refs to the logs, use a value of 0. By default, 0.
+
gerrit.sshCommandTimeout
: Timeout for SSH command execution. If 0, there is no timeout and
the client waits indefinitely. By default, 0.
@@ -177,13 +181,18 @@
local environment. In that case, an alternative SSH url could
be specified to repository creation.
- To enable replication to different Gerrit instance use `gerrit+ssh://`
- as protocol name followed by hostname of another Gerrit server eg.
+ To enable replication to different Gerrit instance use `gerrit+ssh://`,
+ `gerrit+http://` or `gerrit+https://` as protocol name followed
+ by hostname of another Gerrit server eg.
`gerrit+ssh://replica1.my.org/`
+ <br>
+ `gerrit+http://replica2.my.org/`
+ <br>
+ `gerrit+https://replica3.my.org/`
- In this case replication will use Gerrit's SSH API to
- create/remove projects and update repository HEAD references.
+ In this case replication will use Gerrit's SSH API or Gerrit's REST API
+ to create/remove projects and update repository HEAD references.
NOTE: In order to replicate project deletion, the
link:https://gerrit-review.googlesource.com/admin/projects/plugins/delete-project delete-project[delete-project]
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/AbstractConfigTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/AbstractConfigTest.java
new file mode 100644
index 0000000..3525129
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/AbstractConfigTest.java
@@ -0,0 +1,116 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.google.common.truth.Truth.assertThat;
+import static java.nio.file.Files.createTempDirectory;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.getCurrentArguments;
+import static org.easymock.EasyMock.isA;
+import static org.easymock.EasyMock.replay;
+
+import com.google.gerrit.server.config.SitePaths;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.easymock.IAnswer;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
+import org.junit.Before;
+import org.junit.Ignore;
+
+@Ignore
+public abstract class AbstractConfigTest {
+ protected final Path sitePath;
+ protected final SitePaths sitePaths;
+ protected final Destination.Factory destinationFactoryMock;
+ protected final Path pluginDataPath;
+
+ static class FakeDestination extends Destination {
+ public final DestinationConfiguration config;
+
+ protected FakeDestination(DestinationConfiguration config) {
+ super(injectorMock(), null, null, null, null, null, null, null, null, null, config);
+ this.config = config;
+ }
+
+ private static Injector injectorMock() {
+ Injector injector = createNiceMock(Injector.class);
+ Injector childInjectorMock = createNiceMock(Injector.class);
+ expect(injector.createChildInjector((Module) anyObject())).andReturn(childInjectorMock);
+ replay(childInjectorMock);
+ replay(injector);
+ return injector;
+ }
+ }
+
+ AbstractConfigTest() throws IOException {
+ sitePath = createTempPath("site");
+ sitePaths = new SitePaths(sitePath);
+ pluginDataPath = createTempPath("data");
+ destinationFactoryMock = createMock(Destination.Factory.class);
+ }
+
+ @Before
+ public void setup() {
+ expect(destinationFactoryMock.create(isA(DestinationConfiguration.class)))
+ .andAnswer(
+ new IAnswer<Destination>() {
+ @Override
+ public Destination answer() throws Throwable {
+ return new FakeDestination((DestinationConfiguration) getCurrentArguments()[0]);
+ }
+ })
+ .anyTimes();
+ replay(destinationFactoryMock);
+ }
+
+ protected static Path createTempPath(String prefix) throws IOException {
+ return createTempDirectory(prefix);
+ }
+
+ protected FileBasedConfig newReplicationConfig() {
+ FileBasedConfig replicationConfig =
+ new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED);
+ return replicationConfig;
+ }
+
+ protected void assertThatIsDestination(
+ Destination destination, String remoteName, String... remoteUrls) {
+ DestinationConfiguration destinationConfig = ((FakeDestination) destination).config;
+ assertThat(destinationConfig.getRemoteConfig().getName()).isEqualTo(remoteName);
+ assertThat(destinationConfig.getUrls()).containsExactlyElementsIn(remoteUrls);
+ }
+
+ protected void assertThatContainsDestination(
+ List<Destination> destinations, String remoteName, String... remoteUrls) {
+ List<Destination> matchingDestinations =
+ destinations.stream()
+ .filter(
+ (Destination dst) ->
+ ((FakeDestination) dst).config.getRemoteConfig().getName().equals(remoteName))
+ .collect(Collectors.toList());
+
+ assertThat(matchingDestinations).isNotEmpty();
+
+ assertThatIsDestination(matchingDestinations.get(0), remoteName, remoteUrls);
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecoratorTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecoratorTest.java
new file mode 100644
index 0000000..211cafa
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecoratorTest.java
@@ -0,0 +1,252 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.easymock.EasyMock.anyInt;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+
+import com.google.gerrit.server.git.WorkQueue;
+import com.google.inject.util.Providers;
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.junit.Before;
+import org.junit.Test;
+
+public class AutoReloadConfigDecoratorTest extends AbstractConfigTest {
+ private AutoReloadConfigDecorator autoReloadConfig;
+ private ReplicationQueue replicationQueueMock;
+ private WorkQueue workQueueMock;
+ private FakeExecutorService executorService = new FakeExecutorService();
+
+ public class FakeExecutorService implements ScheduledExecutorService {
+ public Runnable refreshCommand;
+
+ @Override
+ public void shutdown() {}
+
+ @Override
+ public List<Runnable> shutdownNow() {
+ return null;
+ }
+
+ @Override
+ public boolean isShutdown() {
+ return false;
+ }
+
+ @Override
+ public boolean isTerminated() {
+ return false;
+ }
+
+ @Override
+ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+ return false;
+ }
+
+ @Override
+ public <T> Future<T> submit(Callable<T> task) {
+ return null;
+ }
+
+ @Override
+ public <T> Future<T> submit(Runnable task, T result) {
+ return null;
+ }
+
+ @Override
+ public Future<?> submit(Runnable task) {
+ return null;
+ }
+
+ @Override
+ public <T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks)
+ throws InterruptedException {
+ return null;
+ }
+
+ @Override
+ public <T> List<Future<T>> invokeAll(
+ Collection<? extends Callable<T>> tasks, long timeout, TimeUnit unit)
+ throws InterruptedException {
+ return null;
+ }
+
+ @Override
+ public <T> T invokeAny(Collection<? extends Callable<T>> tasks)
+ throws InterruptedException, ExecutionException {
+ return null;
+ }
+
+ @Override
+ public <T> T invokeAny(Collection<? extends Callable<T>> tasks, long timeout, TimeUnit unit)
+ throws InterruptedException, ExecutionException, TimeoutException {
+ return null;
+ }
+
+ @Override
+ public void execute(Runnable command) {}
+
+ @Override
+ public ScheduledFuture<?> schedule(Runnable command, long delay, TimeUnit unit) {
+ return null;
+ }
+
+ @Override
+ public <V> ScheduledFuture<V> schedule(Callable<V> callable, long delay, TimeUnit unit) {
+ return null;
+ }
+
+ @Override
+ public ScheduledFuture<?> scheduleAtFixedRate(
+ Runnable command, long initialDelay, long period, TimeUnit unit) {
+ refreshCommand = command;
+ return null;
+ }
+
+ @Override
+ public ScheduledFuture<?> scheduleWithFixedDelay(
+ Runnable command, long initialDelay, long delay, TimeUnit unit) {
+ return null;
+ }
+ }
+
+ public AutoReloadConfigDecoratorTest() throws IOException {
+ super();
+ }
+
+ @Override
+ @Before
+ public void setup() {
+ super.setup();
+
+ setupMocks();
+ }
+
+ private void setupMocks() {
+ replicationQueueMock = createNiceMock(ReplicationQueue.class);
+ expect(replicationQueueMock.isRunning()).andReturn(true);
+ replay(replicationQueueMock);
+
+ workQueueMock = createNiceMock(WorkQueue.class);
+ expect(workQueueMock.createQueue(anyInt(), anyObject(String.class))).andReturn(executorService);
+ replay(workQueueMock);
+ }
+
+ @Test
+ public void shouldLoadNotEmptyInitialReplicationConfig() throws Exception {
+ FileBasedConfig replicationConfig = newReplicationConfig();
+ String remoteName = "foo";
+ String remoteUrl = "ssh://git@git.somewhere.com/${name}";
+ replicationConfig.setString("remote", remoteName, "url", remoteUrl);
+ replicationConfig.save();
+
+ autoReloadConfig =
+ new AutoReloadConfigDecorator(
+ sitePaths,
+ destinationFactoryMock,
+ Providers.of(replicationQueueMock),
+ pluginDataPath,
+ "replication",
+ workQueueMock);
+
+ List<Destination> destinations = autoReloadConfig.getDestinations(FilterType.ALL);
+ assertThat(destinations).hasSize(1);
+ assertThatIsDestination(destinations.get(0), remoteName, remoteUrl);
+ }
+
+ @Test
+ public void shouldAutoReloadReplicationConfig() throws Exception {
+ FileBasedConfig replicationConfig = newReplicationConfig();
+ replicationConfig.setBoolean("gerrit", null, "autoReload", true);
+ String remoteName1 = "foo";
+ String remoteUrl1 = "ssh://git@git.foo.com/${name}";
+ replicationConfig.setString("remote", remoteName1, "url", remoteUrl1);
+ replicationConfig.save();
+
+ autoReloadConfig =
+ new AutoReloadConfigDecorator(
+ sitePaths,
+ destinationFactoryMock,
+ Providers.of(replicationQueueMock),
+ pluginDataPath,
+ "replication",
+ workQueueMock);
+ autoReloadConfig.startup(workQueueMock);
+
+ List<Destination> destinations = autoReloadConfig.getDestinations(FilterType.ALL);
+ assertThat(destinations).hasSize(1);
+ assertThatIsDestination(destinations.get(0), remoteName1, remoteUrl1);
+
+ TimeUnit.SECONDS.sleep(1); // Allow the filesystem to change the update TS
+
+ String remoteName2 = "bar";
+ String remoteUrl2 = "ssh://git@git.bar.com/${name}";
+ replicationConfig.setString("remote", remoteName2, "url", remoteUrl2);
+ replicationConfig.save();
+ executorService.refreshCommand.run();
+
+ destinations = autoReloadConfig.getDestinations(FilterType.ALL);
+ assertThat(destinations).hasSize(2);
+ assertThatContainsDestination(destinations, remoteName1, remoteUrl1);
+ assertThatContainsDestination(destinations, remoteName2, remoteUrl2);
+ }
+
+ @Test
+ public void shouldNotAutoReloadReplicationConfigIfDisabled() throws Exception {
+ String remoteName1 = "foo";
+ String remoteUrl1 = "ssh://git@git.foo.com/${name}";
+ FileBasedConfig replicationConfig = newReplicationConfig();
+ replicationConfig.setBoolean("gerrit", null, "autoReload", false);
+ replicationConfig.setString("remote", remoteName1, "url", remoteUrl1);
+ replicationConfig.save();
+
+ autoReloadConfig =
+ new AutoReloadConfigDecorator(
+ sitePaths,
+ destinationFactoryMock,
+ Providers.of(replicationQueueMock),
+ pluginDataPath,
+ "replication",
+ workQueueMock);
+ autoReloadConfig.startup(workQueueMock);
+
+ List<Destination> destinations = autoReloadConfig.getDestinations(FilterType.ALL);
+ assertThat(destinations).hasSize(1);
+ assertThatIsDestination(destinations.get(0), remoteName1, remoteUrl1);
+
+ TimeUnit.SECONDS.sleep(1); // Allow the filesystem to change the update TS
+
+ replicationConfig.setString("remote", "bar", "url", "ssh://git@git.bar.com/${name}");
+ replicationConfig.save();
+ executorService.refreshCommand.run();
+
+ assertThat(autoReloadConfig.getDestinations(FilterType.ALL)).isEqualTo(destinations);
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
index 5fa7b98..0f6d629 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
@@ -15,20 +15,13 @@
package com.googlesource.gerrit.plugins.replication;
import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.reset;
import static org.easymock.EasyMock.verify;
-import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.permissions.PermissionBackendException;
-import com.google.gwtorm.client.KeyUtil;
-import com.google.gwtorm.server.OrmException;
-import com.google.gwtorm.server.SchemaFactory;
-import com.google.gwtorm.server.StandardKeyEncoder;
import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdateProcessing;
import com.googlesource.gerrit.plugins.replication.ReplicationState.RefPushResult;
import java.net.URISyntaxException;
@@ -37,12 +30,7 @@
import org.junit.Before;
import org.junit.Test;
-@SuppressWarnings("unchecked")
public class GitUpdateProcessingTest {
- static {
- KeyUtil.setEncoderImpl(new StandardKeyEncoder());
- }
-
private EventDispatcher dispatcherMock;
private GitUpdateProcessing gitUpdateProcessing;
@@ -50,17 +38,11 @@
public void setUp() throws Exception {
dispatcherMock = createMock(EventDispatcher.class);
replay(dispatcherMock);
- ReviewDb reviewDbMock = createNiceMock(ReviewDb.class);
- replay(reviewDbMock);
- SchemaFactory<ReviewDb> schemaMock = createMock(SchemaFactory.class);
- expect(schemaMock.open()).andReturn(reviewDbMock).anyTimes();
- replay(schemaMock);
gitUpdateProcessing = new GitUpdateProcessing(dispatcherMock);
}
@Test
- public void headRefReplicated()
- throws URISyntaxException, OrmException, PermissionBackendException {
+ public void headRefReplicated() throws URISyntaxException, PermissionBackendException {
reset(dispatcherMock);
RefReplicatedEvent expectedEvent =
new RefReplicatedEvent(
@@ -83,8 +65,7 @@
}
@Test
- public void changeRefReplicated()
- throws URISyntaxException, OrmException, PermissionBackendException {
+ public void changeRefReplicated() throws URISyntaxException, PermissionBackendException {
reset(dispatcherMock);
RefReplicatedEvent expectedEvent =
new RefReplicatedEvent(
@@ -107,7 +88,7 @@
}
@Test
- public void onAllNodesReplicated() throws OrmException, PermissionBackendException {
+ public void onAllNodesReplicated() throws PermissionBackendException {
reset(dispatcherMock);
RefReplicationDoneEvent expectedDoneEvent =
new RefReplicationDoneEvent("someProject", "refs/heads/master", 5);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java
index af065b3..836da2f 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/PushOneTest.java
@@ -53,6 +53,7 @@
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdRef;
import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.RefDatabase;
import org.eclipse.jgit.lib.RefUpdate;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.storage.file.FileBasedConfig;
@@ -94,10 +95,13 @@
private PushConnection pushConnection;
private ProjectState projectStateMock;
private RefUpdate refUpdateMock;
+ private CreateProjectTask.Factory createProjectTaskFactoryMock;
+ private ReplicationConfig replicationConfigMock;
+ private RefDatabase refDatabaseMock;
private Project.NameKey projectNameKey;
private URIish urish;
- private Map<String, Ref> localRefs;
+ private List<Ref> localRefs;
private Map<String, Ref> remoteRefs;
private CountDownLatch isCallFinished;
@@ -112,8 +116,7 @@
new ObjectIdRef.Unpeeled(
NEW, "foo", ObjectId.fromString("0000000000000000000000000000000000000001"));
- localRefs = new HashMap<>();
- localRefs.put("fooProject", newLocalRef);
+ localRefs = Arrays.asList(newLocalRef);
Ref remoteRef = new ObjectIdRef.Unpeeled(NEW, "foo", ObjectId.zeroId());
remoteRefs = new HashMap<>();
@@ -158,6 +161,8 @@
setupProjectCacheMock();
+ replicationConfigMock = createNiceMock(ReplicationConfig.class);
+
replay(
gitRepositoryManagerMock,
refUpdateMock,
@@ -179,7 +184,9 @@
forProjectMock,
fetchConnection,
pushConnection,
- refSpecMock);
+ refSpecMock,
+ refDatabaseMock,
+ replicationConfigMock);
}
@Test
@@ -260,13 +267,14 @@
permissionBackendMock,
destinationMock,
remoteConfigMock,
+ replicationConfigMock,
credentialsFactory,
threadRequestScoperMock,
- replicationQueueMock,
idGeneratorMock,
replicationStateListenersMock,
replicationMetricsMock,
projectCacheMock,
+ createProjectTaskFactoryMock,
transportFactoryMock,
projectNameKey,
urish);
@@ -358,11 +366,12 @@
expect(gitRepositoryManagerMock.openRepository(projectNameKey)).andReturn(repositoryMock);
}
- @SuppressWarnings("deprecation")
private void setupRepositoryMock(FileBasedConfig config) throws IOException {
repositoryMock = createNiceMock(Repository.class);
+ refDatabaseMock = createNiceMock(RefDatabase.class);
expect(repositoryMock.getConfig()).andReturn(config).anyTimes();
- expect(repositoryMock.getAllRefs()).andReturn(localRefs);
+ expect(repositoryMock.getRefDatabase()).andReturn(refDatabaseMock);
+ expect(refDatabaseMock.getRefs()).andReturn(localRefs);
expect(repositoryMock.updateRef("fooProject")).andReturn(refUpdateMock);
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfigTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfigTest.java
new file mode 100644
index 0000000..36cc209
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfigTest.java
@@ -0,0 +1,72 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
+import java.io.IOException;
+import java.util.List;
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.junit.Test;
+
+public class ReplicationFileBasedConfigTest extends AbstractConfigTest {
+
+ public ReplicationFileBasedConfigTest() throws IOException {
+ super();
+ }
+
+ @Test
+ public void shouldLoadOneDestination() throws Exception {
+ String remoteName = "foo";
+ String remoteUrl = "ssh://git@git.somewhere.com/${name}";
+ FileBasedConfig config = newReplicationConfig();
+ config.setString("remote", remoteName, "url", remoteUrl);
+ config.save();
+
+ ReplicationFileBasedConfig replicationConfig = newReplicationFileBasedConfig();
+ List<Destination> destinations = replicationConfig.getDestinations(FilterType.ALL);
+ assertThat(destinations).hasSize(1);
+
+ assertThatIsDestination(destinations.get(0), remoteName, remoteUrl);
+ }
+
+ @Test
+ public void shouldLoadTwoDestinations() throws Exception {
+ String remoteName1 = "foo";
+ String remoteUrl1 = "ssh://git@git.somewhere.com/${name}";
+ String remoteName2 = "bar";
+ String remoteUrl2 = "ssh://git@git.elsewhere.com/${name}";
+ FileBasedConfig config = newReplicationConfig();
+ config.setString("remote", remoteName1, "url", remoteUrl1);
+ config.setString("remote", remoteName2, "url", remoteUrl2);
+ config.save();
+
+ ReplicationFileBasedConfig replicationConfig = newReplicationFileBasedConfig();
+ List<Destination> destinations = replicationConfig.getDestinations(FilterType.ALL);
+ assertThat(destinations).hasSize(2);
+
+ assertThatIsDestination(destinations.get(0), remoteName1, remoteUrl1);
+ assertThatIsDestination(destinations.get(1), remoteName2, remoteUrl2);
+ }
+
+ private ReplicationFileBasedConfig newReplicationFileBasedConfig()
+ throws ConfigInvalidException, IOException {
+ ReplicationFileBasedConfig replicationConfig =
+ new ReplicationFileBasedConfig(sitePaths, destinationFactoryMock, pluginDataPath);
+ return replicationConfig;
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java
index 7f68238..fe800b0 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java
@@ -21,6 +21,7 @@
import com.google.gerrit.acceptance.PushOneCommit.Result;
import com.google.gerrit.acceptance.TestPlugin;
import com.google.gerrit.acceptance.UseLocalDisk;
+import com.google.gerrit.acceptance.testsuite.project.ProjectOperations;
import com.google.gerrit.extensions.common.ProjectInfo;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.config.SitePaths;
@@ -46,6 +47,7 @@
private static final Duration TEST_TIMEMOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 10);
@Inject private SitePaths sitePaths;
+ @Inject private ProjectOperations projectOperations;
private Path gitPath;
private FileBasedConfig config;
@@ -64,7 +66,7 @@
public void shouldReplicateNewProject() throws Exception {
setReplicationDestination("foo", "replica");
- Project.NameKey sourceProject = createProject("foo");
+ Project.NameKey sourceProject = projectOperations.newProject().name("foo").create();
waitUntil(() -> gitPath.resolve(sourceProject + "replica.git").toFile().isDirectory());
ProjectInfo replicaProject = gApi.projects().name(sourceProject + "replica").get();
@@ -75,7 +77,8 @@
public void shouldReplicateNewBranch() throws Exception {
setReplicationDestination("foo", "replica");
- Project.NameKey targetProject = createProject("projectreplica");
+ Project.NameKey targetProject =
+ projectOperations.newProject().name(project + "replica").create();
Result pushResult = createChange();
RevCommit sourceCommit = pushResult.getCommit();
