Merge branch 'stable-2.16'
* stable-2.16:
Replace DestinationFactory with Guice generated Factory and assisted injection
Revert "DestinationFactory: remove @Singleton annotation"
DestinationFactory: remove @Singleton annotation
Updated docs to clarify that SSH config is not optional
Introduce DynamicSet of ReplicationStateListener
Remove unneeded RemoteSiteUser.Factory
Make SecureCredentialsProvider public for reuse
Change-Id: I52bc2580bef89d40f9b02749550cdbbd6e7a26dc
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApi.java
index ef7e353..79362fd 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApi.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApi.java
@@ -17,7 +17,7 @@
import com.google.gerrit.reviewdb.client.Project;
public interface AdminApi {
- public void createProject(Project.NameKey project, String head);
+ public boolean createProject(Project.NameKey project, String head);
public void deleteProject(Project.NameKey project);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java
index 528aff2..e5a1628 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java
@@ -23,10 +23,12 @@
public class AdminApiFactory {
private final SshHelper sshHelper;
+ private final GerritRestApi.Factory gerritRestApiFactory;
@Inject
- AdminApiFactory(SshHelper sshHelper) {
+ AdminApiFactory(SshHelper sshHelper, GerritRestApi.Factory gerritRestApiFactory) {
this.sshHelper = sshHelper;
+ this.gerritRestApiFactory = gerritRestApiFactory;
}
public Optional<AdminApi> create(URIish uri) {
@@ -36,6 +38,8 @@
return Optional.of(new LocalFS(uri));
} else if (isSSH(uri)) {
return Optional.of(new RemoteSsh(sshHelper, uri));
+ } else if (isGerritHttp(uri)) {
+ return Optional.of(gerritRestApiFactory.create(uri));
}
return Optional.empty();
}
@@ -58,4 +62,9 @@
}
return false;
}
+
+ public static boolean isGerritHttp(URIish uri) {
+ String scheme = uri.getScheme();
+ return scheme != null && scheme.toLowerCase().contains("gerrit+http");
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
index ad6e67d..53f56a7 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
@@ -13,9 +13,11 @@
// limitations under the License.
package com.googlesource.gerrit.plugins.replication;
+import com.google.common.collect.Multimap;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.common.FileUtil;
import com.google.gerrit.extensions.annotations.PluginData;
+import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.git.WorkQueue;
import com.google.inject.Inject;
@@ -24,7 +26,9 @@
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
+import java.util.Optional;
import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.transport.URIish;
@Singleton
public class AutoReloadConfigDecorator implements ReplicationConfig {
@@ -74,6 +78,13 @@
return currentConfig.getDestinations(filterType);
}
+ @Override
+ public synchronized Multimap<Destination, URIish> getURIs(
+ Optional<String> remoteName, Project.NameKey projectName, FilterType filterType) {
+ reloadIfNeeded();
+ return currentConfig.getURIs(remoteName, projectName, filterType);
+ }
+
private void reloadIfNeeded() {
if (isAutoReload()) {
ReplicationQueue queue = replicationQueue.get();
@@ -110,6 +121,11 @@
}
@Override
+ public synchronized int getMaxRefsToLog() {
+ return currentConfig.getMaxRefsToLog();
+ }
+
+ @Override
public synchronized boolean isEmpty() {
return currentConfig.isEmpty();
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
index 29a7ee6..98f364d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
@@ -23,6 +23,7 @@
import java.nio.file.Files;
import java.util.concurrent.atomic.AtomicReference;
import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.transport.CredentialsProvider;
public class AutoReloadSecureCredentialsFactoryDecorator implements CredentialsFactory {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -50,7 +51,7 @@
}
@Override
- public SecureCredentialsProvider create(String remoteName) {
+ public CredentialsProvider create(String remoteName) {
try {
if (needsReload()) {
secureCredentialsFactory.compareAndSet(
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java
new file mode 100644
index 0000000..e3d57de
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java
@@ -0,0 +1,69 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
+import java.util.Optional;
+import org.eclipse.jgit.transport.RemoteConfig;
+import org.eclipse.jgit.transport.URIish;
+
+public class CreateProjectTask {
+ interface Factory {
+ CreateProjectTask create(Project.NameKey project, String head);
+ }
+
+ private final RemoteConfig config;
+ private final ReplicationConfig replicationConfig;
+ private final AdminApiFactory adminApiFactory;
+ private final Project.NameKey project;
+ private final String head;
+
+ @Inject
+ CreateProjectTask(
+ RemoteConfig config,
+ ReplicationConfig replicationConfig,
+ AdminApiFactory adminApiFactory,
+ @Assisted Project.NameKey project,
+ @Assisted String head) {
+ this.config = config;
+ this.replicationConfig = replicationConfig;
+ this.adminApiFactory = adminApiFactory;
+ this.project = project;
+ this.head = head;
+ }
+
+ public boolean create() {
+ return replicationConfig
+ .getURIs(Optional.of(config.getName()), project, FilterType.PROJECT_CREATION).values()
+ .stream()
+ .map(u -> createProject(u, project, head))
+ .reduce(true, (a, b) -> a && b);
+ }
+
+ private boolean createProject(URIish replicateURI, Project.NameKey projectName, String head) {
+ Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI);
+ if (adminApi.isPresent() && adminApi.get().createProject(projectName, head)) {
+ return true;
+ }
+
+ repLog.warn("Cannot create new project {} on remote site {}.", projectName, replicateURI);
+ return false;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java
index 10719c1..3bb64ab 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java
@@ -13,7 +13,9 @@
// limitations under the License.
package com.googlesource.gerrit.plugins.replication;
+import org.eclipse.jgit.transport.CredentialsProvider;
+
public interface CredentialsFactory {
- SecureCredentialsProvider create(String remoteName);
+ CredentialsProvider create(String remoteName);
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java
new file mode 100644
index 0000000..96d8c70
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java
@@ -0,0 +1,65 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.server.ioutil.HexFormat;
+import com.google.gerrit.server.util.IdGenerator;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import java.util.Optional;
+import org.eclipse.jgit.transport.URIish;
+
+public class DeleteProjectTask implements Runnable {
+ interface Factory {
+ DeleteProjectTask create(URIish replicateURI, Project.NameKey project);
+ }
+
+ private final AdminApiFactory adminApiFactory;
+ private final int id;
+ private final URIish replicateURI;
+ private final Project.NameKey project;
+
+ @Inject
+ DeleteProjectTask(
+ AdminApiFactory adminApiFactory,
+ IdGenerator ig,
+ @Assisted URIish replicateURI,
+ @Assisted Project.NameKey project) {
+ this.adminApiFactory = adminApiFactory;
+ this.id = ig.next();
+ this.replicateURI = replicateURI;
+ this.project = project;
+ }
+
+ @Override
+ public void run() {
+ Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI);
+ if (adminApi.isPresent()) {
+ adminApi.get().deleteProject(project);
+ return;
+ }
+
+ repLog.warn("Cannot delete project {} on remote site {}.", project, replicateURI);
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "[%s] delete-project %s at %s", HexFormat.fromInt(id), project.get(), replicateURI);
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
index 5a3b92c..56a341e 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
@@ -15,6 +15,7 @@
package com.googlesource.gerrit.plugins.replication;
import static com.googlesource.gerrit.plugins.replication.PushResultProcessing.resolveNodeName;
+import static com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig.replaceName;
import static org.eclipse.jgit.transport.RemoteRefUpdate.Status.NON_EXISTING;
import static org.eclipse.jgit.transport.RemoteRefUpdate.Status.REJECTED_OTHER_REASON;
@@ -32,14 +33,12 @@
import com.google.gerrit.reviewdb.client.Branch;
import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.reviewdb.client.RefNames;
-import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.CurrentUser;
import com.google.gerrit.server.PluginUser;
import com.google.gerrit.server.account.GroupBackend;
import com.google.gerrit.server.account.GroupBackends;
import com.google.gerrit.server.account.GroupIncludeCache;
import com.google.gerrit.server.account.ListGroupMembership;
-import com.google.gerrit.server.config.RequestScopedReviewDbProvider;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.git.GitRepositoryManager;
import com.google.gerrit.server.git.PerThreadRequestScope;
@@ -56,6 +55,7 @@
import com.google.inject.Injector;
import com.google.inject.Provider;
import com.google.inject.Provides;
+import com.google.inject.Scopes;
import com.google.inject.assistedinject.Assisted;
import com.google.inject.assistedinject.FactoryModuleBuilder;
import com.google.inject.servlet.RequestScoped;
@@ -71,6 +71,7 @@
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FilenameUtils;
+import org.apache.http.impl.client.CloseableHttpClient;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
@@ -92,6 +93,8 @@
private final Map<URIish, PushOne> pending = new HashMap<>();
private final Map<URIish, PushOne> inFlight = new HashMap<>();
private final PushOne.Factory opFactory;
+ private final DeleteProjectTask.Factory deleteProjectFactory;
+ private final UpdateHeadTask.Factory updateHeadFactory;
private final GitRepositoryManager gitManager;
private final PermissionBackend permissionBackend;
private final Provider<CurrentUser> userProvider;
@@ -165,24 +168,20 @@
bind(Destination.class).toInstance(Destination.this);
bind(RemoteConfig.class).toInstance(config.getRemoteConfig());
install(new FactoryModuleBuilder().build(PushOne.Factory.class));
+ install(new FactoryModuleBuilder().build(CreateProjectTask.Factory.class));
+ install(new FactoryModuleBuilder().build(DeleteProjectTask.Factory.class));
+ install(new FactoryModuleBuilder().build(UpdateHeadTask.Factory.class));
+ bind(AdminApiFactory.class);
+ install(new FactoryModuleBuilder().build(GerritRestApi.Factory.class));
+ bind(CloseableHttpClient.class)
+ .toProvider(HttpClientProvider.class)
+ .in(Scopes.SINGLETON);
}
@Provides
public PerThreadRequestScope.Scoper provideScoper(
- final PerThreadRequestScope.Propagator propagator,
- final Provider<RequestScopedReviewDbProvider> dbProvider) {
- final RequestContext requestContext =
- new RequestContext() {
- @Override
- public CurrentUser getUser() {
- return remoteUser;
- }
-
- @Override
- public Provider<ReviewDb> getReviewDbProvider() {
- return dbProvider.get();
- }
- };
+ final PerThreadRequestScope.Propagator propagator) {
+ final RequestContext requestContext = () -> remoteUser;
return new PerThreadRequestScope.Scoper() {
@Override
public <T> Callable<T> scope(Callable<T> callable) {
@@ -193,6 +192,8 @@
});
opFactory = child.getInstance(PushOne.Factory.class);
+ deleteProjectFactory = child.getInstance(DeleteProjectTask.Factory.class);
+ updateHeadFactory = child.getInstance(UpdateHeadTask.Factory.class);
threadScoper = child.getInstance(PerThreadRequestScope.Scoper.class);
}
@@ -255,38 +256,35 @@
try {
return threadScoper
.scope(
- new Callable<Boolean>() {
- @Override
- public Boolean call() throws NoSuchProjectException, PermissionBackendException {
- ProjectState projectState;
- try {
- projectState = projectCache.checkedGet(project);
- } catch (IOException e) {
- return false;
- }
- if (projectState == null) {
- throw new NoSuchProjectException(project);
- }
- if (!projectState.statePermitsRead()) {
- return false;
- }
- if (!shouldReplicate(projectState, userProvider.get())) {
- return false;
- }
- if (PushOne.ALL_REFS.equals(ref)) {
- return true;
- }
- try {
- permissionBackend
- .user(userProvider.get())
- .project(project)
- .ref(ref)
- .check(RefPermission.READ);
- } catch (AuthException e) {
- return false;
- }
+ () -> {
+ ProjectState projectState;
+ try {
+ projectState = projectCache.checkedGet(project);
+ } catch (IOException e) {
+ return false;
+ }
+ if (projectState == null) {
+ throw new NoSuchProjectException(project);
+ }
+ if (!projectState.statePermitsRead()) {
+ return false;
+ }
+ if (!shouldReplicate(projectState, userProvider.get())) {
+ return false;
+ }
+ if (PushOne.ALL_REFS.equals(ref)) {
return true;
}
+ try {
+ permissionBackend
+ .user(userProvider.get())
+ .project(project)
+ .ref(ref)
+ .check(RefPermission.READ);
+ } catch (AuthException e) {
+ return false;
+ }
+ return true;
})
.call();
} catch (NoSuchProjectException err) {
@@ -302,20 +300,17 @@
try {
return threadScoper
.scope(
- new Callable<Boolean>() {
- @Override
- public Boolean call() throws NoSuchProjectException, PermissionBackendException {
- ProjectState projectState;
- try {
- projectState = projectCache.checkedGet(project);
- } catch (IOException e) {
- return false;
- }
- if (projectState == null) {
- throw new NoSuchProjectException(project);
- }
- return shouldReplicate(projectState, userProvider.get());
+ () -> {
+ ProjectState projectState;
+ try {
+ projectState = projectCache.checkedGet(project);
+ } catch (IOException e) {
+ return false;
}
+ if (projectState == null) {
+ throw new NoSuchProjectException(project);
+ }
+ return shouldReplicate(projectState, userProvider.get());
})
.call();
} catch (NoSuchProjectException err) {
@@ -387,6 +382,14 @@
}
}
+ void scheduleDeleteProject(URIish uri, Project.NameKey project) {
+ pool.schedule(deleteProjectFactory.create(uri, project), 0, TimeUnit.SECONDS);
+ }
+
+ void scheduleUpdateHead(URIish uri, Project.NameKey project, String newHead) {
+ pool.schedule(updateHeadFactory.create(uri, project, newHead), 0, TimeUnit.SECONDS);
+ }
+
private void addRef(PushOne e, String ref) {
e.addRef(ref);
postReplicationScheduledEvent(e, ref);
@@ -582,8 +585,7 @@
} else if (!remoteNameStyle.equals("slash")) {
repLog.debug("Unknown remoteNameStyle: {}, falling back to slash", remoteNameStyle);
}
- String replacedPath =
- ReplicationQueue.replaceName(uri.getPath(), name, isSingleProjectMatch());
+ String replacedPath = replaceName(uri.getPath(), name, isSingleProjectMatch());
if (replacedPath != null) {
uri = uri.setPath(replacedPath);
r.add(uri);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java
new file mode 100644
index 0000000..91fe20a
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java
@@ -0,0 +1,132 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.googlesource.gerrit.plugins.replication.GerritSshApi.GERRIT_ADMIN_PROTOCOL_PREFIX;
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
+import com.google.common.base.Charsets;
+import com.google.gerrit.extensions.restapi.Url;
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import java.io.IOException;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicHeader;
+import org.eclipse.jgit.transport.CredentialItem;
+import org.eclipse.jgit.transport.RemoteConfig;
+import org.eclipse.jgit.transport.URIish;
+
+public class GerritRestApi implements AdminApi {
+
+ public interface Factory {
+ GerritRestApi create(URIish uri);
+ }
+
+ private final CredentialsFactory credentials;
+ private final CloseableHttpClient httpClient;
+ private final RemoteConfig remoteConfig;
+ private final URIish uri;
+
+ @Inject
+ GerritRestApi(
+ CredentialsFactory credentials,
+ CloseableHttpClient httpClient,
+ RemoteConfig remoteConfig,
+ @Assisted URIish uri) {
+ this.credentials = credentials;
+ this.httpClient = httpClient;
+ this.remoteConfig = remoteConfig;
+ this.uri = uri;
+ }
+
+ @Override
+ public boolean createProject(Project.NameKey project, String head) {
+ repLog.info("Creating project {} on {}", project, uri);
+ String url = String.format("%s/a/projects/%s", toHttpUri(uri), Url.encode(project.get()));
+ try {
+ return httpClient
+ .execute(new HttpPut(url), new HttpResponseHandler(), getContext())
+ .isSuccessful();
+ } catch (IOException e) {
+ repLog.error("Couldn't perform project creation on {}", uri, e);
+ return false;
+ }
+ }
+
+ @Override
+ public void deleteProject(Project.NameKey project) {
+ repLog.info("Deleting project {} on {}", project, uri);
+ String url = String.format("%s/a/projects/%s", toHttpUri(uri), Url.encode(project.get()));
+ try {
+ httpClient.execute(new HttpDelete(url), new HttpResponseHandler(), getContext());
+ } catch (IOException e) {
+ repLog.error("Couldn't perform project deletion on {}", uri, e);
+ }
+ }
+
+ @Override
+ public void updateHead(Project.NameKey project, String newHead) {
+ repLog.info("Updating head of {} on {}", project, uri);
+ String url = String.format("%s/a/projects/%s/HEAD", toHttpUri(uri), Url.encode(project.get()));
+ try {
+ HttpPut req = new HttpPut(url);
+ req.setEntity(
+ new StringEntity(String.format("{\"ref\": \"%s\"}", newHead), Charsets.UTF_8.name()));
+ req.addHeader(new BasicHeader("Content-Type", "application/json"));
+ httpClient.execute(req, new HttpResponseHandler(), getContext());
+ } catch (IOException e) {
+ repLog.error("Couldn't perform update head on {}", uri, e);
+ }
+ }
+
+ private HttpClientContext getContext() {
+ HttpClientContext ctx = HttpClientContext.create();
+ ctx.setCredentialsProvider(adapt(credentials.create(remoteConfig.getName())));
+ return ctx;
+ }
+
+ private CredentialsProvider adapt(org.eclipse.jgit.transport.CredentialsProvider cp) {
+ CredentialItem.Username user = new CredentialItem.Username();
+ CredentialItem.Password pass = new CredentialItem.Password();
+ if (cp.supports(user, pass) && cp.get(uri, user, pass)) {
+ CredentialsProvider adapted = new BasicCredentialsProvider();
+ adapted.setCredentials(
+ AuthScope.ANY,
+ new UsernamePasswordCredentials(user.getValue(), new String(pass.getValue())));
+ return adapted;
+ }
+ return null;
+ }
+
+ private static String toHttpUri(URIish uri) {
+ String u = uri.toString();
+ if (u.startsWith(GERRIT_ADMIN_PROTOCOL_PREFIX)) {
+ u = u.substring(GERRIT_ADMIN_PROTOCOL_PREFIX.length());
+ }
+ if (u.endsWith("/")) {
+ return u;
+ }
+ return u + "/";
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
index 85b17d0..12b6cbe 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
@@ -28,7 +28,7 @@
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
static int SSH_COMMAND_FAILED = -1;
- private static String GERRIT_ADMIN_PROTOCOL_PREFIX = "gerrit+";
+ static String GERRIT_ADMIN_PROTOCOL_PREFIX = "gerrit+";
private final SshHelper sshHelper;
private final URIish uri;
@@ -41,14 +41,16 @@
}
@Override
- public void createProject(Project.NameKey projectName, String head) {
+ public boolean createProject(Project.NameKey projectName, String head) {
OutputStream errStream = sshHelper.newErrorBufferStream();
String cmd = "gerrit create-project --branch " + head + " " + projectName.get();
try {
execute(uri, cmd, errStream);
} catch (IOException e) {
logError("creating", uri, errStream, cmd, e);
+ return false;
}
+ return true;
}
@Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/HttpClientProvider.java b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpClientProvider.java
new file mode 100644
index 0000000..916059c
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpClientProvider.java
@@ -0,0 +1,108 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.ProvisionException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import javax.net.ssl.SSLContext;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.HttpClientConnectionManager;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.ssl.SSLContexts;
+import org.eclipse.jgit.lib.Config;
+
+/** Provides an HTTP client with SSL capabilities. */
+class HttpClientProvider implements Provider<CloseableHttpClient> {
+ private static final int CONNECTIONS_PER_ROUTE = 100;
+
+ // Up to 2 target instances with the max number of connections per host:
+ private static final int MAX_CONNECTIONS = 2 * CONNECTIONS_PER_ROUTE;
+
+ private static final int MAX_CONNECTION_INACTIVITY = 10000;
+ private static final int DEFAULT_TIMEOUT_MS = 5000;
+
+ private final Config cfg;
+ private final SitePaths site;
+
+ @Inject
+ HttpClientProvider(@GerritServerConfig Config cfg, SitePaths site) {
+ this.cfg = cfg;
+ this.site = site;
+ }
+
+ @Override
+ public CloseableHttpClient get() {
+ try {
+ return HttpClients.custom()
+ .setConnectionManager(customConnectionManager())
+ .setDefaultRequestConfig(customRequestConfig())
+ .build();
+ } catch (Exception e) {
+ throw new ProvisionException("Couldn't create CloseableHttpClient", e);
+ }
+ }
+
+ private RequestConfig customRequestConfig() {
+ return RequestConfig.custom()
+ .setConnectTimeout(DEFAULT_TIMEOUT_MS)
+ .setSocketTimeout(DEFAULT_TIMEOUT_MS)
+ .setConnectionRequestTimeout(DEFAULT_TIMEOUT_MS)
+ .build();
+ }
+
+ private HttpClientConnectionManager customConnectionManager() throws Exception {
+ Registry<ConnectionSocketFactory> socketFactoryRegistry =
+ RegistryBuilder.<ConnectionSocketFactory>create()
+ .register("https", buildSslSocketFactory())
+ .register("http", PlainConnectionSocketFactory.INSTANCE)
+ .build();
+ PoolingHttpClientConnectionManager connManager =
+ new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+ connManager.setDefaultMaxPerRoute(CONNECTIONS_PER_ROUTE);
+ connManager.setMaxTotal(MAX_CONNECTIONS);
+ connManager.setValidateAfterInactivity(MAX_CONNECTION_INACTIVITY);
+ return connManager;
+ }
+
+ private SSLConnectionSocketFactory buildSslSocketFactory() throws Exception {
+ String keyStore = cfg.getString("httpd", null, "sslKeyStore");
+ if (keyStore == null) {
+ keyStore = "etc/keystore";
+ }
+ return new SSLConnectionSocketFactory(createSSLContext(site.resolve(keyStore)));
+ }
+
+ private SSLContext createSSLContext(Path keyStorePath) throws Exception {
+ SSLContext ctx;
+ if (Files.exists(keyStorePath)) {
+ ctx = SSLContexts.custom().loadTrustMaterial(keyStorePath.toFile()).build();
+ } else {
+ ctx = SSLContext.getDefault();
+ }
+ return ctx;
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponse.java b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponse.java
new file mode 100644
index 0000000..f26128c
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponse.java
@@ -0,0 +1,66 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import com.google.common.base.Preconditions;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.ByteBuffer;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.eclipse.jgit.util.IO;
+import org.eclipse.jgit.util.RawParseUtils;
+
+public class HttpResponse implements AutoCloseable {
+
+ protected CloseableHttpResponse response;
+ protected Reader reader;
+
+ HttpResponse(CloseableHttpResponse response) {
+ this.response = response;
+ }
+
+ public Reader getReader() throws IllegalStateException, IOException {
+ if (reader == null && response.getEntity() != null) {
+ reader = new InputStreamReader(response.getEntity().getContent());
+ }
+ return reader;
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ Reader reader = getReader();
+ if (reader != null) {
+ while (reader.read() != -1) {
+ // Empty
+ }
+ }
+ } finally {
+ response.close();
+ }
+ }
+
+ public int getStatusCode() {
+ return response.getStatusLine().getStatusCode();
+ }
+
+ public String getEntityContent() throws IOException {
+ Preconditions.checkNotNull(response, "Response is not initialized.");
+ Preconditions.checkNotNull(response.getEntity(), "Response.Entity is not initialized.");
+ ByteBuffer buf = IO.readWholeStream(response.getEntity().getContent(), 1024);
+ return RawParseUtils.decode(buf.array(), buf.arrayOffset(), buf.limit()).trim();
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponseHandler.java b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponseHandler.java
new file mode 100644
index 0000000..6a7c73e
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponseHandler.java
@@ -0,0 +1,71 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static javax.servlet.http.HttpServletResponse.SC_CREATED;
+import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+
+import com.google.common.flogger.FluentLogger;
+import com.googlesource.gerrit.plugins.replication.HttpResponseHandler.HttpResult;
+import java.io.IOException;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.ResponseHandler;
+import org.apache.http.util.EntityUtils;
+
+class HttpResponseHandler implements ResponseHandler<HttpResult> {
+ private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+ static class HttpResult {
+ private final boolean successful;
+ private final String message;
+
+ HttpResult(boolean successful, String message) {
+ this.successful = successful;
+ this.message = message;
+ }
+
+ boolean isSuccessful() {
+ return successful;
+ }
+
+ String getMessage() {
+ return message;
+ }
+ }
+
+ @Override
+ public HttpResult handleResponse(HttpResponse response) {
+ return new HttpResult(isSuccessful(response), parseResponse(response));
+ }
+
+ private static boolean isSuccessful(HttpResponse response) {
+ int sc = response.getStatusLine().getStatusCode();
+ return sc == SC_CREATED || sc == SC_NO_CONTENT || sc == SC_OK;
+ }
+
+ private static String parseResponse(HttpResponse response) {
+ HttpEntity entity = response.getEntity();
+ if (entity != null) {
+ try {
+ return EntityUtils.toString(entity);
+ } catch (IOException e) {
+ logger.atSevere().withCause(e).log("Error parsing entity");
+ }
+ }
+ return "";
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java b/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java
index 1012cd7..908e7e0 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java
@@ -34,7 +34,7 @@
}
@Override
- public void createProject(Project.NameKey project, String head) {
+ public boolean createProject(Project.NameKey project, String head) {
try (Repository repo = new FileRepository(uri.getPath())) {
repo.create(true /* bare */);
@@ -46,7 +46,9 @@
repLog.info("Created local repository: {}", uri);
} catch (IOException e) {
repLog.error("Error creating local repository {}", uri.getPath(), e);
+ return false;
}
+ return true;
}
@Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
index 09187a0..9c1de54 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
@@ -16,6 +16,7 @@
import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static java.util.stream.Collectors.toMap;
import com.google.common.base.Throwables;
import com.google.common.collect.LinkedListMultimap;
@@ -52,7 +53,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicBoolean;
import org.eclipse.jgit.errors.NoRemoteRepositoryException;
import org.eclipse.jgit.errors.NotSupportedException;
@@ -92,9 +92,9 @@
private final PermissionBackend permissionBackend;
private final Destination pool;
private final RemoteConfig config;
+ private final ReplicationConfig replConfig;
private final CredentialsProvider credentialsProvider;
private final PerThreadRequestScope.Scoper threadScoper;
- private final ReplicationQueue replicationQueue;
private final Project.NameKey projectName;
private final URIish uri;
@@ -112,6 +112,7 @@
private final long createdAt;
private final ReplicationMetrics metrics;
private final ProjectCache projectCache;
+ private final CreateProjectTask.Factory createProjectFactory;
private final AtomicBoolean canceledWhileRunning;
@Inject
@@ -120,22 +121,23 @@
PermissionBackend permissionBackend,
Destination p,
RemoteConfig c,
+ ReplicationConfig rc,
CredentialsFactory cpFactory,
PerThreadRequestScope.Scoper ts,
- ReplicationQueue rq,
IdGenerator ig,
ReplicationStateListeners sl,
ReplicationMetrics m,
ProjectCache pc,
+ CreateProjectTask.Factory cpf,
@Assisted Project.NameKey d,
@Assisted URIish u) {
gitManager = grm;
this.permissionBackend = permissionBackend;
pool = p;
config = c;
+ replConfig = rc;
credentialsProvider = cpFactory.create(c.getName());
threadScoper = ts;
- replicationQueue = rq;
projectName = d;
uri = u;
lockRetryCount = 0;
@@ -145,6 +147,7 @@
createdAt = System.nanoTime();
metrics = m;
projectCache = pc;
+ createProjectFactory = cpf;
canceledWhileRunning = new AtomicBoolean(false);
maxRetries = p.getMaxRetries();
}
@@ -275,12 +278,9 @@
try {
threadScoper
.scope(
- new Callable<Void>() {
- @Override
- public Void call() {
- runPushOperation();
- return null;
- }
+ () -> {
+ runPushOperation();
+ return null;
})
.call();
} catch (Exception e) {
@@ -333,7 +333,8 @@
String msg = e.getMessage();
if (msg.contains("access denied")
|| msg.contains("no such repository")
- || msg.contains("Git repository not found")) {
+ || msg.contains("Git repository not found")
+ || msg.contains("unavailable")) {
createRepository();
} else {
repLog.error("Cannot replicate {}; Remote repository error: {}", projectName, msg);
@@ -396,15 +397,11 @@
if (pool.isCreateMissingRepos()) {
try {
Ref head = git.exactRef(Constants.HEAD);
- if (replicationQueue.createProject(projectName, head != null ? getName(head) : null)) {
+ if (createProject(projectName, head != null ? getName(head) : null)) {
repLog.warn("Missing repository created; retry replication to {}", uri);
pool.reschedule(this, Destination.RetryReason.REPOSITORY_MISSING);
} else {
- repLog.warn(
- "Missing repository could not be created when replicating {}. "
- + "You can only create missing repositories locally, over SSH or when "
- + "using adminUrl in replication.config. See documentation for more information.",
- uri);
+ repLog.warn("Missing repository could not be created when replicating {}", uri);
}
} catch (IOException ioe) {
stateLog.error(
@@ -417,6 +414,10 @@
}
}
+ private boolean createProject(Project.NameKey project, String head) {
+ return createProjectFactory.create(project, head).create();
+ }
+
private String getName(Ref ref) {
Ref target = ref;
while (target.isSymbolic()) {
@@ -447,7 +448,16 @@
return new PushResult();
}
- repLog.info("Push to {} references: {}", uri, todo);
+ if (replConfig.getMaxRefsToLog() == 0 || todo.size() <= replConfig.getMaxRefsToLog()) {
+ repLog.info("Push to {} references: {}", uri, todo);
+ } else {
+ repLog.info(
+ "Push to {} references (first {} of {} listed): {}",
+ uri,
+ replConfig.getMaxRefsToLog(),
+ todo.size(),
+ todo.subList(0, replConfig.getMaxRefsToLog()));
+ }
return tn.push(NullProgressMonitor.INSTANCE, todo);
}
@@ -459,7 +469,8 @@
return Collections.emptyList();
}
- Map<String, Ref> local = git.getAllRefs();
+ Map<String, Ref> local =
+ git.getRefDatabase().getRefs().stream().collect(toMap(Ref::getName, r -> r));
boolean filter;
PermissionBackend.ForProject forProject = permissionBackend.currentUser().project(projectName);
try {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java
index dad1b0b..6dfa117 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java
@@ -33,7 +33,7 @@
}
@Override
- public void createProject(Project.NameKey project, String head) {
+ public boolean createProject(Project.NameKey project, String head) {
String quotedPath = QuotedString.BOURNE.quote(uri.getPath());
String cmd = "mkdir -p " + quotedPath + " && cd " + quotedPath + " && git init --bare";
if (head != null) {
@@ -54,7 +54,9 @@
cmd,
errStream,
e);
+ return false;
}
+ return true;
}
@Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
index 9693e2d..1c62ab7 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
@@ -13,9 +13,13 @@
// limitations under the License.
package com.googlesource.gerrit.plugins.replication;
+import com.google.common.collect.Multimap;
+import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.git.WorkQueue;
import java.nio.file.Path;
import java.util.List;
+import java.util.Optional;
+import org.eclipse.jgit.transport.URIish;
public interface ReplicationConfig {
@@ -27,10 +31,15 @@
List<Destination> getDestinations(FilterType filterType);
+ Multimap<Destination, URIish> getURIs(
+ Optional<String> remoteName, Project.NameKey projectName, FilterType filterType);
+
boolean isReplicateAllOnPluginStart();
boolean isDefaultForceUpdate();
+ int getMaxRefsToLog();
+
boolean isEmpty();
Path getEventsDirectory();
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
index 46d2f66..64eacf9 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
@@ -13,13 +13,22 @@
// limitations under the License.
package com.googlesource.gerrit.plugins.replication;
+import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isGerrit;
+import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isGerritHttp;
+import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isSSH;
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
import static java.util.stream.Collectors.toList;
import com.google.common.base.Strings;
+import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.SetMultimap;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.extensions.annotations.PluginData;
+import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.git.WorkQueue;
import com.google.inject.Inject;
@@ -30,6 +39,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
import org.eclipse.jgit.errors.ConfigInvalidException;
@@ -48,6 +58,7 @@
private Path cfgPath;
private boolean replicateAllOnPluginStart;
private boolean defaultForceUpdate;
+ private int maxRefsToLog;
private final FileBasedConfig config;
private final Path pluginDataDir;
@@ -111,6 +122,8 @@
defaultForceUpdate = config.getBoolean("gerrit", "defaultForceUpdate", false);
+ maxRefsToLog = config.getInt("gerrit", "maxRefsToLog", 0);
+
ImmutableList.Builder<Destination> dest = ImmutableList.builder();
for (RemoteConfig c : allRemotes(config)) {
if (c.getURIs().isEmpty()) {
@@ -149,6 +162,77 @@
return dest.build();
}
+ @Override
+ public Multimap<Destination, URIish> getURIs(
+ Optional<String> remoteName, Project.NameKey projectName, FilterType filterType) {
+ if (getDestinations(filterType).isEmpty()) {
+ return ImmutableMultimap.of();
+ }
+
+ SetMultimap<Destination, URIish> uris = HashMultimap.create();
+ for (Destination config : getDestinations(filterType)) {
+ if (!config.wouldPushProject(projectName)) {
+ continue;
+ }
+
+ if (remoteName.isPresent() && !config.getRemoteConfigName().equals(remoteName.get())) {
+ continue;
+ }
+
+ boolean adminURLUsed = false;
+
+ for (String url : config.getAdminUrls()) {
+ if (Strings.isNullOrEmpty(url)) {
+ continue;
+ }
+
+ URIish uri;
+ try {
+ uri = new URIish(url);
+ } catch (URISyntaxException e) {
+ repLog.warn("adminURL '{}' is invalid: {}", url, e.getMessage());
+ continue;
+ }
+
+ if (!isGerrit(uri) && !isGerritHttp(uri)) {
+ String path =
+ replaceName(uri.getPath(), projectName.get(), config.isSingleProjectMatch());
+ if (path == null) {
+ repLog.warn("adminURL {} does not contain ${name}", uri);
+ continue;
+ }
+
+ uri = uri.setPath(path);
+ if (!isSSH(uri)) {
+ repLog.warn("adminURL '{}' is invalid: only SSH and HTTP are supported", uri);
+ continue;
+ }
+ }
+ uris.put(config, uri);
+ adminURLUsed = true;
+ }
+
+ if (!adminURLUsed) {
+ for (URIish uri : config.getURIs(projectName, "*")) {
+ uris.put(config, uri);
+ }
+ }
+ }
+ return uris;
+ }
+
+ static String replaceName(String in, String name, boolean keyIsOptional) {
+ String key = "${name}";
+ int n = in.indexOf(key);
+ if (0 <= n) {
+ return in.substring(0, n) + name + in.substring(n + key.length());
+ }
+ if (keyIsOptional) {
+ return in;
+ }
+ return null;
+ }
+
/* (non-Javadoc)
* @see com.googlesource.gerrit.plugins.replication.ReplicationConfig#isReplicateAllOnPluginStart()
*/
@@ -165,6 +249,11 @@
return defaultForceUpdate;
}
+ @Override
+ public int getMaxRefsToLog() {
+ return maxRefsToLog;
+ }
+
private static List<RemoteConfig> allRemotes(FileBasedConfig cfg) throws ConfigInvalidException {
Set<String> names = cfg.getSubsections("remote");
List<RemoteConfig> result = Lists.newArrayListWithCapacity(names.size());
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
index 1405a12..9b7e8e6 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
@@ -21,7 +21,6 @@
import com.google.gerrit.extensions.events.GitReferenceUpdatedListener;
import com.google.gerrit.extensions.events.HeadUpdatedListener;
import com.google.gerrit.extensions.events.LifecycleListener;
-import com.google.gerrit.extensions.events.NewProjectCreatedListener;
import com.google.gerrit.extensions.events.ProjectDeletedListener;
import com.google.gerrit.extensions.registration.DynamicSet;
import com.google.gerrit.server.events.EventTypes;
@@ -41,7 +40,6 @@
.to(ReplicationQueue.class);
DynamicSet.bind(binder(), GitReferenceUpdatedListener.class).to(ReplicationQueue.class);
- DynamicSet.bind(binder(), NewProjectCreatedListener.class).to(ReplicationQueue.class);
DynamicSet.bind(binder(), ProjectDeletedListener.class).to(ReplicationQueue.class);
DynamicSet.bind(binder(), HeadUpdatedListener.class).to(ReplicationQueue.class);
@@ -68,7 +66,5 @@
EventTypes.register(RefReplicationDoneEvent.TYPE, RefReplicationDoneEvent.class);
EventTypes.register(ReplicationScheduledEvent.TYPE, ReplicationScheduledEvent.class);
bind(SshSessionFactory.class).toProvider(ReplicationSshSessionFactoryProvider.class);
-
- bind(AdminApiFactory.class);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
index 541a595..d2a34be 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
@@ -14,14 +14,9 @@
package com.googlesource.gerrit.plugins.replication;
-import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isGerrit;
-import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isSSH;
-
-import com.google.common.base.Strings;
import com.google.gerrit.extensions.events.GitReferenceUpdatedListener;
import com.google.gerrit.extensions.events.HeadUpdatedListener;
import com.google.gerrit.extensions.events.LifecycleListener;
-import com.google.gerrit.extensions.events.NewProjectCreatedListener;
import com.google.gerrit.extensions.events.ProjectDeletedListener;
import com.google.gerrit.extensions.registration.DynamicItem;
import com.google.gerrit.reviewdb.client.Project;
@@ -30,11 +25,7 @@
import com.google.inject.Inject;
import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdateProcessing;
import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType;
-import java.net.URISyntaxException;
-import java.util.Collections;
-import java.util.HashSet;
import java.util.Optional;
-import java.util.Set;
import org.eclipse.jgit.transport.URIish;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,7 +34,6 @@
public class ReplicationQueue
implements LifecycleListener,
GitReferenceUpdatedListener,
- NewProjectCreatedListener,
ProjectDeletedListener,
HeadUpdatedListener {
static final String REPLICATION_LOG_NAME = "replication_log";
@@ -51,22 +41,9 @@
private final ReplicationStateListener stateLog;
- static String replaceName(String in, String name, boolean keyIsOptional) {
- String key = "${name}";
- int n = in.indexOf(key);
- if (0 <= n) {
- return in.substring(0, n) + name + in.substring(n + key.length());
- }
- if (keyIsOptional) {
- return in;
- }
- return null;
- }
-
private final WorkQueue workQueue;
private final DynamicItem<EventDispatcher> dispatcher;
private final ReplicationConfig config;
- private final AdminApiFactory adminApiFactory;
private final ReplicationState.Factory replicationStateFactory;
private final EventsStorage eventsStorage;
private volatile boolean running;
@@ -74,7 +51,6 @@
@Inject
ReplicationQueue(
WorkQueue wq,
- AdminApiFactory aaf,
ReplicationConfig rc,
DynamicItem<EventDispatcher> dis,
ReplicationStateListeners sl,
@@ -84,7 +60,6 @@
dispatcher = dis;
config = rc;
stateLog = sl;
- adminApiFactory = aaf;
replicationStateFactory = rsf;
eventsStorage = es;
}
@@ -161,130 +136,16 @@
}
@Override
- public void onNewProjectCreated(NewProjectCreatedListener.Event event) {
- Project.NameKey projectName = new Project.NameKey(event.getProjectName());
- for (URIish uri : getURIs(projectName, FilterType.PROJECT_CREATION)) {
- createProject(uri, projectName, event.getHeadName());
- }
- }
-
- @Override
public void onProjectDeleted(ProjectDeletedListener.Event event) {
- Project.NameKey projectName = new Project.NameKey(event.getProjectName());
- for (URIish uri : getURIs(projectName, FilterType.PROJECT_DELETION)) {
- deleteProject(uri, projectName);
- }
+ Project.NameKey p = new Project.NameKey(event.getProjectName());
+ config.getURIs(Optional.empty(), p, FilterType.PROJECT_DELETION).entries().stream()
+ .forEach(e -> e.getKey().scheduleDeleteProject(e.getValue(), p));
}
@Override
public void onHeadUpdated(HeadUpdatedListener.Event event) {
- Project.NameKey project = new Project.NameKey(event.getProjectName());
- for (URIish uri : getURIs(project, FilterType.ALL)) {
- updateHead(uri, project, event.getNewHeadName());
- }
- }
-
- private Set<URIish> getURIs(Project.NameKey projectName, FilterType filterType) {
- if (config.getDestinations(filterType).isEmpty()) {
- return Collections.emptySet();
- }
- if (!running) {
- repLog.error("Replication plugin did not finish startup before event");
- return Collections.emptySet();
- }
-
- Set<URIish> uris = new HashSet<>();
- for (Destination config : this.config.getDestinations(filterType)) {
- if (!config.wouldPushProject(projectName)) {
- continue;
- }
-
- boolean adminURLUsed = false;
-
- for (String url : config.getAdminUrls()) {
- if (Strings.isNullOrEmpty(url)) {
- continue;
- }
-
- URIish uri;
- try {
- uri = new URIish(url);
- } catch (URISyntaxException e) {
- repLog.warn("adminURL '{}' is invalid: {}", url, e.getMessage());
- continue;
- }
-
- if (!isGerrit(uri)) {
- String path =
- replaceName(uri.getPath(), projectName.get(), config.isSingleProjectMatch());
- if (path == null) {
- repLog.warn("adminURL {} does not contain ${name}", uri);
- continue;
- }
-
- uri = uri.setPath(path);
- if (!isSSH(uri)) {
- repLog.warn("adminURL '{}' is invalid: only SSH is supported", uri);
- continue;
- }
- }
- uris.add(uri);
- adminURLUsed = true;
- }
-
- if (!adminURLUsed) {
- for (URIish uri : config.getURIs(projectName, "*")) {
- uris.add(uri);
- }
- }
- }
- return uris;
- }
-
- public boolean createProject(Project.NameKey project, String head) {
- boolean success = true;
- for (URIish uri : getURIs(project, FilterType.PROJECT_CREATION)) {
- success &= createProject(uri, project, head);
- }
- return success;
- }
-
- private boolean createProject(URIish replicateURI, Project.NameKey projectName, String head) {
- Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI);
- if (adminApi.isPresent()) {
- adminApi.get().createProject(projectName, head);
- return true;
- }
-
- warnCannotPerform("create new project", replicateURI);
- return false;
- }
-
- private void deleteProject(URIish replicateURI, Project.NameKey projectName) {
- Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI);
- if (adminApi.isPresent()) {
- adminApi.get().deleteProject(projectName);
- return;
- }
-
- warnCannotPerform("delete project", replicateURI);
- }
-
- private void updateHead(URIish replicateURI, Project.NameKey projectName, String newHead) {
- Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI);
- if (adminApi.isPresent()) {
- adminApi.get().updateHead(projectName, newHead);
- return;
- }
-
- warnCannotPerform("update HEAD of project", replicateURI);
- }
-
- private void warnCannotPerform(String op, URIish uri) {
- repLog.warn(
- "Cannot {} on remote site {}."
- + "Only local paths and SSH URLs are supported for this operation",
- op,
- uri);
+ Project.NameKey p = new Project.NameKey(event.getProjectName());
+ config.getURIs(Optional.empty(), p, FilterType.ALL).entries().stream()
+ .forEach(e -> e.getKey().scheduleUpdateHead(e.getValue(), p, event.getNewHeadName()));
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java
index 2b0c16b..e511b92 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java
@@ -20,6 +20,8 @@
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.transport.CredentialsProvider;
+import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.eclipse.jgit.util.FS;
/** Looks up a remote's password in secure.config. */
@@ -48,9 +50,9 @@
}
@Override
- public SecureCredentialsProvider create(String remoteName) {
+ public CredentialsProvider create(String remoteName) {
String user = config.getString("remote", remoteName, "username");
String pass = config.getString("remote", remoteName, "password");
- return new SecureCredentialsProvider(user, pass);
+ return new UsernamePasswordCredentialsProvider(user, pass);
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java
deleted file mode 100644
index 62b4036..0000000
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java
+++ /dev/null
@@ -1,80 +0,0 @@
-// Copyright (C) 2011 The Android Open Source Project
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.googlesource.gerrit.plugins.replication;
-
-import org.eclipse.jgit.errors.UnsupportedCredentialItem;
-import org.eclipse.jgit.transport.CredentialItem;
-import org.eclipse.jgit.transport.CredentialsProvider;
-import org.eclipse.jgit.transport.URIish;
-
-/** Looks up a remote's password in secure.config. */
-public class SecureCredentialsProvider extends CredentialsProvider {
- private final String cfgUser;
- private final String cfgPass;
-
- SecureCredentialsProvider(String user, String pass) {
- cfgUser = user;
- cfgPass = pass;
- }
-
- @Override
- public boolean isInteractive() {
- return false;
- }
-
- @Override
- public boolean supports(CredentialItem... items) {
- for (CredentialItem i : items) {
- if (i instanceof CredentialItem.Username) {
- continue;
- } else if (i instanceof CredentialItem.Password) {
- continue;
- } else {
- return false;
- }
- }
- return true;
- }
-
- @Override
- public boolean get(URIish uri, CredentialItem... items) throws UnsupportedCredentialItem {
- String username = uri.getUser();
- if (username == null) {
- username = cfgUser;
- }
- if (username == null) {
- return false;
- }
-
- String password = uri.getPass();
- if (password == null) {
- password = cfgPass;
- }
- if (password == null) {
- return false;
- }
-
- for (CredentialItem i : items) {
- if (i instanceof CredentialItem.Username) {
- ((CredentialItem.Username) i).setValue(username);
- } else if (i instanceof CredentialItem.Password) {
- ((CredentialItem.Password) i).setValue(password.toCharArray());
- } else {
- throw new UnsupportedCredentialItem(uri, i.getPromptText());
- }
- }
- return true;
- }
-}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java
new file mode 100644
index 0000000..db58f49
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java
@@ -0,0 +1,69 @@
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
+
+import com.google.gerrit.reviewdb.client.Project;
+import com.google.gerrit.server.ioutil.HexFormat;
+import com.google.gerrit.server.util.IdGenerator;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import java.util.Optional;
+import org.eclipse.jgit.transport.URIish;
+
+public class UpdateHeadTask implements Runnable {
+ private final AdminApiFactory adminApiFactory;
+ private final int id;
+ private final URIish replicateURI;
+ private final Project.NameKey project;
+ private final String newHead;
+
+ interface Factory {
+ UpdateHeadTask create(URIish uri, Project.NameKey project, String newHead);
+ }
+
+ @Inject
+ UpdateHeadTask(
+ AdminApiFactory adminApiFactory,
+ IdGenerator ig,
+ @Assisted URIish replicateURI,
+ @Assisted Project.NameKey project,
+ @Assisted String newHead) {
+ this.adminApiFactory = adminApiFactory;
+ this.id = ig.next();
+ this.replicateURI = replicateURI;
+ this.project = project;
+ this.newHead = newHead;
+ }
+
+ @Override
+ public void run() {
+ Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI);
+ if (adminApi.isPresent()) {
+ adminApi.get().updateHead(project, newHead);
+ return;
+ }
+
+ repLog.warn("Cannot update HEAD of project {} on remote site {}.", project, replicateURI);
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "[%s] update-head of %s at %s to %s",
+ HexFormat.fromInt(id), project.get(), replicateURI, newHead);
+ }
+}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 2359599..6493761 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -82,6 +82,10 @@
: If true, the default push refspec will be set to use forced
update to the remote when no refspec is given. By default, false.
+gerrit.maxRefsToLog
+: Number of refs, that are pushed during replication, to be logged.
+ For printing all refs to the logs, use a value of 0. By default, 0.
+
replication.lockErrorMaxRetries
: Number of times to retry a replication operation if a lock
error is detected.
@@ -169,13 +173,18 @@
local environment. In that case, an alternative SSH url could
be specified to repository creation.
- To enable replication to different Gerrit instance use `gerrit+ssh://`
- as protocol name followed by hostname of another Gerrit server eg.
+ To enable replication to different Gerrit instance use `gerrit+ssh://`,
+ `gerrit+http://` or `gerrit+https://` as protocol name followed
+ by hostname of another Gerrit server eg.
`gerrit+ssh://replica1.my.org/`
+ <br>
+ `gerrit+http://replica2.my.org/`
+ <br>
+ `gerrit+https://replica3.my.org/`
- In this case replication will use Gerrit's SSH API to
- create/remove projects and update repository HEAD references.
+ In this case replication will use Gerrit's SSH API or Gerrit's REST API
+ to create/remove projects and update repository HEAD references.
NOTE: In order to replicate project deletion, the
link:https://gerrit-review.googlesource.com/admin/projects/plugins/delete-project delete-project[delete-project]
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
index 5fa7b98..549890a 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
@@ -15,20 +15,14 @@
package com.googlesource.gerrit.plugins.replication;
import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.createNiceMock;
-import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.reset;
import static org.easymock.EasyMock.verify;
-import com.google.gerrit.reviewdb.server.ReviewDb;
import com.google.gerrit.server.events.EventDispatcher;
import com.google.gerrit.server.permissions.PermissionBackendException;
-import com.google.gwtorm.client.KeyUtil;
import com.google.gwtorm.server.OrmException;
-import com.google.gwtorm.server.SchemaFactory;
-import com.google.gwtorm.server.StandardKeyEncoder;
import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdateProcessing;
import com.googlesource.gerrit.plugins.replication.ReplicationState.RefPushResult;
import java.net.URISyntaxException;
@@ -37,12 +31,7 @@
import org.junit.Before;
import org.junit.Test;
-@SuppressWarnings("unchecked")
public class GitUpdateProcessingTest {
- static {
- KeyUtil.setEncoderImpl(new StandardKeyEncoder());
- }
-
private EventDispatcher dispatcherMock;
private GitUpdateProcessing gitUpdateProcessing;
@@ -50,11 +39,6 @@
public void setUp() throws Exception {
dispatcherMock = createMock(EventDispatcher.class);
replay(dispatcherMock);
- ReviewDb reviewDbMock = createNiceMock(ReviewDb.class);
- replay(reviewDbMock);
- SchemaFactory<ReviewDb> schemaMock = createMock(SchemaFactory.class);
- expect(schemaMock.open()).andReturn(reviewDbMock).anyTimes();
- replay(schemaMock);
gitUpdateProcessing = new GitUpdateProcessing(dispatcherMock);
}
