Merge branch 'stable-2.16' into stable-3.0 * stable-2.16: Fix NPE when username or password isn't specified for remote Change-Id: I30150b8eebef71b17285591c1f0db4521c1fe4d7
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApi.java index ef7e353..79362fd 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApi.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApi.java
@@ -17,7 +17,7 @@ import com.google.gerrit.reviewdb.client.Project; public interface AdminApi { - public void createProject(Project.NameKey project, String head); + public boolean createProject(Project.NameKey project, String head); public void deleteProject(Project.NameKey project);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java index 528aff2..e5a1628 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApiFactory.java
@@ -23,10 +23,12 @@ public class AdminApiFactory { private final SshHelper sshHelper; + private final GerritRestApi.Factory gerritRestApiFactory; @Inject - AdminApiFactory(SshHelper sshHelper) { + AdminApiFactory(SshHelper sshHelper, GerritRestApi.Factory gerritRestApiFactory) { this.sshHelper = sshHelper; + this.gerritRestApiFactory = gerritRestApiFactory; } public Optional<AdminApi> create(URIish uri) { @@ -36,6 +38,8 @@ return Optional.of(new LocalFS(uri)); } else if (isSSH(uri)) { return Optional.of(new RemoteSsh(sshHelper, uri)); + } else if (isGerritHttp(uri)) { + return Optional.of(gerritRestApiFactory.create(uri)); } return Optional.empty(); } @@ -58,4 +62,9 @@ } return false; } + + public static boolean isGerritHttp(URIish uri) { + String scheme = uri.getScheme(); + return scheme != null && scheme.toLowerCase().contains("gerrit+http"); + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java index ad6e67d..53f56a7 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadConfigDecorator.java
@@ -13,9 +13,11 @@ // limitations under the License. package com.googlesource.gerrit.plugins.replication; +import com.google.common.collect.Multimap; import com.google.common.flogger.FluentLogger; import com.google.gerrit.common.FileUtil; import com.google.gerrit.extensions.annotations.PluginData; +import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.git.WorkQueue; import com.google.inject.Inject; @@ -24,7 +26,9 @@ import java.io.IOException; import java.nio.file.Path; import java.util.List; +import java.util.Optional; import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.transport.URIish; @Singleton public class AutoReloadConfigDecorator implements ReplicationConfig { @@ -74,6 +78,13 @@ return currentConfig.getDestinations(filterType); } + @Override + public synchronized Multimap<Destination, URIish> getURIs( + Optional<String> remoteName, Project.NameKey projectName, FilterType filterType) { + reloadIfNeeded(); + return currentConfig.getURIs(remoteName, projectName, filterType); + } + private void reloadIfNeeded() { if (isAutoReload()) { ReplicationQueue queue = replicationQueue.get(); @@ -110,6 +121,11 @@ } @Override + public synchronized int getMaxRefsToLog() { + return currentConfig.getMaxRefsToLog(); + } + + @Override public synchronized boolean isEmpty() { return currentConfig.isEmpty(); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java index 29a7ee6..98f364d 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AutoReloadSecureCredentialsFactoryDecorator.java
@@ -23,6 +23,7 @@ import java.nio.file.Files; import java.util.concurrent.atomic.AtomicReference; import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.transport.CredentialsProvider; public class AutoReloadSecureCredentialsFactoryDecorator implements CredentialsFactory { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); @@ -50,7 +51,7 @@ } @Override - public SecureCredentialsProvider create(String remoteName) { + public CredentialsProvider create(String remoteName) { try { if (needsReload()) { secureCredentialsFactory.compareAndSet(
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java new file mode 100644 index 0000000..e3d57de --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java
@@ -0,0 +1,69 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog; + +import com.google.gerrit.reviewdb.client.Project; +import com.google.inject.Inject; +import com.google.inject.assistedinject.Assisted; +import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType; +import java.util.Optional; +import org.eclipse.jgit.transport.RemoteConfig; +import org.eclipse.jgit.transport.URIish; + +public class CreateProjectTask { + interface Factory { + CreateProjectTask create(Project.NameKey project, String head); + } + + private final RemoteConfig config; + private final ReplicationConfig replicationConfig; + private final AdminApiFactory adminApiFactory; + private final Project.NameKey project; + private final String head; + + @Inject + CreateProjectTask( + RemoteConfig config, + ReplicationConfig replicationConfig, + AdminApiFactory adminApiFactory, + @Assisted Project.NameKey project, + @Assisted String head) { + this.config = config; + this.replicationConfig = replicationConfig; + this.adminApiFactory = adminApiFactory; + this.project = project; + this.head = head; + } + + public boolean create() { + return replicationConfig + .getURIs(Optional.of(config.getName()), project, FilterType.PROJECT_CREATION).values() + .stream() + .map(u -> createProject(u, project, head)) + .reduce(true, (a, b) -> a && b); + } + + private boolean createProject(URIish replicateURI, Project.NameKey projectName, String head) { + Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI); + if (adminApi.isPresent() && adminApi.get().createProject(projectName, head)) { + return true; + } + + repLog.warn("Cannot create new project {} on remote site {}.", projectName, replicateURI); + return false; + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java index 10719c1..3bb64ab 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/CredentialsFactory.java
@@ -13,7 +13,9 @@ // limitations under the License. package com.googlesource.gerrit.plugins.replication; +import org.eclipse.jgit.transport.CredentialsProvider; + public interface CredentialsFactory { - SecureCredentialsProvider create(String remoteName); + CredentialsProvider create(String remoteName); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java new file mode 100644 index 0000000..96d8c70 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DeleteProjectTask.java
@@ -0,0 +1,65 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog; + +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.ioutil.HexFormat; +import com.google.gerrit.server.util.IdGenerator; +import com.google.inject.Inject; +import com.google.inject.assistedinject.Assisted; +import java.util.Optional; +import org.eclipse.jgit.transport.URIish; + +public class DeleteProjectTask implements Runnable { + interface Factory { + DeleteProjectTask create(URIish replicateURI, Project.NameKey project); + } + + private final AdminApiFactory adminApiFactory; + private final int id; + private final URIish replicateURI; + private final Project.NameKey project; + + @Inject + DeleteProjectTask( + AdminApiFactory adminApiFactory, + IdGenerator ig, + @Assisted URIish replicateURI, + @Assisted Project.NameKey project) { + this.adminApiFactory = adminApiFactory; + this.id = ig.next(); + this.replicateURI = replicateURI; + this.project = project; + } + + @Override + public void run() { + Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI); + if (adminApi.isPresent()) { + adminApi.get().deleteProject(project); + return; + } + + repLog.warn("Cannot delete project {} on remote site {}.", project, replicateURI); + } + + @Override + public String toString() { + return String.format( + "[%s] delete-project %s at %s", HexFormat.fromInt(id), project.get(), replicateURI); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java index 5a3b92c..56a341e 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
@@ -15,6 +15,7 @@ package com.googlesource.gerrit.plugins.replication; import static com.googlesource.gerrit.plugins.replication.PushResultProcessing.resolveNodeName; +import static com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig.replaceName; import static org.eclipse.jgit.transport.RemoteRefUpdate.Status.NON_EXISTING; import static org.eclipse.jgit.transport.RemoteRefUpdate.Status.REJECTED_OTHER_REASON; @@ -32,14 +33,12 @@ import com.google.gerrit.reviewdb.client.Branch; import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.reviewdb.client.RefNames; -import com.google.gerrit.reviewdb.server.ReviewDb; import com.google.gerrit.server.CurrentUser; import com.google.gerrit.server.PluginUser; import com.google.gerrit.server.account.GroupBackend; import com.google.gerrit.server.account.GroupBackends; import com.google.gerrit.server.account.GroupIncludeCache; import com.google.gerrit.server.account.ListGroupMembership; -import com.google.gerrit.server.config.RequestScopedReviewDbProvider; import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.server.git.GitRepositoryManager; import com.google.gerrit.server.git.PerThreadRequestScope; @@ -56,6 +55,7 @@ import com.google.inject.Injector; import com.google.inject.Provider; import com.google.inject.Provides; +import com.google.inject.Scopes; import com.google.inject.assistedinject.Assisted; import com.google.inject.assistedinject.FactoryModuleBuilder; import com.google.inject.servlet.RequestScoped; @@ -71,6 +71,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.commons.io.FilenameUtils; +import org.apache.http.impl.client.CloseableHttpClient; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; @@ -92,6 +93,8 @@ private final Map<URIish, PushOne> pending = new HashMap<>(); private final Map<URIish, PushOne> inFlight = new HashMap<>(); private final PushOne.Factory opFactory; + private final DeleteProjectTask.Factory deleteProjectFactory; + private final UpdateHeadTask.Factory updateHeadFactory; private final GitRepositoryManager gitManager; private final PermissionBackend permissionBackend; private final Provider<CurrentUser> userProvider; @@ -165,24 +168,20 @@ bind(Destination.class).toInstance(Destination.this); bind(RemoteConfig.class).toInstance(config.getRemoteConfig()); install(new FactoryModuleBuilder().build(PushOne.Factory.class)); + install(new FactoryModuleBuilder().build(CreateProjectTask.Factory.class)); + install(new FactoryModuleBuilder().build(DeleteProjectTask.Factory.class)); + install(new FactoryModuleBuilder().build(UpdateHeadTask.Factory.class)); + bind(AdminApiFactory.class); + install(new FactoryModuleBuilder().build(GerritRestApi.Factory.class)); + bind(CloseableHttpClient.class) + .toProvider(HttpClientProvider.class) + .in(Scopes.SINGLETON); } @Provides public PerThreadRequestScope.Scoper provideScoper( - final PerThreadRequestScope.Propagator propagator, - final Provider<RequestScopedReviewDbProvider> dbProvider) { - final RequestContext requestContext = - new RequestContext() { - @Override - public CurrentUser getUser() { - return remoteUser; - } - - @Override - public Provider<ReviewDb> getReviewDbProvider() { - return dbProvider.get(); - } - }; + final PerThreadRequestScope.Propagator propagator) { + final RequestContext requestContext = () -> remoteUser; return new PerThreadRequestScope.Scoper() { @Override public <T> Callable<T> scope(Callable<T> callable) { @@ -193,6 +192,8 @@ }); opFactory = child.getInstance(PushOne.Factory.class); + deleteProjectFactory = child.getInstance(DeleteProjectTask.Factory.class); + updateHeadFactory = child.getInstance(UpdateHeadTask.Factory.class); threadScoper = child.getInstance(PerThreadRequestScope.Scoper.class); } @@ -255,38 +256,35 @@ try { return threadScoper .scope( - new Callable<Boolean>() { - @Override - public Boolean call() throws NoSuchProjectException, PermissionBackendException { - ProjectState projectState; - try { - projectState = projectCache.checkedGet(project); - } catch (IOException e) { - return false; - } - if (projectState == null) { - throw new NoSuchProjectException(project); - } - if (!projectState.statePermitsRead()) { - return false; - } - if (!shouldReplicate(projectState, userProvider.get())) { - return false; - } - if (PushOne.ALL_REFS.equals(ref)) { - return true; - } - try { - permissionBackend - .user(userProvider.get()) - .project(project) - .ref(ref) - .check(RefPermission.READ); - } catch (AuthException e) { - return false; - } + () -> { + ProjectState projectState; + try { + projectState = projectCache.checkedGet(project); + } catch (IOException e) { + return false; + } + if (projectState == null) { + throw new NoSuchProjectException(project); + } + if (!projectState.statePermitsRead()) { + return false; + } + if (!shouldReplicate(projectState, userProvider.get())) { + return false; + } + if (PushOne.ALL_REFS.equals(ref)) { return true; } + try { + permissionBackend + .user(userProvider.get()) + .project(project) + .ref(ref) + .check(RefPermission.READ); + } catch (AuthException e) { + return false; + } + return true; }) .call(); } catch (NoSuchProjectException err) { @@ -302,20 +300,17 @@ try { return threadScoper .scope( - new Callable<Boolean>() { - @Override - public Boolean call() throws NoSuchProjectException, PermissionBackendException { - ProjectState projectState; - try { - projectState = projectCache.checkedGet(project); - } catch (IOException e) { - return false; - } - if (projectState == null) { - throw new NoSuchProjectException(project); - } - return shouldReplicate(projectState, userProvider.get()); + () -> { + ProjectState projectState; + try { + projectState = projectCache.checkedGet(project); + } catch (IOException e) { + return false; } + if (projectState == null) { + throw new NoSuchProjectException(project); + } + return shouldReplicate(projectState, userProvider.get()); }) .call(); } catch (NoSuchProjectException err) { @@ -387,6 +382,14 @@ } } + void scheduleDeleteProject(URIish uri, Project.NameKey project) { + pool.schedule(deleteProjectFactory.create(uri, project), 0, TimeUnit.SECONDS); + } + + void scheduleUpdateHead(URIish uri, Project.NameKey project, String newHead) { + pool.schedule(updateHeadFactory.create(uri, project, newHead), 0, TimeUnit.SECONDS); + } + private void addRef(PushOne e, String ref) { e.addRef(ref); postReplicationScheduledEvent(e, ref); @@ -582,8 +585,7 @@ } else if (!remoteNameStyle.equals("slash")) { repLog.debug("Unknown remoteNameStyle: {}, falling back to slash", remoteNameStyle); } - String replacedPath = - ReplicationQueue.replaceName(uri.getPath(), name, isSingleProjectMatch()); + String replacedPath = replaceName(uri.getPath(), name, isSingleProjectMatch()); if (replacedPath != null) { uri = uri.setPath(replacedPath); r.add(uri);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java new file mode 100644 index 0000000..91fe20a --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritRestApi.java
@@ -0,0 +1,132 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import static com.googlesource.gerrit.plugins.replication.GerritSshApi.GERRIT_ADMIN_PROTOCOL_PREFIX; +import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog; + +import com.google.common.base.Charsets; +import com.google.gerrit.extensions.restapi.Url; +import com.google.gerrit.reviewdb.client.Project; +import com.google.inject.Inject; +import com.google.inject.assistedinject.Assisted; +import java.io.IOException; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.message.BasicHeader; +import org.eclipse.jgit.transport.CredentialItem; +import org.eclipse.jgit.transport.RemoteConfig; +import org.eclipse.jgit.transport.URIish; + +public class GerritRestApi implements AdminApi { + + public interface Factory { + GerritRestApi create(URIish uri); + } + + private final CredentialsFactory credentials; + private final CloseableHttpClient httpClient; + private final RemoteConfig remoteConfig; + private final URIish uri; + + @Inject + GerritRestApi( + CredentialsFactory credentials, + CloseableHttpClient httpClient, + RemoteConfig remoteConfig, + @Assisted URIish uri) { + this.credentials = credentials; + this.httpClient = httpClient; + this.remoteConfig = remoteConfig; + this.uri = uri; + } + + @Override + public boolean createProject(Project.NameKey project, String head) { + repLog.info("Creating project {} on {}", project, uri); + String url = String.format("%s/a/projects/%s", toHttpUri(uri), Url.encode(project.get())); + try { + return httpClient + .execute(new HttpPut(url), new HttpResponseHandler(), getContext()) + .isSuccessful(); + } catch (IOException e) { + repLog.error("Couldn't perform project creation on {}", uri, e); + return false; + } + } + + @Override + public void deleteProject(Project.NameKey project) { + repLog.info("Deleting project {} on {}", project, uri); + String url = String.format("%s/a/projects/%s", toHttpUri(uri), Url.encode(project.get())); + try { + httpClient.execute(new HttpDelete(url), new HttpResponseHandler(), getContext()); + } catch (IOException e) { + repLog.error("Couldn't perform project deletion on {}", uri, e); + } + } + + @Override + public void updateHead(Project.NameKey project, String newHead) { + repLog.info("Updating head of {} on {}", project, uri); + String url = String.format("%s/a/projects/%s/HEAD", toHttpUri(uri), Url.encode(project.get())); + try { + HttpPut req = new HttpPut(url); + req.setEntity( + new StringEntity(String.format("{\"ref\": \"%s\"}", newHead), Charsets.UTF_8.name())); + req.addHeader(new BasicHeader("Content-Type", "application/json")); + httpClient.execute(req, new HttpResponseHandler(), getContext()); + } catch (IOException e) { + repLog.error("Couldn't perform update head on {}", uri, e); + } + } + + private HttpClientContext getContext() { + HttpClientContext ctx = HttpClientContext.create(); + ctx.setCredentialsProvider(adapt(credentials.create(remoteConfig.getName()))); + return ctx; + } + + private CredentialsProvider adapt(org.eclipse.jgit.transport.CredentialsProvider cp) { + CredentialItem.Username user = new CredentialItem.Username(); + CredentialItem.Password pass = new CredentialItem.Password(); + if (cp.supports(user, pass) && cp.get(uri, user, pass)) { + CredentialsProvider adapted = new BasicCredentialsProvider(); + adapted.setCredentials( + AuthScope.ANY, + new UsernamePasswordCredentials(user.getValue(), new String(pass.getValue()))); + return adapted; + } + return null; + } + + private static String toHttpUri(URIish uri) { + String u = uri.toString(); + if (u.startsWith(GERRIT_ADMIN_PROTOCOL_PREFIX)) { + u = u.substring(GERRIT_ADMIN_PROTOCOL_PREFIX.length()); + } + if (u.endsWith("/")) { + return u; + } + return u + "/"; + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java index 85b17d0..12b6cbe 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/GerritSshApi.java
@@ -28,7 +28,7 @@ private static final FluentLogger logger = FluentLogger.forEnclosingClass(); static int SSH_COMMAND_FAILED = -1; - private static String GERRIT_ADMIN_PROTOCOL_PREFIX = "gerrit+"; + static String GERRIT_ADMIN_PROTOCOL_PREFIX = "gerrit+"; private final SshHelper sshHelper; private final URIish uri; @@ -41,14 +41,16 @@ } @Override - public void createProject(Project.NameKey projectName, String head) { + public boolean createProject(Project.NameKey projectName, String head) { OutputStream errStream = sshHelper.newErrorBufferStream(); String cmd = "gerrit create-project --branch " + head + " " + projectName.get(); try { execute(uri, cmd, errStream); } catch (IOException e) { logError("creating", uri, errStream, cmd, e); + return false; } + return true; } @Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/HttpClientProvider.java b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpClientProvider.java new file mode 100644 index 0000000..916059c --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpClientProvider.java
@@ -0,0 +1,108 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import com.google.gerrit.server.config.GerritServerConfig; +import com.google.gerrit.server.config.SitePaths; +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.google.inject.ProvisionException; +import java.nio.file.Files; +import java.nio.file.Path; +import javax.net.ssl.SSLContext; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.HttpClientConnectionManager; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.ssl.SSLContexts; +import org.eclipse.jgit.lib.Config; + +/** Provides an HTTP client with SSL capabilities. */ +class HttpClientProvider implements Provider<CloseableHttpClient> { + private static final int CONNECTIONS_PER_ROUTE = 100; + + // Up to 2 target instances with the max number of connections per host: + private static final int MAX_CONNECTIONS = 2 * CONNECTIONS_PER_ROUTE; + + private static final int MAX_CONNECTION_INACTIVITY = 10000; + private static final int DEFAULT_TIMEOUT_MS = 5000; + + private final Config cfg; + private final SitePaths site; + + @Inject + HttpClientProvider(@GerritServerConfig Config cfg, SitePaths site) { + this.cfg = cfg; + this.site = site; + } + + @Override + public CloseableHttpClient get() { + try { + return HttpClients.custom() + .setConnectionManager(customConnectionManager()) + .setDefaultRequestConfig(customRequestConfig()) + .build(); + } catch (Exception e) { + throw new ProvisionException("Couldn't create CloseableHttpClient", e); + } + } + + private RequestConfig customRequestConfig() { + return RequestConfig.custom() + .setConnectTimeout(DEFAULT_TIMEOUT_MS) + .setSocketTimeout(DEFAULT_TIMEOUT_MS) + .setConnectionRequestTimeout(DEFAULT_TIMEOUT_MS) + .build(); + } + + private HttpClientConnectionManager customConnectionManager() throws Exception { + Registry<ConnectionSocketFactory> socketFactoryRegistry = + RegistryBuilder.<ConnectionSocketFactory>create() + .register("https", buildSslSocketFactory()) + .register("http", PlainConnectionSocketFactory.INSTANCE) + .build(); + PoolingHttpClientConnectionManager connManager = + new PoolingHttpClientConnectionManager(socketFactoryRegistry); + connManager.setDefaultMaxPerRoute(CONNECTIONS_PER_ROUTE); + connManager.setMaxTotal(MAX_CONNECTIONS); + connManager.setValidateAfterInactivity(MAX_CONNECTION_INACTIVITY); + return connManager; + } + + private SSLConnectionSocketFactory buildSslSocketFactory() throws Exception { + String keyStore = cfg.getString("httpd", null, "sslKeyStore"); + if (keyStore == null) { + keyStore = "etc/keystore"; + } + return new SSLConnectionSocketFactory(createSSLContext(site.resolve(keyStore))); + } + + private SSLContext createSSLContext(Path keyStorePath) throws Exception { + SSLContext ctx; + if (Files.exists(keyStorePath)) { + ctx = SSLContexts.custom().loadTrustMaterial(keyStorePath.toFile()).build(); + } else { + ctx = SSLContext.getDefault(); + } + return ctx; + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponse.java b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponse.java new file mode 100644 index 0000000..f26128c --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponse.java
@@ -0,0 +1,66 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import com.google.common.base.Preconditions; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.ByteBuffer; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.eclipse.jgit.util.IO; +import org.eclipse.jgit.util.RawParseUtils; + +public class HttpResponse implements AutoCloseable { + + protected CloseableHttpResponse response; + protected Reader reader; + + HttpResponse(CloseableHttpResponse response) { + this.response = response; + } + + public Reader getReader() throws IllegalStateException, IOException { + if (reader == null && response.getEntity() != null) { + reader = new InputStreamReader(response.getEntity().getContent()); + } + return reader; + } + + @Override + public void close() throws IOException { + try { + Reader reader = getReader(); + if (reader != null) { + while (reader.read() != -1) { + // Empty + } + } + } finally { + response.close(); + } + } + + public int getStatusCode() { + return response.getStatusLine().getStatusCode(); + } + + public String getEntityContent() throws IOException { + Preconditions.checkNotNull(response, "Response is not initialized."); + Preconditions.checkNotNull(response.getEntity(), "Response.Entity is not initialized."); + ByteBuffer buf = IO.readWholeStream(response.getEntity().getContent(), 1024); + return RawParseUtils.decode(buf.array(), buf.arrayOffset(), buf.limit()).trim(); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponseHandler.java b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponseHandler.java new file mode 100644 index 0000000..6a7c73e --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/HttpResponseHandler.java
@@ -0,0 +1,71 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import static javax.servlet.http.HttpServletResponse.SC_CREATED; +import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT; +import static javax.servlet.http.HttpServletResponse.SC_OK; + +import com.google.common.flogger.FluentLogger; +import com.googlesource.gerrit.plugins.replication.HttpResponseHandler.HttpResult; +import java.io.IOException; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.ResponseHandler; +import org.apache.http.util.EntityUtils; + +class HttpResponseHandler implements ResponseHandler<HttpResult> { + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + static class HttpResult { + private final boolean successful; + private final String message; + + HttpResult(boolean successful, String message) { + this.successful = successful; + this.message = message; + } + + boolean isSuccessful() { + return successful; + } + + String getMessage() { + return message; + } + } + + @Override + public HttpResult handleResponse(HttpResponse response) { + return new HttpResult(isSuccessful(response), parseResponse(response)); + } + + private static boolean isSuccessful(HttpResponse response) { + int sc = response.getStatusLine().getStatusCode(); + return sc == SC_CREATED || sc == SC_NO_CONTENT || sc == SC_OK; + } + + private static String parseResponse(HttpResponse response) { + HttpEntity entity = response.getEntity(); + if (entity != null) { + try { + return EntityUtils.toString(entity); + } catch (IOException e) { + logger.atSevere().withCause(e).log("Error parsing entity"); + } + } + return ""; + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java b/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java index 1012cd7..908e7e0 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java
@@ -34,7 +34,7 @@ } @Override - public void createProject(Project.NameKey project, String head) { + public boolean createProject(Project.NameKey project, String head) { try (Repository repo = new FileRepository(uri.getPath())) { repo.create(true /* bare */); @@ -46,7 +46,9 @@ repLog.info("Created local repository: {}", uri); } catch (IOException e) { repLog.error("Error creating local repository {}", uri.getPath(), e); + return false; } + return true; } @Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java index 09187a0..9c1de54 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
@@ -16,6 +16,7 @@ import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog; import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.stream.Collectors.toMap; import com.google.common.base.Throwables; import com.google.common.collect.LinkedListMultimap; @@ -52,7 +53,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicBoolean; import org.eclipse.jgit.errors.NoRemoteRepositoryException; import org.eclipse.jgit.errors.NotSupportedException; @@ -92,9 +92,9 @@ private final PermissionBackend permissionBackend; private final Destination pool; private final RemoteConfig config; + private final ReplicationConfig replConfig; private final CredentialsProvider credentialsProvider; private final PerThreadRequestScope.Scoper threadScoper; - private final ReplicationQueue replicationQueue; private final Project.NameKey projectName; private final URIish uri; @@ -112,6 +112,7 @@ private final long createdAt; private final ReplicationMetrics metrics; private final ProjectCache projectCache; + private final CreateProjectTask.Factory createProjectFactory; private final AtomicBoolean canceledWhileRunning; @Inject @@ -120,22 +121,23 @@ PermissionBackend permissionBackend, Destination p, RemoteConfig c, + ReplicationConfig rc, CredentialsFactory cpFactory, PerThreadRequestScope.Scoper ts, - ReplicationQueue rq, IdGenerator ig, ReplicationStateListeners sl, ReplicationMetrics m, ProjectCache pc, + CreateProjectTask.Factory cpf, @Assisted Project.NameKey d, @Assisted URIish u) { gitManager = grm; this.permissionBackend = permissionBackend; pool = p; config = c; + replConfig = rc; credentialsProvider = cpFactory.create(c.getName()); threadScoper = ts; - replicationQueue = rq; projectName = d; uri = u; lockRetryCount = 0; @@ -145,6 +147,7 @@ createdAt = System.nanoTime(); metrics = m; projectCache = pc; + createProjectFactory = cpf; canceledWhileRunning = new AtomicBoolean(false); maxRetries = p.getMaxRetries(); } @@ -275,12 +278,9 @@ try { threadScoper .scope( - new Callable<Void>() { - @Override - public Void call() { - runPushOperation(); - return null; - } + () -> { + runPushOperation(); + return null; }) .call(); } catch (Exception e) { @@ -333,7 +333,8 @@ String msg = e.getMessage(); if (msg.contains("access denied") || msg.contains("no such repository") - || msg.contains("Git repository not found")) { + || msg.contains("Git repository not found") + || msg.contains("unavailable")) { createRepository(); } else { repLog.error("Cannot replicate {}; Remote repository error: {}", projectName, msg); @@ -396,15 +397,11 @@ if (pool.isCreateMissingRepos()) { try { Ref head = git.exactRef(Constants.HEAD); - if (replicationQueue.createProject(projectName, head != null ? getName(head) : null)) { + if (createProject(projectName, head != null ? getName(head) : null)) { repLog.warn("Missing repository created; retry replication to {}", uri); pool.reschedule(this, Destination.RetryReason.REPOSITORY_MISSING); } else { - repLog.warn( - "Missing repository could not be created when replicating {}. " - + "You can only create missing repositories locally, over SSH or when " - + "using adminUrl in replication.config. See documentation for more information.", - uri); + repLog.warn("Missing repository could not be created when replicating {}", uri); } } catch (IOException ioe) { stateLog.error( @@ -417,6 +414,10 @@ } } + private boolean createProject(Project.NameKey project, String head) { + return createProjectFactory.create(project, head).create(); + } + private String getName(Ref ref) { Ref target = ref; while (target.isSymbolic()) { @@ -447,7 +448,16 @@ return new PushResult(); } - repLog.info("Push to {} references: {}", uri, todo); + if (replConfig.getMaxRefsToLog() == 0 || todo.size() <= replConfig.getMaxRefsToLog()) { + repLog.info("Push to {} references: {}", uri, todo); + } else { + repLog.info( + "Push to {} references (first {} of {} listed): {}", + uri, + replConfig.getMaxRefsToLog(), + todo.size(), + todo.subList(0, replConfig.getMaxRefsToLog())); + } return tn.push(NullProgressMonitor.INSTANCE, todo); } @@ -459,7 +469,8 @@ return Collections.emptyList(); } - Map<String, Ref> local = git.getAllRefs(); + Map<String, Ref> local = + git.getRefDatabase().getRefs().stream().collect(toMap(Ref::getName, r -> r)); boolean filter; PermissionBackend.ForProject forProject = permissionBackend.currentUser().project(projectName); try {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java index ae0662d..ad68d42 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushResultProcessing.java
@@ -15,10 +15,10 @@ package com.googlesource.gerrit.plugins.replication; import com.google.common.flogger.FluentLogger; +import com.google.gerrit.exceptions.StorageException; import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.server.events.RefEvent; import com.google.gerrit.server.permissions.PermissionBackendException; -import com.google.gwtorm.server.OrmException; import com.googlesource.gerrit.plugins.replication.ReplicationState.RefPushResult; import java.lang.ref.WeakReference; import java.util.concurrent.atomic.AtomicBoolean; @@ -187,7 +187,7 @@ private void postEvent(RefEvent event) { try { dispatcher.postEvent(event); - } catch (OrmException | PermissionBackendException e) { + } catch (StorageException | PermissionBackendException e) { logger.atSevere().withCause(e).log("Cannot post event"); } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java index dad1b0b..6dfa117 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteSsh.java
@@ -33,7 +33,7 @@ } @Override - public void createProject(Project.NameKey project, String head) { + public boolean createProject(Project.NameKey project, String head) { String quotedPath = QuotedString.BOURNE.quote(uri.getPath()); String cmd = "mkdir -p " + quotedPath + " && cd " + quotedPath + " && git init --bare"; if (head != null) { @@ -54,7 +54,9 @@ cmd, errStream, e); + return false; } + return true; } @Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java index 9693e2d..1c62ab7 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationConfig.java
@@ -13,9 +13,13 @@ // limitations under the License. package com.googlesource.gerrit.plugins.replication; +import com.google.common.collect.Multimap; +import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.server.git.WorkQueue; import java.nio.file.Path; import java.util.List; +import java.util.Optional; +import org.eclipse.jgit.transport.URIish; public interface ReplicationConfig { @@ -27,10 +31,15 @@ List<Destination> getDestinations(FilterType filterType); + Multimap<Destination, URIish> getURIs( + Optional<String> remoteName, Project.NameKey projectName, FilterType filterType); + boolean isReplicateAllOnPluginStart(); boolean isDefaultForceUpdate(); + int getMaxRefsToLog(); + boolean isEmpty(); Path getEventsDirectory();
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java index 46d2f66..64eacf9 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationFileBasedConfig.java
@@ -13,13 +13,22 @@ // limitations under the License. package com.googlesource.gerrit.plugins.replication; +import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isGerrit; +import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isGerritHttp; +import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isSSH; +import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog; import static java.util.stream.Collectors.toList; import com.google.common.base.Strings; +import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.common.collect.SetMultimap; import com.google.common.flogger.FluentLogger; import com.google.gerrit.extensions.annotations.PluginData; +import com.google.gerrit.reviewdb.client.Project; import com.google.gerrit.server.config.SitePaths; import com.google.gerrit.server.git.WorkQueue; import com.google.inject.Inject; @@ -30,6 +39,7 @@ import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.function.Predicate; import org.eclipse.jgit.errors.ConfigInvalidException; @@ -48,6 +58,7 @@ private Path cfgPath; private boolean replicateAllOnPluginStart; private boolean defaultForceUpdate; + private int maxRefsToLog; private final FileBasedConfig config; private final Path pluginDataDir; @@ -111,6 +122,8 @@ defaultForceUpdate = config.getBoolean("gerrit", "defaultForceUpdate", false); + maxRefsToLog = config.getInt("gerrit", "maxRefsToLog", 0); + ImmutableList.Builder<Destination> dest = ImmutableList.builder(); for (RemoteConfig c : allRemotes(config)) { if (c.getURIs().isEmpty()) { @@ -149,6 +162,77 @@ return dest.build(); } + @Override + public Multimap<Destination, URIish> getURIs( + Optional<String> remoteName, Project.NameKey projectName, FilterType filterType) { + if (getDestinations(filterType).isEmpty()) { + return ImmutableMultimap.of(); + } + + SetMultimap<Destination, URIish> uris = HashMultimap.create(); + for (Destination config : getDestinations(filterType)) { + if (!config.wouldPushProject(projectName)) { + continue; + } + + if (remoteName.isPresent() && !config.getRemoteConfigName().equals(remoteName.get())) { + continue; + } + + boolean adminURLUsed = false; + + for (String url : config.getAdminUrls()) { + if (Strings.isNullOrEmpty(url)) { + continue; + } + + URIish uri; + try { + uri = new URIish(url); + } catch (URISyntaxException e) { + repLog.warn("adminURL '{}' is invalid: {}", url, e.getMessage()); + continue; + } + + if (!isGerrit(uri) && !isGerritHttp(uri)) { + String path = + replaceName(uri.getPath(), projectName.get(), config.isSingleProjectMatch()); + if (path == null) { + repLog.warn("adminURL {} does not contain ${name}", uri); + continue; + } + + uri = uri.setPath(path); + if (!isSSH(uri)) { + repLog.warn("adminURL '{}' is invalid: only SSH and HTTP are supported", uri); + continue; + } + } + uris.put(config, uri); + adminURLUsed = true; + } + + if (!adminURLUsed) { + for (URIish uri : config.getURIs(projectName, "*")) { + uris.put(config, uri); + } + } + } + return uris; + } + + static String replaceName(String in, String name, boolean keyIsOptional) { + String key = "${name}"; + int n = in.indexOf(key); + if (0 <= n) { + return in.substring(0, n) + name + in.substring(n + key.length()); + } + if (keyIsOptional) { + return in; + } + return null; + } + /* (non-Javadoc) * @see com.googlesource.gerrit.plugins.replication.ReplicationConfig#isReplicateAllOnPluginStart() */ @@ -165,6 +249,11 @@ return defaultForceUpdate; } + @Override + public int getMaxRefsToLog() { + return maxRefsToLog; + } + private static List<RemoteConfig> allRemotes(FileBasedConfig cfg) throws ConfigInvalidException { Set<String> names = cfg.getSubsections("remote"); List<RemoteConfig> result = Lists.newArrayListWithCapacity(names.size());
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java index 1405a12..9b7e8e6 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationModule.java
@@ -21,7 +21,6 @@ import com.google.gerrit.extensions.events.GitReferenceUpdatedListener; import com.google.gerrit.extensions.events.HeadUpdatedListener; import com.google.gerrit.extensions.events.LifecycleListener; -import com.google.gerrit.extensions.events.NewProjectCreatedListener; import com.google.gerrit.extensions.events.ProjectDeletedListener; import com.google.gerrit.extensions.registration.DynamicSet; import com.google.gerrit.server.events.EventTypes; @@ -41,7 +40,6 @@ .to(ReplicationQueue.class); DynamicSet.bind(binder(), GitReferenceUpdatedListener.class).to(ReplicationQueue.class); - DynamicSet.bind(binder(), NewProjectCreatedListener.class).to(ReplicationQueue.class); DynamicSet.bind(binder(), ProjectDeletedListener.class).to(ReplicationQueue.class); DynamicSet.bind(binder(), HeadUpdatedListener.class).to(ReplicationQueue.class); @@ -68,7 +66,5 @@ EventTypes.register(RefReplicationDoneEvent.TYPE, RefReplicationDoneEvent.class); EventTypes.register(ReplicationScheduledEvent.TYPE, ReplicationScheduledEvent.class); bind(SshSessionFactory.class).toProvider(ReplicationSshSessionFactoryProvider.class); - - bind(AdminApiFactory.class); } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java index 541a595..d2a34be 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationQueue.java
@@ -14,14 +14,9 @@ package com.googlesource.gerrit.plugins.replication; -import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isGerrit; -import static com.googlesource.gerrit.plugins.replication.AdminApiFactory.isSSH; - -import com.google.common.base.Strings; import com.google.gerrit.extensions.events.GitReferenceUpdatedListener; import com.google.gerrit.extensions.events.HeadUpdatedListener; import com.google.gerrit.extensions.events.LifecycleListener; -import com.google.gerrit.extensions.events.NewProjectCreatedListener; import com.google.gerrit.extensions.events.ProjectDeletedListener; import com.google.gerrit.extensions.registration.DynamicItem; import com.google.gerrit.reviewdb.client.Project; @@ -30,11 +25,7 @@ import com.google.inject.Inject; import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdateProcessing; import com.googlesource.gerrit.plugins.replication.ReplicationConfig.FilterType; -import java.net.URISyntaxException; -import java.util.Collections; -import java.util.HashSet; import java.util.Optional; -import java.util.Set; import org.eclipse.jgit.transport.URIish; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +34,6 @@ public class ReplicationQueue implements LifecycleListener, GitReferenceUpdatedListener, - NewProjectCreatedListener, ProjectDeletedListener, HeadUpdatedListener { static final String REPLICATION_LOG_NAME = "replication_log"; @@ -51,22 +41,9 @@ private final ReplicationStateListener stateLog; - static String replaceName(String in, String name, boolean keyIsOptional) { - String key = "${name}"; - int n = in.indexOf(key); - if (0 <= n) { - return in.substring(0, n) + name + in.substring(n + key.length()); - } - if (keyIsOptional) { - return in; - } - return null; - } - private final WorkQueue workQueue; private final DynamicItem<EventDispatcher> dispatcher; private final ReplicationConfig config; - private final AdminApiFactory adminApiFactory; private final ReplicationState.Factory replicationStateFactory; private final EventsStorage eventsStorage; private volatile boolean running; @@ -74,7 +51,6 @@ @Inject ReplicationQueue( WorkQueue wq, - AdminApiFactory aaf, ReplicationConfig rc, DynamicItem<EventDispatcher> dis, ReplicationStateListeners sl, @@ -84,7 +60,6 @@ dispatcher = dis; config = rc; stateLog = sl; - adminApiFactory = aaf; replicationStateFactory = rsf; eventsStorage = es; } @@ -161,130 +136,16 @@ } @Override - public void onNewProjectCreated(NewProjectCreatedListener.Event event) { - Project.NameKey projectName = new Project.NameKey(event.getProjectName()); - for (URIish uri : getURIs(projectName, FilterType.PROJECT_CREATION)) { - createProject(uri, projectName, event.getHeadName()); - } - } - - @Override public void onProjectDeleted(ProjectDeletedListener.Event event) { - Project.NameKey projectName = new Project.NameKey(event.getProjectName()); - for (URIish uri : getURIs(projectName, FilterType.PROJECT_DELETION)) { - deleteProject(uri, projectName); - } + Project.NameKey p = new Project.NameKey(event.getProjectName()); + config.getURIs(Optional.empty(), p, FilterType.PROJECT_DELETION).entries().stream() + .forEach(e -> e.getKey().scheduleDeleteProject(e.getValue(), p)); } @Override public void onHeadUpdated(HeadUpdatedListener.Event event) { - Project.NameKey project = new Project.NameKey(event.getProjectName()); - for (URIish uri : getURIs(project, FilterType.ALL)) { - updateHead(uri, project, event.getNewHeadName()); - } - } - - private Set<URIish> getURIs(Project.NameKey projectName, FilterType filterType) { - if (config.getDestinations(filterType).isEmpty()) { - return Collections.emptySet(); - } - if (!running) { - repLog.error("Replication plugin did not finish startup before event"); - return Collections.emptySet(); - } - - Set<URIish> uris = new HashSet<>(); - for (Destination config : this.config.getDestinations(filterType)) { - if (!config.wouldPushProject(projectName)) { - continue; - } - - boolean adminURLUsed = false; - - for (String url : config.getAdminUrls()) { - if (Strings.isNullOrEmpty(url)) { - continue; - } - - URIish uri; - try { - uri = new URIish(url); - } catch (URISyntaxException e) { - repLog.warn("adminURL '{}' is invalid: {}", url, e.getMessage()); - continue; - } - - if (!isGerrit(uri)) { - String path = - replaceName(uri.getPath(), projectName.get(), config.isSingleProjectMatch()); - if (path == null) { - repLog.warn("adminURL {} does not contain ${name}", uri); - continue; - } - - uri = uri.setPath(path); - if (!isSSH(uri)) { - repLog.warn("adminURL '{}' is invalid: only SSH is supported", uri); - continue; - } - } - uris.add(uri); - adminURLUsed = true; - } - - if (!adminURLUsed) { - for (URIish uri : config.getURIs(projectName, "*")) { - uris.add(uri); - } - } - } - return uris; - } - - public boolean createProject(Project.NameKey project, String head) { - boolean success = true; - for (URIish uri : getURIs(project, FilterType.PROJECT_CREATION)) { - success &= createProject(uri, project, head); - } - return success; - } - - private boolean createProject(URIish replicateURI, Project.NameKey projectName, String head) { - Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI); - if (adminApi.isPresent()) { - adminApi.get().createProject(projectName, head); - return true; - } - - warnCannotPerform("create new project", replicateURI); - return false; - } - - private void deleteProject(URIish replicateURI, Project.NameKey projectName) { - Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI); - if (adminApi.isPresent()) { - adminApi.get().deleteProject(projectName); - return; - } - - warnCannotPerform("delete project", replicateURI); - } - - private void updateHead(URIish replicateURI, Project.NameKey projectName, String newHead) { - Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI); - if (adminApi.isPresent()) { - adminApi.get().updateHead(projectName, newHead); - return; - } - - warnCannotPerform("update HEAD of project", replicateURI); - } - - private void warnCannotPerform(String op, URIish uri) { - repLog.warn( - "Cannot {} on remote site {}." - + "Only local paths and SSH URLs are supported for this operation", - op, - uri); + Project.NameKey p = new Project.NameKey(event.getProjectName()); + config.getURIs(Optional.empty(), p, FilterType.ALL).entries().stream() + .forEach(e -> e.getKey().scheduleUpdateHead(e.getValue(), p, event.getNewHeadName())); } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java index c518091..18a4cc2 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsFactory.java
@@ -21,6 +21,8 @@ import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.util.FS; /** Looks up a remote's password in secure.config. */ @@ -49,9 +51,9 @@ } @Override - public SecureCredentialsProvider create(String remoteName) { + public CredentialsProvider create(String remoteName) { String user = Objects.toString(config.getString("remote", remoteName, "username"), ""); String pass = Objects.toString(config.getString("remote", remoteName, "password"), ""); - return new SecureCredentialsProvider(user, pass); + return new UsernamePasswordCredentialsProvider(user, pass); } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java b/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java deleted file mode 100644 index 62b4036..0000000 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/SecureCredentialsProvider.java +++ /dev/null
@@ -1,80 +0,0 @@ -// Copyright (C) 2011 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.googlesource.gerrit.plugins.replication; - -import org.eclipse.jgit.errors.UnsupportedCredentialItem; -import org.eclipse.jgit.transport.CredentialItem; -import org.eclipse.jgit.transport.CredentialsProvider; -import org.eclipse.jgit.transport.URIish; - -/** Looks up a remote's password in secure.config. */ -public class SecureCredentialsProvider extends CredentialsProvider { - private final String cfgUser; - private final String cfgPass; - - SecureCredentialsProvider(String user, String pass) { - cfgUser = user; - cfgPass = pass; - } - - @Override - public boolean isInteractive() { - return false; - } - - @Override - public boolean supports(CredentialItem... items) { - for (CredentialItem i : items) { - if (i instanceof CredentialItem.Username) { - continue; - } else if (i instanceof CredentialItem.Password) { - continue; - } else { - return false; - } - } - return true; - } - - @Override - public boolean get(URIish uri, CredentialItem... items) throws UnsupportedCredentialItem { - String username = uri.getUser(); - if (username == null) { - username = cfgUser; - } - if (username == null) { - return false; - } - - String password = uri.getPass(); - if (password == null) { - password = cfgPass; - } - if (password == null) { - return false; - } - - for (CredentialItem i : items) { - if (i instanceof CredentialItem.Username) { - ((CredentialItem.Username) i).setValue(username); - } else if (i instanceof CredentialItem.Password) { - ((CredentialItem.Password) i).setValue(password.toCharArray()); - } else { - throw new UnsupportedCredentialItem(uri, i.getPromptText()); - } - } - return true; - } -}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java new file mode 100644 index 0000000..db58f49 --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/UpdateHeadTask.java
@@ -0,0 +1,69 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication; + +import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog; + +import com.google.gerrit.reviewdb.client.Project; +import com.google.gerrit.server.ioutil.HexFormat; +import com.google.gerrit.server.util.IdGenerator; +import com.google.inject.Inject; +import com.google.inject.assistedinject.Assisted; +import java.util.Optional; +import org.eclipse.jgit.transport.URIish; + +public class UpdateHeadTask implements Runnable { + private final AdminApiFactory adminApiFactory; + private final int id; + private final URIish replicateURI; + private final Project.NameKey project; + private final String newHead; + + interface Factory { + UpdateHeadTask create(URIish uri, Project.NameKey project, String newHead); + } + + @Inject + UpdateHeadTask( + AdminApiFactory adminApiFactory, + IdGenerator ig, + @Assisted URIish replicateURI, + @Assisted Project.NameKey project, + @Assisted String newHead) { + this.adminApiFactory = adminApiFactory; + this.id = ig.next(); + this.replicateURI = replicateURI; + this.project = project; + this.newHead = newHead; + } + + @Override + public void run() { + Optional<AdminApi> adminApi = adminApiFactory.create(replicateURI); + if (adminApi.isPresent()) { + adminApi.get().updateHead(project, newHead); + return; + } + + repLog.warn("Cannot update HEAD of project {} on remote site {}.", project, replicateURI); + } + + @Override + public String toString() { + return String.format( + "[%s] update-head of %s at %s to %s", + HexFormat.fromInt(id), project.get(), replicateURI, newHead); + } +}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index 2359599..6493761 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -82,6 +82,10 @@ : If true, the default push refspec will be set to use forced update to the remote when no refspec is given. By default, false. +gerrit.maxRefsToLog +: Number of refs, that are pushed during replication, to be logged. + For printing all refs to the logs, use a value of 0. By default, 0. + replication.lockErrorMaxRetries : Number of times to retry a replication operation if a lock error is detected. @@ -169,13 +173,18 @@ local environment. In that case, an alternative SSH url could be specified to repository creation. - To enable replication to different Gerrit instance use `gerrit+ssh://` - as protocol name followed by hostname of another Gerrit server eg. + To enable replication to different Gerrit instance use `gerrit+ssh://`, + `gerrit+http://` or `gerrit+https://` as protocol name followed + by hostname of another Gerrit server eg. `gerrit+ssh://replica1.my.org/` + <br> + `gerrit+http://replica2.my.org/` + <br> + `gerrit+https://replica3.my.org/` - In this case replication will use Gerrit's SSH API to - create/remove projects and update repository HEAD references. + In this case replication will use Gerrit's SSH API or Gerrit's REST API + to create/remove projects and update repository HEAD references. NOTE: In order to replicate project deletion, the link:https://gerrit-review.googlesource.com/admin/projects/plugins/delete-project delete-project[delete-project]
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java index 5fa7b98..0f6d629 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/GitUpdateProcessingTest.java
@@ -15,20 +15,13 @@ package com.googlesource.gerrit.plugins.replication; import static org.easymock.EasyMock.createMock; -import static org.easymock.EasyMock.createNiceMock; -import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.reset; import static org.easymock.EasyMock.verify; -import com.google.gerrit.reviewdb.server.ReviewDb; import com.google.gerrit.server.events.EventDispatcher; import com.google.gerrit.server.permissions.PermissionBackendException; -import com.google.gwtorm.client.KeyUtil; -import com.google.gwtorm.server.OrmException; -import com.google.gwtorm.server.SchemaFactory; -import com.google.gwtorm.server.StandardKeyEncoder; import com.googlesource.gerrit.plugins.replication.PushResultProcessing.GitUpdateProcessing; import com.googlesource.gerrit.plugins.replication.ReplicationState.RefPushResult; import java.net.URISyntaxException; @@ -37,12 +30,7 @@ import org.junit.Before; import org.junit.Test; -@SuppressWarnings("unchecked") public class GitUpdateProcessingTest { - static { - KeyUtil.setEncoderImpl(new StandardKeyEncoder()); - } - private EventDispatcher dispatcherMock; private GitUpdateProcessing gitUpdateProcessing; @@ -50,17 +38,11 @@ public void setUp() throws Exception { dispatcherMock = createMock(EventDispatcher.class); replay(dispatcherMock); - ReviewDb reviewDbMock = createNiceMock(ReviewDb.class); - replay(reviewDbMock); - SchemaFactory<ReviewDb> schemaMock = createMock(SchemaFactory.class); - expect(schemaMock.open()).andReturn(reviewDbMock).anyTimes(); - replay(schemaMock); gitUpdateProcessing = new GitUpdateProcessing(dispatcherMock); } @Test - public void headRefReplicated() - throws URISyntaxException, OrmException, PermissionBackendException { + public void headRefReplicated() throws URISyntaxException, PermissionBackendException { reset(dispatcherMock); RefReplicatedEvent expectedEvent = new RefReplicatedEvent( @@ -83,8 +65,7 @@ } @Test - public void changeRefReplicated() - throws URISyntaxException, OrmException, PermissionBackendException { + public void changeRefReplicated() throws URISyntaxException, PermissionBackendException { reset(dispatcherMock); RefReplicatedEvent expectedEvent = new RefReplicatedEvent( @@ -107,7 +88,7 @@ } @Test - public void onAllNodesReplicated() throws OrmException, PermissionBackendException { + public void onAllNodesReplicated() throws PermissionBackendException { reset(dispatcherMock); RefReplicationDoneEvent expectedDoneEvent = new RefReplicationDoneEvent("someProject", "refs/heads/master", 5);