Adopt new replication configuration structure Pull-replication plugin is capable to load both replication.config file and all *.config files from etc/replication directory. Move configuration parsing out of SourcesCollection to ConfigParser class. Feature: Issue 12560 Change-Id: I07a18898c121d149fd70b4d4bed2b961c1cb67f5
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ConfigParser.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ConfigParser.java new file mode 100644 index 0000000..7440e0f --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/ConfigParser.java
@@ -0,0 +1,96 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.flogger.FluentLogger; +import com.googlesource.gerrit.plugins.replication.RemoteConfiguration; +import java.net.URISyntaxException; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.transport.RemoteConfig; +import org.eclipse.jgit.transport.URIish; + +public class ConfigParser { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + /** + * parse the new replication config + * + * @param config new configuration to parse + * @return List of parsed {@link RemoteConfiguration} + * @throws ConfigInvalidException if the new configuration is not valid. + */ + public List<RemoteConfiguration> parseRemotes(Config config) throws ConfigInvalidException { + + if (config.getSections().isEmpty()) { + logger.atWarning().log("Replication config does not exist or it's empty; not replicating"); + return Collections.emptyList(); + } + + ImmutableList.Builder<RemoteConfiguration> sourceConfigs = ImmutableList.builder(); + for (RemoteConfig c : allFetchRemotes(config)) { + if (c.getURIs().isEmpty()) { + continue; + } + + // fetch source has to be specified. + if (c.getFetchRefSpecs().isEmpty()) { + throw new ConfigInvalidException( + String.format("You must specify a valid refSpec for this remote")); + } + + SourceConfiguration sourceConfig = new SourceConfiguration(c, config); + + if (!sourceConfig.isSingleProjectMatch()) { + for (URIish u : c.getURIs()) { + if (u.getPath() == null || !u.getPath().contains("${name}")) { + throw new ConfigInvalidException( + String.format("remote.%s.url \"%s\" lacks ${name} placeholder", c.getName(), u)); + } + } + } + sourceConfigs.add(sourceConfig); + } + return sourceConfigs.build(); + } + + private static List<RemoteConfig> allFetchRemotes(Config cfg) throws ConfigInvalidException { + + Set<String> names = cfg.getSubsections("remote"); + List<RemoteConfig> result = Lists.newArrayListWithCapacity(names.size()); + for (String name : names) { + try { + final RemoteConfig remoteConfig = new RemoteConfig(cfg, name); + if (!remoteConfig.getFetchRefSpecs().isEmpty()) { + result.add(remoteConfig); + } else { + logger.atWarning().log( + "Skip loading of remote [remote \"%s\"], since it has no 'fetch' configuration", + name); + } + } catch (URISyntaxException e) { + throw new ConfigInvalidException( + String.format("remote %s has invalid URL in %s", name, cfg)); + } + } + return result; + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java index 1b079dd..5541da6 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationModule.java
@@ -33,9 +33,10 @@ import com.googlesource.gerrit.plugins.replication.AutoReloadConfigDecorator; import com.googlesource.gerrit.plugins.replication.AutoReloadSecureCredentialsFactoryDecorator; import com.googlesource.gerrit.plugins.replication.CredentialsFactory; +import com.googlesource.gerrit.plugins.replication.FanoutReplicationConfig; +import com.googlesource.gerrit.plugins.replication.MainReplicationConfig; import com.googlesource.gerrit.plugins.replication.ObservableQueue; import com.googlesource.gerrit.plugins.replication.ReplicationConfig; -import com.googlesource.gerrit.plugins.replication.ReplicationConfigValidator; import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig; import com.googlesource.gerrit.plugins.replication.StartReplicationCapability; import com.googlesource.gerrit.plugins.replication.pull.api.PullReplicationApiModule; @@ -43,6 +44,7 @@ import com.googlesource.gerrit.plugins.replication.pull.client.HttpClientProvider; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import org.apache.http.impl.client.CloseableHttpClient; import org.eclipse.jgit.errors.ConfigInvalidException; @@ -50,10 +52,12 @@ import org.eclipse.jgit.util.FS; class PullReplicationModule extends AbstractModule { + private final SitePaths site; private final Path cfgPath; @Inject public PullReplicationModule(SitePaths site) { + this.site = site; cfgPath = site.etc_dir.resolve("replication.config"); } @@ -94,15 +98,18 @@ DynamicSet.bind(binder(), GitReferenceUpdatedListener.class).to(ReplicationQueue.class); - bind(ReplicationConfigValidator.class).to(SourcesCollection.class); + bind(ConfigParser.class).in(Scopes.SINGLETON); if (getReplicationConfig().getBoolean("gerrit", "autoReload", false)) { - bind(ReplicationConfig.class).to(AutoReloadConfigDecorator.class); + bind(ReplicationConfig.class) + .annotatedWith(MainReplicationConfig.class) + .to(getReplicationConfigClass()); + bind(ReplicationConfig.class).to(AutoReloadConfigDecorator.class).in(Scopes.SINGLETON); bind(LifecycleListener.class) .annotatedWith(UniqueAnnotations.create()) .to(AutoReloadConfigDecorator.class); } else { - bind(ReplicationConfig.class).to(ReplicationFileBasedConfig.class); + bind(ReplicationConfig.class).to(getReplicationConfigClass()).in(Scopes.SINGLETON); } DynamicSet.setOf(binder(), ReplicationStateListener.class); @@ -122,4 +129,11 @@ } return config; } + + private Class<? extends ReplicationConfig> getReplicationConfigClass() { + if (Files.exists(site.etc_dir.resolve("replication"))) { + return FanoutReplicationConfig.class; + } + return ReplicationFileBasedConfig.class; + } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourcesCollection.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourcesCollection.java index f24d25e..acdac0d 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourcesCollection.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/SourcesCollection.java
@@ -16,8 +16,6 @@ import static java.util.stream.Collectors.toList; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; import com.google.common.eventbus.EventBus; import com.google.common.eventbus.Subscribe; import com.google.common.flogger.FluentLogger; @@ -25,20 +23,13 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import com.googlesource.gerrit.plugins.replication.RemoteConfiguration; -import com.googlesource.gerrit.plugins.replication.ReplicationConfigValidator; -import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig; -import java.io.IOException; -import java.net.URISyntaxException; +import com.googlesource.gerrit.plugins.replication.ReplicationConfig; import java.util.List; import java.util.Objects; -import java.util.Set; import org.eclipse.jgit.errors.ConfigInvalidException; -import org.eclipse.jgit.storage.file.FileBasedConfig; -import org.eclipse.jgit.transport.RemoteConfig; -import org.eclipse.jgit.transport.URIish; @Singleton -public class SourcesCollection implements ReplicationSources, ReplicationConfigValidator { +public class SourcesCollection implements ReplicationSources { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private final Source.Factory sourceFactory; @@ -47,10 +38,14 @@ @Inject public SourcesCollection( - ReplicationFileBasedConfig replicationConfig, Source.Factory sourceFactory, EventBus eventBus) + ReplicationConfig replicationConfig, + ConfigParser configParser, + Source.Factory sourceFactory, + EventBus eventBus) throws ConfigInvalidException { this.sourceFactory = sourceFactory; - this.sources = allSources(sourceFactory, validateConfig(replicationConfig)); + this.sources = + allSources(sourceFactory, configParser.parseRemotes(replicationConfig.getConfig())); eventBus.register(this); } @@ -117,65 +112,4 @@ sources = allSources(sourceFactory, sourceConfigurations); logger.atInfo().log("Configuration reloaded: %d sources", getAll().size()); } - - @Override - public List<RemoteConfiguration> validateConfig(ReplicationFileBasedConfig newConfig) - throws ConfigInvalidException { - - try { - newConfig.getConfig().load(); - } catch (IOException e) { - throw new ConfigInvalidException( - String.format("Cannot read %s: %s", newConfig.getConfig().getFile(), e.getMessage()), e); - } - - ImmutableList.Builder<RemoteConfiguration> sourceConfigs = ImmutableList.builder(); - for (RemoteConfig c : allFetchRemotes(newConfig.getConfig())) { - if (c.getURIs().isEmpty()) { - continue; - } - - // fetch source has to be specified. - if (c.getFetchRefSpecs().isEmpty()) { - throw new ConfigInvalidException( - String.format("You must specify a valid refSpec for this remote")); - } - - SourceConfiguration sourceConfig = new SourceConfiguration(c, newConfig.getConfig()); - - if (!sourceConfig.isSingleProjectMatch()) { - for (URIish u : c.getURIs()) { - if (u.getPath() == null || !u.getPath().contains("${name}")) { - throw new ConfigInvalidException( - String.format("remote.%s.url \"%s\" lacks ${name} placeholder", c.getName(), u)); - } - } - } - sourceConfigs.add(sourceConfig); - } - return sourceConfigs.build(); - } - - private static List<RemoteConfig> allFetchRemotes(FileBasedConfig cfg) - throws ConfigInvalidException { - - Set<String> names = cfg.getSubsections("remote"); - List<RemoteConfig> result = Lists.newArrayListWithCapacity(names.size()); - for (String name : names) { - try { - final RemoteConfig remoteConfig = new RemoteConfig(cfg, name); - if (!remoteConfig.getFetchRefSpecs().isEmpty()) { - result.add(remoteConfig); - } else { - logger.atWarning().log( - "Skip loading of remote [remote \"%s\"], since it has no 'fetch' configuration", - name); - } - } catch (URISyntaxException e) { - throw new ConfigInvalidException( - String.format("remote %s has invalid URL in %s", name, cfg.getFile())); - } - } - return result; - } }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java index 4296f67..da263aa 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/pull/client/FetchRestApiClient.java
@@ -20,7 +20,7 @@ import com.google.inject.Inject; import com.google.inject.assistedinject.Assisted; import com.googlesource.gerrit.plugins.replication.CredentialsFactory; -import com.googlesource.gerrit.plugins.replication.ReplicationFileBasedConfig; +import com.googlesource.gerrit.plugins.replication.ReplicationConfig; import com.googlesource.gerrit.plugins.replication.pull.Source; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -59,7 +59,7 @@ FetchRestApiClient( CredentialsFactory credentials, CloseableHttpClient httpClient, - ReplicationFileBasedConfig replicationConfig, + ReplicationConfig replicationConfig, @Assisted Source source) { this.credentials = credentials; this.httpClient = httpClient;
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index c8c53df..8fac1d9 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -291,6 +291,82 @@ By default, replicates without matching, i.e. replicates everything from all remotes. +Directory `replication` +-------------------- +The optional directory `$site_path/etc/replication` contains Git-style +config files that controls the replication settings for the pull replication +plugin. When present all `remote` sections from `replication.config` file are +ignored. + +Files are composed of one `remote` section. Multiple `remote` sections or any +other section makes the file invalid and skipped by the pull replication plugin. +File name defines remote section name. Each section provides common configuration +settings for one or more destination URLs. For more details how to setup `remote` +sections please refer to the `replication.config` section. + +### Configuration example: + +Static configuration in `$site_path/etc/replication.config`: + +``` +[gerrit] + autoReload = true + replicateOnStartup = false +[replication] + instanceLabel = host-one + lockErrorMaxRetries = 5 + maxRetries = 5 +``` + +Remote sections in `$site_path/etc/replication` directory: + +* File `$site_path/etc/replication/host-two.config` + + ``` + [remote] + url = gerrit2@host-two.example.com:/some/path/${name}.git + apiUrl = http://host-two + fetch = +refs/*:refs/* + ``` + + +* File `$site_path/etc/replication/host-three.config` + + ``` + [remote] + url = mirror1.host-three:/pub/git/${name}.git + url = mirror2.host-three:/pub/git/${name}.git + url = mirror3.host-three:/pub/git/${name}.git + apiUrl = http://host-three + fetch = +refs/heads/*:refs/heads/* + fetch = +refs/tags/*:refs/tags/* + ``` + +Pull replication plugin resolves config files to the following configuration: + +``` +[gerrit] + autoReload = true + replicateOnStartup = false +[replication] + instanceLabel = host-one + lockErrorMaxRetries = 5 + maxRetries = 5 + +[remote "host-two"] + url = gerrit2@host-two.example.com:/some/path/${name}.git + apiUrl = http://host-two + fetch = +refs/*:refs/* + +[remote "host-three"] + url = mirror1.host-three:/pub/git/${name}.git + url = mirror2.host-three:/pub/git/${name}.git + url = mirror3.host-three:/pub/git/${name}.git + apiUrl = http://host-three + fetch = +refs/heads/*:refs/heads/* + fetch = +refs/tags/*:refs/tags/* +``` + File `secure.config` --------------------
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigIT.java new file mode 100644 index 0000000..1badb5c --- /dev/null +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/pull/PullReplicationFanoutConfigIT.java
@@ -0,0 +1,211 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.replication.pull; + +import static com.google.common.truth.Truth.assertThat; +import static java.util.stream.Collectors.toList; + +import com.google.common.flogger.FluentLogger; +import com.google.gerrit.acceptance.LightweightPluginDaemonTest; +import com.google.gerrit.acceptance.PushOneCommit.Result; +import com.google.gerrit.acceptance.SkipProjectClone; +import com.google.gerrit.acceptance.TestPlugin; +import com.google.gerrit.acceptance.UseLocalDisk; +import com.google.gerrit.acceptance.testsuite.project.ProjectOperations; +import com.google.gerrit.entities.Project; +import com.google.gerrit.extensions.api.projects.BranchInput; +import com.google.gerrit.extensions.events.GitReferenceUpdatedListener; +import com.google.gerrit.server.config.SitePaths; +import com.google.inject.Inject; +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.function.Supplier; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.transport.ReceiveCommand; +import org.eclipse.jgit.util.FS; +import org.junit.Test; + +@SkipProjectClone +@UseLocalDisk +@TestPlugin( + name = "pull-replication", + sysModule = "com.googlesource.gerrit.plugins.replication.pull.PullReplicationModule") +public class PullReplicationFanoutConfigIT extends LightweightPluginDaemonTest { + private static final Optional<String> ALL_PROJECTS = Optional.empty(); + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private static final int TEST_REPLICATION_DELAY = 60; + private static final Duration TEST_TIMEOUT = Duration.ofSeconds(TEST_REPLICATION_DELAY * 2); + private static final String TEST_REPLICATION_SUFFIX = "suffix1"; + private static final String TEST_REPLICATION_REMOTE = "remote1"; + + @Inject private SitePaths sitePaths; + @Inject private ProjectOperations projectOperations; + private Path gitPath; + private FileBasedConfig config; + private FileBasedConfig remoteConfig; + private FileBasedConfig secureConfig; + + @Override + public void setUpTestPlugin() throws Exception { + gitPath = sitePaths.site_path.resolve("git"); + + config = + new FileBasedConfig(sitePaths.etc_dir.resolve("replication.config").toFile(), FS.DETECTED); + remoteConfig = + new FileBasedConfig( + sitePaths + .etc_dir + .resolve("replication/" + TEST_REPLICATION_REMOTE + ".config") + .toFile(), + FS.DETECTED); + + setReplicationSource( + TEST_REPLICATION_REMOTE); // Simulates a full replication.config initialization + + setRemoteConfig(TEST_REPLICATION_SUFFIX, ALL_PROJECTS); + + secureConfig = + new FileBasedConfig(sitePaths.etc_dir.resolve("secure.config").toFile(), FS.DETECTED); + setReplicationCredentials(TEST_REPLICATION_REMOTE, admin.username(), admin.httpPassword()); + secureConfig.save(); + + super.setUpTestPlugin(); + } + + @Test + public void shouldReplicateNewChangeRef() throws Exception { + testRepo = cloneProject(createTestProject(project + TEST_REPLICATION_SUFFIX)); + + Result pushResult = createChange(); + RevCommit sourceCommit = pushResult.getCommit(); + String sourceRef = pushResult.getPatchSet().refName(); + + ReplicationQueue pullReplicationQueue = getInstance(ReplicationQueue.class); + GitReferenceUpdatedListener.Event event = + new FakeGitReferenceUpdatedEvent( + project, + sourceRef, + ObjectId.zeroId().getName(), + sourceCommit.getId().getName(), + ReceiveCommand.Type.CREATE); + pullReplicationQueue.onGitReferenceUpdated(event); + + try (Repository repo = repoManager.openRepository(project)) { + waitUntil(() -> checkedGetRef(repo, sourceRef) != null); + + Ref targetBranchRef = getRef(repo, sourceRef); + assertThat(targetBranchRef).isNotNull(); + assertThat(targetBranchRef.getObjectId()).isEqualTo(sourceCommit.getId()); + } + } + + @Test + public void shouldReplicateNewBranch() throws Exception { + String testProjectName = project + TEST_REPLICATION_SUFFIX; + createTestProject(testProjectName); + + String newBranch = "refs/heads/mybranch"; + String master = "refs/heads/master"; + BranchInput input = new BranchInput(); + input.revision = master; + gApi.projects().name(testProjectName).branch(newBranch).create(input); + String branchRevision = gApi.projects().name(testProjectName).branch(newBranch).get().revision; + + ReplicationQueue pullReplicationQueue = + plugin.getSysInjector().getInstance(ReplicationQueue.class); + GitReferenceUpdatedListener.Event event = + new FakeGitReferenceUpdatedEvent( + project, + newBranch, + ObjectId.zeroId().getName(), + branchRevision, + ReceiveCommand.Type.CREATE); + pullReplicationQueue.onGitReferenceUpdated(event); + + try (Repository repo = repoManager.openRepository(project); + Repository sourceRepo = repoManager.openRepository(project)) { + waitUntil(() -> checkedGetRef(repo, newBranch) != null); + + Ref targetBranchRef = getRef(repo, newBranch); + assertThat(targetBranchRef).isNotNull(); + assertThat(targetBranchRef.getObjectId().getName()).isEqualTo(branchRevision); + } + } + + private Ref getRef(Repository repo, String branchName) throws IOException { + return repo.getRefDatabase().exactRef(branchName); + } + + private Ref checkedGetRef(Repository repo, String branchName) { + try { + return repo.getRefDatabase().exactRef(branchName); + } catch (Exception e) { + logger.atSevere().withCause(e).log("failed to get ref %s in repo %s", branchName, repo); + return null; + } + } + + private void setReplicationSource(String remoteName) throws IOException { + config.setString("replication", null, "instanceLabel", remoteName); + config.setBoolean("gerrit", null, "autoReload", true); + config.save(); + } + + private void setRemoteConfig(String replicaSuffix, Optional<String> project) throws IOException { + setRemoteConfig(Arrays.asList(replicaSuffix), project); + } + + private void setRemoteConfig(List<String> replicaSuffixes, Optional<String> project) + throws IOException { + List<String> replicaUrls = + replicaSuffixes.stream() + .map(suffix -> gitPath.resolve("${name}" + suffix + ".git").toString()) + .collect(toList()); + remoteConfig.setStringList("remote", null, "url", replicaUrls); + remoteConfig.setString("remote", null, "apiUrl", adminRestSession.url()); + remoteConfig.setString("remote", null, "fetch", "+refs/*:refs/*"); + remoteConfig.setInt("remote", null, "timeout", 600); + remoteConfig.setInt("remote", null, "replicationDelay", TEST_REPLICATION_DELAY); + project.ifPresent(prj -> remoteConfig.setString("remote", null, "projects", prj)); + remoteConfig.save(); + } + + private void setReplicationCredentials(String remoteName, String username, String password) + throws IOException { + secureConfig.setString("remote", remoteName, "username", username); + secureConfig.setString("remote", remoteName, "password", password); + secureConfig.save(); + } + + private void waitUntil(Supplier<Boolean> waitCondition) throws InterruptedException { + WaitUtil.waitUntil(waitCondition, TEST_TIMEOUT); + } + + private <T> T getInstance(Class<T> classObj) { + return plugin.getSysInjector().getInstance(classObj); + } + + private Project.NameKey createTestProject(String name) throws Exception { + return projectOperations.newProject().name(name).create(); + } +}