Merge branch 'stable-3.12' into stable-3.13 * stable-3.12: Add --remote switch to replication start command Change-Id: Ifa94395155956f48dae777e0e41222a79554afbd
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApi.java b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApi.java index f3d2103..f3bc7cc 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApi.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/AdminApi.java
@@ -17,9 +17,32 @@ import com.google.gerrit.entities.Project; public interface AdminApi { - public boolean createProject(Project.NameKey project, String head); - public boolean deleteProject(Project.NameKey project); + /** + * Create a new project without honouring the reflog configuration. + * + * @param project the new project to create + * @param head initial value for the HEAD + * @return true if the project was created + * @deprecated this method should not be used as it does not respect the reflog settings + * <p>Use {@link AdminApi#createProject(Project.NameKey,String,boolean)} instead. + */ + @Deprecated(since = "3.14", forRemoval = true) + boolean createProject(Project.NameKey project, String head); - public boolean updateHead(Project.NameKey project, String newHead); + /** + * Create a new project honouring the reflog configuration. + * + * @param project the new project to create + * @param head initial value for the HEAD + * @param enableRefLog true if the reflog tracking should be enabled + * @return true if the project was created + */ + default boolean createProject(Project.NameKey project, String head, boolean enableRefLog) { + return createProject(project, head); + } + + boolean deleteProject(Project.NameKey project); + + boolean updateHead(Project.NameKey project, String newHead); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java index 8cbb2a2..32903ab 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/CreateProjectTask.java
@@ -53,15 +53,16 @@ public boolean create() { return destinations .getURIs(Optional.of(config.getName()), project, FilterType.PROJECT_CREATION) - .values() + .entries() .stream() - .map(u -> createProject(u, project, head)) + .map(entry -> createProject(entry.getValue(), project, head, entry.getKey().storeRefLog())) .reduce(true, (a, b) -> a && b); } - private boolean createProject(URIish replicateURI, Project.NameKey projectName, String head) { + private boolean createProject( + URIish replicateURI, Project.NameKey projectName, String head, boolean storeRefLog) { Optional<AdminApi> adminApi = adminApiFactory.get().create(replicateURI); - if (adminApi.isPresent() && adminApi.get().createProject(projectName, head)) { + if (adminApi.isPresent() && adminApi.get().createProject(projectName, head, storeRefLog)) { return true; }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java index 41a9a49..f0b26f9 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
@@ -894,6 +894,10 @@ return config.excludedRefsPattern(); } + boolean storeRefLog() { + return config.storeRefLog(); + } + private static boolean matches(URIish uri, String urlMatch) { if (urlMatch == null || urlMatch.equals("") || urlMatch.equals("*")) { return true;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfigParser.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfigParser.java index 9b6d431..31bd7d8 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfigParser.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfigParser.java
@@ -37,9 +37,7 @@ private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - /* (non-Javadoc) - * @see com.googlesource.gerrit.plugins.replication.ConfigParser#parseRemotes(org.eclipse.jgit.lib.Config) - */ + /** {@inheritDoc} */ @Override public List<RemoteConfiguration> parseRemotes(Config config) throws ConfigInvalidException {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java index 41ab4c4..b9e1be9 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java
@@ -56,6 +56,7 @@ private final int slowLatencyThreshold; private final Supplier<Integer> pushBatchSize; private final ImmutableList<Pattern> excludedRefsPattern; + private final boolean storeRefLog; protected DestinationConfiguration(RemoteConfig remoteConfig, Config cfg) { this.remoteConfig = remoteConfig; @@ -122,6 +123,7 @@ return 0; }); excludedRefsPattern = getExcludedRefsPattern(cfg, name); + storeRefLog = cfg.getBoolean("remote", name, "storeRefLog", false); } @Override @@ -227,6 +229,11 @@ return excludedRefsPattern; } + @Override + public boolean storeRefLog() { + return storeRefLog; + } + private ImmutableList<Pattern> getExcludedRefsPattern(Config cfg, String name) { List<Pattern> patterns = new ArrayList<>(); for (String regex : cfg.getStringList("remote", name, "excludedRefsPattern")) {
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java b/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java index b092363..932d907 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/LocalFS.java
@@ -20,9 +20,11 @@ import java.io.File; import java.io.IOException; import org.eclipse.jgit.internal.storage.file.FileRepository; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.transport.URIish; public class LocalFS implements AdminApi { @@ -35,14 +37,33 @@ @Override public boolean createProject(Project.NameKey project, String head) { + return createProject(project, head, false); + } + + @Override + public boolean createProject(Project.NameKey project, String head, boolean enableRefLog) { try (Repository repo = new FileRepository(uri.getPath())) { repo.create(true /* bare */); if (head != null && head.startsWith(Constants.R_REFS)) { + // It is unclear why the reflog is disabled when updating the HEAD. It has been like that + // for over a decade and does not cause issues so far. + // One educated guess is that it follows the default of the git config option + // core.logAllRefUpdates which is false by default for bare repositories and by default + // stores reflogs only for repositories which have a working tree. RefUpdate u = repo.updateRef(Constants.HEAD); u.disableRefLog(); u.link(head); } + + StoredConfig config = repo.getConfig(); + config.setBoolean( + ConfigConstants.CONFIG_CORE_SECTION, + null, + ConfigConstants.CONFIG_KEY_LOGALLREFUPDATES, + enableRefLog); + config.save(); + repLog.atInfo().log("Created local repository: %s", uri); } catch (IOException e) { repLog.atSevere().withCause(e).log("Error creating local repository %s", uri.getPath());
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/MergedConfigResource.java b/src/main/java/com/googlesource/gerrit/plugins/replication/MergedConfigResource.java index 58a10f0..541b0f4 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/MergedConfigResource.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/MergedConfigResource.java
@@ -34,8 +34,11 @@ public class MergedConfigResource { @VisibleForTesting @UsedAt(Project.PLUGIN_PULL_REPLICATION) + @SuppressWarnings("UnnecessaryAssignment") public static MergedConfigResource withBaseOnly(ConfigResource base) { MergedConfigResource mergedConfigResource = new MergedConfigResource(); + // This assigns an @Inject field explicitly, which is intentional here. + // Suppress UnnecessaryAssignment as DI is bypassed on this code path. mergedConfigResource.baseConfigProvider = Providers.of(base); return mergedConfigResource; }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteConfiguration.java b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteConfiguration.java index 67d23b0..c725d40 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteConfiguration.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/RemoteConfiguration.java
@@ -152,4 +152,11 @@ default ImmutableList<Pattern> excludedRefsPattern() { return ImmutableList.of(); } + + /** + * reflog storage flag for newly created repositories + * + * @return true if new repositories should store ref-updates in their reflog + */ + boolean storeRefLog(); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationLogFile.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationLogFile.java index 60a9523..622034a 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationLogFile.java +++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationLogFile.java
@@ -15,19 +15,47 @@ package com.googlesource.gerrit.plugins.replication; import com.google.gerrit.extensions.systemstatus.ServerInformation; +import com.google.gerrit.server.config.LogConfig; import com.google.gerrit.server.util.PluginLogFile; import com.google.gerrit.server.util.SystemLog; +import com.google.gerrit.util.logging.JsonLayout; +import com.google.gerrit.util.logging.JsonLogEntry; +import com.google.gson.annotations.SerializedName; import com.google.inject.Inject; import org.apache.log4j.PatternLayout; +import org.apache.log4j.spi.LoggingEvent; public class ReplicationLogFile extends PluginLogFile { @Inject - public ReplicationLogFile(SystemLog systemLog, ServerInformation serverInfo) { + public ReplicationLogFile(SystemLog systemLog, ServerInformation serverInfo, LogConfig config) { super( systemLog, serverInfo, ReplicationQueue.REPLICATION_LOG_NAME, - new PatternLayout("[%d] %m%n")); + new PatternLayout("[%d] %m%n"), + new ReplicationJsonLayout(), + config); + } + + static class ReplicationJsonLayout extends JsonLayout { + @SuppressWarnings("unused") + private class ReplicationJsonLogEntry extends JsonLogEntry { + public String timestamp; + public String message; + + @SerializedName("@version") + public final int version = 1; + + public ReplicationJsonLogEntry(LoggingEvent event) { + timestamp = timestampFormatter.format(event.getTimeStamp()); + message = (String) event.getMessage(); + } + } + + @Override + public JsonLogEntry toJsonLogEntry(LoggingEvent event) { + return new ReplicationJsonLogEntry(event); + } } }
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index 67c201c..150810e 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -344,6 +344,17 @@ Defaults to `git-receive-pack`. +remote.NAME.storeRefLog +: `true` if the remote repositories should be enabled for storing + ref updates in the reflog once they are created by replication. + + NOTE: Enabling the reflog would prevent the unreferenced objects + from being garbage collected, potentially impacting the post-gc + repository size. Also, the ref updates may take additional time + because of the need to store the old and new SHA1s in the reflog. + + Defaults to `false`. + remote.NAME.uploadpack : Path of the `git-upload-pack` executable on the remote system, if using the SSH transport. @@ -546,10 +557,13 @@ they do in Gerrit. <a name="remote.NAME.projects">remote.NAME.projects</a> -: Specifies which repositories should be replicated to the - remote. It can be provided more than once, and supports three - formats: regular expressions, wildcard matching, and single - project matching. All three formats match case-sensitive. +: Specifies which repositories should be replicated to the remote. It + can be provided more than once, if any matches, the repository will + be replicated. + + Projects supports three formats: regular expressions, wildcard + matching, and single project matching. All three formats match + case-sensitive. Values starting with a caret `^` are treated as regular expressions. `^foo/(bar|baz)` would match the projects @@ -559,7 +573,12 @@ Projects may be excluded from replication by using a regular expression with inverse match. `^(?:(?!PATTERN).)*$` will - exclude any project that matches. + exclude any project that matches. Beware: if another project pattern + matches the project, it will be replicated. You can not provide + multiple inverted matches since they will cancel each other (a + project `foo` would not be matched by `^(?:(?!foo).)*$` but would be + matched if a second inverted match `^(?:(?!bar).)*$` is also provided + since it matches `foo`). Values that are not regular expressions and end in `*` are treated as wildcard matches. Wildcards match projects whose
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java index 93b131d..5013014 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationIT.java
@@ -29,6 +29,7 @@ import com.google.gerrit.extensions.registration.DynamicSet; import com.google.gerrit.server.git.WorkQueue; import com.google.inject.Inject; +import java.io.IOException; import java.time.Duration; import java.util.Arrays; import java.util.List; @@ -39,13 +40,16 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import java.util.function.Supplier; import java.util.stream.Collectors; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; import org.junit.Test; @@ -73,18 +77,53 @@ } @Test - public void shouldReplicateNewProject() throws Exception { + public void shouldReplicateNewProjectWithoutRefLog() throws Exception { setReplicationDestination("foo", "replica", ALL_PROJECTS); reloadConfig(); - Project.NameKey sourceProject = createTestProject("foo"); + Project.NameKey sourceProject = createTestProject("no_reflog_project"); + Project.NameKey replicaProject = Project.nameKey(sourceProject + "replica.git"); - WaitUtil.waitUntil( - () -> nonEmptyProjectExists(Project.nameKey(sourceProject + "replica.git")), - TEST_NEW_PROJECT_TIMEOUT); + waitForProjectCreated(replicaProject); - ProjectInfo replicaProject = gApi.projects().name(sourceProject + "replica").get(); - assertThat(replicaProject).isNotNull(); + ProjectInfo replicaProjectInfo = gApi.projects().name(replicaProject.get()).get(); + assertThat(replicaProjectInfo).isNotNull(); + applyToRepositoryConfig(replicaProject, assertStoreRefLog(false)); + } + + @Test + public void shouldCreateNewProjectWithRefLog() throws Exception { + config.setBoolean("remote", "foo", "storeRefLog", true); + setReplicationDestination("foo", "replica", ALL_PROJECTS); + reloadConfig(); + + Project.NameKey sourceProject = createTestProject("reflog_project"); + Project.NameKey replicaProject = Project.nameKey(sourceProject + "replica.git"); + + waitForProjectCreated(replicaProject); + + applyToRepositoryConfig(replicaProject, assertStoreRefLog(true)); + } + + private void waitForProjectCreated(Project.NameKey replicaProject) throws InterruptedException { + WaitUtil.waitUntil(() -> nonEmptyProjectExists(replicaProject), TEST_NEW_PROJECT_TIMEOUT); + } + + private static Consumer<StoredConfig> assertStoreRefLog(boolean expectedValue) { + return conf -> + assertThat( + conf.getBoolean( + ConfigConstants.CONFIG_CORE_SECTION, + null, + ConfigConstants.CONFIG_KEY_LOGALLREFUPDATES)) + .isEqualTo(expectedValue); + } + + private void applyToRepositoryConfig( + Project.NameKey projectName, Consumer<StoredConfig> functionToApply) throws IOException { + try (Repository repo = repoManager.openRepository(projectName)) { + functionToApply.accept(repo.getConfig()); + } } @Test
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStorageIT.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStorageIT.java index 94646c3..643a781 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStorageIT.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationStorageIT.java
@@ -26,7 +26,6 @@ import com.googlesource.gerrit.plugins.replication.ReplicationTasksStorage.ReplicateRefUpdate; import com.googlesource.gerrit.plugins.replication.api.ReplicationConfig.FilterType; import java.io.IOException; -import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; @@ -96,12 +95,9 @@ changeReplicationTasksForRemote(tasksStorage.streamWaiting(), changeRef, remote1) .forEach( (update) -> { - try { - UriUpdates uriUpdates = new TestUriUpdates(update); - tasksStorage.start(uriUpdates); - tasksStorage.finish(uriUpdates); - } catch (URISyntaxException e) { - } + UriUpdates uriUpdates = new TestUriUpdates(update); + tasksStorage.start(uriUpdates); + tasksStorage.finish(uriUpdates); }); reloadConfig(); @@ -125,12 +121,9 @@ changeReplicationTasksForRemote(tasksStorage.streamWaiting(), changeRef, remote1) .forEach( (update) -> { - try { - UriUpdates uriUpdates = new TestUriUpdates(update); - tasksStorage.start(uriUpdates); - tasksStorage.finish(uriUpdates); - } catch (URISyntaxException e) { - } + UriUpdates uriUpdates = new TestUriUpdates(update); + tasksStorage.start(uriUpdates); + tasksStorage.finish(uriUpdates); }); setReplicationDestination(remote1, suffix1, ALL_PROJECTS);
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorageMPTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorageMPTest.java index cf5168f..1f1c512 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorageMPTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorageMPTest.java
@@ -21,7 +21,6 @@ import com.google.common.jimfs.Configuration; import com.google.common.jimfs.Jimfs; import com.googlesource.gerrit.plugins.replication.ReplicationTasksStorage.ReplicateRefUpdate; -import java.net.URISyntaxException; import java.nio.file.FileSystem; import java.nio.file.Path; import java.util.Set; @@ -40,7 +39,7 @@ ReplicateRefUpdate.create(PROJECT, Set.of(REF), URISH, REMOTE); protected static final ReplicateRefUpdate STORED_REF_UPDATE = ReplicateRefUpdate.create(REF_UPDATE, REF_UPDATE.sha1()); - protected static final UriUpdates URI_UPDATES = getUriUpdates(REF_UPDATE); + protected static final UriUpdates URI_UPDATES = new TestUriUpdates(REF_UPDATE); protected ReplicationTasksStorage nodeA; protected ReplicationTasksStorage nodeB; @@ -209,12 +208,4 @@ assertThat(nodeB.start(URI_UPDATES)).isEmpty(); assertThatStream(persistedView.streamRunning()).containsExactly(STORED_REF_UPDATE); } - - public static UriUpdates getUriUpdates(ReplicationTasksStorage.ReplicateRefUpdate refUpdate) { - try { - return new TestUriUpdates(refUpdate); - } catch (URISyntaxException e) { - throw new RuntimeException("Cannot instantiate UriUpdates object", e); - } - } }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorageTaskTest.java b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorageTaskTest.java index 43c0b3e..b996938 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorageTaskTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorageTaskTest.java
@@ -378,8 +378,8 @@ new URIish("git://host1/someRepo.git"), "someRemote"); assertEquals( - gson.toJson(update), - "{\"project\":\"someProject\",\"refs\":[\"ref1\"],\"uri\":\"git://host1/someRepo.git\",\"remote\":\"someRemote\"}"); + /* expected= */ "{\"project\":\"someProject\",\"refs\":[\"ref1\"],\"uri\":\"git://host1/someRepo.git\",\"remote\":\"someRemote\"}", + /* actual= */ gson.toJson(update)); ReplicateRefUpdate update2 = ReplicateRefUpdate.create( "someProject", @@ -387,8 +387,8 @@ new URIish("git://host1/someRepo.git"), "someRemote"); assertEquals( - gson.toJson(update2), - "{\"project\":\"someProject\",\"refs\":[\"ref1\",\"ref2\"],\"uri\":\"git://host1/someRepo.git\",\"remote\":\"someRemote\"}"); + /* expected= */ "{\"project\":\"someProject\",\"refs\":[\"ref1\",\"ref2\"],\"uri\":\"git://host1/someRepo.git\",\"remote\":\"someRemote\"}", + /* actual= */ gson.toJson(update2)); } @Test
diff --git a/src/test/java/com/googlesource/gerrit/plugins/replication/TestUriUpdates.java b/src/test/java/com/googlesource/gerrit/plugins/replication/TestUriUpdates.java index f6eec83..74b2082 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/replication/TestUriUpdates.java +++ b/src/test/java/com/googlesource/gerrit/plugins/replication/TestUriUpdates.java
@@ -27,11 +27,15 @@ private final String remote; private final Set<ImmutableSet<String>> refs; - public TestUriUpdates(ReplicateRefUpdate update) throws URISyntaxException { - project = Project.nameKey(update.project()); - uri = new URIish(update.uri()); - remote = update.remote(); - refs = Set.of(update.refs()); + public TestUriUpdates(ReplicateRefUpdate update) { + try { + project = Project.nameKey(update.project()); + uri = new URIish(update.uri()); + remote = update.remote(); + refs = Set.of(update.refs()); + } catch (URISyntaxException e) { + throw new RuntimeException("TestUriUpdates init failed.", e); + } } @Override