Merge branch 'stable-3.10' into stable-3.11 * stable-3.10: Implement filterAndLock in ReplicationFetchFilter Delete duplicate DisabledSharedRefLogger class Reformat with GJF 1.24.0 Remove obsolete LockWrapper.Factory Adding the pull-replication extension for multi-site Change-Id: I30e306c2f4dc017842137615083cc760d26c3e7c
diff --git a/setup_local_env/configs/gerrit.config b/setup_local_env/configs/gerrit.config index c92c60d..a4d68c2 100644 --- a/setup_local_env/configs/gerrit.config +++ b/setup_local_env/configs/gerrit.config
@@ -5,6 +5,7 @@ serverId = 69ec38f0-350e-4d9c-96d4-bc956f2faaac canonicalWebUrl = $GERRIT_CANONICAL_WEB_URL installModule = com.gerritforge.gerrit.eventbroker.BrokerApiModule # events-broker module to setup BrokerApi dynamic item + installModule = com.googlesource.gerrit.plugins.replication.pull.ReplicationExtensionPointModule installModule = com.googlesource.gerrit.plugins.multisite.Module # multi-site needs to be a gerrit lib installDbModule = com.googlesource.gerrit.plugins.multisite.GitModule instanceId = $INSTANCE_ID
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/Configuration.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/Configuration.java index a7a05d3..4415aea 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/multisite/Configuration.java +++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/Configuration.java
@@ -58,6 +58,8 @@ private static final String REPLICATION_LAG_REFRESH_INTERVAL = "replicationLagRefreshInterval"; private static final String REPLICATION_LAG_ENABLED = "replicationLagEnabled"; private static final Duration REPLICATION_LAG_REFRESH_INTERVAL_DEFAULT = Duration.ofSeconds(60); + private static final String LOCAL_REF_LOCK_TIMEOUT = "localRefLockTimeout"; + private static final Duration LOCAL_REF_LOCK_TIMEOUT_DEFAULT = Duration.ofSeconds(30); private static final String REPLICATION_CONFIG = "replication.config"; // common parameters to cache and index sections @@ -79,6 +81,7 @@ private final Config multiSiteConfig; private final Supplier<Duration> replicationLagRefreshInterval; private final Supplier<Boolean> replicationLagEnabled; + private final Supplier<Long> localRefLockTimeoutMsec; @Inject Configuration(SitePaths sitePaths) { @@ -118,6 +121,16 @@ lazyMultiSiteCfg .get() .getBoolean(REF_DATABASE, null, REPLICATION_LAG_ENABLED, true)); + localRefLockTimeoutMsec = + memoize( + () -> + ConfigUtil.getTimeUnit( + lazyMultiSiteCfg.get(), + REF_DATABASE, + null, + LOCAL_REF_LOCK_TIMEOUT, + LOCAL_REF_LOCK_TIMEOUT_DEFAULT.toMillis(), + TimeUnit.MILLISECONDS)); } public Config getMultiSiteConfig() { @@ -160,6 +173,10 @@ return replicationLagEnabled.get(); } + public long localRefLockTimeoutMsec() { + return localRefLockTimeoutMsec.get(); + } + public Collection<Message> validate() { return replicationConfigValidation.get(); }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationFetchFilter.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationFetchFilter.java index d67492d..89310bf 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationFetchFilter.java +++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultisiteReplicationFetchFilter.java
@@ -17,7 +17,9 @@ import static com.googlesource.gerrit.plugins.replication.pull.PullReplicationLogger.repLog; import com.gerritforge.gerrit.globalrefdb.GlobalRefDbLockException; +import com.gerritforge.gerrit.globalrefdb.RefDbLockException; import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDatabaseWrapper; +import com.google.common.collect.Sets; import com.google.common.flogger.FluentLogger; import com.google.gerrit.entities.Project; import com.google.gerrit.server.git.GitRepositoryManager; @@ -27,6 +29,9 @@ import com.googlesource.gerrit.plugins.replication.pull.ReplicationFetchFilter; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -89,6 +94,37 @@ } } + @Override + public Map<String, AutoCloseable> filterAndLock(String projectName, Set<String> fetchRefs) + throws RefDbLockException { + Project.NameKey projectKey = Project.nameKey(projectName); + Set<String> filteredRefs = new HashSet<>(); + Map<String, AutoCloseable> refLocks = new HashMap<>(); + try { + for (String ref : fetchRefs) { + refLocks.put(ref, sharedRefDb.lockLocalRef(projectKey, ref)); + } + filteredRefs.addAll(filter(projectName, fetchRefs)); + } catch (RefDbLockException lockException) { + filteredRefs.clear(); + throw lockException; + } finally { + for (String excludedRef : Sets.difference(fetchRefs, filteredRefs)) { + AutoCloseable excludedLock = refLocks.remove(excludedRef); + if (excludedLock != null) { + try { + excludedLock.close(); + } catch (Exception e) { + logger.atWarning().withCause(e).log( + "Error whilst unlocking ref %s:%s", projectName, excludedRef); + } + } + } + } + + return refLocks; + } + private Optional<ObjectId> getLocalSha1IfEqualsToExistingGlobalRefDb( Repository repository, String projectName,
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ReentrantRefDbLocker.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ReentrantRefDbLocker.java new file mode 100644 index 0000000..b2085ae --- /dev/null +++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ReentrantRefDbLocker.java
@@ -0,0 +1,74 @@ +// Copyright (C) 2025 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.googlesource.gerrit.plugins.multisite.validation; + +import com.gerritforge.gerrit.globalrefdb.RefDbLockException; +import com.gerritforge.gerrit.globalrefdb.validation.RefLocker; +import com.google.gerrit.entities.Project; +import com.google.inject.Inject; +import com.googlesource.gerrit.plugins.multisite.Configuration; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +class ReentrantRefDbLocker implements RefLocker { + private final ConcurrentHashMap<String, RefLock> refsLocks; + private final long timeoutMsec; + + private class RefLock implements AutoCloseable { + private final Lock lock; + + RefLock() { + lock = new ReentrantLock(); + } + + boolean lock() throws InterruptedException { + return lock.tryLock(timeoutMsec, TimeUnit.MILLISECONDS); + } + + @Override + public void close() throws Exception { + lock.unlock(); + } + } + + @Inject + public ReentrantRefDbLocker(Configuration configuration) { + this.timeoutMsec = configuration.localRefLockTimeoutMsec(); + refsLocks = new ConcurrentHashMap<>(); + } + + @Override + public AutoCloseable lockRef(Project.NameKey project, String refName) throws RefDbLockException { + RefLock lock = refsLocks.computeIfAbsent(getKey(project, refName), ref -> new RefLock()); + try { + if (lock.lock()) { + return lock; + } + + throw new RefDbLockException( + project.get(), + refName, + String.format("Unable to acquire local ref lock after %s msec", timeoutMsec)); + } catch (InterruptedException e) { + throw new RefDbLockException(project.get(), refName, e); + } + } + + private String getKey(Project.NameKey project, String refName) { + return String.format("%s:%s", project.get(), refName); + } +}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationModule.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationModule.java index ed22855..958f265 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationModule.java +++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationModule.java
@@ -15,8 +15,8 @@ package com.googlesource.gerrit.plugins.multisite.validation; import com.gerritforge.gerrit.globalrefdb.validation.BatchRefUpdateValidator; -import com.gerritforge.gerrit.globalrefdb.validation.LockWrapper; import com.gerritforge.gerrit.globalrefdb.validation.Log4jSharedRefLogger; +import com.gerritforge.gerrit.globalrefdb.validation.RefLocker; import com.gerritforge.gerrit.globalrefdb.validation.RefUpdateValidator; import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDatabaseWrapper; import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDbBatchRefUpdate; @@ -50,9 +50,9 @@ @Override protected void configure() { + bind(RefLocker.class).to(ReentrantRefDbLocker.class).in(Scopes.SINGLETON); bind(SharedRefDatabaseWrapper.class).in(Scopes.SINGLETON); bind(SharedRefLogger.class).to(Log4jSharedRefLogger.class); - factory(LockWrapper.Factory.class); factory(SharedRefDbRepository.Factory.class); factory(SharedRefDbRefDatabase.Factory.class);
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md index 7547fd9..55e5041 100644 --- a/src/main/resources/Documentation/config.md +++ b/src/main/resources/Documentation/config.md
@@ -95,6 +95,11 @@ : Enable the use of a shared ref-database Defaults: true +```ref-database.localRefLockTimeout``` +: Timeout waiting for a local ref to become available to accept + updates or for starting a replication task. + Defaults: 30 sec + ```ref-database.replicationLagEnabled``` : Enable the metrics to trace the auto-replication lag between sites updating the `refs/multi-site/version/*` to the _epoch_ timestamp in
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java deleted file mode 100644 index fc2259f..0000000 --- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java +++ /dev/null
@@ -1,42 +0,0 @@ -// Copyright (C) 2022 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.googlesource.gerrit.plugins.multisite.validation; - -import com.gerritforge.gerrit.globalrefdb.validation.SharedRefLogger; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Ref; -import org.junit.Ignore; - -@Ignore -public class DisabledSharedRefLogger implements SharedRefLogger { - - @Override - public void logRefUpdate(String project, Ref currRef, ObjectId newRefValue) {} - - @Override - public void logProjectDelete(String project) {} - - @Override - public void logLockAcquisition(String project, String refName) {} - - @Override - public void logLockRelease(String project, String refName) {} - - @Override - public <T> void logRefUpdate(String project, String refName, T currRef, T newRefValue) {} - - @Override - public <T> void logRefUpdate(String project, String refName, T newRefValue) {} -}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/FakeSharedRefDatabaseWrapper.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/FakeSharedRefDatabaseWrapper.java index bf98a4b..74bd41b 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/FakeSharedRefDatabaseWrapper.java +++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/FakeSharedRefDatabaseWrapper.java
@@ -17,6 +17,7 @@ import com.gerritforge.gerrit.globalrefdb.GlobalRefDatabase; import com.gerritforge.gerrit.globalrefdb.GlobalRefDbLockException; import com.gerritforge.gerrit.globalrefdb.GlobalRefDbSystemError; +import com.gerritforge.gerrit.globalrefdb.validation.DisabledSharedRefLogger; import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDBMetrics; import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDatabaseWrapper; import com.google.gerrit.entities.Project; @@ -79,6 +80,7 @@ } }), new DisabledSharedRefLogger(), - new SharedRefDBMetrics(new DisabledMetricMaker())); + new SharedRefDBMetrics(new DisabledMetricMaker()), + ((project, refName) -> () -> {})); } }