Merge branch 'stable-3.9' * stable-3.9: Use put operation for project version update Remove project version SHA1 from global-refdb Consume global-refdb directly from source Do not download events-broker from maven during local setup Change-Id: Ia935d02e61f3a3c141c3b74bd284a4976e7bb3a4
diff --git a/BUILD b/BUILD index f9cfcee..d95dc33 100644 --- a/BUILD +++ b/BUILD
@@ -19,9 +19,9 @@ resources = glob(["src/main/resources/**/*"]), deps = [ ":events-broker-neverlink", + ":global-refdb-neverlink", ":pull-replication-neverlink", ":replication-neverlink", - "@global-refdb//jar:neverlink", ], ) @@ -43,6 +43,12 @@ exports = ["//plugins/events-broker"], ) +java_library( + name = "global-refdb-neverlink", + neverlink = 1, + exports = ["//plugins/global-refdb"], +) + junit_tests( name = "multi_site_tests", srcs = glob(["src/test/java/**/*.java"]), @@ -62,8 +68,8 @@ visibility = ["//visibility:public"], exports = PLUGIN_DEPS + PLUGIN_TEST_DEPS + [ ":multi-site__plugin", - "@global-refdb//jar", "//plugins/events-broker", + "//plugins/global-refdb", "//plugins/pull-replication", "//plugins/replication", ], @@ -100,7 +106,6 @@ "//plugins/multi-site", "//plugins/multi-site:e2e_multi_site_test_dir", "//plugins/multi-site:e2e_multi_site_setup_local_env_dir", - "external_plugin_deps.bzl", ] + glob(["setup_local_env/**/*"]) + glob(["e2e-tests/**/*"]), tags = [ "e2e-multi-site",
diff --git a/Jenkinsfile b/Jenkinsfile index 5b02057..6ef43af 100644 --- a/Jenkinsfile +++ b/Jenkinsfile
@@ -1,4 +1,4 @@ pluginPipeline(formatCheckId: 'gerritforge:multi-site-format-47168e90078b0b3f11401610930e82830e76bff7', buildCheckId: 'gerritforge:multi-site-47168e90078b0b3f11401610930e82830e76bff7', extraPlugins: [ 'pull-replication' ], - extraModules: [ 'events-broker' ]) + extraModules: [ 'events-broker', 'global-refdb' ])
diff --git a/README.md b/README.md index fe75d36..1e6882c 100644 --- a/README.md +++ b/README.md
@@ -40,8 +40,6 @@ cd gerrit/plugins ln -s ../../multi-site . -rm external_plugin_deps.bzl -ln -s multi-site/external_plugin_deps.bzl . ``` Example of building the multi-site plugin:
diff --git a/e2e-tests/test.sh b/e2e-tests/test.sh index b103720..fab3c7e 100755 --- a/e2e-tests/test.sh +++ b/e2e-tests/test.sh
@@ -210,7 +210,6 @@ done # Defaults -GLOBAL_REFDB_VER=`grep 'com.gerritforge:global-refdb' ${LOCATION}/../external_plugin_deps.bzl | cut -d '"' -f 2 | cut -d ':' -f 3` DEPLOYMENT_LOCATION=$(mktemp -d || $(echo >&2 "Could not create temp dir" && exit 1)) MULTISITE_LIB_LOCATION=${MULTISITE_LIB_LOCATION:-${DEF_MULTISITE_LOCATION}} BROKER_TYPE=${BROKER_TYPE:-"kafka"} @@ -261,9 +260,8 @@ docker cp ${CONTAINER_NAME}:/var/gerrit/plugins/replication.jar $COMMON_LIBS/ docker rm -fv ${CONTAINER_NAME} -echo "Downloading global-refdb library $GERRIT_BRANCH" -wget https://repo1.maven.org/maven2/com/gerritforge/global-refdb/$GLOBAL_REFDB_VER/global-refdb-$GLOBAL_REFDB_VER.jar \ - -O $COMMON_LIBS/global-refdb.jar || { echo >&2 "Cannot download global-refdb library: Check internet connection. Aborting"; exit 1; } +echo "Copying global-refdb library $GERRIT_BRANCH" +cp bazel-bin/plugins/global-refdb/global-refdb.jar $COMMON_LIBS/global-refdb.jar echo "Downloading events-broker library $GERRIT_BRANCH" cp bazel-bin/plugins/events-broker/events-broker.jar $COMMON_LIBS/events-broker.jar
diff --git a/external_plugin_deps.bzl b/external_plugin_deps.bzl deleted file mode 100644 index 7d855cf..0000000 --- a/external_plugin_deps.bzl +++ /dev/null
@@ -1,8 +0,0 @@ -load("//tools/bzl:maven_jar.bzl", "maven_jar") - -def external_plugin_deps(): - maven_jar( - name = "global-refdb", - artifact = "com.gerritforge:global-refdb:3.8.0-rc5", - sha1 = "3f54f99e4ddf454c119e8ac3183bba7e9d7c7897", - )
diff --git a/setup_local_env/setup.sh b/setup_local_env/setup.sh index 32880a2..0e84603 100755 --- a/setup_local_env/setup.sh +++ b/setup_local_env/setup.sh
@@ -242,6 +242,8 @@ echo echo "[--release-war-file] Location to release.war file" echo "[--multisite-lib-file] Location to lib multi-site.jar file" + echo "[--eventsbroker-lib-file] Location to lib events-broker.jar file" + echo "[--globalrefdb-lib-file] Location to lib global-refdb.jar file" echo echo "[--new-deployment] Cleans up previous gerrit deployment and re-installs it. default true" echo "[--get-websession-plugin] Download websession-broker plugin from CI lastSuccessfulBuild; default true" @@ -295,6 +297,16 @@ shift shift ;; + "--eventsbroker-lib-file" ) + EVENTS_BROKER_LIB_LOCATION=$2 + shift + shift + ;; + "--globalrefdb-lib-file" ) + GLOBAL_REFDB_LIB_LOCATION=$2 + shift + shift + ;; "--gerrit-canonical-host" ) export GERRIT_CANONICAL_HOSTNAME=$2 shift @@ -396,7 +408,8 @@ RELEASE_WAR_FILE_LOCATION=${RELEASE_WAR_FILE_LOCATION:-bazel-bin/release.war} MULTISITE_LIB_LOCATION=${MULTISITE_LIB_LOCATION:-bazel-bin/plugins/multi-site/multi-site.jar} - +EVENTS_BROKER_LIB_LOCATION=${EVENTS_BROKER_LIB_LOCATION:-bazel-bin/plugins/events-broker/events-broker.jar} +GLOBAL_REFDB_LIB_LOCATION=${GLOBAL_REFDB_LIB_LOCATION:-bazel-bin/plugins/global-refdb/global-refdb.jar} export FAKE_NFS=$COMMON_LOCATION/fake_nfs @@ -417,6 +430,13 @@ else cp -f $MULTISITE_LIB_LOCATION $DEPLOYMENT_LOCATION/multi-site.jar >/dev/null 2>&1 || { echo >&2 "$MULTISITE_LIB_LOCATION: Not able to copy the file. Aborting"; exit 1; } fi + +echo "Copying events-broker library" + cp -f $EVENTS_BROKER_LIB_LOCATION $DEPLOYMENT_LOCATION/events-broker.jar >/dev/null 2>&1 || { echo >&2 "$EVENTS_BROKER_LIB_LOCATION: Not able to copy the file. Aborting"; exit 1; } + +echo "Copying global-refdb library" + cp -f $GLOBAL_REFDB_LIB_LOCATION $DEPLOYMENT_LOCATION/global-refdb.jar >/dev/null 2>&1 || { echo >&2 "$GLOBAL_REFDB_LIB_LOCATION: Not able to copy the file. Aborting"; exit 1; } + if [ $DOWNLOAD_WEBSESSION_PLUGIN = "true" ];then echo "Downloading websession-broker plugin $GERRIT_BRANCH" download_artifact_from_ci websession-broker @@ -429,12 +449,6 @@ echo "Downloading zookeeper plugin $GERRIT_BRANCH" download_artifact_from_ci zookeeper-refdb -echo "Downloading global-refdb library $GERRIT_BRANCH" - download_artifact_from_ci global-refdb "module" - -echo "Downloading events-broker library $GERRIT_BRANCH" - download_artifact_from_ci events-broker "module" - if [ "$BROKER_TYPE" = "kafka" ]; then echo "Downloading events-kafka plugin $GERRIT_BRANCH" download_artifact_from_ci events-kafka
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateImpl.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateImpl.java index 03271aa..aa589b6 100644 --- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateImpl.java +++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateImpl.java
@@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.flogger.FluentLogger; import com.google.gerrit.entities.Project; +import com.google.gerrit.entities.Project.NameKey; import com.google.gerrit.entities.RefNames; import com.google.gerrit.server.config.GerritInstanceId; import com.google.gerrit.server.events.Event; @@ -40,9 +41,7 @@ import java.util.Set; import org.eclipse.jgit.errors.RepositoryNotFoundException; import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.Repository; @@ -106,8 +105,7 @@ if (newProjectVersionRefUpdate.isPresent()) { verLogger.log(projectNameKey, newVersion, 0L); - if (updateSharedProjectVersion( - projectNameKey, newProjectVersionRefUpdate.get().getNewObjectId(), newVersion)) { + if (updateSharedProjectVersion(projectNameKey, newVersion)) { gitReferenceUpdated.fire(projectNameKey, newProjectVersionRefUpdate.get(), null); } } else { @@ -138,20 +136,9 @@ } @SuppressWarnings("FloggerLogString") - private boolean updateSharedProjectVersion( - Project.NameKey projectNameKey, ObjectId newObjectId, Long newVersion) + private boolean updateSharedProjectVersion(Project.NameKey projectNameKey, Long newVersion) throws SharedProjectVersionUpdateException { - Ref sharedRef = - sharedRefDb - .get(projectNameKey, MULTI_SITE_VERSIONING_REF, String.class) - .map( - (String objectId) -> - new ObjectIdRef.Unpeeled( - Ref.Storage.NEW, MULTI_SITE_VERSIONING_REF, ObjectId.fromString(objectId))) - .orElse( - new ObjectIdRef.Unpeeled( - Ref.Storage.NEW, MULTI_SITE_VERSIONING_REF, ObjectId.zeroId())); Optional<Long> sharedVersion = sharedRefDb .get(projectNameKey, MULTI_SITE_VERSIONING_VALUE_REF, String.class) @@ -160,58 +147,49 @@ try { if (sharedVersion.isPresent() && sharedVersion.get() >= newVersion) { logger.atWarning().log( - "NOT Updating project %s version %s (value=%d) in shared ref-db because is more recent than the local one %s (value=%d) ", - projectNameKey.get(), - newObjectId, - newVersion, - sharedRef.getObjectId().getName(), - sharedVersion.get()); + "NOT Updating project %s value=%d in shared ref-db because is more recent than the local value=%d", + projectNameKey.get(), newVersion, sharedVersion.get()); return false; } logger.atFine().log( - "Updating shared project %s version to %s (value=%d)", - projectNameKey.get(), newObjectId, newVersion); + "Updating shared project %s value to %d", projectNameKey.get(), newVersion); - boolean success = sharedRefDb.compareAndPut(projectNameKey, sharedRef, newObjectId); - if (!success) { - String message = - String.format( - "Project version blob update failed for %s. Current value %s, new value: %s", - projectNameKey.get(), safeGetObjectId(sharedRef), newObjectId); - logger.atSevere().log(message); - throw new SharedProjectVersionUpdateException(message); - } - - success = - sharedRefDb.compareAndPut( - projectNameKey, - MULTI_SITE_VERSIONING_VALUE_REF, - sharedVersion.map(Object::toString).orElse(null), - newVersion.toString()); - if (!success) { - String message = - String.format( - "Project version update failed for %s. Current value %s, new value: %s", - projectNameKey.get(), safeGetObjectId(sharedRef), newObjectId); - logger.atSevere().log(message); - throw new SharedProjectVersionUpdateException(message); - } - + updateProjectVersionValue(projectNameKey, newVersion, sharedVersion); return true; } catch (GlobalRefDbSystemError refDbSystemError) { String message = String.format( - "Error while updating shared project version for %s. Current value %s, new value: %s. Error: %s", + "Error while updating shared project value for %s. Current value %s, new value: %s. Error: %s", projectNameKey.get(), - sharedRef.getObjectId(), - newObjectId, + sharedVersion.map(Object::toString).orElse(null), + newVersion, refDbSystemError.getMessage()); logger.atSevere().withCause(refDbSystemError).log(message); throw new SharedProjectVersionUpdateException(message); } } + private void updateProjectVersionValue( + NameKey projectNameKey, Long newVersion, Optional<Long> sharedVersion) { + try { + if (sharedRefDb.isSetOperationSupported()) { + sharedRefDb.put(projectNameKey, MULTI_SITE_VERSIONING_VALUE_REF, newVersion.toString()); + return; + } + } catch (NoSuchMethodError e) { + logger.atSevere().log( + "Global-refdb library is outdated and is not supporting " + + "'put' method, update global-refdb to the newest version. Falling back to 'compareAndPut'"); + } + + sharedRefDb.compareAndPut( + projectNameKey, + MULTI_SITE_VERSIONING_VALUE_REF, + sharedVersion.map(Object::toString).orElse(null), + newVersion.toString()); + } + /* (non-Javadoc) * @see com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate#getProjectLocalVersion(java.lang.String) */ @@ -260,10 +238,6 @@ return globalVersion.flatMap(longString -> getLongValueOf(longString)); } - private Object safeGetObjectId(Ref currentRef) { - return currentRef == null ? "null" : currentRef.getObjectId(); - } - private Optional<Long> getLongValueOf(String longString) { try { return Optional.ofNullable(Long.parseLong(longString));
diff --git a/src/main/resources/Documentation/build.md b/src/main/resources/Documentation/build.md index ec44918..146341e 100644 --- a/src/main/resources/Documentation/build.md +++ b/src/main/resources/Documentation/build.md
@@ -3,14 +3,7 @@ This plugin can be built with Bazel in the Gerrit tree. Clone or link this plugin to the plugins directory of Gerrit's -source tree. Put the external dependency Bazel build file into -the Gerrit /plugins directory, replacing the existing empty one. - -``` - cd gerrit/plugins - rm external_plugin_deps.bzl - ln -s @PLUGIN@/external_plugin_deps.bzl . -``` +source tree. Clone the [pull-replication](https://gerrit.googlesource.com/plugins/pull-replication) on the same branch of the @PLUGIN@ plugin and link it to the `gerrit/plugins` directory. @@ -23,6 +16,15 @@ ln -s @PLUGIN@/external_plugin_deps.bzl . ``` +Clone the [global-refdb](git clone "https://gerrit.googlesource.com/modules/global-refdb") on +the same branch of the @PLUGIN@ plugin and link it to the `gerrit/plugins` directory. + +``` + export BRANCH=$(git --git-dir=@PLUGIN@ branch) + git clone "https://gerrit.googlesource.com/modules/global-refdb" + cd gerrit/plugins + ln -s ../../global-refdb . +``` From the Gerrit source tree issue the command:
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java index b7ad98c..fc2259f 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java +++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java
@@ -36,4 +36,7 @@ @Override public <T> void logRefUpdate(String project, String refName, T currRef, T newRefValue) {} + + @Override + public <T> void logRefUpdate(String project, String refName, T newRefValue) {} }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateTest.java index a997f9f..1cfb2cd 100644 --- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateTest.java +++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateTest.java
@@ -18,15 +18,20 @@ import static com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF; import static com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_VALUE_REF; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.atMost; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDatabaseWrapper; +import com.google.common.base.Suppliers; import com.google.gerrit.entities.Project; import com.google.gerrit.entities.RefNames; +import com.google.gerrit.server.data.RefUpdateAttribute; import com.google.gerrit.server.events.RefUpdatedEvent; import com.google.gerrit.server.extensions.events.GitReferenceUpdated; import com.google.gerrit.server.project.ProjectConfig; @@ -85,16 +90,9 @@ public void producerShouldUpdateProjectVersionUponRefUpdatedEvent() throws IOException { when(sharedRefDb.get( A_TEST_PROJECT_NAME_KEY, - ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF, - String.class)) - .thenReturn(Optional.of("26f7ee61bf0e470e8393c884526eec8a9b943a63")); - when(sharedRefDb.get( - A_TEST_PROJECT_NAME_KEY, ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_VALUE_REF, String.class)) .thenReturn(Optional.of("" + (masterCommit.getCommitTime() - 1))); - when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class))) - .thenReturn(true); when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(String.class), any(), any())) .thenReturn(true); when(refUpdatedEvent.getProjectNameKey()).thenReturn(A_TEST_PROJECT_NAME_KEY); @@ -119,25 +117,47 @@ } @Test + public void producerShouldUsePutInsteadOfCompareAndPutWhenExtendedGlobalRefDb() + throws IOException { + when(sharedRefDb.isSetOperationSupported()).thenReturn(true); + RefUpdatedEvent refUpdatedEvent = new RefUpdatedEvent(); + refUpdatedEvent.instanceId = DEFAULT_INSTANCE_ID; + RefUpdateAttribute refUpdatedAttribute = new RefUpdateAttribute(); + refUpdatedAttribute.project = A_TEST_PROJECT_NAME_KEY.get(); + refUpdatedAttribute.refName = A_TEST_REF_NAME; + refUpdatedEvent.refUpdate = Suppliers.memoize(() -> refUpdatedAttribute); + + new ProjectVersionRefUpdateImpl( + repoManager, sharedRefDb, gitReferenceUpdated, verLogger, DEFAULT_INSTANCE_ID) + .onEvent(refUpdatedEvent); + + Ref ref = repo.getRepository().findRef(MULTI_SITE_VERSIONING_REF); + + verify(sharedRefDb, never()) + .compareAndPut(any(Project.NameKey.class), anyString(), anyLong(), anyLong()); + + verify(sharedRefDb).put(any(Project.NameKey.class), any(String.class), any(String.class)); + assertThat(ref).isNotNull(); + + ObjectLoader loader = repo.getRepository().open(ref.getObjectId()); + long storedVersion = readLongObject(loader); + assertThat(storedVersion).isGreaterThan((long) masterCommit.getCommitTime()); + + verify(verLogger).log(A_TEST_PROJECT_NAME_KEY, storedVersion, 0); + } + + @Test public void producerShouldUpdateProjectVersionUponForcedPushRefUpdatedEvent() throws Exception { Thread.sleep(1000L); RevCommit masterPlusOneCommit = repo.branch("master").commit().create(); Thread.sleep(1000L); repo.branch("master").update(masterCommit); - - when(sharedRefDb.get( - A_TEST_PROJECT_NAME_KEY, - ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF, - String.class)) - .thenReturn(Optional.of("26f7ee61bf0e470e8393c884526eec8a9b943a63")); when(sharedRefDb.get( A_TEST_PROJECT_NAME_KEY, ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_VALUE_REF, String.class)) .thenReturn(Optional.of("" + (masterCommit.getCommitTime() - 1))); - when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class))) - .thenReturn(true); when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(String.class), any(), any())) .thenReturn(true); when(refUpdatedEvent.getProjectNameKey()).thenReturn(A_TEST_PROJECT_NAME_KEY); @@ -171,17 +191,10 @@ throws IOException { when(sharedRefDb.get( A_TEST_PROJECT_NAME_KEY, - ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF, - String.class)) - .thenReturn(Optional.empty()); - when(sharedRefDb.get( - A_TEST_PROJECT_NAME_KEY, ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_VALUE_REF, String.class)) .thenReturn(Optional.empty()); - when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(Ref.class), any(ObjectId.class))) - .thenReturn(true); when(sharedRefDb.compareAndPut(any(Project.NameKey.class), any(String.class), any(), any())) .thenReturn(true); when(refUpdatedEvent.getProjectNameKey()).thenReturn(A_TEST_PROJECT_NAME_KEY);