Merge branch 'stable-3.1'

* stable-3.1:
  Update subscriber metrics at every replication stage
  Enable HTTP log in test environment
  Add and configure Prometheus with the multi-site environment
  Fix typo in setup.sh messages
  Dry out Gerrit CI URL in setup.sh script
  Tidy up replication lag metrics publishing logs
  Fix replication lag unit: seconds
  SubscriberMetrics: update only when replication is complete
  Store version value in shared db

Change-Id: I2035133401508ccfdf72e2e88ca4c474327eccb6
diff --git a/setup_local_env/configs/gerrit.config b/setup_local_env/configs/gerrit.config
index f05184e..6f49cd7 100644
--- a/setup_local_env/configs/gerrit.config
+++ b/setup_local_env/configs/gerrit.config
@@ -32,6 +32,7 @@
     advertisedAddress = *:$SSH_ADVERTISED_PORT
 [httpd]
     listenUrl = proxy-$HTTP_PROTOCOL://*:$GERRIT_HTTPD_PORT/
+    requestLog = true
 [cache]
     directory = cache
 [plugins]
@@ -47,3 +48,5 @@
     enableAutoCommit = true
     autoCommitIntervalMs = 1000
     autoOffsetReset = latest
+[plugin "metrics-reporter-prometheus"]
+    prometheusBearerToken = token
diff --git a/setup_local_env/setup.sh b/setup_local_env/setup.sh
index 5d4ca83..affa535 100755
--- a/setup_local_env/setup.sh
+++ b/setup_local_env/setup.sh
@@ -16,9 +16,14 @@
 
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+GERRIT_BRANCH=stable-3.1
+GERRIT_CI=https://gerrit-ci.gerritforge.com/view/Plugins-$GERRIT_BRANCH/job
+LAST_BUILD=lastSuccessfulBuild/artifact/bazel-bin/plugins
+EVENTS_BROKER_VER=`grep 'com.gerritforge:events-broker' $(dirname $0)/../external_plugin_deps.bzl | cut -d '"' -f 2 | cut -d ':' -f 3`
 
 function check_application_requirements {
   type haproxy >/dev/null 2>&1 || { echo >&2 "Require haproxy but it's not installed. Aborting."; exit 1; }
+  type prometheus >/dev/null 2>&1 || { echo >&2 "Require prometheus but it's not installed. Aborting."; exit 1; }
   type java >/dev/null 2>&1 || { echo >&2 "Require java but it's not installed. Aborting."; exit 1; }
   type docker >/dev/null 2>&1 || { echo >&2 "Require docker but it's not installed. Aborting."; exit 1; }
   type docker-compose >/dev/null 2>&1 || { echo >&2 "Require docker-compose but it's not installed. Aborting."; exit 1; }
@@ -70,6 +75,7 @@
     cat $file | envsubst | sed 's/#{name}#/${name}/g' > $CONFIG_TEST_SITE/$file_name
   done
 }
+
 function start_ha_proxy {
 
   export HA_GERRIT_CANONICAL_HOSTNAME=$GERRIT_CANONICAL_HOSTNAME
@@ -93,6 +99,16 @@
   haproxy -f $HA_PROXY_CONFIG_DIR/haproxy.cfg &
 }
 
+function start_prometheus {
+  mkdir -p $PROMETHEUS_CONFIG_DIR
+  cat $SCRIPT_DIR/prometheus-config/prometheus.yml | envsubst > $PROMETHEUS_CONFIG_DIR/prometheus.yml
+
+  echo "Starting Prometheus..."
+  echo "THE SCRIPT LOCATION $SCRIPT_DIR"
+  echo "THE HA SCRIPT_LOCATION $PROMETHEUS_SCRIPT_DIR"
+  prometheus --config.file $PROMETHEUS_CONFIG_DIR/prometheus.yml &
+}
+
 function deploy_config_files {
   # KAFKA configuration
   export KAFKA_PORT=9092
@@ -127,6 +143,8 @@
 function cleanup_environment {
   echo "Killing existing HA-PROXY setup"
   kill $(ps -ax | grep haproxy | grep "gerrit_setup/ha-proxy-config" | awk '{print $1}') 2> /dev/null
+  echo "Killing existing Prometheus setup"
+  kill $(ps -ax | grep prometheus | grep "gerrit_setup/prometheus-config" | awk '{print $1}') 2> /dev/null
   echo "Stopping kafka and zk"
   docker-compose -f $SCRIPT_DIR/docker-compose.kafka-broker.yaml down 2> /dev/null
 
@@ -295,6 +313,7 @@
 LOCATION_TEST_SITE_2=$COMMON_LOCATION/instance-2
 HA_PROXY_CONFIG_DIR=$COMMON_LOCATION/ha-proxy-config
 HA_PROXY_CERTIFICATES_DIR="$HA_PROXY_CONFIG_DIR/certificates"
+PROMETHEUS_CONFIG_DIR=$COMMON_LOCATION/prometheus-config
 
 RELEASE_WAR_FILE_LOCATION=${RELEASE_WAR_FILE_LOCATION:-bazel-bin/release.war}
 MULTISITE_LIB_LOCATION=${MULTISITE_LIB_LOCATION:-bazel-bin/plugins/multi-site/multi-site.jar}
@@ -320,32 +339,37 @@
   cp -f $MULTISITE_LIB_LOCATION $DEPLOYMENT_LOCATION/multi-site.jar  >/dev/null 2>&1 || { echo >&2 "$MULTISITE_LIB_LOCATION: Not able to copy the file. Aborting"; exit 1; }
 fi
 if [ $DOWNLOAD_WEBSESSION_PLUGIN = "true" ];then
-  echo "Downloading websession-broker plugin"
-  wget https://gerrit-ci.gerritforge.com/view/Plugins-master/job/plugin-websession-broker-bazel-master/lastSuccessfulBuild/artifact/bazel-bin/plugins/websession-broker/websession-broker.jar \
+  echo "Downloading websession-broker plugin $GERRIT_BRANCH"
+  wget $GERRIT_CI/plugin-websession-broker-bazel-$GERRIT_BRANCH/$LAST_BUILD/websession-broker/websession-broker.jar \
   -O $DEPLOYMENT_LOCATION/websession-broker.jar || { echo >&2 "Cannot download websession-broker plugin: Check internet connection. Abort\
 ing"; exit 1; }
-  wget https://gerrit-ci.gerritforge.com/view/Plugins-master/job/plugin-healthcheck-bazel-master/lastSuccessfulBuild/artifact/bazel-bin/plugins/healthcheck/healthcheck.jar \
+  wget $GERRIT_CI/plugin-healthcheck-bazel-$GERRIT_BRANCH/$LAST_BUILD/healthcheck/healthcheck.jar \
   -O $DEPLOYMENT_LOCATION/healthcheck.jar || { echo >&2 "Cannot download healthcheck plugin: Check internet connection. Abort\
 ing"; exit 1; }
 else
   echo "Without the websession-broker; user login via haproxy will fail."
 fi
 
-echo "Downloading zookeeper plugin master"
-  wget https://gerrit-ci.gerritforge.com/view/Plugins-master/job/plugin-zookeeper-gh-bazel-master/lastSuccessfulBuild/artifact/bazel-bin/plugins/zookeeper/zookeeper.jar \
+echo "Downloading zookeeper plugin $GERRIT_BRANCH"
+  wget $GERRIT_CI/plugin-zookeeper-gh-bazel-$GERRIT_BRANCH/$LAST_BUILD/zookeeper/zookeeper.jar \
   -O $DEPLOYMENT_LOCATION/zookeeper.jar || { echo >&2 "Cannot download zookeeper plugin: Check internet connection. Abort\
 ing"; exit 1; }
 
-echo "Downloading events-broker library"
-  wget https://repo1.maven.org/maven2/com/gerritforge/events-broker/3.1.4/events-broker-3.1.4.jar \
+echo "Downloading events-broker library $GERRIT_BRANCH"
+  wget https://repo1.maven.org/maven2/com/gerritforge/events-broker/$EVENTS_BROKER_VER/events-broker-$EVENTS_BROKER_VER.jar \
   -O $DEPLOYMENT_LOCATION/events-broker.jar || { echo >&2 "Cannot download events-broker library: Check internet connection. Abort\
 ing"; exit 1; }
 
-echo "Downloading kafka-events plugin master"
-  wget https://gerrit-ci.gerritforge.com/view/Plugins-master/job/plugin-kafka-events-bazel-master/lastSuccessfulBuild/artifact/bazel-bin/plugins/kafka-events/kafka-events.jar \
+echo "Downloading kafka-events plugin $GERRIT_BRANCH"
+  wget $GERRIT_CI/plugin-kafka-events-bazel-$GERRIT_BRANCH/$LAST_BUILD/kafka-events/kafka-events.jar \
   -O $DEPLOYMENT_LOCATION/kafka-events.jar || { echo >&2 "Cannot download kafka-events plugin: Check internet connection. Abort\
 ing"; exit 1; }
 
+echo "Downloading metrics-reporter-prometheus plugin $GERRIT_BRANCH"
+  wget $GERRIT_CI/plugin-metrics-reporter-prometheus-bazel-master-$GERRIT_BRANCH/$LAST_BUILD/metrics-reporter-prometheus/metrics-reporter-prometheus.jar \
+  -O $DEPLOYMENT_LOCATION/metrics-reporter-prometheus.jar || { echo >&2 "Cannot download metrics-reporter-prometheus plugin: Check internet connection. Abort\
+ing"; exit 1; }
+
 if [ "$REPLICATION_TYPE" = "ssh" ];then
   echo "Using 'SSH' replication type"
   echo "Make sure ~/.ssh/authorized_keys and ~/.ssh/known_hosts are configured correctly"
@@ -391,6 +415,9 @@
   echo "Copy kafka events plugin"
   cp -f $DEPLOYMENT_LOCATION/kafka-events.jar $LOCATION_TEST_SITE_1/plugins/kafka-events.jar
 
+  echo "Copy metrics-reporter-prometheus plugin"
+  cp -f $DEPLOYMENT_LOCATION/metrics-reporter-prometheus.jar $LOCATION_TEST_SITE_1/plugins/metrics-reporter-prometheus.jar
+
   echo "Re-indexing"
   java -jar $DEPLOYMENT_LOCATION/gerrit.war reindex -d $LOCATION_TEST_SITE_1
   # Replicating environment
@@ -429,6 +456,11 @@
   start_ha_proxy
 fi
 
+if [[ $(ps -ax | grep promtheus | grep "gerrit_setup/prometheus-config" | awk '{print $1}' | wc -l) -lt 1 ]];then
+  echo "Starting prometheus"
+  start_prometheus
+fi
+
 echo "==============================="
 echo "Current gerrit multi-site setup"
 echo "==============================="
@@ -442,6 +474,7 @@
 echo "GERRIT HA-PROXY: $GERRIT_CANONICAL_WEB_URL"
 echo "GERRIT-1: http://$GERRIT_1_HOSTNAME:$GERRIT_1_HTTPD_PORT"
 echo "GERRIT-2: http://$GERRIT_2_HOSTNAME:$GERRIT_2_HTTPD_PORT"
+echo "Prometheus: http://localhost:9090"
 echo
 echo "Site-1: $LOCATION_TEST_SITE_1"
 echo "Site-2: $LOCATION_TEST_SITE_2"
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jSharedRefLogger.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jSharedRefLogger.java
index 6a73d19..003ce5b 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jSharedRefLogger.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/Log4jSharedRefLogger.java
@@ -100,6 +100,19 @@
   }
 
   @Override
+  public <T> void logRefUpdate(String project, String refName, T currRef, T newRefValue) {
+    if (newRefValue != null) {
+      sharedRefDBLog.info(
+          gson.toJson(
+              new SharedRefLogEntry.UpdateRef(
+                  project, refName, safeToString(currRef), safeToString(newRefValue), null, null)));
+    } else {
+      sharedRefDBLog.info(
+          gson.toJson(new SharedRefLogEntry.DeleteRef(project, refName, safeToString(currRef))));
+    }
+  }
+
+  @Override
   public void logProjectDelete(String project) {
     sharedRefDBLog.info(gson.toJson(new SharedRefLogEntry.DeleteProject(project)));
   }
@@ -118,4 +131,11 @@
   public void setLogger(Logger logger) {
     this.sharedRefDBLog = logger;
   }
+
+  private <T> String safeToString(T currRef) {
+    if (currRef == null) {
+      return "<null>";
+    }
+    return currRef.toString();
+  }
 }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefDatabaseWrapper.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefDatabaseWrapper.java
index 9827002..b56f2ea 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefDatabaseWrapper.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefDatabaseWrapper.java
@@ -62,7 +62,11 @@
   @Override
   public <T> boolean compareAndPut(Project.NameKey project, String refName, T currValue, T newValue)
       throws GlobalRefDbSystemError {
-    throw new UnsupportedOperationException();
+    boolean succeeded = sharedRefDb().compareAndPut(project, refName, currValue, newValue);
+    if (succeeded) {
+      sharedRefLogger.logRefUpdate(project.get(), refName, currValue, newValue);
+    }
+    return succeeded;
   }
 
   @Override
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefLogger.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefLogger.java
index 51f9ff0..3853af6 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefLogger.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/SharedRefLogger.java
@@ -21,6 +21,8 @@
 
   void logRefUpdate(String project, Ref currRef, ObjectId newRefValue);
 
+  <T> void logRefUpdate(String project, String refName, T currRef, T newRefValue);
+
   void logProjectDelete(String project);
 
   void logLockAcquisition(String project, String refName);
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetrics.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetrics.java
index d325fd8..b4e5c94 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetrics.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/SubscriberMetrics.java
@@ -16,15 +16,19 @@
 
 import com.gerritforge.gerrit.eventbroker.EventMessage;
 import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.entities.Project;
 import com.google.gerrit.metrics.Counter1;
 import com.google.gerrit.metrics.Description;
 import com.google.gerrit.metrics.MetricMaker;
 import com.google.gerrit.server.events.Event;
+import com.google.gerrit.server.events.RefUpdatedEvent;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import com.googlesource.gerrit.plugins.multisite.MultiSiteMetrics;
 import com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate;
 import com.googlesource.gerrit.plugins.replication.RefReplicatedEvent;
+import com.googlesource.gerrit.plugins.replication.RefReplicationDoneEvent;
+import com.googlesource.gerrit.plugins.replication.ReplicationScheduledEvent;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -37,8 +41,8 @@
   private static final String SUBSCRIBER_SUCCESS_COUNTER = "subscriber_msg_consumer_counter";
   private static final String SUBSCRIBER_FAILURE_COUNTER =
       "subscriber_msg_consumer_failure_counter";
-  private static final String REPLICATION_LAG_MS =
-      "multi_site/subscriber/subscriber_replication_status/ms_behind";
+  private static final String REPLICATION_LAG_SEC =
+      "multi_site/subscriber/subscriber_replication_status/sec_behind";
 
   private final Counter1<String> subscriberSuccessCounter;
   private final Counter1<String> subscriberFailureCounter;
@@ -67,9 +71,9 @@
                 .setUnit("errors"),
             stringField(SUBSCRIBER_FAILURE_COUNTER, "Subscriber failed to consume messages count"));
     metricMaker.newCallbackMetric(
-        REPLICATION_LAG_MS,
+        REPLICATION_LAG_SEC,
         Long.class,
-        new Description("Replication lag (ms)").setGauge().setUnit(Description.Units.MILLISECONDS),
+        new Description("Replication lag (sec)").setGauge().setUnit(Description.Units.SECONDS),
         () -> {
           Collection<Long> lags = replicationStatusPerProject.values();
           if (lags.isEmpty()) {
@@ -89,23 +93,36 @@
 
   public void updateReplicationStatusMetrics(EventMessage eventMessage) {
     Event event = eventMessage.getEvent();
-    if (event instanceof RefReplicatedEvent) {
-      RefReplicatedEvent refReplicatedEvent = (RefReplicatedEvent) event;
-      String projectName = refReplicatedEvent.getProjectNameKey().get();
-      logger.atFine().log("Updating replication lag for %s", projectName);
-      Optional<Long> remoteVersion = projectVersionRefUpdate.getProjectRemoteVersion(projectName);
-      Optional<Long> localVersion = projectVersionRefUpdate.getProjectLocalVersion(projectName);
-      if (remoteVersion.isPresent() && localVersion.isPresent()) {
-        long lag = remoteVersion.get() - localVersion.get();
-        logger.atFine().log("Calculated lag for project '%s' [%d]", projectName, lag);
-        replicationStatusPerProject.put(projectName, lag);
-      } else {
-        logger.atFine().log(
-            "Didn't update metric for %s. Local [%b] or remote [%b] version is not defined",
-            projectName, localVersion.isPresent(), remoteVersion.isPresent());
-      }
+    if (event instanceof RefReplicationDoneEvent) {
+      RefReplicationDoneEvent replicationDone = (RefReplicationDoneEvent) event;
+      updateReplicationLagMetrics(
+          replicationDone.getProjectNameKey(), replicationDone.getRefName());
+    } else if (event instanceof RefReplicatedEvent) {
+      RefReplicatedEvent replicated = (RefReplicatedEvent) event;
+      updateReplicationLagMetrics(replicated.getProjectNameKey(), replicated.getRefName());
+    } else if (event instanceof ReplicationScheduledEvent) {
+      ReplicationScheduledEvent updated = (ReplicationScheduledEvent) event;
+      updateReplicationLagMetrics(updated.getProjectNameKey(), updated.getRefName());
+    } else if (event instanceof RefUpdatedEvent) {
+      RefUpdatedEvent updated = (RefUpdatedEvent) event;
+      updateReplicationLagMetrics(updated.getProjectNameKey(), updated.getRefName());
+    }
+  }
+
+  private void updateReplicationLagMetrics(Project.NameKey projectName, String ref) {
+    Optional<Long> remoteVersion =
+        projectVersionRefUpdate.getProjectRemoteVersion(projectName.get());
+    Optional<Long> localVersion = projectVersionRefUpdate.getProjectLocalVersion(projectName.get());
+    if (remoteVersion.isPresent() && localVersion.isPresent()) {
+      long lag = remoteVersion.get() - localVersion.get();
+      logger.atFine().log(
+          "Published replication lag metric for project '%s' of %d sec(s) [local-ref=%d global-ref=%d]",
+          projectName, lag, localVersion.get(), remoteVersion.get());
+      replicationStatusPerProject.put(projectName.get(), lag);
     } else {
-      logger.atFine().log("Not a ref-replicated-event event [%s], skipping", event.type);
+      logger.atFine().log(
+          "Did not publish replication lag metric for %s because the %s version is not defined",
+          projectName, localVersion.isPresent() ? "remote" : "local");
     }
   }
 }
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdate.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdate.java
index bcba594..45d0806 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdate.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdate.java
@@ -55,6 +55,7 @@
       ImmutableSet.of(RefUpdate.Result.NEW, RefUpdate.Result.FORCED, RefUpdate.Result.NO_CHANGE);
 
   public static final String MULTI_SITE_VERSIONING_REF = "refs/multi-site/version";
+  public static final String MULTI_SITE_VERSIONING_VALUE_REF = "refs/multi-site/version/value";
   public static final Ref NULL_PROJECT_VERSION_REF =
       new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, MULTI_SITE_VERSIONING_REF, ObjectId.zeroId());
 
@@ -93,7 +94,8 @@
       return;
     }
     try {
-      updateLocalProjectVersion(projectNameKey, refReplicationDoneEvent.getRefName());
+      updateLocalProjectVersion(
+          projectNameKey, getLastRefUpdatedTimestamp(projectNameKey, refName));
     } catch (LocalProjectVersionUpdateException e) {
       logger.atSevere().withCause(e).log(
           "Issue encountered when updating version for project " + projectNameKey);
@@ -117,12 +119,20 @@
     try {
       Project.NameKey projectNameKey = refUpdatedEvent.getProjectNameKey();
       Ref currentProjectVersionRef = getLocalProjectVersionRef(refUpdatedEvent.getProjectNameKey());
+      Optional<Long> currentProjectVersionValue =
+          getLongFromObjectId(projectNameKey.get(), currentProjectVersionRef.getObjectId());
+
+      Optional<Long> lastRefUpdatedTimestamp = getLastRefUpdatedTimestamp(projectNameKey, refName);
       Optional<ObjectId> newProjectVersionObjectId =
-          updateLocalProjectVersion(projectNameKey, refUpdatedEvent.getRefName());
+          updateLocalProjectVersion(projectNameKey, lastRefUpdatedTimestamp);
 
       if (newProjectVersionObjectId.isPresent()) {
         updateSharedProjectVersion(
-            projectNameKey, currentProjectVersionRef, newProjectVersionObjectId.get());
+            projectNameKey,
+            currentProjectVersionRef,
+            newProjectVersionObjectId.get(),
+            currentProjectVersionValue,
+            lastRefUpdatedTimestamp);
       } else {
         logger.atWarning().log(
             "Ref %s not found on projet %s: skipping project version update",
@@ -157,7 +167,7 @@
       return ref != null ? ref : NULL_PROJECT_VERSION_REF;
     } catch (IOException e) {
       String message =
-          String.format("Error while getting current version for %s", projectNameKey.get());
+          String.format("Error while getting current version ref for %s", projectNameKey.get());
       logger.atSevere().withCause(e).log(message);
       throw new LocalProjectVersionUpdateException(message);
     }
@@ -189,19 +199,38 @@
   }
 
   private void updateSharedProjectVersion(
-      Project.NameKey projectNameKey, Ref currentRef, ObjectId newObjectId)
+      Project.NameKey projectNameKey,
+      Ref currentRef,
+      ObjectId newObjectId,
+      Optional<Long> currentVersion,
+      Optional<Long> newVersion)
       throws SharedProjectVersionUpdateException {
+
     logger.atFine().log(
         String.format(
             "Updating shared project version for %s. Current value %s, new value: %s",
             projectNameKey.get(), currentRef.getObjectId(), newObjectId));
     try {
       boolean success = sharedRefDb.compareAndPut(projectNameKey, currentRef, newObjectId);
-      String message =
-          String.format(
-              "Project version update failed for %s. Current value %s, new value: %s",
-              projectNameKey.get(), currentRef.getObjectId(), newObjectId);
       if (!success) {
+        String message =
+            String.format(
+                "Project version blob update failed for %s. Current value %s, new value: %s",
+                projectNameKey.get(), currentRef.getObjectId(), newObjectId);
+        logger.atSevere().log(message);
+        throw new SharedProjectVersionUpdateException(message);
+      }
+      success =
+          sharedRefDb.compareAndPut(
+              projectNameKey,
+              MULTI_SITE_VERSIONING_VALUE_REF,
+              currentVersion.map(Object::toString).orElse(null),
+              newVersion.map(Object::toString).orElse(null));
+      if (!success) {
+        String message =
+            String.format(
+                "Project version update failed for %s. Current value %s, new value: %s",
+                projectNameKey.get(), currentRef.getObjectId(), newObjectId);
         logger.atSevere().log(message);
         throw new SharedProjectVersionUpdateException(message);
       }
@@ -234,17 +263,26 @@
   }
 
   public Optional<Long> getProjectRemoteVersion(String projectName) {
-    Optional<ObjectId> remoteObjectId =
+    Optional<String> globalVersion =
         sharedRefDb.get(
-            Project.NameKey.parse(projectName), MULTI_SITE_VERSIONING_REF, ObjectId.class);
-    if (remoteObjectId.isPresent()) {
-      return getLongFromObjectId(projectName, remoteObjectId.get());
+            Project.NameKey.parse(projectName), MULTI_SITE_VERSIONING_VALUE_REF, String.class);
+    return globalVersion.flatMap(longString -> getLongValueOf(longString));
+  }
+
+  private Optional<Long> getLongValueOf(String longString) {
+    try {
+      return Optional.ofNullable(Long.parseLong(longString));
+    } catch (NumberFormatException e) {
+      logger.atSevere().withCause(e).log(
+          "Unable to parse timestamp value %s into Long", longString);
+      return Optional.empty();
     }
-    logger.atFine().log("Didn't find remote version for %s", projectName);
-    return Optional.empty();
   }
 
   private Optional<Long> getLongFromObjectId(String projectName, ObjectId objectId) {
+    if (objectId.equals(ObjectId.zeroId())) {
+      return Optional.empty();
+    }
     try (Repository repository =
         gitRepositoryManager.openRepository(Project.NameKey.parse(projectName))) {
       ObjectReader or = repository.newObjectReader();
@@ -268,8 +306,8 @@
   }
 
   private Optional<ObjectId> updateLocalProjectVersion(
-      Project.NameKey projectNameKey, String refName) throws LocalProjectVersionUpdateException {
-    Optional<Long> lastRefUpdatedTimestamp = getLastRefUpdatedTimestamp(projectNameKey, refName);
+      Project.NameKey projectNameKey, Optional<Long> lastRefUpdatedTimestamp)
+      throws LocalProjectVersionUpdateException {
     if (!lastRefUpdatedTimestamp.isPresent()) {
       return Optional.empty();
     }
diff --git a/src/main/resources/Documentation/about.md b/src/main/resources/Documentation/about.md
index 5ac3dec..b6de0a9 100644
--- a/src/main/resources/Documentation/about.md
+++ b/src/main/resources/Documentation/about.md
@@ -93,6 +93,6 @@
 
 `metric=plugins/multi-site/multi_site/subscriber/subscriber_message_consumer_failure_counter/subscriber_msg_consumer_poll_failure_counter, type=com.codahale.metrics.Meter`
 
-* Subscriber replication status (ms behind the producer)
+* Subscriber replication lag (sec behind the producer)
 
-`metric=site/multi_site/subscriber/subscriber_replication_status/ms_behind, type=com.google.gerrit.metrics.dropwizard.CallbackMetricImpl`
\ No newline at end of file
+`metric=site/multi_site/subscriber/subscriber_replication_status/sec_behind, type=com.google.gerrit.metrics.dropwizard.CallbackMetricImpl`
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java
index 37d5008..047a1c5 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/DisabledSharedRefLogger.java
@@ -33,4 +33,7 @@
 
   @Override
   public void logLockRelease(String project, String refName) {}
+
+  @Override
+  public <T> void logRefUpdate(String project, String refName, T currRef, T newRefValue) {}
 }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateTest.java
index 5ea7b15..9a4020d 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/ProjectVersionRefUpdateTest.java
@@ -16,6 +16,7 @@
 
 import static com.google.common.truth.Truth.assertThat;
 import static com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_REF;
+import static com.googlesource.gerrit.plugins.multisite.validation.ProjectVersionRefUpdate.MULTI_SITE_VERSIONING_VALUE_REF;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.atMost;
 import static org.mockito.Mockito.never;
@@ -187,18 +188,16 @@
   }
 
   @Test
-  public void getRemoteProjectVersionShouldReturnCorrectValue() throws IOException {
-    updateLocalVersion();
-    Ref ref = repo.getRepository().findRef(MULTI_SITE_VERSIONING_REF);
-    when(sharedRefDb.get(A_TEST_PROJECT_NAME_KEY, MULTI_SITE_VERSIONING_REF, ObjectId.class))
-        .thenReturn(Optional.of(ref.getObjectId()));
+  public void getRemoteProjectVersionShouldReturnCorrectValue() {
+    when(sharedRefDb.get(A_TEST_PROJECT_NAME_KEY, MULTI_SITE_VERSIONING_VALUE_REF, String.class))
+        .thenReturn(Optional.of("123"));
 
     Optional<Long> version =
         new ProjectVersionRefUpdate(repoManager, sharedRefDb)
             .getProjectRemoteVersion(A_TEST_PROJECT_NAME);
 
     assertThat(version.isPresent()).isTrue();
-    assertThat(version.get()).isEqualTo(masterCommit.getCommitTime());
+    assertThat(version.get()).isEqualTo(123L);
   }
 
   @Test