Record delay, latency and retry metrics at project level
Replication latency was recorded only for remote destinations
but it was not possible to have visibility at project level.
Latency metric will now be recorded also at a project level, when
the latency was greater than a configured threshold, configurable in
replication.config.
Feature: Issue 11196
Change-Id: I4d42705b311d693d42cfab6b6476c5234a0e2069
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
index cbe2bc6..996cb2a 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Destination.java
@@ -731,6 +731,10 @@
return config.getDelay() * 1000;
}
+ int getSlowLatencyThreshold() {
+ return config.getSlowLatencyThreshold();
+ }
+
private static boolean matches(URIish uri, String urlMatch) {
if (urlMatch == null || urlMatch.equals("") || urlMatch.equals("*")) {
return true;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java
index f688cfc..1c20892 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/DestinationConfiguration.java
@@ -16,13 +16,17 @@
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableList;
+import com.google.gerrit.server.config.ConfigUtil;
import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.transport.RemoteConfig;
+import java.util.concurrent.TimeUnit;
+
public class DestinationConfiguration {
static final int DEFAULT_REPLICATION_DELAY = 15;
static final int DEFAULT_RESCHEDULE_DELAY = 3;
static final int DEFAULT_DRAIN_QUEUE_ATTEMPTS = 0;
+ private static final int DEFAULT_SLOW_LATENCY_THRESHOLD_SECS = 900;
private final int delay;
private final int rescheduleDelay;
@@ -41,6 +45,7 @@
private final ImmutableList<String> authGroupNames;
private final RemoteConfig remoteConfig;
private final int maxRetries;
+ private final int slowLatencyThreshold;
protected DestinationConfiguration(RemoteConfig remoteConfig, Config cfg) {
this.remoteConfig = remoteConfig;
@@ -67,6 +72,9 @@
maxRetries =
getInt(
remoteConfig, cfg, "replicationMaxRetries", cfg.getInt("replication", "maxRetries", 0));
+
+ slowLatencyThreshold = (int) ConfigUtil.getTimeUnit(
+ cfg, "remote", remoteConfig.getName(), "slowLatencyThreshold", DEFAULT_SLOW_LATENCY_THRESHOLD_SECS, TimeUnit.SECONDS);
}
public int getDelay() {
@@ -140,4 +148,8 @@
private static int getInt(RemoteConfig rc, Config cfg, String name, int defValue) {
return cfg.getInt("remote", rc.getName(), name, defValue);
}
+
+ public int getSlowLatencyThreshold() {
+ return slowLatencyThreshold;
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
index d4e1148..5324d57 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/PushOne.java
@@ -16,6 +16,7 @@
import static com.googlesource.gerrit.plugins.replication.ReplicationQueue.repLog;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toMap;
import com.google.common.base.Throwables;
@@ -329,14 +330,19 @@
}
repLog.info("Replication to {} started...", uri);
- Timer1.Context<String> context = metrics.start(config.getName());
+ Timer1.Context destinationContext = metrics.start(config.getName());
try {
- long startedAt = context.getStartTime();
+ long startedAt = destinationContext.getStartTime();
long delay = NANOSECONDS.toMillis(startedAt - createdAt);
metrics.record(config.getName(), delay, retryCount);
git = gitManager.openRepository(projectName);
runImpl();
- long elapsed = NANOSECONDS.toMillis(context.stop());
+ long elapsed = NANOSECONDS.toMillis(destinationContext.stop());
+
+ if (elapsed > SECONDS.toMillis(pool.getSlowLatencyThreshold())) {
+ metrics.recordSlowProjectReplication(
+ config.getName(), projectName.get(), pool.getSlowLatencyThreshold(), elapsed);
+ }
repLog.info(
"Replication to {} completed in {}ms, {}ms delay, {} retries",
uri,
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationMetrics.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationMetrics.java
index be215b2..1bc17ec 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationMetrics.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationMetrics.java
@@ -18,6 +18,7 @@
import com.google.gerrit.metrics.Description;
import com.google.gerrit.metrics.Field;
import com.google.gerrit.metrics.Histogram1;
+import com.google.gerrit.metrics.Histogram3;
import com.google.gerrit.metrics.MetricMaker;
import com.google.gerrit.metrics.Timer1;
import com.google.gerrit.server.logging.PluginMetadata;
@@ -29,6 +30,7 @@
private final Timer1<String> executionTime;
private final Histogram1<String> executionDelay;
private final Histogram1<String> executionRetries;
+ private final Histogram3<Integer, String, String> slowProjectReplicationLatency;
@Inject
ReplicationMetrics(@PluginName String pluginName, MetricMaker metricMaker) {
@@ -41,6 +43,25 @@
.addPluginMetadata(PluginMetadata.create("destination", fieldValue)))
.build();
+ Field<String> PROJECT_FIELD =
+ Field.ofString(
+ "project",
+ (metadataBuilder, fieldValue) ->
+ metadataBuilder
+ .pluginName(pluginName)
+ .addPluginMetadata(PluginMetadata.create("project", fieldValue)))
+ .build();
+
+ Field<Integer> SLOW_THRESHOLD_FIELD =
+ Field.ofInteger(
+ "slow_threshold",
+ (metadataBuilder, fieldValue) ->
+ metadataBuilder
+ .pluginName(pluginName)
+ .addPluginMetadata(
+ PluginMetadata.create("slow_threshold", fieldValue.toString())))
+ .build();
+
executionTime =
metricMaker.newTimer(
"replication_latency",
@@ -64,6 +85,17 @@
.setCumulative()
.setUnit("retries"),
DEST_FIELD);
+
+ slowProjectReplicationLatency =
+ metricMaker.newHistogram(
+ "latency_slower_than_threshold" + "",
+ new Description(
+ "latency for project to destination, where latency was slower than threshold")
+ .setCumulative()
+ .setUnit(Description.Units.MILLISECONDS),
+ SLOW_THRESHOLD_FIELD,
+ PROJECT_FIELD,
+ DEST_FIELD);
}
/**
@@ -87,4 +119,17 @@
executionDelay.record(name, delay);
executionRetries.record(name, retries);
}
+
+ /**
+ * Record replication latency for project to destination, where latency was slower than threshold
+ *
+ * @param destinationName the destination name.
+ * @param projectName the project name.
+ * @param slowThreshold replication initialDelay in milliseconds.
+ * @param latency number of retries.
+ */
+ void recordSlowProjectReplication(
+ String destinationName, String projectName, Integer slowThreshold, long latency) {
+ slowProjectReplicationLatency.record(slowThreshold, destinationName, projectName, latency);
+ }
}
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index 32bc630..d9d4bae 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -419,6 +419,15 @@
By default, replicates without matching, i.e. replicates
everything to all remotes.
+remote.NAME.slowLatencyThreshold
+: the time duration after which the replication of a project to this
+ destination will be considered "slow". A slow project replication
+ will cause additional metrics to be exposed for further investigation.
+ See [metrics.md](metrics.md) for further details.
+
+ default: 15 minutes
+
+
File `secure.config`
--------------------
diff --git a/src/main/resources/Documentation/metrics.md b/src/main/resources/Documentation/metrics.md
new file mode 100644
index 0000000..ef8b150
--- /dev/null
+++ b/src/main/resources/Documentation/metrics.md
@@ -0,0 +1,61 @@
+# Metrics
+
+Some metrics are emitted when replication occurs to a remote destination.
+The granularity of the metrics recorded is at destination level, however when a particular project replication is flagged
+as slow. This happens when the replication took longer than allowed threshold (see _remote.NAME.slowLatencyThreshold_ in [config.md](config.md))
+
+The reason only slow metrics are published, rather than all, is to contain their number, which, on a big Gerrit installation
+could potentially be considerably big.
+
+### Project level
+
+* plugins_replication_latency_slower_than_<threshold>_<destinationName>_<ProjectName> - Time spent pushing <ProjectName> to remote <destinationName> (in ms)
+
+### Destination level
+
+* plugins_replication_replication_delay_<destinationName> - Time spent waiting before pushing to remote <destinationName> (in ms)
+* plugins_replication_replication_retries_<destinationName> - Number of retries when pushing to remote <destinationName>
+* plugins_replication_replication_latency_<destinationName> - Time spent pushing to remote <destinationName> (in ms)
+
+### Example
+```
+# HELP plugins_replication_replication_delay_destination Generated from Dropwizard metric import (metric=plugins/replication/replication_delay/destination, type=com.codahale.metrics.Histogram)
+# TYPE plugins_replication_replication_delay_destination summary
+plugins_replication_replication_delay_destinationName{quantile="0.5",} 65726.0
+plugins_replication_replication_delay_destinationName{quantile="0.75",} 65726.0
+plugins_replication_replication_delay_destinationName{quantile="0.95",} 65726.0
+plugins_replication_replication_delay_destinationName{quantile="0.98",} 65726.0
+plugins_replication_replication_delay_destinationName{quantile="0.99",} 65726.0
+plugins_replication_replication_delay_destinationName{quantile="0.999",} 65726.0
+plugins_replication_replication_delay_destinationName_count 3.0
+
+# HELP plugins_replication_replication_retries_destination Generated from Dropwizard metric import (metric=plugins/replication/replication_retries/destination, type=com.codahale.metrics.Histogram)
+# TYPE plugins_replication_replication_retries_destination summary
+plugins_replication_replication_retries_destinationName{quantile="0.5",} 1.0
+plugins_replication_replication_retries_destinationName{quantile="0.75",} 1.0
+plugins_replication_replication_retries_destinationName{quantile="0.95",} 1.0
+plugins_replication_replication_retries_destinationName{quantile="0.98",} 1.0
+plugins_replication_replication_retries_destinationName{quantile="0.99",} 1.0
+plugins_replication_replication_retries_destinationName{quantile="0.999",} 1.0
+plugins_replication_replication_retries_destinationName_count 3.0
+
+# HELP plugins_replication_replication_latency_destinationName Generated from Dropwizard metric import (metric=plugins/replication/replication_latency/destinationName, type=com.codahale.metrics.Timer)
+# TYPE plugins_replication_replication_latency_destinationName summary
+plugins_replication_replication_latency_destinationName{quantile="0.5",} 0.21199641400000002
+plugins_replication_replication_latency_destinationName{quantile="0.75",} 0.321083881
+plugins_replication_replication_latency_destinationName{quantile="0.95",} 0.321083881
+plugins_replication_replication_latency_destinationName{quantile="0.98",} 0.321083881
+plugins_replication_replication_latency_destinationName{quantile="0.99",} 0.321083881
+plugins_replication_replication_latency_destinationName{quantile="0.999",} 0.321083881
+plugins_replication_replication_latency_destinationName_count 2.0
+
+# HELP plugins_replication_latency_slower_than_60_destinationName_projectName Generated from Dropwizard metric import (metric=plugins/replication/latency_slower_than/60/destinationName/projectName, type=com.codahale.metrics.Histogram)
+# TYPE plugins_replication_latency_slower_than_60_destinationName_projectName summary
+plugins_replication_latency_slower_than_60_destinationName_projectName{quantile="0.5",} 278.0
+plugins_replication_latency_slower_than_60_destinationName_projectName{quantile="0.75",} 278.0
+plugins_replication_latency_slower_than_60_destinationName_projectName{quantile="0.95",} 278.0
+plugins_replication_latency_slower_than_60_destinationName_projectName{quantile="0.98",} 278.0
+plugins_replication_latency_slower_than_60_destinationName_projectName{quantile="0.99",} 278.0
+plugins_replication_latency_slower_than_60_destinationName_projectName{quantile="0.999",} 278.0
+plugins_replication_latency_slower_than_60_destinationName_projectName 1.0
+```
\ No newline at end of file
