Add metric to count number of detected split-brain ref-updates
Adds a new metric to count number of detected ref-update alteration blocked by
split-brain validation
Feature: Issue 10706
Change-Id: I82d3fca9aa11a87b10ca9d3da8b6ffea075bc11c
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteBatchRefUpdate.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteBatchRefUpdate.java
index 783f6a2..ef89f4d 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteBatchRefUpdate.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteBatchRefUpdate.java
@@ -52,6 +52,7 @@
private final RefDatabase refDb;
private final SharedRefDatabase sharedRefDb;
private final String projectName;
+ private final ValidationMetrics validationMetrics;
public static class RefPair {
final Ref oldRef;
@@ -81,13 +82,17 @@
@Inject
public MultiSiteBatchRefUpdate(
- SharedRefDatabase sharedRefDb, @Assisted String projectName, @Assisted RefDatabase refDb) {
+ SharedRefDatabase sharedRefDb,
+ ValidationMetrics validationMetrics,
+ @Assisted String projectName,
+ @Assisted RefDatabase refDb) {
super(refDb);
this.sharedRefDb = sharedRefDb;
this.projectName = projectName;
this.refDb = refDb;
this.batchRefUpdate = refDb.newBatchUpdate();
+ this.validationMetrics = validationMetrics;
}
@Override
@@ -235,6 +240,8 @@
boolean compareAndPutResult =
sharedRefDb.compareAndPut(projectName, refPair.oldRef, refPair.newRef);
if (!compareAndPutResult) {
+ validationMetrics.incrementSplitBrainRefUpdates();
+
throw new IOException(
String.format(
"This repos is out of sync for project %s. old_ref=%s, new_ref=%s",
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefUpdate.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefUpdate.java
index d60082e..b543a37 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefUpdate.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefUpdate.java
@@ -47,7 +47,7 @@
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
protected final RefUpdate refUpdateBase;
-
+ private final ValidationMetrics validationMetrics;
private final SharedRefDatabase sharedDb;
private final String projectName;
@@ -57,9 +57,13 @@
@Inject
public MultiSiteRefUpdate(
- SharedRefDatabase db, @Assisted String projectName, @Assisted RefUpdate refUpdate) {
+ SharedRefDatabase db,
+ ValidationMetrics validationMetrics,
+ @Assisted String projectName,
+ @Assisted RefUpdate refUpdate) {
super(refUpdate.getRef());
refUpdateBase = refUpdate;
+ this.validationMetrics = validationMetrics;
this.sharedDb = db;
this.projectName = projectName;
}
@@ -81,6 +85,8 @@
+ "Trying to update it cannot extract the existing one on DB",
refUpdateBase.getName());
+ validationMetrics.incrementSplitBrainRefUpdates();
+
throw new IOException(
String.format(
"Unable to update ref '%s', cannot open the local ref on the local DB",
@@ -105,6 +111,8 @@
+ "Trying to delete it but it is not in the DB",
oldRef.getName());
+ validationMetrics.incrementSplitBrainRefUpdates();
+
throw new IOException(
String.format(
"Unable to delete ref '%s', cannot find it in the shared ref database",
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationMetrics.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationMetrics.java
new file mode 100644
index 0000000..fb6e08c
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/validation/ValidationMetrics.java
@@ -0,0 +1,56 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// Copyright (C) 2018 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.validation;
+
+import com.google.gerrit.metrics.Counter1;
+import com.google.gerrit.metrics.Description;
+import com.google.gerrit.metrics.Field;
+import com.google.gerrit.metrics.MetricMaker;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+@Singleton
+public class ValidationMetrics {
+ private static final String REF_UPDATES = "ref_updates";
+
+ private final Counter1<String> splitBrain;
+
+ @Inject
+ public ValidationMetrics(MetricMaker metricMaker) {
+ this.splitBrain =
+ metricMaker.newCounter(
+ "multi_site/validation/split_brain",
+ new Description("Rate of REST API error responses").setRate().setUnit("errors"),
+ Field.ofString(
+ REF_UPDATES, "Ref-update operations detected as leading to split-brain"));
+ }
+
+ public void incrementSplitBrainRefUpdates() {
+ splitBrain.increment(REF_UPDATES);
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteBatchRefUpdateTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteBatchRefUpdateTest.java
index bc558d5..8d16687 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteBatchRefUpdateTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteBatchRefUpdateTest.java
@@ -27,7 +27,10 @@
package com.googlesource.gerrit.plugins.multisite.validation;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefDatabase;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper.RefFixture;
@@ -56,6 +59,7 @@
@Mock RefDatabase refDatabase;
@Mock RevWalk revWalk;
@Mock ProgressMonitor progressMonitor;
+ @Mock ValidationMetrics validationMetrics;
private final Ref oldRef =
new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, A_TEST_REF_NAME, AN_OBJECT_ID_1);
@@ -79,7 +83,11 @@
doReturn(oldRef).when(refDatabase).getRef(A_TEST_REF_NAME);
doReturn(newRef).when(sharedRefDb).newRef(A_TEST_REF_NAME, AN_OBJECT_ID_2);
- multiSiteRefUpdate = new MultiSiteBatchRefUpdate(sharedRefDb, A_TEST_PROJECT_NAME, refDatabase);
+ multiSiteRefUpdate =
+ new MultiSiteBatchRefUpdate(
+ sharedRefDb, validationMetrics, A_TEST_PROJECT_NAME, refDatabase);
+
+ verifyZeroInteractions(validationMetrics);
}
@Test
@@ -91,13 +99,18 @@
multiSiteRefUpdate.execute(revWalk, progressMonitor, Collections.emptyList());
}
- @Test(expected = IOException.class)
+ @Test
public void executeAndFailsWithExceptions() throws IOException {
setMockRequiredReturnValues();
// When compareAndPut against sharedDb fails
doReturn(false).when(sharedRefDb).compareAndPut(A_TEST_PROJECT_NAME, oldRef, newRef);
- multiSiteRefUpdate.execute(revWalk, progressMonitor, Collections.emptyList());
+ try {
+ multiSiteRefUpdate.execute(revWalk, progressMonitor, Collections.emptyList());
+ fail("Expecting an IOException to be thrown");
+ } catch (IOException e) {
+ verify(validationMetrics).incrementSplitBrainRefUpdates();
+ }
}
@Test
@@ -105,7 +118,9 @@
doReturn(batchRefUpdate).when(refDatabase).newBatchUpdate();
doReturn(Collections.emptyList()).when(batchRefUpdate).getCommands();
- multiSiteRefUpdate = new MultiSiteBatchRefUpdate(sharedRefDb, A_TEST_PROJECT_NAME, refDatabase);
+ multiSiteRefUpdate =
+ new MultiSiteBatchRefUpdate(
+ sharedRefDb, validationMetrics, A_TEST_PROJECT_NAME, refDatabase);
multiSiteRefUpdate.execute(revWalk, progressMonitor, Collections.emptyList());
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefUpdateTest.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefUpdateTest.java
index 9b8acde..e88dffb 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefUpdateTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/validation/MultiSiteRefUpdateTest.java
@@ -28,7 +28,10 @@
package com.googlesource.gerrit.plugins.multisite.validation;
import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.SharedRefDatabase;
import com.googlesource.gerrit.plugins.multisite.validation.dfsrefdb.zookeeper.RefFixture;
@@ -49,6 +52,7 @@
@Mock SharedRefDatabase sharedRefDb;
@Mock RefUpdate refUpdate;
+ @Mock ValidationMetrics validationMetrics;
private final Ref oldRef =
new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, A_TEST_REF_NAME, AN_OBJECT_ID_1);
private final Ref newRef =
@@ -77,9 +81,11 @@
doReturn(Result.NEW).when(refUpdate).update();
MultiSiteRefUpdate multiSiteRefUpdate =
- new MultiSiteRefUpdate(sharedRefDb, A_TEST_PROJECT_NAME, refUpdate);
+ new MultiSiteRefUpdate(sharedRefDb, validationMetrics, A_TEST_PROJECT_NAME, refUpdate);
assertThat(multiSiteRefUpdate.update()).isEqualTo(Result.NEW);
+
+ verifyZeroInteractions(validationMetrics);
}
@Test(expected = IOException.class)
@@ -90,11 +96,29 @@
doReturn(false).when(sharedRefDb).compareAndPut(A_TEST_PROJECT_NAME, oldRef, newRef);
MultiSiteRefUpdate multiSiteRefUpdate =
- new MultiSiteRefUpdate(sharedRefDb, A_TEST_PROJECT_NAME, refUpdate);
+ new MultiSiteRefUpdate(sharedRefDb, validationMetrics, A_TEST_PROJECT_NAME, refUpdate);
multiSiteRefUpdate.update();
}
@Test
+ public void newUpdateShouldIncreaseRefUpdateFailureCountWhenFailing() throws IOException {
+ setMockRequiredReturnValues();
+
+ // When compareAndPut fails
+ doReturn(false).when(sharedRefDb).compareAndPut(A_TEST_PROJECT_NAME, oldRef, newRef);
+
+ MultiSiteRefUpdate multiSiteRefUpdate =
+ new MultiSiteRefUpdate(sharedRefDb, validationMetrics, A_TEST_PROJECT_NAME, refUpdate);
+
+ try {
+ multiSiteRefUpdate.update();
+ fail("Expecting an IOException to be thrown");
+ } catch (IOException e) {
+ verify(validationMetrics).incrementSplitBrainRefUpdates();
+ }
+ }
+
+ @Test
public void deleteShouldValidateAndSucceed() throws IOException {
setMockRequiredReturnValues();
@@ -103,9 +127,10 @@
doReturn(Result.FORCED).when(refUpdate).delete();
MultiSiteRefUpdate multiSiteRefUpdate =
- new MultiSiteRefUpdate(sharedRefDb, A_TEST_PROJECT_NAME, refUpdate);
+ new MultiSiteRefUpdate(sharedRefDb, validationMetrics, A_TEST_PROJECT_NAME, refUpdate);
assertThat(multiSiteRefUpdate.delete()).isEqualTo(Result.FORCED);
+ verifyZeroInteractions(validationMetrics);
}
@Test(expected = IOException.class)
@@ -116,7 +141,24 @@
doReturn(false).when(sharedRefDb).compareAndRemove(A_TEST_PROJECT_NAME, oldRef);
MultiSiteRefUpdate multiSiteRefUpdate =
- new MultiSiteRefUpdate(sharedRefDb, A_TEST_PROJECT_NAME, refUpdate);
+ new MultiSiteRefUpdate(sharedRefDb, validationMetrics, A_TEST_PROJECT_NAME, refUpdate);
multiSiteRefUpdate.delete();
}
+
+ @Test
+ public void deleteShouldIncreaseRefUpdateFailureCountWhenFailing() throws IOException {
+ setMockRequiredReturnValues();
+
+ // When compareAndPut fails
+ doReturn(false).when(sharedRefDb).compareAndRemove(A_TEST_PROJECT_NAME, oldRef);
+
+ MultiSiteRefUpdate multiSiteRefUpdate =
+ new MultiSiteRefUpdate(sharedRefDb, validationMetrics, A_TEST_PROJECT_NAME, refUpdate);
+ try {
+ multiSiteRefUpdate.delete();
+ fail("Expecting an IOException to be thrown");
+ } catch (IOException e) {
+ verify(validationMetrics).incrementSplitBrainRefUpdates();
+ }
+ }
}
