Test spanner-refdb using spanner emulator testcontainer Implement a SpannerEmulatorContainer for running the upstream Spanner emulator container for tests: - start the emulator container - create a Spanner instance - create a Spanner database - use a separate plugin config file spanner-refdb.config to configure the plugin - mock the PluginConfigFactory so that we can start the plugin from test code without loading the plugin in a running Gerrit server - create the SpannerRefDatabase schema - implement first tests Change-Id: I9290c4458299aa27bcf6e1d036b2c2245d948ba3

diff --git a/BUILD b/BUILD
index bb0e739..6ae07a1 100644
--- a/BUILD
+++ b/BUILD

@@ -1,9 +1,9 @@
 load("//tools/bzl:junit.bzl", "junit_tests")
 load(
     "//tools/bzl:plugin.bzl",
-    "gerrit_plugin",
     "PLUGIN_DEPS",
-    "PLUGIN_TEST_DEPS"
+    "PLUGIN_TEST_DEPS",
+    "gerrit_plugin",
 )
 
 gerrit_plugin(
@@ -114,6 +114,14 @@
     visibility = ["//visibility:public"],
     exports = PLUGIN_DEPS + PLUGIN_TEST_DEPS + [
         ":spanner-refdb__plugin",
+        "@docker-java-api//jar",
+        "@docker-java-transport-zerodep//jar",
+        "@docker-java-transport//jar",
+        "@duct-tape//jar",
         "@global-refdb//jar",
+        "@google-cloud-core//jar",
+        "@google-cloud-spanner//jar",
+        "@testcontainer-localstack//jar",
+        "@testcontainers//jar",
     ],
 )

diff --git a/external_plugin_deps.bzl b/external_plugin_deps.bzl
index c10bb0d..bf9d40d 100644
--- a/external_plugin_deps.bzl
+++ b/external_plugin_deps.bzl

@@ -396,3 +396,43 @@
         sha1 = "00b6b0f39b3c8fc280a19d91fb0681954ebccd02",
         artifact = "com.gerritforge:global-refdb:3.3.2.1",
     )
+
+    TESTCONTAINERS_VERSION = "1.18.3"
+
+    maven_jar(
+        name = "testcontainers",
+        artifact = "org.testcontainers:testcontainers:" + TESTCONTAINERS_VERSION,
+        sha1 = "a82f6258f92d50d278b9c67bdf5eabcaa5c08654",
+    )
+
+    maven_jar(
+        name = "testcontainer-localstack",
+        artifact = "org.testcontainers:localstack:" + TESTCONTAINERS_VERSION,
+        sha1 = "2b7a8d4522330217545c4234b916b6b77f5c6f95",
+    )
+
+    DOCKER_JAVA_VERS = "3.3.2"
+
+    maven_jar(
+        name = "docker-java-api",
+        artifact = "com.github.docker-java:docker-java-api:" + DOCKER_JAVA_VERS,
+        sha1 = "0de6345d2f69638a224f73d9e62de83c7692e436",
+    )
+
+    maven_jar(
+        name = "docker-java-transport",
+        artifact = "com.github.docker-java:docker-java-transport:" + DOCKER_JAVA_VERS,
+        sha1 = "a4c2cba248ccfefe9c5c8d8d4726f3e0b2b51104",
+    )
+
+    maven_jar(
+        name = "docker-java-transport-zerodep",
+        artifact = "com.github.docker-java:docker-java-transport-zerodep:" + DOCKER_JAVA_VERS,
+        sha1 = "36ef508e5e48613afb7fafbf7e89184243738e96",
+    )
+
+    maven_jar(
+        name = "duct-tape",
+        artifact = "org.rnorth.duct-tape:duct-tape:1.0.8",
+        sha1 = "92edc22a9ab2f3e17c9bf700aaee377d50e8b530",
+    )

diff --git a/src/main/java/com/googlesource/gerrit/plugins/spannerrefdb/Configuration.java b/src/main/java/com/googlesource/gerrit/plugins/spannerrefdb/Configuration.java
index bbdea23..e6653a3 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/spannerrefdb/Configuration.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/spannerrefdb/Configuration.java

@@ -16,31 +16,53 @@
 
 import com.google.cloud.spanner.DatabaseId;
 import com.google.cloud.spanner.SpannerOptions;
+import com.google.common.base.Strings;
 import com.google.gerrit.extensions.annotations.PluginName;
-import com.google.gerrit.server.config.PluginConfig;
 import com.google.gerrit.server.config.PluginConfigFactory;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
+import org.eclipse.jgit.lib.Config;
 
 @Singleton
 class Configuration {
-  private final SpannerOptions options;
-  private final DatabaseId databaseId;
+  public static final String DATABASE_KEY = "database";
+  public static final String INSTANCE_KEY = "instance";
+  public static final String SECTION = "ref-database";
+  public static final String SUBSECTION = "spanner";
+  private final String spannerInstance;
+  private final String spannerDatabase;
+  private SpannerOptions options;
 
   @Inject
   Configuration(PluginConfigFactory configFactory, @PluginName String pluginName) {
-    PluginConfig pluginConfig = configFactory.getFromGerritConfig(pluginName);
-    String spannerInstance = pluginConfig.getString("spannerInstance", "test-instance");
-    String spannerDatabase = pluginConfig.getString("spannerDatabase", "global-refdb");
+    Config cfg = configFactory.getGlobalPluginConfig(pluginName);
+    this.spannerInstance = getString(cfg, SECTION, SUBSECTION, INSTANCE_KEY, "test-instance");
+    this.spannerDatabase = getString(cfg, SECTION, SUBSECTION, DATABASE_KEY, "global-refdb");
     this.options = SpannerOptions.newBuilder().build();
-    this.databaseId = DatabaseId.of(options.getProjectId(), spannerInstance, spannerDatabase);
+  }
+
+  final String getSpannerInstanceName() {
+    return spannerInstance;
+  }
+
+  final DatabaseId getDatabaseId() {
+    return DatabaseId.of(options.getProjectId(), spannerInstance, spannerDatabase);
   }
 
   final SpannerOptions getOptions() {
     return options;
   }
 
-  final DatabaseId getDatabaseId() {
-    return databaseId;
+  void setOptions(SpannerOptions options) {
+    this.options = options;
+  }
+
+  private String getString(
+      Config cfg, String section, String subsection, String name, String defaultValue) {
+    String value = cfg.getString(section, subsection, name);
+    if (!Strings.isNullOrEmpty(value)) {
+      return value;
+    }
+    return defaultValue;
   }
 }

diff --git a/src/main/java/com/googlesource/gerrit/plugins/spannerrefdb/SpannerRefDatabase.java b/src/main/java/com/googlesource/gerrit/plugins/spannerrefdb/SpannerRefDatabase.java
index 94a34f8..940743e 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/spannerrefdb/SpannerRefDatabase.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/spannerrefdb/SpannerRefDatabase.java

@@ -24,6 +24,7 @@
 import com.google.cloud.spanner.ResultSet;
 import com.google.cloud.spanner.Statement;
 import com.google.cloud.spanner.Struct;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.flogger.FluentLogger;
 import com.google.gerrit.entities.Project;
 import com.google.inject.Inject;
@@ -213,7 +214,8 @@
             Mutation.delete("refs", KeySet.prefixRange(Key.of(project.get())))));
   }
 
-  private String get(Project.NameKey project, String refName) throws GlobalRefDbSystemError {
+  @VisibleForTesting
+  String get(Project.NameKey project, String refName) throws GlobalRefDbSystemError {
     try (ResultSet resultSet =
         dbClient
             .singleUse()
@@ -242,4 +244,12 @@
     // with this, while the aws implementation opts for this.
     return Optional.ofNullable((T) get(project, refName));
   }
+
+  public Optional<ObjectId> getObjectId(Project.NameKey project, String refName) {
+    String idName = get(project, refName);
+    if (idName == null) {
+      return Optional.empty();
+    }
+    return Optional.of(ObjectId.fromString(idName));
+  }
 }

diff --git a/src/main/resources/Documentation/build.md b/src/main/resources/Documentation/build.md
index fb978d1..2268819 100644
--- a/src/main/resources/Documentation/build.md
+++ b/src/main/resources/Documentation/build.md

@@ -17,4 +17,72 @@
 
 ```
 gerrit/bazel-bin/plugins/spanner-refdb/spanner-refdb.jar
-```
\ No newline at end of file
+```
+
+## Eclipse project setup
+
+This project can be imported into the Eclipse IDE:
+
+- Add the plugin name to the `CUSTOM_PLUGINS_TEST_DEPS`
+set in Gerrit core in `tools/bzl/plugins.bzl`,
+- execute:
+
+```
+./tools/eclipse/project.py
+```
+
+## Run tests
+
+To execute the tests run
+
+```
+bazelisk test --test_tag_filters=spanner-refdb //...
+```
+or
+```
+bazelisk test //plugins/@PLUGIN@/...
+```
+
+On MacOS you may need to access the docker daemon via TCP.
+
+Run [socat](https://linux.die.net/man/1/socat) to expose the docker daemon socket via TCP
+
+```
+socat TCP-LISTEN:2375,fork UNIX-CONNECT:/var/run/docker.sock
+```
+
+Execute the tests over TCP
+
+```
+bazelisk test --test_env='DOCKER_HOST=tcp://127.0.0.1:2375' //plugins/@PLUGIN@/...
+```
+
+### Debugging tests
+
+```
+bazelisk test --test_output=streamed //plugins/@PLUGIN@/...
+```
+
+If necessary increase log levels in `src/test/resources/log4j.properties`
+to trace testcontainers and docker java API.
+
+### Tracing traffic to docker daemon
+
+If you face issue you can trace traffic to the docker daemon using
+[socat](https://linux.die.net/man/1/socat) exposing the docker daemon via TCP.
+
+Run socat to log diagnostics and show the traffic to the docker daemon
+
+```
+socat -dd -v TCP-LISTEN:2375,fork UNIX-CONNECT:/var/run/docker.sock
+```
+
+Execute the tests over TCP
+
+```
+bazelisk test --test_env='DOCKER_HOST=tcp://127.0.0.1:2375' //plugins/@PLUGIN@/...
+```
+
+[Back to @PLUGIN@ documentation index][index]
+
+[index]: index.html

diff --git a/src/test/java/com/googlesource/gerrit/plugins/spannerrefdb/RefFixture.java b/src/test/java/com/googlesource/gerrit/plugins/spannerrefdb/RefFixture.java
new file mode 100644
index 0000000..67e8428
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/spannerrefdb/RefFixture.java

@@ -0,0 +1,52 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.spannerrefdb;
+
+import com.google.gerrit.entities.Project;
+import com.google.gerrit.entities.RefNames;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdRef;
+import org.eclipse.jgit.lib.Ref;
+import org.junit.Ignore;
+
+@Ignore
+public interface RefFixture {
+  static final String PROJECT_NAME = "A_TEST_PROJECT_NAME";
+  static final Project.NameKey PROJECT_NAME_KEY = Project.nameKey(PROJECT_NAME);
+  static final ObjectId OBJECT_ID_1 = new ObjectId(1, 2, 3, 4, 5);
+  static final ObjectId OBJECT_ID_2 = new ObjectId(1, 2, 3, 4, 6);
+  static final ObjectId OBJECT_ID_3 = new ObjectId(1, 2, 3, 4, 7);
+  static final Long UNIX_TIMESTAMP_1 = 1690379900L;
+  static final Long UNIX_TIMESTAMP_2 = 1690815810L;
+  static final String REF_NAME = "refs/heads/master";
+  static final String PATCHSET_REF_NAME = "refs/changes/01/1/1";
+  static final String MULTISITE_VERSION_VALUE_REF_NAME = "refs/multi-site/version/value";
+
+  default String aBranchRef() {
+    return RefNames.REFS_HEADS + testBranch();
+  }
+
+  default String testBranch() {
+    return "foo";
+  }
+
+  default Ref newRef(String refName, ObjectId objectId) {
+    return new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, refName, objectId);
+  }
+
+  default Ref nullRef(String refName) {
+    return newRef(refName, ObjectId.zeroId());
+  }
+}

diff --git a/src/test/java/com/googlesource/gerrit/plugins/spannerrefdb/SpannerEmulatorContainer.java b/src/test/java/com/googlesource/gerrit/plugins/spannerrefdb/SpannerEmulatorContainer.java
new file mode 100644
index 0000000..a331e2c
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/spannerrefdb/SpannerEmulatorContainer.java

@@ -0,0 +1,172 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.spannerrefdb;
+
+import static com.googlesource.gerrit.plugins.spannerrefdb.Configuration.DATABASE_KEY;
+import static com.googlesource.gerrit.plugins.spannerrefdb.Configuration.INSTANCE_KEY;
+import static com.googlesource.gerrit.plugins.spannerrefdb.Configuration.SECTION;
+import static com.googlesource.gerrit.plugins.spannerrefdb.Configuration.SUBSECTION;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.cloud.NoCredentials;
+import com.google.cloud.spanner.Database;
+import com.google.cloud.spanner.DatabaseAdminClient;
+import com.google.cloud.spanner.DatabaseClient;
+import com.google.cloud.spanner.Instance;
+import com.google.cloud.spanner.InstanceAdminClient;
+import com.google.cloud.spanner.InstanceConfigId;
+import com.google.cloud.spanner.InstanceId;
+import com.google.cloud.spanner.InstanceInfo;
+import com.google.cloud.spanner.Spanner;
+import com.google.cloud.spanner.SpannerOptions;
+import com.google.gerrit.server.config.PluginConfigFactory;
+import com.google.inject.Inject;
+import java.util.Collections;
+import java.util.concurrent.ExecutionException;
+import org.eclipse.jgit.lib.Config;
+import org.junit.Ignore;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
+
+@Ignore
+public class SpannerEmulatorContainer {
+  public static class Container extends GenericContainer<Container> {
+    public static String EMULATOR_VERSION = "1.5.7";
+
+    public Container() {
+      super("gcr.io/cloud-spanner-emulator/emulator:" + EMULATOR_VERSION);
+    }
+  }
+
+  public static final String PROJECT_ID = "test";
+  public static final String SPANNER_INSTANCE_ID = "instance";
+  public static final String SPANNER_DATABASE_ID = "refdb";
+  private static final int GRPC_PORT = 9010;
+  private static final int REST_PORT = 9020;
+  private static final String pluginName = "spanner-refdb";
+
+  private final Container container;
+  private final Integer grpcPort;
+  private final Integer restPort;
+
+  private final Spanner spanner;
+  private final InstanceAdminClient instanceAdminClient;
+  private final Instance spannerInstance;
+  private final Database spannerDatabase;
+  private final DatabaseClient databaseClient;
+  private final SpannerRefDatabase spannerRefDb;
+  private final Configuration pluginConfig;
+
+  @Inject
+  @SuppressWarnings("resource")
+  public SpannerEmulatorContainer() throws Exception {
+    container =
+        new Container().withExposedPorts(GRPC_PORT, REST_PORT).waitingFor(Wait.forListeningPort());
+    container.start();
+    grpcPort = container.getMappedPort(GRPC_PORT);
+    restPort = container.getMappedPort(REST_PORT);
+    System.out.println(
+        "Spanner emulator container started and is listening on ports "
+            + grpcPort
+            + ", "
+            + restPort);
+
+    spanner = getEmulatorOptions().getService();
+    instanceAdminClient = spanner.getInstanceAdminClient();
+    spannerInstance = createSpannerInstance();
+    spannerDatabase =
+        spannerInstance.createDatabase(SPANNER_DATABASE_ID, Collections.emptyList()).get();
+    pluginConfig = createEmulatorConfiguration();
+    createSchema();
+    databaseClient = createDatabaseClient();
+    spannerRefDb = new SpannerRefDatabase(databaseClient);
+  }
+
+  private void createSchema() {
+    DatabaseAdminClient dbAdminClient = spanner.getDatabaseAdminClient();
+    SpannerLifeCycleManager lcm = new SpannerLifeCycleManager(pluginConfig, dbAdminClient);
+    lcm.start();
+  }
+
+  public void cleanup() {
+    spannerDatabase.drop();
+    instanceAdminClient.deleteInstance(SPANNER_INSTANCE_ID);
+    spanner.close();
+    container.stop();
+    container.close();
+    System.out.println("Spanner emulator container was stopped");
+  }
+
+  public Container getContainer() {
+    return container;
+  }
+
+  public Integer getGrpcPort() {
+    return grpcPort;
+  }
+
+  public Integer getRestPort() {
+    return restPort;
+  }
+
+  public Database getSpannerDatabase() {
+    return spannerDatabase;
+  }
+
+  public SpannerRefDatabase getSpannerRefDatabase() {
+    return spannerRefDb;
+  }
+
+  private SpannerOptions getEmulatorOptions() {
+    return SpannerOptions.newBuilder()
+        .setEmulatorHost("localhost:" + getGrpcPort())
+        .setCredentials(NoCredentials.getInstance())
+        .setProjectId(PROJECT_ID)
+        .build();
+  }
+
+  private Instance createSpannerInstance() throws InterruptedException, ExecutionException {
+    InstanceConfigId instanceConfig = InstanceConfigId.of(PROJECT_ID, "emulator-config");
+    InstanceId instanceId = InstanceId.of(PROJECT_ID, SPANNER_INSTANCE_ID);
+    InstanceInfo instanceInfo =
+        InstanceInfo.newBuilder(instanceId)
+            .setInstanceConfigId(instanceConfig)
+            .setNodeCount(1)
+            .setDisplayName("test instance")
+            .build();
+    return instanceAdminClient.createInstance(instanceInfo).get();
+  }
+
+  private DatabaseClient createDatabaseClient() {
+    return createDatabaseClient(pluginConfig);
+  }
+
+  private Configuration createEmulatorConfiguration() {
+    Config refDbConfig = new Config();
+    refDbConfig.setString(SECTION, SUBSECTION, INSTANCE_KEY, SPANNER_INSTANCE_ID);
+    refDbConfig.setString(SECTION, SUBSECTION, DATABASE_KEY, SPANNER_DATABASE_ID);
+
+    PluginConfigFactory cfgFactory = mock(PluginConfigFactory.class);
+    when(cfgFactory.getGlobalPluginConfig(pluginName)).thenReturn(refDbConfig);
+    Configuration spannerConfig = new Configuration(cfgFactory, pluginName);
+    spannerConfig.setOptions(getEmulatorOptions());
+    return spannerConfig;
+  }
+
+  private DatabaseClient createDatabaseClient(Configuration configuration) {
+    return configuration.getOptions().getService().getDatabaseClient(configuration.getDatabaseId());
+  }
+}

diff --git a/src/test/java/com/googlesource/gerrit/plugins/spannerrefdb/SpannerRefDatabaseTest.java b/src/test/java/com/googlesource/gerrit/plugins/spannerrefdb/SpannerRefDatabaseTest.java
new file mode 100644
index 0000000..751e7bc
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/spannerrefdb/SpannerRefDatabaseTest.java

@@ -0,0 +1,138 @@
+// Copyright (C) 2023 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.spannerrefdb;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import com.google.gerrit.entities.Project;
+import java.util.Optional;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Ref;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class SpannerRefDatabaseTest implements RefFixture {
+
+  private SpannerEmulatorContainer emulator;
+  private SpannerRefDatabase refdb;
+
+  @Before
+  public void setup() throws Exception {
+    emulator = new SpannerEmulatorContainer();
+    refdb = emulator.getSpannerRefDatabase();
+  }
+
+  @After
+  public void cleanup() {
+    emulator.cleanup();
+  }
+
+  @Test
+  public void testCreateNewRef() throws Exception {
+    createNewRef(PROJECT_NAME_KEY, REF_NAME, OBJECT_ID_1);
+
+    createNewRef(PROJECT_NAME_KEY, PATCHSET_REF_NAME, OBJECT_ID_2);
+
+    createNewRef(PROJECT_NAME_KEY, MULTISITE_VERSION_VALUE_REF_NAME, UNIX_TIMESTAMP_1.toString());
+  }
+
+  @Test
+  public void updateRef() throws Exception {
+    createNewRef(PROJECT_NAME_KEY, REF_NAME, OBJECT_ID_1);
+    updateRef(PROJECT_NAME_KEY, REF_NAME, OBJECT_ID_1, OBJECT_ID_2);
+
+    createNewRef(PROJECT_NAME_KEY, PATCHSET_REF_NAME, OBJECT_ID_2);
+    updateRef(PROJECT_NAME_KEY, PATCHSET_REF_NAME, OBJECT_ID_2, OBJECT_ID_3);
+
+    createNewRef(PROJECT_NAME_KEY, MULTISITE_VERSION_VALUE_REF_NAME, UNIX_TIMESTAMP_1.toString());
+    updateRef(
+        PROJECT_NAME_KEY,
+        MULTISITE_VERSION_VALUE_REF_NAME,
+        UNIX_TIMESTAMP_1.toString(),
+        UNIX_TIMESTAMP_2.toString());
+  }
+
+  @Test
+  public void testDeleteRef() throws Exception {
+    createNewRef(PROJECT_NAME_KEY, REF_NAME, OBJECT_ID_1);
+    deleteRef(PROJECT_NAME_KEY, REF_NAME, OBJECT_ID_1);
+
+    createNewRef(PROJECT_NAME_KEY, PATCHSET_REF_NAME, OBJECT_ID_2);
+    deleteRef(PROJECT_NAME_KEY, PATCHSET_REF_NAME, OBJECT_ID_2);
+
+    createNewRef(PROJECT_NAME_KEY, MULTISITE_VERSION_VALUE_REF_NAME, UNIX_TIMESTAMP_1.toString());
+    deleteRef(PROJECT_NAME_KEY, MULTISITE_VERSION_VALUE_REF_NAME, UNIX_TIMESTAMP_1.toString());
+  }
+
+  private void createNewRef(Project.NameKey project, String refName, ObjectId newValue)
+      throws Exception {
+    Ref ref = nullRef(refName);
+    try (AutoCloseable lock = refdb.lockRef(project, refName)) {
+      assertTrue(refdb.compareAndPut(project, ref, newValue));
+      Optional<ObjectId> oid = refdb.getObjectId(project, refName);
+      assertFalse(oid.isEmpty());
+      assertEquals(newValue, oid.get());
+    }
+  }
+
+  private void updateRef(
+      Project.NameKey project, String refName, ObjectId oldValue, ObjectId newValue)
+      throws Exception {
+    Ref expecRef = newRef(refName, oldValue);
+    try (AutoCloseable lock = refdb.lockRef(project, refName)) {
+      assertTrue(refdb.compareAndPut(project, expecRef, newValue));
+      assertEquals(newValue, refdb.getObjectId(project, refName).get());
+    }
+  }
+
+  private void deleteRef(Project.NameKey project, String refName, ObjectId oldValue)
+      throws Exception {
+    Ref expecRef = newRef(refName, oldValue);
+    try (AutoCloseable lock = refdb.lockRef(project, refName)) {
+      assertTrue(refdb.compareAndPut(project, expecRef, ObjectId.zeroId()));
+      assertTrue(refdb.getObjectId(project, refName).isEmpty());
+    }
+  }
+
+  private void createNewRef(Project.NameKey project, String refName, String newValue)
+      throws Exception {
+    try (AutoCloseable lock = refdb.lockRef(project, refName)) {
+      assertTrue(refdb.compareAndPut(project, refName, ObjectId.zeroId().name(), newValue));
+      assertNotNull(refdb.get(project, refName));
+      assertEquals(newValue, refdb.get(project, refName));
+    }
+  }
+
+  private void updateRef(Project.NameKey project, String refName, String oldValue, String newValue)
+      throws Exception {
+    try (AutoCloseable lock = refdb.lockRef(project, refName)) {
+      assertTrue(refdb.compareAndPut(project, refName, oldValue, newValue));
+      assertEquals(newValue, refdb.get(project, refName));
+    }
+  }
+
+  private void deleteRef(Project.NameKey project, String refName, String oldValue)
+      throws Exception {
+    try (AutoCloseable lock = refdb.lockRef(project, refName)) {
+      assertTrue(refdb.compareAndPut(project, refName, oldValue, ObjectId.zeroId().name()));
+      assertNull(refdb.get(project, refName));
+    }
+  }
+}