Introduce endpoint to query replication-lag
Introduce an HTTP endpoint that can be called by admin users in order to
understand the replication lag status by project.
Bug: Issue 15309
Change-Id: Iec196d7aed4f585353b8a6549cb89406e95715ac
diff --git a/BUILD b/BUILD
index 39e1bc3..d7bd3d8 100644
--- a/BUILD
+++ b/BUILD
@@ -12,6 +12,7 @@
manifest_entries = [
"Gerrit-PluginName: multi-site",
"Gerrit-Module: com.googlesource.gerrit.plugins.multisite.PluginModule",
+ "Gerrit-HttpModule: com.googlesource.gerrit.plugins.multisite.http.HttpModule",
"Implementation-Title: multi-site plugin",
"Implementation-URL: https://review.gerrithub.io/admin/repos/GerritForge/plugins_multi-site",
],
diff --git a/README.md b/README.md
index 34f7709..79094d3 100644
--- a/README.md
+++ b/README.md
@@ -86,3 +86,8 @@
You also need to setup the Git-level replication between nodes, for more details
please refer to the
[replication plugin documentation](https://gerrit.googlesource.com/plugins/replication/+/refs/heads/master/src/main/resources/Documentation/config.md).
+
+# HTTP endpoints
+
+For information about available HTTP endpoints please refer to
+the [documentation](src/main/resources/Documentation/http-endpoints.md).
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ReplicationStatus.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ReplicationStatus.java
index cac9913..45e66a6 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ReplicationStatus.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/consumer/ReplicationStatus.java
@@ -14,6 +14,7 @@
package com.googlesource.gerrit.plugins.multisite.consumer;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.flogger.FluentLogger;
import com.google.gerrit.entities.Project;
import com.google.inject.Inject;
@@ -23,8 +24,10 @@
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Optional;
+import java.util.stream.Collectors;
@Singleton
public class ReplicationStatus {
@@ -50,7 +53,19 @@
return Collections.max(lags);
}
- void updateReplicationLag(Project.NameKey projectName) {
+ public Map<String, Long> getReplicationLags(Integer limit) {
+ return replicationStatusPerProject.entrySet().stream()
+ .sorted((c1, c2) -> c2.getValue().compareTo(c1.getValue()))
+ .limit(limit)
+ .collect(
+ Collectors.toMap(
+ Map.Entry::getKey,
+ Map.Entry::getValue,
+ (oldValue, newValue) -> oldValue,
+ LinkedHashMap::new));
+ }
+
+ public void updateReplicationLag(Project.NameKey projectName) {
Optional<Long> remoteVersion =
projectVersionRefUpdate.getProjectRemoteVersion(projectName.get());
Optional<Long> localVersion = projectVersionRefUpdate.getProjectLocalVersion(projectName.get());
@@ -62,7 +77,7 @@
logger.atFine().log(
"Updated replication lag for project '%s' of %d sec(s) [local-ref=%d global-ref=%d]",
projectName, lag, localVersion.get(), remoteVersion.get());
- replicationStatusPerProject.put(projectName.get(), lag);
+ doUpdateLag(projectName, lag);
localVersionPerProject.put(projectName.get(), localVersion.get());
verLogger.log(projectName, localVersion.get(), lag);
}
@@ -72,4 +87,9 @@
projectName, localVersion.isPresent() ? "remote" : "local");
}
}
+
+ @VisibleForTesting
+ public void doUpdateLag(Project.NameKey projectName, Long lag) {
+ replicationStatusPerProject.put(projectName.get(), lag);
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/http/HttpModule.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/http/HttpModule.java
new file mode 100644
index 0000000..2ae2d9b
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/http/HttpModule.java
@@ -0,0 +1,27 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.http;
+
+import com.google.inject.servlet.ServletModule;
+
+public class HttpModule extends ServletModule {
+
+ public static final String LAG_ENDPOINT_SEGMENT = "replication-lag";
+
+ @Override
+ protected void configureServlets() {
+ serve(String.format("/%s", LAG_ENDPOINT_SEGMENT)).with(ReplicationStatusServlet.class);
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/multisite/http/ReplicationStatusServlet.java b/src/main/java/com/googlesource/gerrit/plugins/multisite/http/ReplicationStatusServlet.java
new file mode 100644
index 0000000..e2b3e18
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/multisite/http/ReplicationStatusServlet.java
@@ -0,0 +1,84 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.http;
+
+import static com.google.gerrit.server.permissions.GlobalPermission.ADMINISTRATE_SERVER;
+
+import com.google.common.flogger.FluentLogger;
+import com.google.gerrit.httpd.restapi.RestApiServlet;
+import com.google.gerrit.server.permissions.PermissionBackend;
+import com.google.gson.Gson;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.multisite.consumer.ReplicationStatus;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Optional;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+@Singleton
+public class ReplicationStatusServlet extends HttpServlet {
+ protected static final FluentLogger logger = FluentLogger.forEnclosingClass();
+ private static final String LIMIT_RESULT_PARAMETER = "limit";
+ private static final long serialVersionUID = 1L;
+ private static final Integer DEFAULT_LIMIT_RESULT_PARAMETER = 10;
+
+ private final Gson gson;
+ private final ReplicationStatus replicationStatus;
+ private final PermissionBackend permissionBackend;
+
+ @Inject
+ ReplicationStatusServlet(
+ Gson gson, ReplicationStatus replicationStatus, PermissionBackend permissionBackend) {
+ this.gson = gson;
+ this.replicationStatus = replicationStatus;
+ this.permissionBackend = permissionBackend;
+ }
+
+ @Override
+ protected void doGet(
+ HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse)
+ throws ServletException, IOException {
+ if (!permissionBackend.currentUser().testOrFalse(ADMINISTRATE_SERVER)) {
+ setResponse(
+ httpServletResponse,
+ HttpServletResponse.SC_FORBIDDEN,
+ String.format("%s permissions required. Operation not permitted", ADMINISTRATE_SERVER));
+ return;
+ }
+
+ int limitResult =
+ Optional.ofNullable(httpServletRequest.getParameter(LIMIT_RESULT_PARAMETER))
+ .map(Integer::parseInt)
+ .orElse(DEFAULT_LIMIT_RESULT_PARAMETER);
+
+ setResponse(
+ httpServletResponse,
+ HttpServletResponse.SC_OK,
+ gson.toJson(replicationStatus.getReplicationLags(limitResult)));
+ }
+
+ static void setResponse(HttpServletResponse httpResponse, int statusCode, String value)
+ throws IOException {
+ httpResponse.setContentType("application/json");
+ httpResponse.setStatus(statusCode);
+ PrintWriter writer = httpResponse.getWriter();
+ writer.print(new String(RestApiServlet.JSON_MAGIC));
+ writer.print(value);
+ }
+}
diff --git a/src/main/resources/Documentation/http-endpoints.md b/src/main/resources/Documentation/http-endpoints.md
new file mode 100644
index 0000000..ff56fdf
--- /dev/null
+++ b/src/main/resources/Documentation/http-endpoints.md
@@ -0,0 +1,42 @@
+HTTP endpoints
+=========================
+
+The @PLUGIN@ plugin also provides HTTP endpoints, as described here below:
+
+## replication-lag
+
+Admin users can query for the replication lag in order to understand what
+projects' replication is running behind and by how much (in milliseconds).
+
+The results are returned in a map ordered in descending order by the replication
+lag, so that the most behind projects are returned first.
+
+Whilst some lag information is also available as a metric (see
+the [documentation](./about.md#metrics)), this endpoint provides more
+information since it shows _which_ project is associated to _which specific lag_.
+
+You can query the endpoint (at the receiving end of the replication) as follows:
+
+```bash
+curl -v -XGET -u <admin> '<gerrit>/a/plugins/multi-site/replication-lag?[limit=LIMIT]'
+```
+
+Output example:
+
+```
+)]}'
+{
+ "All-Users": 62,
+ "bar": 13,
+ "foo": 0,
+ "some/other/project": 1451,
+ "baz": 6432
+}
+```
+
+Optionally the REST endpoint can receive the following additional arguments:
+
+* limit=LIMIT
+
+maximum number of projects to return
+*default:10*
\ No newline at end of file
diff --git a/src/test/java/com/googlesource/gerrit/plugins/multisite/http/ReplicationStatusServletIT.java b/src/test/java/com/googlesource/gerrit/plugins/multisite/http/ReplicationStatusServletIT.java
new file mode 100644
index 0000000..3b5f9a6
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/multisite/http/ReplicationStatusServletIT.java
@@ -0,0 +1,125 @@
+// Copyright (C) 2021 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.multisite.http;
+
+import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
+import static com.google.common.truth.Truth.assertThat;
+import static com.googlesource.gerrit.plugins.multisite.http.HttpModule.LAG_ENDPOINT_SEGMENT;
+
+import com.gerritforge.gerrit.globalrefdb.validation.Log4jSharedRefLogger;
+import com.gerritforge.gerrit.globalrefdb.validation.SharedRefDbConfiguration;
+import com.gerritforge.gerrit.globalrefdb.validation.SharedRefLogger;
+import com.google.gerrit.acceptance.LightweightPluginDaemonTest;
+import com.google.gerrit.acceptance.RestResponse;
+import com.google.gerrit.acceptance.TestPlugin;
+import com.google.gerrit.entities.Project;
+import com.google.gerrit.httpd.restapi.RestApiServlet;
+import com.google.inject.AbstractModule;
+import com.googlesource.gerrit.plugins.multisite.Log4jProjectVersionLogger;
+import com.googlesource.gerrit.plugins.multisite.ProjectVersionLogger;
+import com.googlesource.gerrit.plugins.multisite.cache.CacheModule;
+import com.googlesource.gerrit.plugins.multisite.consumer.ReplicationStatus;
+import com.googlesource.gerrit.plugins.multisite.forwarder.ForwarderModule;
+import com.googlesource.gerrit.plugins.multisite.forwarder.router.RouterModule;
+import com.googlesource.gerrit.plugins.multisite.index.IndexModule;
+import java.io.IOException;
+import org.eclipse.jgit.lib.Config;
+import org.junit.Before;
+import org.junit.Test;
+
+@TestPlugin(
+ name = "multi-site",
+ sysModule =
+ "com.googlesource.gerrit.plugins.multisite.http.ReplicationStatusServletIT$TestModule",
+ httpModule = "com.googlesource.gerrit.plugins.multisite.http.HttpModule")
+public class ReplicationStatusServletIT extends LightweightPluginDaemonTest {
+ private static final String APPLICATION_JSON = "application/json";
+ private static final String LAG_ENDPOINT =
+ String.format("/plugins/multi-site/%s", LAG_ENDPOINT_SEGMENT);
+ private ReplicationStatus replicationStatus;
+
+ public static class TestModule extends AbstractModule {
+ @Override
+ protected void configure() {
+ install(new ForwarderModule());
+ install(new CacheModule());
+ install(new RouterModule());
+ install(new IndexModule());
+ SharedRefDbConfiguration sharedRefDbConfig =
+ new SharedRefDbConfiguration(new Config(), "multi-site");
+ bind(SharedRefDbConfiguration.class).toInstance(sharedRefDbConfig);
+ bind(ProjectVersionLogger.class).to(Log4jProjectVersionLogger.class);
+ bind(SharedRefLogger.class).to(Log4jSharedRefLogger.class);
+ }
+ }
+
+ @Before
+ public void setUp() throws IOException {
+ replicationStatus = plugin.getSysInjector().getInstance(ReplicationStatus.class);
+ }
+
+ @Test
+ public void shouldSucceedForAdminUsers() throws Exception {
+ RestResponse result = adminRestSession.get(LAG_ENDPOINT);
+
+ result.assertOK();
+ assertThat(result.getHeader(CONTENT_TYPE)).contains(APPLICATION_JSON);
+ }
+
+ @Test
+ public void shouldFailWhenUserHasNoAdminServerCapability() throws Exception {
+ RestResponse result = userRestSession.get(LAG_ENDPOINT);
+ result.assertForbidden();
+ assertThat(result.getEntityContent()).contains("not permitted");
+ }
+
+ @Test
+ public void shouldReturnCurrentProjectLag() throws Exception {
+ replicationStatus.doUpdateLag(Project.nameKey("foo"), 123L);
+
+ RestResponse result = adminRestSession.get(LAG_ENDPOINT);
+
+ result.assertOK();
+ assertThat(contentWithoutMagicJson(result)).isEqualTo("{\"foo\":123}");
+ }
+
+ @Test
+ public void shouldReturnProjectsOrderedDescendinglyByLag() throws Exception {
+ replicationStatus.doUpdateLag(Project.nameKey("bar"), 123L);
+ replicationStatus.doUpdateLag(Project.nameKey("foo"), 3L);
+ replicationStatus.doUpdateLag(Project.nameKey("baz"), 52300L);
+
+ RestResponse result = adminRestSession.get(LAG_ENDPOINT);
+
+ result.assertOK();
+ assertThat(contentWithoutMagicJson(result)).isEqualTo("{\"baz\":52300,\"bar\":123,\"foo\":3}");
+ }
+
+ @Test
+ public void shouldHonourTheLimitParameter() throws Exception {
+ replicationStatus.doUpdateLag(Project.nameKey("bar"), 1L);
+ replicationStatus.doUpdateLag(Project.nameKey("foo"), 2L);
+ replicationStatus.doUpdateLag(Project.nameKey("baz"), 3L);
+
+ RestResponse result = adminRestSession.get(String.format("%s?limit=2", LAG_ENDPOINT));
+
+ result.assertOK();
+ assertThat(contentWithoutMagicJson(result)).isEqualTo("{\"baz\":3,\"foo\":2}");
+ }
+
+ private String contentWithoutMagicJson(RestResponse response) throws IOException {
+ return response.getEntityContent().substring(RestApiServlet.JSON_MAGIC.length);
+ }
+}
