Merge branch 'stable-3.6' into stable-3.7
* stable-3.6:
Rename config.md to metrics.md
Add the ability to fail healthcheck by creating a fail file
Change-Id: I210391682ccee6ce6ae3127cbd32f9e46af8130a
diff --git a/README.md b/README.md
index 8be0b33..bd49524 100644
--- a/README.md
+++ b/README.md
@@ -126,6 +126,25 @@
}
```
+It's also possible to artificially make the healthcheck fail by placing a file
+at a configurable path specified like:
+
+```
+[healtcheck]
+ failFileFlaPath="data/healthcheck/fail"
+```
+
+This will make the healthcheck endpoint return 500 even if the node is otherwise
+healthy. This is useful when a node needs to be removed from the pool of
+available Gerrit instance while it undergoes maintenance.
+
+**NOTE**: If the path starts with `/` then even paths outside of Gerrit's home
+will be checked. If the path starts WITHOUT `/` then the path is relative to
+Gerrit's home.
+
+**NOTE**: The file needs to be a real file rather than a symlink.
+
+
## Metrics
As for all other endpoints in Gerrit, some metrics are automatically emitted when the `/config/server/healthcheck~status`
@@ -134,4 +153,4 @@
Some additional metrics are also produced to give extra insights on their result about results and latency of healthcheck
sub component, such as jgit, reviewdb, etc.
-More information can be found in the [config.md](resources/Documentation/config.md) file.
+More information can be found in the [metrics.md](resources/Documentation/metrics.md) file.
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java
index 651dbe9..1dc143c 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java
@@ -47,6 +47,7 @@
private static final int ACTIVE_WORKERS_THRESHOLD_DEFAULT = 80;
private static final String USERNAME_DEFAULT = "healthcheck";
private static final String PASSWORD_DEFAULT = "";
+ private static final String FAIL_FILE_FLAG_DEFAULT = "data/healthcheck/fail";
private static final boolean HEALTH_CHECK_ENABLED_DEFAULT = true;
private final AllProjectsName allProjectsName;
private final AllUsersName allUsersName;
@@ -135,6 +136,10 @@
return getStringWithFallback("password", healthCheckName, PASSWORD_DEFAULT);
}
+ public String getFailFileFlagPath() {
+ return getStringWithFallback("failFileFlagPath", null, FAIL_FILE_FLAG_DEFAULT);
+ }
+
public boolean healthCheckEnabled(String healthCheckName) {
if (isReplica && HEALTH_CHECK_DISABLED_FOR_REPLICAS.contains(healthCheckName)) {
return false;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/api/HealthCheckStatusEndpoint.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/api/HealthCheckStatusEndpoint.java
index 2d5f7de..6819ae1 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/api/HealthCheckStatusEndpoint.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/api/HealthCheckStatusEndpoint.java
@@ -18,9 +18,14 @@
import com.google.gerrit.server.config.ConfigResource;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.healthcheck.HealthCheckConfig;
import com.googlesource.gerrit.plugins.healthcheck.check.GlobalHealthCheck;
import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheck;
import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheck.Result;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
import java.util.Map;
import javax.servlet.http.HttpServletResponse;
@@ -29,14 +34,21 @@
private final GlobalHealthCheck healthChecks;
+ private final String failedFileFlagPath;
+
@Inject
- public HealthCheckStatusEndpoint(GlobalHealthCheck healthChecks) {
+ public HealthCheckStatusEndpoint(GlobalHealthCheck healthChecks, HealthCheckConfig config) {
this.healthChecks = healthChecks;
+ this.failedFileFlagPath = config.getFailFileFlagPath();
}
@Override
public Response<Map<String, Object>> apply(ConfigResource resource)
throws AuthException, BadRequestException, ResourceConflictException, Exception {
+ if (failFlagFileExists()) {
+ return Response.withStatusCode(
+ HttpServletResponse.SC_INTERNAL_SERVER_ERROR, Map.of("reason", "Fail Flag File exists"));
+ }
HealthCheck.StatusSummary globalHealthCheckStatus = healthChecks.run();
Map<String, Object> result = globalHealthCheckStatus.subChecks;
@@ -50,4 +62,13 @@
? HttpServletResponse.SC_INTERNAL_SERVER_ERROR
: HttpServletResponse.SC_OK;
}
+
+ private boolean failFlagFileExists() throws IOException {
+ File file = new File(failedFileFlagPath);
+ try (InputStream targetStream = new FileInputStream(file)) {
+ return true;
+ } catch (Exception e) {
+ return false;
+ }
+ }
}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckIT.java b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckIT.java
index e9901a9..1cd37f4 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckIT.java
@@ -31,7 +31,9 @@
import com.google.gson.JsonObject;
import com.google.inject.Key;
import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames;
+import java.io.File;
import java.io.IOException;
+import javax.servlet.http.HttpServletResponse;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.junit.Before;
import org.junit.Test;
@@ -45,6 +47,7 @@
Gson gson = new Gson();
HealthCheckConfig config;
String healthCheckUriPath;
+ String failFilePath;
@Override
@Before
@@ -52,6 +55,9 @@
super.setUpTestPlugin();
config = plugin.getSysInjector().getInstance(HealthCheckConfig.class);
+ failFilePath = "/tmp/fail";
+ new File(failFilePath).delete();
+
int numChanges = config.getLimit(HealthCheckNames.QUERYCHANGES);
for (int i = 0; i < numChanges; i++) {
createChange("refs/for/master");
@@ -195,6 +201,25 @@
assertCheckResult(getResponseJson(resp), ACTIVEWORKERS, "passed");
}
+ @Test
+ public void shouldReturnFailedIfFailFlagFileExists() throws Exception {
+ setFailFlagFilePath(failFilePath);
+ createFailFileFlag(failFilePath);
+ getHealthCheckStatus();
+ RestResponse resp = getHealthCheckStatus();
+ resp.assertStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+
+ JsonObject respBody = getResponseJson(resp);
+ assertThat(respBody.has("reason")).isTrue();
+ assertThat(respBody.get("reason").getAsString()).isEqualTo("Fail Flag File exists");
+ }
+
+ private void createFailFileFlag(String path) throws IOException {
+ File file = new File(path);
+ file.createNewFile();
+ file.deleteOnExit();
+ }
+
private RestResponse getHealthCheckStatus() throws IOException {
return adminRestSession.get(healthCheckUriPath);
}
@@ -214,6 +239,10 @@
config.fromText(String.format("[healthcheck \"%s\"]\n" + "enabled = false", check));
}
+ private void setFailFlagFilePath(String path) throws ConfigInvalidException {
+ config.fromText(String.format("[healthcheck]\n" + "failFileFlagPath = %s", path));
+ }
+
private JsonObject getResponseJson(RestResponse resp) throws IOException {
JsonObject respPayload = gson.fromJson(resp.getReader(), JsonObject.class);
return respPayload;
