Add the ability to fail healthcheck by creating a fail file

This will enable the Gerrit Admin to cause healthcheck failures by
simply creating a file at a configurable location.
This is useful when a node needs to be taken  out of the pool of
available nodes, for example while it undergoes maintanence and should
not be serving requests.

This approach is instantaneous, unlike enabling/disabling the plugin as
Gerrit only unloads/reloads plugins once every 'plugins.frequency'.
The file approach gives more immediate feedback as the healthcheck
will start failing as soon as the file is created, even if we're
already processing the request.

Change-Id: I999ca049cff9213d3720a982530a5b03f2d02e44
diff --git a/README.md b/README.md
index 8be0b33..d273fc6 100644
--- a/README.md
+++ b/README.md
@@ -126,6 +126,25 @@
 }
 ```
 
+It's also possible to artificially make the healthcheck fail by placing a file
+at a configurable path specified like:
+
+```
+[healtcheck]
+  failFileFlaPath="data/healthcheck/fail"
+```
+
+This will make the healthcheck endpoint return 500 even if the node is otherwise
+healthy. This is useful when a node needs to be removed from the pool of
+available Gerrit instance while it undergoes maintenance.
+
+**NOTE**: If the path starts with `/` then even paths outside of Gerrit's home
+will be checked. If the path starts WITHOUT `/` then the path is relative to
+Gerrit's home.
+
+**NOTE**: The file needs to be a real file rather than a symlink.
+
+
 ## Metrics
 
 As for all other endpoints in Gerrit, some metrics are automatically emitted when the  `/config/server/healthcheck~status`
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java
index 651dbe9..1dc143c 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckConfig.java
@@ -47,6 +47,7 @@
   private static final int ACTIVE_WORKERS_THRESHOLD_DEFAULT = 80;
   private static final String USERNAME_DEFAULT = "healthcheck";
   private static final String PASSWORD_DEFAULT = "";
+  private static final String FAIL_FILE_FLAG_DEFAULT = "data/healthcheck/fail";
   private static final boolean HEALTH_CHECK_ENABLED_DEFAULT = true;
   private final AllProjectsName allProjectsName;
   private final AllUsersName allUsersName;
@@ -135,6 +136,10 @@
     return getStringWithFallback("password", healthCheckName, PASSWORD_DEFAULT);
   }
 
+  public String getFailFileFlagPath() {
+    return getStringWithFallback("failFileFlagPath", null, FAIL_FILE_FLAG_DEFAULT);
+  }
+
   public boolean healthCheckEnabled(String healthCheckName) {
     if (isReplica && HEALTH_CHECK_DISABLED_FOR_REPLICAS.contains(healthCheckName)) {
       return false;
diff --git a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/api/HealthCheckStatusEndpoint.java b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/api/HealthCheckStatusEndpoint.java
index 2d5f7de..6819ae1 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/healthcheck/api/HealthCheckStatusEndpoint.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/healthcheck/api/HealthCheckStatusEndpoint.java
@@ -18,9 +18,14 @@
 import com.google.gerrit.server.config.ConfigResource;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
+import com.googlesource.gerrit.plugins.healthcheck.HealthCheckConfig;
 import com.googlesource.gerrit.plugins.healthcheck.check.GlobalHealthCheck;
 import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheck;
 import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheck.Result;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.util.Map;
 import javax.servlet.http.HttpServletResponse;
 
@@ -29,14 +34,21 @@
 
   private final GlobalHealthCheck healthChecks;
 
+  private final String failedFileFlagPath;
+
   @Inject
-  public HealthCheckStatusEndpoint(GlobalHealthCheck healthChecks) {
+  public HealthCheckStatusEndpoint(GlobalHealthCheck healthChecks, HealthCheckConfig config) {
     this.healthChecks = healthChecks;
+    this.failedFileFlagPath = config.getFailFileFlagPath();
   }
 
   @Override
   public Response<Map<String, Object>> apply(ConfigResource resource)
       throws AuthException, BadRequestException, ResourceConflictException, Exception {
+    if (failFlagFileExists()) {
+      return Response.withStatusCode(
+          HttpServletResponse.SC_INTERNAL_SERVER_ERROR, Map.of("reason", "Fail Flag File exists"));
+    }
     HealthCheck.StatusSummary globalHealthCheckStatus = healthChecks.run();
 
     Map<String, Object> result = globalHealthCheckStatus.subChecks;
@@ -50,4 +62,13 @@
         ? HttpServletResponse.SC_INTERNAL_SERVER_ERROR
         : HttpServletResponse.SC_OK;
   }
+
+  private boolean failFlagFileExists() throws IOException {
+    File file = new File(failedFileFlagPath);
+    try (InputStream targetStream = new FileInputStream(file)) {
+      return true;
+    } catch (Exception e) {
+      return false;
+    }
+  }
 }
diff --git a/src/test/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckIT.java b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckIT.java
index e9901a9..1cd37f4 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckIT.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/healthcheck/HealthCheckIT.java
@@ -31,7 +31,9 @@
 import com.google.gson.JsonObject;
 import com.google.inject.Key;
 import com.googlesource.gerrit.plugins.healthcheck.check.HealthCheckNames;
+import java.io.File;
 import java.io.IOException;
+import javax.servlet.http.HttpServletResponse;
 import org.eclipse.jgit.errors.ConfigInvalidException;
 import org.junit.Before;
 import org.junit.Test;
@@ -45,6 +47,7 @@
   Gson gson = new Gson();
   HealthCheckConfig config;
   String healthCheckUriPath;
+  String failFilePath;
 
   @Override
   @Before
@@ -52,6 +55,9 @@
     super.setUpTestPlugin();
 
     config = plugin.getSysInjector().getInstance(HealthCheckConfig.class);
+    failFilePath = "/tmp/fail";
+    new File(failFilePath).delete();
+
     int numChanges = config.getLimit(HealthCheckNames.QUERYCHANGES);
     for (int i = 0; i < numChanges; i++) {
       createChange("refs/for/master");
@@ -195,6 +201,25 @@
     assertCheckResult(getResponseJson(resp), ACTIVEWORKERS, "passed");
   }
 
+  @Test
+  public void shouldReturnFailedIfFailFlagFileExists() throws Exception {
+    setFailFlagFilePath(failFilePath);
+    createFailFileFlag(failFilePath);
+    getHealthCheckStatus();
+    RestResponse resp = getHealthCheckStatus();
+    resp.assertStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+
+    JsonObject respBody = getResponseJson(resp);
+    assertThat(respBody.has("reason")).isTrue();
+    assertThat(respBody.get("reason").getAsString()).isEqualTo("Fail Flag File exists");
+  }
+
+  private void createFailFileFlag(String path) throws IOException {
+    File file = new File(path);
+    file.createNewFile();
+    file.deleteOnExit();
+  }
+
   private RestResponse getHealthCheckStatus() throws IOException {
     return adminRestSession.get(healthCheckUriPath);
   }
@@ -214,6 +239,10 @@
     config.fromText(String.format("[healthcheck \"%s\"]\n" + "enabled = false", check));
   }
 
+  private void setFailFlagFilePath(String path) throws ConfigInvalidException {
+    config.fromText(String.format("[healthcheck]\n" + "failFileFlagPath = %s", path));
+  }
+
   private JsonObject getResponseJson(RestResponse resp) throws IOException {
     JsonObject respPayload = gson.fromJson(resp.getReader(), JsonObject.class);
     return respPayload;