Check for RequestCancelledException in throwable chain
RequestCancelledException may also be the cause of another
RuntimeException. For example ChangeIndexer#indexImpl(ChangeData)
catches all RuntimeExceptions, including RequestCancelledException, and
wraps them into a StorageException. If this happened request
cancellations were not properly recognized. With this change, we are now
catching any RuntimeException and inspect the causual chain to see if it
was caused by a request cancellation.
Signed-off-by: Edwin Kempin <ekempin@google.com>
Change-Id: Ieb9d4fef65709dc10d4f84f98f4c43319c9f6077
diff --git a/java/com/google/gerrit/httpd/restapi/RestApiServlet.java b/java/com/google/gerrit/httpd/restapi/RestApiServlet.java
index 7e6ab58..92e542a 100644
--- a/java/com/google/gerrit/httpd/restapi/RestApiServlet.java
+++ b/java/com/google/gerrit/httpd/restapi/RestApiServlet.java
@@ -712,50 +712,58 @@
messageOr(e, "Quota limit reached"),
e.caching(),
e);
- } catch (RequestCancelledException e) {
- cause = Optional.of(e);
- switch (e.getCancellationReason()) {
- case CLIENT_CLOSED_REQUEST:
- statusCode = SC_CLIENT_CLOSED_REQUEST;
- break;
- case CLIENT_PROVIDED_DEADLINE_EXCEEDED:
- case SERVER_DEADLINE_EXCEEDED:
- statusCode = SC_REQUEST_TIMEOUT;
- break;
- }
-
- StringBuilder msg = new StringBuilder(e.formatCancellationReason());
- if (e.getCancellationMessage().isPresent()) {
- msg.append("\n\n");
- msg.append(e.getCancellationMessage().get());
- }
-
- responseBytes = replyError(req, res, statusCode, msg.toString(), e);
} catch (Exception e) {
cause = Optional.of(e);
- statusCode = SC_INTERNAL_SERVER_ERROR;
- Optional<ExceptionHook.Status> status = getStatus(e);
- statusCode = status.map(ExceptionHook.Status::statusCode).orElse(SC_INTERNAL_SERVER_ERROR);
-
- if (res.isCommitted()) {
- responseBytes = 0;
- if (statusCode == SC_INTERNAL_SERVER_ERROR) {
- logger.atSevere().withCause(e).log(
- "Error in %s %s, response already committed", req.getMethod(), uriForLogging(req));
- } else {
- logger.atWarning().log(
- "Response for %s %s already committed, wanted to set status %d",
- req.getMethod(), uriForLogging(req), statusCode);
+ Optional<RequestCancelledException> requestCancelledException =
+ RequestCancelledException.getFromCausalChain(e);
+ if (requestCancelledException.isPresent()) {
+ switch (requestCancelledException.get().getCancellationReason()) {
+ case CLIENT_CLOSED_REQUEST:
+ statusCode = SC_CLIENT_CLOSED_REQUEST;
+ break;
+ case CLIENT_PROVIDED_DEADLINE_EXCEEDED:
+ case SERVER_DEADLINE_EXCEEDED:
+ statusCode = SC_REQUEST_TIMEOUT;
+ break;
}
- } else {
- res.reset();
- traceContext.getTraceId().ifPresent(traceId -> res.addHeader(X_GERRIT_TRACE, traceId));
- if (status.isPresent()) {
- responseBytes = reply(req, res, e, status.get(), getUserMessages(traceContext, e));
+ StringBuilder msg =
+ new StringBuilder(requestCancelledException.get().formatCancellationReason());
+ if (requestCancelledException.get().getCancellationMessage().isPresent()) {
+ msg.append("\n\n");
+ msg.append(requestCancelledException.get().getCancellationMessage().get());
+ }
+
+ responseBytes = replyError(req, res, statusCode, msg.toString(), e);
+ } else {
+ statusCode = SC_INTERNAL_SERVER_ERROR;
+
+ Optional<ExceptionHook.Status> status = getStatus(e);
+ statusCode =
+ status.map(ExceptionHook.Status::statusCode).orElse(SC_INTERNAL_SERVER_ERROR);
+
+ if (res.isCommitted()) {
+ responseBytes = 0;
+ if (statusCode == SC_INTERNAL_SERVER_ERROR) {
+ logger.atSevere().withCause(e).log(
+ "Error in %s %s, response already committed",
+ req.getMethod(), uriForLogging(req));
+ } else {
+ logger.atWarning().log(
+ "Response for %s %s already committed, wanted to set status %d",
+ req.getMethod(), uriForLogging(req), statusCode);
+ }
} else {
- responseBytes = replyInternalServerError(req, res, e, getUserMessages(traceContext, e));
+ res.reset();
+ traceContext.getTraceId().ifPresent(traceId -> res.addHeader(X_GERRIT_TRACE, traceId));
+
+ if (status.isPresent()) {
+ responseBytes = reply(req, res, e, status.get(), getUserMessages(traceContext, e));
+ } else {
+ responseBytes =
+ replyInternalServerError(req, res, e, getUserMessages(traceContext, e));
+ }
}
}
} finally {
diff --git a/java/com/google/gerrit/server/cancellation/RequestCancelledException.java b/java/com/google/gerrit/server/cancellation/RequestCancelledException.java
index 3c668fb..d89701f 100644
--- a/java/com/google/gerrit/server/cancellation/RequestCancelledException.java
+++ b/java/com/google/gerrit/server/cancellation/RequestCancelledException.java
@@ -14,6 +14,7 @@
package com.google.gerrit.server.cancellation;
+import com.google.common.base.Throwables;
import com.google.gerrit.common.Nullable;
import java.util.Optional;
import org.apache.commons.lang.WordUtils;
@@ -22,6 +23,17 @@
public class RequestCancelledException extends RuntimeException {
private static final long serialVersionUID = 1L;
+ /**
+ * Checks whether the given exception was caused by {@link RequestCancelledException}. If yes, the
+ * {@link RequestCancelledException} is returned. If not, {@link Optional#empty()} is returned.
+ */
+ public static Optional<RequestCancelledException> getFromCausalChain(Throwable e) {
+ return Throwables.getCausalChain(e).stream()
+ .filter(RequestCancelledException.class::isInstance)
+ .map(RequestCancelledException.class::cast)
+ .findFirst();
+ }
+
private final RequestStateProvider.Reason cancellationReason;
private final Optional<String> cancellationMessage;
diff --git a/java/com/google/gerrit/server/git/receive/ReceiveCommits.java b/java/com/google/gerrit/server/git/receive/ReceiveCommits.java
index d074f1e..29c2698 100644
--- a/java/com/google/gerrit/server/git/receive/ReceiveCommits.java
+++ b/java/com/google/gerrit/server/git/receive/ReceiveCommits.java
@@ -46,6 +46,7 @@
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
+import com.google.common.base.Throwables;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import com.google.common.collect.ImmutableList;
@@ -645,10 +646,18 @@
try {
processCommandsUnsafe(commands, progress);
rejectRemaining(commands, INTERNAL_SERVER_ERROR);
- } catch (RequestCancelledException e) {
- StringBuilder msg = new StringBuilder(e.formatCancellationReason());
- if (e.getCancellationMessage().isPresent()) {
- msg.append(String.format(" (%s)", e.getCancellationMessage().get()));
+ } catch (RuntimeException e) {
+ Optional<RequestCancelledException> requestCancelledException =
+ RequestCancelledException.getFromCausalChain(e);
+ if (!requestCancelledException.isPresent()) {
+ Throwables.throwIfUnchecked(e);
+ }
+ StringBuilder msg =
+ new StringBuilder(requestCancelledException.get().formatCancellationReason());
+ if (requestCancelledException.get().getCancellationMessage().isPresent()) {
+ msg.append(
+ String.format(
+ " (%s)", requestCancelledException.get().getCancellationMessage().get()));
}
rejectRemaining(commands, msg.toString());
}
diff --git a/java/com/google/gerrit/sshd/SshCommand.java b/java/com/google/gerrit/sshd/SshCommand.java
index 93c6c2c..e58040f 100644
--- a/java/com/google/gerrit/sshd/SshCommand.java
+++ b/java/com/google/gerrit/sshd/SshCommand.java
@@ -14,6 +14,7 @@
package com.google.gerrit.sshd;
+import com.google.common.base.Throwables;
import com.google.gerrit.extensions.registration.DynamicSet;
import com.google.gerrit.server.AccessPath;
import com.google.gerrit.server.DynamicOptions;
@@ -28,6 +29,7 @@
import com.google.inject.Inject;
import java.io.IOException;
import java.io.PrintWriter;
+import java.util.Optional;
import org.apache.sshd.server.Environment;
import org.apache.sshd.server.channel.ChannelSession;
import org.eclipse.jgit.lib.Config;
@@ -62,10 +64,18 @@
RequestInfo.builder(RequestInfo.RequestType.SSH, user, traceContext).build();
requestListeners.runEach(l -> l.onRequest(requestInfo));
SshCommand.this.run();
- } catch (RequestCancelledException e) {
- StringBuilder msg = new StringBuilder(e.formatCancellationReason());
- if (e.getCancellationMessage().isPresent()) {
- msg.append(String.format(" (%s)", e.getCancellationMessage().get()));
+ } catch (RuntimeException e) {
+ Optional<RequestCancelledException> requestCancelledException =
+ RequestCancelledException.getFromCausalChain(e);
+ if (!requestCancelledException.isPresent()) {
+ Throwables.throwIfUnchecked(e);
+ }
+ StringBuilder msg =
+ new StringBuilder(requestCancelledException.get().formatCancellationReason());
+ if (requestCancelledException.get().getCancellationMessage().isPresent()) {
+ msg.append(
+ String.format(
+ " (%s)", requestCancelledException.get().getCancellationMessage().get()));
}
stderr.println(msg.toString());
} finally {
diff --git a/javatests/com/google/gerrit/acceptance/rest/CancellationIT.java b/javatests/com/google/gerrit/acceptance/rest/CancellationIT.java
index 29d54cc..0713fe6 100644
--- a/javatests/com/google/gerrit/acceptance/rest/CancellationIT.java
+++ b/javatests/com/google/gerrit/acceptance/rest/CancellationIT.java
@@ -121,6 +121,27 @@
}
@Test
+ public void handleWrappedRequestCancelledException() throws Exception {
+ ProjectCreationValidationListener projectCreationListener =
+ new ProjectCreationValidationListener() {
+ @Override
+ public void validateNewProject(CreateProjectArgs args) throws ValidationException {
+ // Simulate an exceeded deadline by throwing RequestCancelledException.
+ throw new RuntimeException(
+ new RequestCancelledException(
+ RequestStateProvider.Reason.SERVER_DEADLINE_EXCEEDED, "deadline = 10m"));
+ }
+ };
+ try (Registration registration =
+ extensionRegistry.newRegistration().add(projectCreationListener)) {
+ RestResponse response = adminRestSession.put("/projects/" + name("new"));
+ assertThat(response.getStatusCode()).isEqualTo(SC_REQUEST_TIMEOUT);
+ assertThat(response.getEntityContent())
+ .isEqualTo("Server Deadline Exceeded\n\ndeadline = 10m");
+ }
+ }
+
+ @Test
public void handleClientDisconnectedForPush() throws Exception {
CommitValidationListener commitValidationListener =
new CommitValidationListener() {
@@ -185,6 +206,27 @@
}
@Test
+ public void handleWrappedRequestCancelledExceptionForPush() throws Exception {
+ CommitValidationListener commitValidationListener =
+ new CommitValidationListener() {
+ @Override
+ public List<CommitValidationMessage> onCommitReceived(CommitReceivedEvent receiveEvent)
+ throws CommitValidationException {
+ // Simulate an exceeded deadline by throwing RequestCancelledException.
+ throw new RuntimeException(
+ new RequestCancelledException(
+ RequestStateProvider.Reason.SERVER_DEADLINE_EXCEEDED, "deadline = 10m"));
+ }
+ };
+ try (Registration registration =
+ extensionRegistry.newRegistration().add(commitValidationListener)) {
+ PushOneCommit push = pushFactory.create(admin.newIdent(), testRepo);
+ PushOneCommit.Result r = push.to("refs/heads/master");
+ r.assertErrorStatus("Server Deadline Exceeded (deadline = 10m)");
+ }
+ }
+
+ @Test
public void handleRequestCancellationWithMessageForPush() throws Exception {
CommitValidationListener commitValidationListener =
new CommitValidationListener() {
diff --git a/javatests/com/google/gerrit/acceptance/ssh/SshCancellationIT.java b/javatests/com/google/gerrit/acceptance/ssh/SshCancellationIT.java
index 2cb9637..3e31c16 100644
--- a/javatests/com/google/gerrit/acceptance/ssh/SshCancellationIT.java
+++ b/javatests/com/google/gerrit/acceptance/ssh/SshCancellationIT.java
@@ -99,4 +99,22 @@
adminSshSession.assertFailure("Server Deadline Exceeded (deadline = 10m)");
}
}
+
+ @Test
+ public void handleWrappedRequestCancelledException() throws Exception {
+ ProjectCreationValidationListener projectCreationListener =
+ new ProjectCreationValidationListener() {
+ @Override
+ public void validateNewProject(CreateProjectArgs args) throws ValidationException {
+ throw new RuntimeException(
+ new RequestCancelledException(
+ RequestStateProvider.Reason.SERVER_DEADLINE_EXCEEDED, "deadline = 10m"));
+ }
+ };
+ try (Registration registration =
+ extensionRegistry.newRegistration().add(projectCreationListener)) {
+ adminSshSession.exec("gerrit create-project " + name("new"));
+ adminSshSession.assertFailure("Server Deadline Exceeded (deadline = 10m)");
+ }
+ }
}
