ReceiveCommits: Increment reject count metric for exceptions
Errors that are happening in the Git layer when updating refs are
causing exceptions (e.g. GitUpdateFailureException for us at Google).
insertChangesAndPatchSets catches the exceptions and wraps them first in
a RestApiException, then in a StorageException. StorageException is a
RuntimeException, hence to increase the reject count metric we catch
RuntimeExceptions in processCommands and then we rethrow the exception.
To format the cause we use ExceptionHook, the same as we do in
RetryHelper and RestApiServlet.
Bug: Google b/151127672
Release-Notes: skip
Change-Id: Iaba330ae9614198a879371b1f6fc378424a6b310
Signed-off-by: Edwin Kempin <ekempin@google.com>
diff --git a/Documentation/metrics.txt b/Documentation/metrics.txt
index e7e09e6..3b56391 100644
--- a/Documentation/metrics.txt
+++ b/Documentation/metrics.txt
@@ -100,7 +100,8 @@
DELETE).
* `receivecommits/reject_count`: number of rejected pushes
** `kind`:
- The push kind (magic or direct).
+ The push kind ('magic' if it was a push for code review, 'direct' if it was
+ a direct push or 'n/a' if the push kind couldn't be detected).
** `reason`:
The rejection reason.
** `status`:
diff --git a/java/com/google/gerrit/server/git/receive/ReceiveCommits.java b/java/com/google/gerrit/server/git/receive/ReceiveCommits.java
index 040e316..6b7f1f0 100644
--- a/java/com/google/gerrit/server/git/receive/ReceiveCommits.java
+++ b/java/com/google/gerrit/server/git/receive/ReceiveCommits.java
@@ -38,6 +38,7 @@
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.joining;
import static java.util.stream.Collectors.toList;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
import static org.eclipse.jgit.lib.Constants.R_HEADS;
import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED;
import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK;
@@ -112,6 +113,7 @@
import com.google.gerrit.server.CreateGroupPermissionSyncer;
import com.google.gerrit.server.DeadlineChecker;
import com.google.gerrit.server.DraftCommentsReader;
+import com.google.gerrit.server.ExceptionHook;
import com.google.gerrit.server.IdentifiedUser;
import com.google.gerrit.server.InvalidDeadlineException;
import com.google.gerrit.server.PatchSetUtil;
@@ -387,6 +389,7 @@
private final BatchUpdates batchUpdates;
private final CancellationMetrics cancellationMetrics;
private final ChangeEditUtil editUtil;
+ private final PluginSetContext<ExceptionHook> exceptionHooks;
private final ChangeIndexer indexer;
private final ChangeInserter.Factory changeInserterFactory;
private final ChangeNotes.Factory notesFactory;
@@ -478,6 +481,7 @@
ProjectConfig.Factory projectConfigFactory,
@GerritServerConfig Config config,
ChangeEditUtil editUtil,
+ PluginSetContext<ExceptionHook> exceptionHooks,
ChangeIndexer indexer,
ChangeInserter.Factory changeInserterFactory,
ChangeNotes.Factory notesFactory,
@@ -545,6 +549,7 @@
this.deadlineCheckerFactory = deadlineCheckerFactory;
this.diffOperationsForCommitValidationFactory = diffOperationsForCommitValidationFactory;
this.editUtil = editUtil;
+ this.exceptionHooks = exceptionHooks;
this.hashtagsFactory = hashtagsFactory;
this.setTopicFactory = setTopicFactory;
this.indexer = indexer;
@@ -758,11 +763,24 @@
commandProgress.end();
loggingTags = traceContext.getTags();
logger.atFine().log("Processing commands done.");
+ } catch (RuntimeException e) {
+ String formattedCause = getFormattedCause(e).orElse(e.getClass().getSimpleName());
+ logger.atFine().withCause(e).log("ReceiveCommits failed due to %s", formattedCause);
+ metrics.rejectCount.increment("n/a", formattedCause, SC_INTERNAL_SERVER_ERROR);
+ throw e;
}
progress.end();
return result.build();
}
+ private Optional<String> getFormattedCause(Throwable t) {
+ return exceptionHooks.stream()
+ .map(h -> h.formatCause(t))
+ .filter(Optional::isPresent)
+ .map(Optional::get)
+ .findFirst();
+ }
+
// Process as many commands as possible, but may leave some commands in state NOT_ATTEMPTED.
private void processCommandsUnsafe(
Collection<ReceiveCommand> commands, MultiProgressMonitor progress) {
