ReplicationTasksStorage: Handle DirectoryIteratorExceptions
These exceptions wrap IOExceptions for methods whose contract doesn't
allow them to throw an IOException. In situations where storage is NFS
based, it's common to have Stale NFS File Handle errors when iterating a
directory that can be moved and undesirable to log each time that
occurs. Reuse some code from JGit and the events plugin to check for NFS
SFH conditions.
Bug: Issue 12687
Change-Id: If833975cf12c195c8039f825e61bee13b6107c0d
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Nfs.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Nfs.java
new file mode 100644
index 0000000..a347f3a
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Nfs.java
@@ -0,0 +1,55 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import java.io.IOException;
+import java.util.Locale;
+
+/** Some NFS utilities */
+public class Nfs {
+ /**
+ * Determine if a throwable or a cause in its causal chain is a Stale NFS File Handle
+ *
+ * @param throwable
+ * @return a boolean true if the throwable or a cause in its causal chain is a Stale NFS File
+ * Handle
+ */
+ public static boolean isStaleFileHandleInCausalChain(Throwable throwable) {
+ while (throwable != null) {
+ if (throwable instanceof IOException && isStaleFileHandle((IOException) throwable)) {
+ return true;
+ }
+ throwable = throwable.getCause();
+ }
+ return false;
+ }
+
+ /**
+ * Determine if an IOException is a Stale NFS File Handle
+ *
+ * @param ioe
+ * @return a boolean true if the IOException is a Stale NFS FIle Handle
+ */
+ public static boolean isStaleFileHandle(IOException ioe) {
+ String msg = ioe.getMessage();
+ return msg != null && msg.toLowerCase(Locale.ROOT).matches(".*stale .*file .*handle.*");
+ }
+
+ public static <T extends Throwable> void throwIfNotStaleFileHandle(T e) throws T {
+ if (!isStaleFileHandleInCausalChain(e)) {
+ throw e;
+ }
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
index 3e6c4d4..8d4b10a 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
@@ -24,6 +24,7 @@
import com.google.inject.ProvisionException;
import com.google.inject.Singleton;
import java.io.IOException;
+import java.nio.file.DirectoryIteratorException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -158,7 +159,12 @@
String json = new String(Files.readAllBytes(e), UTF_8);
results.add(GSON.fromJson(json, ReplicateRefUpdate.class));
} else if (Files.isDirectory(e)) {
- results.addAll(list(e));
+ try {
+ results.addAll(list(e));
+ } catch (DirectoryIteratorException d) {
+ // iterating over the sub-directories is expected to have dirs disappear
+ Nfs.throwIfNotStaleFileHandle(d.getCause());
+ }
}
}
} catch (IOException e) {
