Merge branch 'stable-3.1'

* stable-3.1:
  ReplicationTasksStorage: Handle DirectoryIteratorExceptions

Change-Id: Idcb01987a8f3226d971da82fd530bf09ca8ccfa7
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/Nfs.java b/src/main/java/com/googlesource/gerrit/plugins/replication/Nfs.java
new file mode 100644
index 0000000..a347f3a
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/Nfs.java
@@ -0,0 +1,55 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.replication;
+
+import java.io.IOException;
+import java.util.Locale;
+
+/** Some NFS utilities */
+public class Nfs {
+  /**
+   * Determine if a throwable or a cause in its causal chain is a Stale NFS File Handle
+   *
+   * @param throwable
+   * @return a boolean true if the throwable or a cause in its causal chain is a Stale NFS File
+   *     Handle
+   */
+  public static boolean isStaleFileHandleInCausalChain(Throwable throwable) {
+    while (throwable != null) {
+      if (throwable instanceof IOException && isStaleFileHandle((IOException) throwable)) {
+        return true;
+      }
+      throwable = throwable.getCause();
+    }
+    return false;
+  }
+
+  /**
+   * Determine if an IOException is a Stale NFS File Handle
+   *
+   * @param ioe
+   * @return a boolean true if the IOException is a Stale NFS FIle Handle
+   */
+  public static boolean isStaleFileHandle(IOException ioe) {
+    String msg = ioe.getMessage();
+    return msg != null && msg.toLowerCase(Locale.ROOT).matches(".*stale .*file .*handle.*");
+  }
+
+  public static <T extends Throwable> void throwIfNotStaleFileHandle(T e) throws T {
+    if (!isStaleFileHandleInCausalChain(e)) {
+      throw e;
+    }
+  }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
index 554f8bb..c764161 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/replication/ReplicationTasksStorage.java
@@ -24,6 +24,7 @@
 import com.google.inject.ProvisionException;
 import com.google.inject.Singleton;
 import java.io.IOException;
+import java.nio.file.DirectoryIteratorException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
@@ -221,7 +222,12 @@
           String json = new String(Files.readAllBytes(e), UTF_8);
           results.add(GSON.fromJson(json, ReplicateRefUpdate.class));
         } else if (Files.isDirectory(e)) {
-          results.addAll(list(e));
+          try {
+            results.addAll(list(e));
+          } catch (DirectoryIteratorException d) {
+            // iterating over the sub-directories is expected to have dirs disappear
+            Nfs.throwIfNotStaleFileHandle(d.getCause());
+          }
         }
       }
     } catch (IOException e) {