Use a multi-master safe persistent store ID Initialize the FsStore Id with a transaction. This is a first (little) step towards making the store MM safe. Change-Id: I3e4f1fac950ad6de43e7ff81e2b13b884d5c3557

diff --git a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FileValue.java b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FileValue.java
new file mode 100644
index 0000000..2572c31
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FileValue.java

@@ -0,0 +1,97 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.events.fsstore;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/** Helper file for serialzing and storing a single type in a file */
+public class FileValue<T> {
+  protected final Path path;
+  protected Serializer<T> serializer;
+
+  /**
+   * Use this constructor to use a builtin serializer (String or Long), and be sure to call init(T)
+   * with a value of your type for the serializer auto identification to happen.
+   */
+  public FileValue(Path path) {
+    this(path, (Serializer<T>) null);
+  }
+
+  public FileValue(Path path, Serializer<T> serializer) {
+    this.path = path;
+    this.serializer = serializer;
+  }
+
+  /** Should be called by subclasses when initializing their file to a value. */
+  protected void init(T init) {
+    initSerializer(init);
+  }
+
+  /**
+   * Auto setup the serializer based on the type used to initialize the class.
+   *
+   * <p>Must be called with a supported type before use if a serializer has been set manualy. Safe
+   * to call if the Serializer was already set.
+   */
+  protected void initSerializer(T init) {
+    if (serializer == null) {
+      if (init instanceof String) {
+        serializer = (Serializer<T>) new Serializer.String();
+      } else if (init instanceof Long) {
+        serializer = (Serializer<T>) new Serializer.Long();
+      }
+    }
+  }
+
+  /** get (read) the unserialized object from the file with retries for stale file handles (NFS). */
+  public T spinGet(long maxTries) throws IOException {
+    for (long i = 0; i < maxTries; i++) {
+      try {
+        return get();
+      } catch (IOException e) {
+        Nfs.throwIfNotStaleFileHandle(e);
+      }
+    }
+    throw new IOException(
+        "Cannot read file " + path + " after " + maxTries + " Stale file handle tries.");
+  }
+
+  /** get (read) the unserialized object from the file */
+  protected T get() throws IOException {
+    return serializer.fromString(read());
+  }
+
+  /** The lowest level raw String read of the file */
+  protected String read() throws IOException {
+    return Fs.readFile(path);
+  }
+
+  /** Serialize object to given tmp file in preparation to call update() */
+  protected void prepareT(Path tmp, T t) throws IOException {
+    prepare(tmp, serializer.fromGeneric(t));
+  }
+
+  /** Atmoically update (replace) file with src file. */
+  protected boolean update(Path src) throws IOException {
+    return Fs.tryAtomicMove(src, path);
+  }
+
+  /** Low level raw string write to given tmp file in preparation to call update(). */
+  protected static void prepare(Path tmp, String s) throws IOException {
+    Files.write(tmp, s.getBytes());
+  }
+}

diff --git a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/Fs.java b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/Fs.java
index 6c586b8..8697198 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/Fs.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/Fs.java

@@ -17,15 +17,29 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
+import java.nio.file.StandardCopyOption;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileTime;
+import java.util.concurrent.TimeUnit;
 
 /** Some Filesystem utilities */
 public class Fs {
+  /** Try to move a file/dir atomically. Do NOT throw IOExceptions. */
+  public static boolean tryAtomicMove(Path src, Path dst) {
+    try {
+      Files.move(src, dst, StandardCopyOption.ATOMIC_MOVE);
+      return true;
+    } catch (IOException e) {
+      return false;
+    }
+  }
+
   /** Try to recursively delete a dir. Do NOT throw IOExceptions. */
   public static void tryRecursiveDelete(Path dir) {
     try {
@@ -49,6 +63,36 @@
     }
   }
 
+  /**
+   * Try to recursively delete entries, up to max count, in a dir older than expiry. Do NOT throw
+   * IOExceptions.
+   *
+   * @return whether all entries were deleted
+   */
+  public static boolean tryRecursiveDeleteEntriesOlderThan(Path dir, FileTime expiry, int max) {
+    try (DirectoryStream<Path> dirEntries = Files.newDirectoryStream(dir)) {
+      for (Path path : dirEntries) {
+        if (isOlderThan(path, expiry)) {
+          if (max-- < 1) {
+            return false;
+          }
+          tryRecursiveDelete(path);
+        }
+      }
+    } catch (IOException e) {
+    }
+    return true;
+  }
+
+  /** Is an entry older than expiry. Do NOT throw IOExceptions. */
+  public static boolean isOlderThan(Path path, FileTime expiry) {
+    try {
+      return expiry.compareTo(Files.getLastModifiedTime(path)) > 0;
+    } catch (IOException e) {
+      return false;
+    }
+  }
+
   /** Try to delete a path. Do NOT throw IOExceptions. */
   public static void tryDelete(Path path) {
     try {
@@ -98,4 +142,20 @@
       return Files.createDirectories(Files.readSymbolicLink(path));
     }
   }
+
+  /** Get the reparented path as if a path were moved to a new location */
+  public static Path reparent(Path src, Path dst) {
+    return dst.resolve(basename(src));
+  }
+
+  /** Get the tail of a path (similar to the unix basename command) */
+  public static Path basename(Path p) {
+    return p.getName(p.getNameCount() - 1);
+  }
+
+  /** Get a FileTime indicating a certain amount of time beforehand (ago). */
+  public static FileTime getFileTimeAgo(long ago, TimeUnit unit) {
+    long ms = TimeUnit.MILLISECONDS.convert(ago, unit);
+    return FileTime.fromMillis(System.currentTimeMillis() - ms);
+  }
 }

diff --git a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsId.java b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsId.java
new file mode 100644
index 0000000..5658d79
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsId.java

@@ -0,0 +1,68 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.events.fsstore;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.UUID;
+
+/**
+ * Store a non changing String ID to a file.
+ *
+ * <p>This class is multi node/process (Multi-Master) safe. The ID will only ever get written once,
+ * the first writer will win.
+ */
+public class FsId extends FileValue<String> {
+  public static final Path VALUE = Paths.get("value");
+
+  public static class BasePaths extends FsTransaction.BasePaths {
+    public final Path valueDir;
+
+    public BasePaths(Path base) {
+      super(base);
+      valueDir = base.resolve(VALUE);
+    }
+  }
+
+  protected static class Builder extends FsTransaction.Builder {
+    public Builder(BasePaths paths, String value) throws IOException {
+      super(paths);
+      FileValue.prepare(dir.resolve(VALUE), value); // build/<tmp>/value(val)
+    }
+  }
+
+  protected final BasePaths paths;
+
+  public FsId(Path base) {
+    super(base.resolve(VALUE).resolve(VALUE)); // value/value(val)
+    this.paths = new BasePaths(base);
+  }
+
+  public void initFs() throws IOException {
+    initFs(UUID.randomUUID().toString());
+  }
+
+  public void initFs(String init) throws IOException {
+    super.init(init);
+    while (!Files.exists(path)) {
+      try (Builder b = new Builder(paths, init)) {
+        // mv build/<tmp>/value(val) value/value(val)
+        Fs.tryAtomicMove(b.dir, paths.valueDir);
+      }
+    }
+  }
+}

diff --git a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsStore.java b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsStore.java
index 25264a6..40e0401 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsStore.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsStore.java

@@ -54,20 +54,6 @@
     }
   }
 
-  public static class StringFsValue extends FsValue<String> {
-    public StringFsValue(Path path) {
-      super(path);
-    }
-
-    public String get() throws IOException {
-      return load();
-    }
-
-    public void set(String value) throws IOException {
-      store(value + "\n");
-    }
-  }
-
   public static class FsSequence extends FsValue<Long> {
     public FsSequence(Path path) {
       super(path);
@@ -114,18 +100,18 @@
   }
 
   protected static class Stores {
-    final StringFsValue uuid;
+    final FsId uuid;
     final FsSequence head;
     final FsSequence tail;
 
     public Stores(BasePaths bases) {
-      uuid = new StringFsValue(bases.uuid);
+      uuid = new FsId(bases.uuid);
       head = new FsSequence(bases.head);
       tail = new FsSequence(bases.tail);
     }
 
     public void initFs() throws IOException {
-      uuid.initFs(UUID.randomUUID().toString());
+      uuid.initFs();
       head.initFs((long) 0);
       tail.initFs((long) 1);
     }
@@ -137,7 +123,7 @@
 
   @Inject
   public FsStore(SitePaths site) throws IOException {
-    this(site.data_dir.toPath().resolve("plugin").resolve("events").resolve("fstore-v1"));
+    this(site.data_dir.toPath().resolve("plugin").resolve("events").resolve("fstore-v1.1"));
   }
 
   public FsStore(Path base) throws IOException {

diff --git a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsTransaction.java b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsTransaction.java
new file mode 100644
index 0000000..d2bf8cd
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsTransaction.java

@@ -0,0 +1,135 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.events.fsstore;
+
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.FileTime;
+import java.util.concurrent.TimeUnit;
+
+public class FsTransaction {
+  /**
+   * A class to keep track of scratch pads to safely build proposals, and to safely delete them
+   * during cleanup.
+   *
+   * <p>The first assumption is that unique dirs under the build dir will be used for building, and
+   * that these may be deleted at any time to keep the filesystem clean under the assumption that
+   * they may be stale. The contract however, is that all deleting must be done by first moving the
+   * toplevel dir to the delete directory. This ensures that the processes creating entries under
+   * the build dir will always have their entries intact or non-existing, but never partially what
+   * they expect.
+   *
+   * <p>The next assumption is that all entries under the build dir are not only safe to delete at
+   * any time, but that they should be deleted by helping processes to ensure that interrupted
+   * processes do not lead to entry build up in the filesystem.
+   */
+  public static class BasePaths {
+    public final Path base;
+    public final Path build;
+    public final Path delete;
+
+    // Stale entries should be designed to be rare, and only happen during
+    // unclean shutdowns.
+    public long cleanInterval = TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS);
+    public int maxDelete = 5; // keep low to not slowdown current update much.
+    protected long lastClean;
+
+    public BasePaths(Path base) {
+      this.base = base;
+      build = base.resolve("build");
+      delete = base.resolve("delete");
+    }
+
+    public void autoClean() {
+      if (needsClean()) {
+        FileTime expiry = Fs.getFileTimeAgo(1, TimeUnit.DAYS);
+        // maxDelete spreads a large cleaning burden over multiple updates
+        if (clean(expiry, maxDelete)) {
+          lastClean = System.currentTimeMillis();
+        }
+      }
+    }
+
+    /** Clean up to 'max' expired (presumably stale) entries */
+    public boolean clean(FileTime expiry, int max) {
+      try {
+        return Fs.tryRecursiveDeleteEntriesOlderThan(delete, expiry, max)
+            || renameAndDeleteEntriesOlderThan(build, delete, expiry, max);
+      } catch (IOException e) {
+        // If we knew if it was a repeat offender, we could consider logging it.
+        return true; // Don't keep retrying failures.
+      }
+    }
+
+    protected boolean needsClean() {
+      return System.currentTimeMillis() - cleanInterval > lastClean;
+    }
+  }
+
+  /** A tempdirectory builder that gets automatically cleaned up. */
+  protected static class Builder implements AutoCloseable {
+    public final BasePaths paths;
+    public final Path dir;
+
+    public Builder(BasePaths paths) throws IOException {
+      this.paths = paths;
+      Files.createDirectories(paths.build);
+      Files.createDirectories(paths.delete);
+      dir = Files.createTempDirectory(paths.build, null);
+    }
+
+    public void close() throws IOException {
+      FsTransaction.renameAndDelete(dir, paths.delete);
+      paths.autoClean();
+    }
+  }
+
+  /**
+   * Used to atomically delete a directory tree. Avoids name collisions with other processes
+   * potentially using the same source name directory. Collisions could prevent the move to the
+   * delete directory from succeeding.
+   */
+  public static void renameAndDelete(Path src, Path del) throws IOException {
+    if (Files.exists(src)) {
+      Path tmp = Files.createTempDirectory(del, null);
+      Path reparented = Fs.reparent(src, tmp);
+      Fs.tryAtomicMove(src, reparented);
+      Fs.tryRecursiveDelete(tmp);
+    }
+  }
+
+  /**
+   * Used to atomically delete entries in a directory tree older than expiry, up to max count. Do
+   * NOT throw IOExceptions.
+   *
+   * @return whether all entries were deleted
+   */
+  public static boolean renameAndDeleteEntriesOlderThan(
+      Path dir, Path del, FileTime expiry, int max) throws IOException {
+    try (DirectoryStream<Path> dirEntries = Files.newDirectoryStream(dir)) {
+      for (Path path : dirEntries) {
+        if (expiry.compareTo(Files.getLastModifiedTime(path)) > 0) {
+          if (max-- < 1) {
+            return false;
+          }
+          renameAndDelete(path, del);
+        }
+      }
+      return true;
+    }
+  }
+}

diff --git a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/Nfs.java b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/Nfs.java
new file mode 100644
index 0000000..99c33af
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/Nfs.java

@@ -0,0 +1,56 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.events.fsstore;
+
+import java.io.IOException;
+import java.util.Locale;
+
+/** Some NFS utilities */
+public class Nfs {
+  /**
+   * Determine if a throwable or a cause in its causal chain is a Stale NFS
+   * File Handle
+   *
+   * @param throwable
+   * @return a boolean true if the throwable or a cause in its causal chain is
+   *         a Stale NFS File Handle
+   */
+  public static boolean isStaleFileHandleInCausalChain(Throwable throwable) {
+    while (throwable != null) {
+      if (throwable instanceof IOException && isStaleFileHandle((IOException) throwable)) {
+        return true;
+      }
+      throwable = throwable.getCause();
+    }
+    return false;
+  }
+
+  /**
+   * Determine if an IOException is a Stale NFS File Handle
+   *
+   * @param ioe
+   * @return a boolean true if the IOException is a Stale NFS FIle Handle
+   */
+  public static boolean isStaleFileHandle(IOException ioe) {
+    String msg = ioe.getMessage();
+    return msg != null && msg.toLowerCase(Locale.ROOT).matches(".*stale .*file .*handle.*");
+  }
+
+  public static <T extends Throwable> void throwIfNotStaleFileHandle(T e) throws T {
+    if (!isStaleFileHandleInCausalChain(e)) {
+      throw e;
+    }
+  }
+}

diff --git a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/Serializer.java b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/Serializer.java
new file mode 100644
index 0000000..83864f2
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/Serializer.java

@@ -0,0 +1,48 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.events.fsstore;
+
+/** Simple serializers for writing data types to Strings. */
+public interface Serializer<G> {
+  public static class String implements Serializer<java.lang.String> {
+    @Override
+    public java.lang.String fromString(java.lang.String s) {
+      return s;
+    }
+
+    @Override
+    public java.lang.String fromGeneric(java.lang.String g) {
+      return g;
+    }
+  }
+
+  public static class Long implements Serializer<java.lang.Long> {
+    @Override
+    public java.lang.Long fromString(java.lang.String s) {
+      return s == null ? null : java.lang.Long.parseLong(s);
+    }
+
+    @Override
+    public java.lang.String fromGeneric(java.lang.Long g) {
+      return g == null ? null : java.lang.Long.toString(g) + "\n";
+    }
+  }
+
+  /* -----  Interface starts here ----- */
+
+  G fromString(java.lang.String s);
+
+  java.lang.String fromGeneric(G g);
+}

diff --git a/src/test/java/com/googlesource/gerrit/plugins/events/fsstore/FsIdTest.java b/src/test/java/com/googlesource/gerrit/plugins/events/fsstore/FsIdTest.java
new file mode 100644
index 0000000..41f2e92
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/events/fsstore/FsIdTest.java

@@ -0,0 +1,64 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.events.fsstore;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import junit.framework.TestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class FsIdTest extends TestCase {
+  private static String dir = "events-FsId";
+  private static Path base;
+  private FsId val;
+
+  @Override
+  @Before
+  public void setUp() throws IOException {
+    if (base == null) {
+      base = Files.createTempDirectory(dir);
+    }
+    val = new FsId(base);
+    val.initFs("init");
+  }
+
+  @After
+  public void tearDown() throws IOException {
+    Fs.tryRecursiveDelete(base);
+  }
+
+  @Test
+  public void testGetInit() throws IOException {
+    assertEquals("init", val.get());
+  }
+
+  @Test
+  public void testReInit() throws IOException {
+    val.initFs("init2");
+    assertEquals("init", val.get());
+  }
+
+  @Test
+  public void testGetInitUUID() throws IOException {
+    tearDown();
+    val.initFs();
+    String id = val.get();
+    val.initFs();
+    assertEquals(id, val.get());
+  }
+}

diff --git a/src/test/java/com/googlesource/gerrit/plugins/events/fsstore/FsStoreTest.java b/src/test/java/com/googlesource/gerrit/plugins/events/fsstore/FsStoreTest.java
index 8da8fba..ddb2c3e 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/events/fsstore/FsStoreTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/events/fsstore/FsStoreTest.java

@@ -150,7 +150,7 @@
     }
   }
 
-  public void verify(String id, long head) throws Exception {
+  public boolean verify(String id, long head) throws Exception {
     Set<Long> found = new HashSet<Long>();
     long stop = store.getHead();
     long mine = 1;
@@ -178,10 +178,13 @@
         report("missing", i, -1);
       }
     }
-    reportTotal("duplicate");
-    reportTotal("out of order");
-    reportTotal("missing");
-    reportTotal("gap");
+
+    boolean error = false;
+    error |= reportTotal("duplicate");
+    error |= reportTotal("out of order");
+    error |= reportTotal("missing");
+    error |= reportTotal("gap");
+    return error;
   }
 
   private void report(String type, long n, long i) {
@@ -193,11 +196,13 @@
     reported.put(type, ++cnt);
   }
 
-  private void reportTotal(String type) {
+  private boolean reportTotal(String type) {
     Long cnt = reported.get(type);
     if (cnt != null) {
       System.out.println("Total " + type + "s: " + cnt);
+      return true;
     }
+    return false;
   }
 
   /**
@@ -211,6 +216,10 @@
    * com.googlesource.gerrit.plugins.events.fsstore.FsStoreTest \ [dir [count [store-id]]]
    *
    * <p>Note: if you do not specify <dir>, it will create a directory under /tmp
+   *
+   * <p>Performance: NFS(Fast,Lowlatency,SSDs), 1 worker, count=1000, ~6s ~6ms/event
+   *
+   * <p>Local(spinning) 5 workers count=100K 11m38.512s find events|wc -l 2s rm -rf 10s
    */
   public static void main(String[] argv) throws Exception {
     if (argv.length > 0) {
@@ -231,6 +240,10 @@
     t.setUp();
     long head = t.store.getHead();
     t.count(id);
-    t.verify(id, head);
+    if (t.verify(id, head)) {
+      System.out.println("\nFAIL");
+      System.exit(1);
+    }
+    System.out.println("\nPASS");
   }
 }