Use a multi-master safes persistent sequence
Use the new FsSequence for both the tail and head making them MM safe.
The event storage still needs to be synchronized with the head sequence
for it to be MM safe. This is a big step towards making the store fully
MM safe.
Change-Id: I95b378c799bdb698d865cf17c6de7709e9a79760
diff --git a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/Fs.java b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/Fs.java
index 8697198..4b162ce 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/Fs.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/Fs.java
@@ -21,15 +21,27 @@
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
+import java.nio.file.NotDirectoryException;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileTime;
+import java.util.Iterator;
import java.util.concurrent.TimeUnit;
/** Some Filesystem utilities */
public class Fs {
+ /** Try to create a link. Do NOT throw IOExceptions. */
+ public static boolean tryCreateLink(Path link, Path existing) {
+ try {
+ Files.createLink(link, existing);
+ return true;
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
/** Try to move a file/dir atomically. Do NOT throw IOExceptions. */
public static boolean tryAtomicMove(Path src, Path dst) {
try {
@@ -41,7 +53,7 @@
}
/** Try to recursively delete a dir. Do NOT throw IOExceptions. */
- public static void tryRecursiveDelete(Path dir) {
+ public static boolean tryRecursiveDelete(Path dir) {
try {
Files.walkFileTree(
dir,
@@ -61,6 +73,7 @@
});
} catch (IOException e) { // Intent of 'try' function is to ignore these.
}
+ return !Files.exists(dir);
}
/**
@@ -79,7 +92,25 @@
tryRecursiveDelete(path);
}
}
+ } catch (IOException e) { // Intent of 'try' function is to ignore these.
+ }
+ return true;
+ }
+
+ /** Are all entries in a directory tree older than expiry? Do NOT throw IOExceptions. */
+ public static boolean isAllEntriesOlderThan(Path dir, FileTime expiry) {
+ if (!isOlderThan(dir, expiry)) {
+ return false;
+ }
+ try (DirectoryStream<Path> dirEntries = Files.newDirectoryStream(dir)) {
+ for (Path path : dirEntries) {
+ if (!isAllEntriesOlderThan(path, expiry)) {
+ return false;
+ }
+ }
+ } catch (NotDirectoryException e) { // can't recurse if not a directory
} catch (IOException e) {
+ return false; // Modified after start, so not older
}
return true;
}
@@ -131,6 +162,14 @@
return buffer.toString();
}
+ /** Get the first entry in a directory. */
+ public static Path getFirstDirEntry(Path dir) throws IOException {
+ try (DirectoryStream<Path> dirEntries = Files.newDirectoryStream(dir)) {
+ Iterator<Path> it = dirEntries.iterator();
+ return it.hasNext() ? it.next() : null;
+ }
+ }
+
/**
* A drop in replacement for Files.createDirectories that works even when the tail of `path` is a
* link.
diff --git a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsSequence.java b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsSequence.java
new file mode 100644
index 0000000..76d4046
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsSequence.java
@@ -0,0 +1,109 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.events.fsstore;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+/**
+ * Use a file to store a sequence in a multi node/process (Multi-Master) safe way.
+ *
+ * <p>Any actor may perform any/all of the 6 UpdatableFileValue transaction phases. The actor
+ * performing the commit will be considered to have performed the increment to the new sequence
+ * value.
+ */
+public class FsSequence extends UpdatableFileValue<Long> {
+ protected class UniqueUpdate extends UpdatableFileValue.UniqueUpdate<Long> {
+ UniqueUpdate(String uuid, boolean ours, long maxTries) throws IOException {
+ super(FsSequence.this, uuid, ours, maxTries);
+ spinFinish();
+ }
+ }
+
+ public long totalSpins;
+ public long totalUpdates;
+
+ public FsSequence(Path base) {
+ super(base);
+ }
+
+ public void initFs() throws IOException {
+ initFs((long) 0);
+ }
+
+ /**
+ * Attempt up to maxTries to increment the sequence
+ *
+ * @param maxTries How many times to attempt to increment the sequence
+ * @return the new sequence value after this increment.
+ */
+ public long spinIncrement(long maxTries) throws IOException {
+ long tries = 0;
+ for (; tries < maxTries; tries++) {
+ try (UpdateBuilder b = new UpdateBuilder(paths)) {
+ for (; tries < maxTries; tries++) {
+ UniqueUpdate update = null;
+ if (Fs.tryAtomicMove(b.dir, paths.update)) { // build/<tmp>/ -> update/
+ update = new UniqueUpdate(b.uuid, true, maxTries);
+ // update/<uuid>/
+ } else {
+ update = (UniqueUpdate) completeOngoing(maxTries);
+ }
+ if (update != null) {
+ tries += update.tries - 1;
+ if (update.myCommit) {
+ spun(tries);
+ return update.next;
+ }
+ if (update.ours && update.finished) {
+ break; // usurped -> outer loop can make a new transaction
+ }
+ }
+ }
+ }
+ }
+ spun(tries - 1);
+ throw new IOException(
+ "Cannot increment sequence file " + path + " after " + maxTries + " tries.");
+ }
+
+ /** Do NOT spin on this, it creates a new transaction every time. */
+ protected Long increment() throws IOException {
+ try (UpdateBuilder b = new UpdateBuilder(paths)) {
+ if (Fs.tryAtomicMove(b.dir, paths.update)) { // build/<tmp>/ -> update/
+ // update/<uuid>/
+ UniqueUpdate update = new UniqueUpdate(b.uuid, true, 1);
+ if (update.myCommit) {
+ return update.next;
+ }
+ }
+ }
+ return null;
+ }
+
+ protected synchronized void spun(long spins) {
+ totalUpdates++;
+ totalSpins += spins;
+ }
+
+ protected Long getToValue(Long currentValue) {
+ return currentValue + 1;
+ }
+
+ protected UniqueUpdate createUniqueUpdate(String uuid, boolean ours, long maxTries)
+ throws IOException {
+ return new UniqueUpdate(uuid, ours, maxTries);
+ }
+}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsStore.java b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsStore.java
index 40e0401..01488f4 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsStore.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsStore.java
@@ -27,63 +27,13 @@
/** This class is only Thread, but not process (Multi-Master) safe */
@Singleton
public class FsStore implements EventStore {
- public abstract static class FsValue<T> {
- protected final Path path;
-
- public FsValue(Path path) {
- this.path = path;
- }
-
- /** Only Thread safe, but not process (Multi-Master) safe */
- public synchronized void initFs(T value) throws IOException {
- if (!Files.isRegularFile(path)) {
- Fs.createDirectories(path.getParent());
- set(value);
- }
- }
-
- protected abstract void set(T value) throws IOException;
-
- protected synchronized String load() throws IOException {
- return Fs.readFile(path);
- }
-
- /** Only Thread, but not process (Multi-Master) safe */
- protected synchronized void store(String value) throws IOException {
- Files.write(path, value.getBytes());
- }
- }
-
- public static class FsSequence extends FsValue<Long> {
- public FsSequence(Path path) {
- super(path);
- }
-
- public Long get() throws IOException {
- return Long.parseLong(load());
- }
-
- protected void set(Long value) throws IOException {
- store("" + value + "\n");
- }
-
- /** Only Thread safe, but not process (Multi-Master) safe */
- public synchronized Long increment() throws IOException {
- Long next = get() + 1;
- set(next);
- return next;
- }
- }
-
protected static class BasePaths {
- final Path base;
final Path uuid;
final Path head;
final Path tail;
final DynamicRangeSharder events;
public BasePaths(Path base) {
- this.base = base;
uuid = base.resolve("uuid");
events = new DynamicRangeSharder(base.resolve("events"));
head = base.resolve("head");
@@ -117,13 +67,16 @@
}
}
+ public static final long MAX_GET_SPINS = 1000;
+ public static final long MAX_INCREMENT_SPINS = 1000;
+
protected final BasePaths paths;
protected final Stores stores;
protected final UUID uuid;
@Inject
public FsStore(SitePaths site) throws IOException {
- this(site.data_dir.toPath().resolve("plugin").resolve("events").resolve("fstore-v1.1"));
+ this(site.data_dir.toPath().resolve("plugin").resolve("events").resolve("fstore-v1.2"));
}
public FsStore(Path base) throws IOException {
@@ -145,12 +98,12 @@
Path epath = paths.event(next);
Fs.createDirectories(epath.getParent());
Files.write(epath, (event + "\n").getBytes());
- stores.head.increment();
+ stores.head.spinIncrement(MAX_INCREMENT_SPINS);
}
@Override
public long getHead() throws IOException {
- return stores.head.get();
+ return stores.head.spinGet(MAX_GET_SPINS);
}
@Override
@@ -169,22 +122,22 @@
if (getHead() == 0) {
return 0;
}
- long tail = stores.tail.get();
+ long tail = stores.tail.spinGet(MAX_GET_SPINS);
return tail < 1 ? 1 : tail;
}
@Override
- public synchronized void trim(long trim) throws IOException {
+ public void trim(long trim) throws IOException {
long head = getHead();
if (trim >= head) {
trim = head - 1;
}
if (trim > 0) {
for (long i = getTail(); i <= trim; i++) {
- stores.tail.increment();
- Path event = paths.event(i);
+ long delete = stores.tail.spinIncrement(MAX_INCREMENT_SPINS) - 1;
+ Path event = paths.event(delete);
Fs.tryRecursiveDelete(event);
- if (paths.isEventLastDirEntry(i)) {
+ if (paths.isEventLastDirEntry(delete)) {
Fs.unsafeRecursiveRmdir(event.getParent().toFile());
}
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsTransaction.java b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsTransaction.java
index d2bf8cd..6884daa 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsTransaction.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/FsTransaction.java
@@ -113,6 +113,16 @@
}
/**
+ * Used to atomically delete a directory tree when the src directory name is guaranteed to be
+ * unique.
+ */
+ public static void renameAndDeleteUnique(Path src, Path del) throws IOException {
+ Path reparented = Fs.reparent(src, del);
+ Fs.tryAtomicMove(src, reparented);
+ Fs.tryRecursiveDelete(reparented);
+ }
+
+ /**
* Used to atomically delete entries in a directory tree older than expiry, up to max count. Do
* NOT throw IOExceptions.
*
diff --git a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/Nfs.java b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/Nfs.java
index 99c33af..aacb3a9 100644
--- a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/Nfs.java
+++ b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/Nfs.java
@@ -15,6 +15,9 @@
package com.googlesource.gerrit.plugins.events.fsstore;
import java.io.IOException;
+import java.nio.file.DirectoryIteratorException;
+import java.nio.file.Path;
+import java.nio.file.attribute.FileTime;
import java.util.Locale;
/** Some NFS utilities */
@@ -53,4 +56,28 @@
throw e;
}
}
+
+ /**
+ * Is any entry in a directory tree older than expiry. Do NOT throw IOExceptions, or
+ * DirectoryIteratorExceptions.
+ */
+ public static boolean isAllEntriesOlderThan(Path dir, FileTime expiry) {
+ try {
+ return Fs.isAllEntriesOlderThan(dir, expiry);
+ } catch (DirectoryIteratorException e) {
+ return false; // Modified after start, so not older
+ }
+ }
+
+ /** Get the first entry in a directory. */
+ public static Path getFirstDirEntry(Path dir) throws IOException {
+ try {
+ return Fs.getFirstDirEntry(dir);
+ } catch (DirectoryIteratorException e) {
+ throwIfNotStaleFileHandle(e);
+ } catch (IOException e) {
+ throwIfNotStaleFileHandle(e);
+ }
+ return null;
+ }
}
diff --git a/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/UpdatableFileValue.java b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/UpdatableFileValue.java
new file mode 100644
index 0000000..6f6f922
--- /dev/null
+++ b/src/main/java/com/googlesource/gerrit/plugins/events/fsstore/UpdatableFileValue.java
@@ -0,0 +1,282 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.events.fsstore;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.FileTime;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * An infrastructure to create updates to a FileValue using a 6 phase transaction which is multi
+ * node/process (Multi-Master) safe. The 6 phases of the transaction are:
+ *
+ * <p>1) Intiate a unique ownerless transaction, locking and preventing the value from changing
+ * while the transaction is still open. 2) Read the current locked file value 3) Create a proposed
+ * update file containing the new proposed value based on the value read in phase 2. 4) Close the
+ * transaction with the new proposed value, locking in the new value in the transaction. 5)
+ * Atomically commit the new value to the locked file from the transaction. 6) Clean up the
+ * transaction
+ *
+ * <p>Any actor may perform any/all of the above phases.
+ */
+public abstract class UpdatableFileValue<T> extends FileValue<T> {
+ public static final Path CLOSED = Paths.get("closed");
+ public static final Path INIT = Paths.get("init");
+ public static final Path PRESERVED = Paths.get("preserved");
+ public static final Path UPDATE = Paths.get("update");
+ public static final Path VALUE = Paths.get("value");
+
+ public static class BasePaths extends FsTransaction.BasePaths {
+ public final Path update;
+ public final Path preserved;
+
+ public BasePaths(Path base) {
+ super(base);
+ update = base.resolve(UPDATE);
+ preserved = base.resolve(PRESERVED);
+ }
+ }
+
+ protected static class UpdateBuilder extends FsTransaction.Builder {
+ String uuid = UUID.randomUUID().toString();
+ Path udir = dir.resolve(uuid);
+
+ public UpdateBuilder(BasePaths paths) throws IOException {
+ super(paths);
+ Files.createDirectories(udir);
+ // build/<tmp>/uuid/
+ }
+ }
+
+ protected static class NextBuilder extends FsTransaction.Builder {
+ public NextBuilder(BasePaths paths, String next) throws IOException {
+ super(paths);
+ Path closed = dir.resolve(CLOSED);
+ Files.createDirectory(closed);
+ FileValue.prepare(closed.resolve(VALUE), next);
+ // build/<tmp>/closed/value(next)
+ }
+ }
+
+ protected static class InitBuilder extends FsTransaction.Builder {
+ public InitBuilder(BasePaths paths, String init) throws IOException {
+ super(paths);
+ FileValue.prepare(dir.resolve(INIT), init);
+ // build/<tmp>/init(init)
+ }
+ }
+
+ protected static class UpdatePaths {
+ public final Path udir;
+ public final Path closed;
+ public final Path value;
+
+ UpdatePaths(Path base, String uuid) {
+ udir = base.resolve(uuid);
+ closed = udir.resolve(CLOSED);
+ value = closed.resolve(VALUE);
+ }
+ }
+
+ /** This helper class may only be used by one thread. */
+ protected static class UniqueUpdate<T> {
+ final UpdatableFileValue<T> updatable;
+ final String uuid;
+ final UpdatePaths upaths;
+ final boolean ours;
+ final T currentValue;
+ final T next;
+
+ long maxTries;
+
+ long tries;
+ boolean closed;
+ boolean preserved;
+ boolean committed;
+ boolean finished;
+
+ boolean myCommit;
+
+ UniqueUpdate(UpdatableFileValue<T> updatable, String uuid, boolean ours, long maxTries)
+ throws IOException {
+ this.updatable = updatable;
+ this.uuid = uuid;
+ this.ours = ours;
+ this.maxTries = maxTries;
+
+ upaths = new UpdatePaths(updatable.paths.update, uuid);
+
+ currentValue = spinGet();
+ next = currentValue == null ? null : updatable.getToValue(currentValue);
+ }
+
+ protected void spinFinish() throws IOException {
+ for (; tries < maxTries && !finished; tries++) {
+ finish();
+ }
+ }
+
+ protected void finish() throws IOException {
+ createAndProposeNext();
+ commit();
+ clean();
+ }
+
+ protected T spinGet() throws IOException {
+ IOException ioe = new IOException("No chance to read " + updatable.path);
+ for (; tries < maxTries; tries++) {
+ try {
+ return updatable.get();
+ } catch (IOException e) {
+ Nfs.throwIfNotStaleFileHandle(e);
+ finished = !Files.exists(upaths.udir);
+ if (finished) {
+ // stale handle must have been caused by completion by another
+ return null;
+ }
+ ioe = e;
+ }
+ }
+ throw ioe;
+ }
+
+ protected void createAndProposeNext() throws IOException {
+ if (!closed && !ours) {
+ // In the default fast path (!closed && ours), we would not expect
+ // it to be closed, so skip this check to get to the building faster.
+ // Conversely, if not ours, a quick check here might allow us
+ // to skip the slow building phase
+ closed = Files.exists(upaths.closed);
+ }
+ if (!closed) {
+ try (NextBuilder b =
+ new NextBuilder(updatable.paths, updatable.serializer.fromGeneric(next))) {
+ // build/<tmp>/ -> update/<uuid>/
+ Fs.tryAtomicMove(b.dir, upaths.udir);
+ // update/<uuid>/closed/value(next)
+ }
+
+ // Do not use the result of the move to determine if it is closed.
+ // The move result could provide false positives (a second move
+ // could succeed after the transaction has been finished and the
+ // first "closed" has been deleted under the "delete" dir).
+ // Additionally, this check allows us to be able to detect closes
+ // by other actors, not just ourselves.
+ closed = Files.exists(upaths.closed);
+ }
+ }
+
+ protected void commit() throws IOException {
+ if (!committed) {
+ // Safe to perform this block (for performance reasons) even if we
+ // have not detected "closed yet", since it can only actually succeed
+ // when closed (operations depend on "closed" in paths).
+ perserve();
+
+ // mv update/<uuid>/closed/value(next) -> value
+ committed = myCommit = Fs.tryAtomicMove(upaths.value, updatable.path);
+ }
+ if (!committed && closed) {
+ committed = !Files.exists(upaths.value);
+ }
+ }
+
+ protected void clean() throws IOException {
+ if (committed) {
+ FsTransaction.renameAndDeleteUnique(upaths.udir, updatable.paths.delete);
+ updatable.cleanPreserved();
+ }
+ finished = !Files.exists(upaths.udir);
+ }
+
+ /**
+ * Creating an extra hard link to future "value" files keeps a filesystem reference to them
+ * after the "value" file is replaced with a new "value" file. Keeping the reference around
+ * allows readers on other nodes to still read the contents of the file without experiencing a
+ * stale file handle exception over NFS. This can reduce the amount of spinning required for
+ * readers.
+ */
+ protected void perserve() {
+ if (!preserved) {
+ preserved = Fs.tryCreateLink(updatable.paths.preserved.resolve(uuid), upaths.value);
+ }
+ }
+ }
+
+ protected final BasePaths paths;
+
+ public UpdatableFileValue(Path base) {
+ super(base.resolve(VALUE)); // value(val)
+ this.paths = new BasePaths(base);
+ }
+
+ public void initFs(T init) throws IOException {
+ super.init(init);
+ Files.createDirectories(paths.preserved);
+ while (!Files.exists(path)) {
+ try (InitBuilder b = new InitBuilder(paths, serializer.fromGeneric(init))) {
+ Fs.tryAtomicMove(b.dir, paths.update); // mv build/<tmp>/ -> update/
+ // update/init(init) using a non unique name, "init", to allow recovery
+ if (!Files.exists(path)) {
+ // mv update/init(init) -> value
+ Fs.tryAtomicMove(paths.update.resolve(INIT), path);
+ }
+ }
+ }
+ Fs.tryDelete(paths.update.resolve(INIT)); // cleanup
+ }
+
+ protected abstract T getToValue(T currentValue);
+
+ protected UniqueUpdate<T> completeOngoing(long maxTries) throws IOException {
+ if (shouldCompleteOngoing()) {
+ Path ongoing = Nfs.getFirstDirEntry(paths.update);
+ if (ongoing != null) {
+ // Attempt to complete previous updates;
+ return createUniqueUpdate(Fs.basename(ongoing).toString(), false, maxTries);
+ }
+ }
+ return null;
+ }
+
+ protected boolean shouldCompleteOngoing() {
+ // Collisions are expected, and we don't actually want to
+ // complete them too often since it affects fairness
+ // by potentially preventing slower actors from ever
+ // committing. We do however need to prevent deadlock from
+ // a stale proposal, so we do need to complete proposals
+ // that stay around too long.
+
+ // Maximum delay incurred due to a server crash.
+ FileTime expiry = Fs.getFileTimeAgo(10, TimeUnit.SECONDS);
+ return Fs.isAllEntriesOlderThan(paths.update, expiry);
+ }
+
+ protected abstract UniqueUpdate<T> createUniqueUpdate(String uuid, boolean ours, long maxTries)
+ throws IOException;
+
+ /**
+ * 1 second seems to be long enough even for slow readers (over a WAN) under high contention
+ * ("value" file being updated by a fast writer), to avoid spinning on reads most of the time.
+ */
+ protected void cleanPreserved() {
+ FileTime expiry = Fs.getFileTimeAgo(1, TimeUnit.SECONDS);
+ Fs.tryRecursiveDeleteEntriesOlderThan(paths.preserved, expiry, 5);
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/events/fsstore/FsSequenceTest.java b/src/test/java/com/googlesource/gerrit/plugins/events/fsstore/FsSequenceTest.java
new file mode 100644
index 0000000..4d809c1
--- /dev/null
+++ b/src/test/java/com/googlesource/gerrit/plugins/events/fsstore/FsSequenceTest.java
@@ -0,0 +1,124 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.googlesource.gerrit.plugins.events.fsstore;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import junit.framework.TestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class FsSequenceTest extends TestCase {
+ private static String dir = "events-FsSequence";
+ private static Path base;
+ private Path myBase;
+ private FsSequence seq;
+ private long count = 1000;
+ private long maxSpins = 1;
+ private String incrementMarker = "";
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ myBase = base;
+ if (myBase == null) {
+ myBase = Files.createTempDirectory(dir);
+ }
+ seq = new FsSequence(myBase);
+ seq.initFs((long) 0);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ Fs.tryRecursiveDelete(myBase);
+ }
+
+ @Test
+ public void testGetZero() throws IOException {
+ assertEquals((long) 0, (long) seq.get());
+ }
+
+ @Test
+ public void testIncrement() throws IOException {
+ Long next = seq.get() + (long) 1;
+ assertEquals(next, seq.increment());
+ }
+
+ @Test
+ public void testSpinIncrement() throws IOException {
+ long next = seq.get() + (long) 1;
+ assertEquals(next, (long) seq.spinIncrement(1));
+ }
+
+ @Test
+ public void testCount() throws Exception {
+ long previous = -1;
+ for (long i = 0; i < count; i++) {
+ Long v = seq.spinIncrement(maxSpins);
+ if (v != null) {
+ long val = v;
+ if (val <= previous) {
+ throw new Exception("val(" + val + ") <= previous(" + previous + ")");
+ }
+ }
+ System.out.print(incrementMarker);
+ }
+ long average = seq.totalSpins / seq.totalUpdates;
+ System.out.println("Average update submission spins: " + average);
+ }
+
+ /**
+ * First, make the junit jar easily available
+ *
+ * <p>ln -s ~/.m2/repository/junit/junit/4.8.1/junit-4.8.1.jar target
+ *
+ * <p>To run type:
+ *
+ * <p>java -cp target/classes:target/test-classes:target/junit-4.8.1.jar \
+ * com.googlesource.gerrit.plugins.events.fsstore.FsSequenceTest \ [dir [count [spin_retries]]]
+ *
+ * <p>Note: if you do not specify <dir>, it will create a directory under /tmp
+ *
+ * <p>NFS, 1 worker, count=1000, retries=1000 ~30s ~30ms/event avgspins 0 NFS, 2 workers,
+ * count=1000, retries=1000 ~37s ~18ms/event avgspins 0 NFS, 3 workers, count=1000, retries=1000
+ * ~50s ~17ms/event avgspins 18 NFS, 4 workers, count=1000, retries=1000 ~1m25 ~21ms/event
+ * avgspins 26 scales best with 2 workers!
+ *
+ * <p>NFS(WAN), 1 worker, count=10, retries=1000 19s ~2s/event avgspins 0 NFS(WAN), 2 workers,
+ * count=10, retries=1000 23s ~1s/event avgspins 1 NFS(WAN), 3 workers, count=10, retries=1000 38s
+ * ~1.3s/event avgspins 12
+ *
+ * <p>Our main production server does 220K/week ~22/min events ~ <3s/event
+ */
+ public static void main(String[] argv) throws Exception {
+ if (argv.length > 0) {
+ base = Paths.get(argv[0]);
+ }
+ FsSequenceTest t = new FsSequenceTest();
+ if (argv.length > 1) {
+ t.count = Long.parseLong(argv[1]);
+ }
+ if (argv.length > 2) {
+ t.maxSpins = Long.parseLong(argv[2]);
+ }
+
+ t.incrementMarker = ".";
+ t.setUp();
+ t.testCount();
+ }
+}
diff --git a/src/test/java/com/googlesource/gerrit/plugins/events/fsstore/FsStoreTest.java b/src/test/java/com/googlesource/gerrit/plugins/events/fsstore/FsStoreTest.java
index ddb2c3e..5885c05 100644
--- a/src/test/java/com/googlesource/gerrit/plugins/events/fsstore/FsStoreTest.java
+++ b/src/test/java/com/googlesource/gerrit/plugins/events/fsstore/FsStoreTest.java
@@ -217,9 +217,10 @@
*
* <p>Note: if you do not specify <dir>, it will create a directory under /tmp
*
- * <p>Performance: NFS(Fast,Lowlatency,SSDs), 1 worker, count=1000, ~6s ~6ms/event
+ * <p>Performance: NFS(Lowlatency,SSDs), 1 worker 10K, 2m33s ~153ms/event find events|wc -l .4s rm
+ * -rf 1.3s
*
- * <p>Local(spinning) 5 workers count=100K 11m38.512s find events|wc -l 2s rm -rf 10s
+ * <p>Local(spinning) 1 workers 1M 16m7s ~1ms/event find events|wc -l 1.6s rm -rf 36s
*/
public static void main(String[] argv) throws Exception {
if (argv.length > 0) {
