Merge branch 'stable-3.0' into stable-3.1
* stable-3.0:
Serve reindexing simulating a GET request for /meta ref caching
Auto-reload the indexTs file for auto-reindexing
Download plugins from archive-ci.gerritforge.com
Pin haproxy to 1.8.30-buster and fix associated issues
Fix issue with change indexing during the NoteDb online migration
Remove references to ReviewDb in README.md
Change-Id: I0861860028071010adc40d0ddd19c5ca5120f60c
diff --git a/README.md b/README.md
index 8e7ef27..c8ac4b4 100644
--- a/README.md
+++ b/README.md
@@ -1,13 +1,12 @@
# Gerrit high-availability plugin
This plugin allows deploying a cluster of multiple Gerrit masters
-on the same data-center sharing the same ReviewDb and Git repositories.
+on the same data-center sharing the same Git repositories.
Requirements for the Gerrit masters are:
- Gerrit v2.14.20 or later
- Externally mounted filesystem shared among the cluster
-- ReviewDb on an external DataBase Server
- Load-balancer (HAProxy or similar)
## License
@@ -25,12 +24,12 @@
`gerrit-02.mycompany.com`, listening on the HTTP port 8080, with a shared volume
mounted under `/shared`, see below the minimal configuration steps.
-1. Install one Gerrit master on the first node (e.g. `gerrit-01.mycompany.com`) using an external
- ReviewDb on a DB server and the repositories location under the shared volume (e.g. `/shared/git`).
- Init the site in order to create the DB Schema and the initial repositories.
+1. Install one Gerrit master on the first node (e.g. `gerrit-01.mycompany.com`)
+ with the repositories location under the shared volume (e.g. `/shared/git`).
+ Init the site in order to create the initial repositories.
2. Copy all the files of the first Gerrit master onto the second node (e.g. `gerrit-02.mycompany.com`)
- so that it points to the same ReviewDb and the same repositories location.
+ so that it points to the same repositories location.
3. Install the high-availability plugin into the `$GERRIT_SITE/plugins` directory of both
the Gerrit servers.
diff --git a/src/main/java/com/ericsson/gerrit/plugins/highavailability/autoreindex/IndexTs.java b/src/main/java/com/ericsson/gerrit/plugins/highavailability/autoreindex/IndexTs.java
index 561bb19..b4db602 100644
--- a/src/main/java/com/ericsson/gerrit/plugins/highavailability/autoreindex/IndexTs.java
+++ b/src/main/java/com/ericsson/gerrit/plugins/highavailability/autoreindex/IndexTs.java
@@ -36,6 +36,7 @@
import java.time.format.DateTimeFormatter;
import java.util.Optional;
import java.util.concurrent.ScheduledExecutorService;
+import org.eclipse.jgit.internal.storage.file.FileSnapshot;
@Singleton
public class IndexTs
@@ -62,15 +63,17 @@
class FlusherRunner implements Runnable {
private final AbstractIndexRestApiServlet.IndexName index;
+ private final Path tsPath;
+ private FileSnapshot tsPathSnapshot;
@Override
public void run() {
+ reloadIndexTimeStampIfChanged();
LocalDateTime latestTs = getIndexTimeStamp();
Optional<LocalDateTime> currTs = getUpdateTs(index);
if (!currTs.isPresent() || latestTs.isAfter(currTs.get())) {
- Path indexTsFile = dataDir.resolve(index.name().toLowerCase());
try {
- Files.write(indexTsFile, latestTs.format(formatter).getBytes(StandardCharsets.UTF_8));
+ Files.write(tsPath, latestTs.format(formatter).getBytes(StandardCharsets.UTF_8));
} catch (IOException e) {
log.atSevere().withCause(e).log("Unable to update last timestamp for index %s", index);
}
@@ -79,6 +82,8 @@
FlusherRunner(AbstractIndexRestApiServlet.IndexName index) {
this.index = index;
+ this.tsPath = getIndexTsPath(index);
+ this.tsPathSnapshot = FileSnapshot.save(tsPath.toFile());
}
private LocalDateTime getIndexTimeStamp() {
@@ -95,6 +100,31 @@
throw new IllegalArgumentException("Unsupported index " + index);
}
}
+
+ private void reloadIndexTimeStampIfChanged() {
+ if (tsPathSnapshot.isModified(tsPath.toFile())) {
+ getUpdateTs(index).ifPresent(this::setIndexTimeStamp);
+ }
+ }
+
+ private void setIndexTimeStamp(LocalDateTime newTs) {
+ switch (index) {
+ case CHANGE:
+ changeTs = newTs;
+ break;
+ case GROUP:
+ groupTs = newTs;
+ break;
+ case ACCOUNT:
+ accountTs = newTs;
+ break;
+ case PROJECT:
+ projectTs = newTs;
+ break;
+ }
+
+ tsPathSnapshot = FileSnapshot.save(tsPath.toFile());
+ }
}
@Inject
@@ -152,7 +182,7 @@
public Optional<LocalDateTime> getUpdateTs(AbstractIndexRestApiServlet.IndexName index) {
try {
- Path indexTsFile = dataDir.resolve(index.name().toLowerCase());
+ Path indexTsFile = getIndexTsPath(index);
if (indexTsFile.toFile().exists()) {
String tsString = Files.readAllLines(indexTsFile).get(0);
return Optional.of(LocalDateTime.parse(tsString, formatter));
@@ -185,4 +215,8 @@
throw new IllegalArgumentException("Unsupported index " + index);
}
}
+
+ private Path getIndexTsPath(AbstractIndexRestApiServlet.IndexName index) {
+ return dataDir.resolve(index.name().toLowerCase());
+ }
}
diff --git a/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/ForwardedIndexChangeHandler.java b/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/ForwardedIndexChangeHandler.java
index 91e5a02..770259d 100644
--- a/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/ForwardedIndexChangeHandler.java
+++ b/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/ForwardedIndexChangeHandler.java
@@ -21,6 +21,7 @@
import com.ericsson.gerrit.plugins.highavailability.index.ForwardedIndexExecutor;
import com.google.common.base.Splitter;
import com.google.gerrit.entities.Change;
+import com.google.gerrit.entities.Project;
import com.google.gerrit.exceptions.StorageException;
import com.google.gerrit.server.index.change.ChangeIndexer;
import com.google.gerrit.server.notedb.ChangeNotes;
@@ -30,6 +31,7 @@
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.io.IOException;
+import java.util.List;
import java.util.Optional;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -157,7 +159,15 @@
}
private static Change.Id parseChangeId(String id) {
- return Change.id(Integer.parseInt(Splitter.on("~").splitToList(id).get(1)));
+ return Change.id(Integer.parseInt(getChangeIdParts(id).get(1)));
+ }
+
+ private static Project.NameKey parseProject(String id) {
+ return Project.nameKey(getChangeIdParts(id).get(0));
+ }
+
+ private static List<String> getChangeIdParts(String id) {
+ return Splitter.on("~").splitToList(id);
}
private static boolean isCausedByNoSuchChangeException(Throwable throwable) {
diff --git a/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/AbstractIndexRestApiServlet.java b/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/AbstractIndexRestApiServlet.java
index f9ec516..6b23588 100644
--- a/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/AbstractIndexRestApiServlet.java
+++ b/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/AbstractIndexRestApiServlet.java
@@ -22,6 +22,7 @@
import com.ericsson.gerrit.plugins.highavailability.forwarder.ForwardedIndexingHandler.Operation;
import com.ericsson.gerrit.plugins.highavailability.forwarder.IndexEvent;
import com.google.common.base.Charsets;
+import com.google.gerrit.server.cache.PerThreadCache;
import com.google.gerrit.server.events.EventGson;
import com.google.gson.Gson;
import java.io.IOException;
@@ -29,6 +30,7 @@
import java.io.Reader;
import java.util.Optional;
import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletRequestWrapper;
import javax.servlet.http.HttpServletResponse;
public abstract class AbstractIndexRestApiServlet<T> extends AbstractRestApiServlet {
@@ -88,7 +90,26 @@
setHeaders(rsp);
String path = req.getRequestURI();
T id = parse(path.substring(path.lastIndexOf('/') + 1));
- try {
+
+ /**
+ * Since [1] the change notes /meta refs are cached for all the incoming GET/HEAD REST APIs;
+ * however, the high-availability indexing API is a POST served by a regular servlet and
+ * therefore won't have any caching, which is problematic because of the high number of
+ * associated refs lookups generated.
+ *
+ * <p>Simulate an incoming GET request for allowing caching of the /meta refs lookups.
+ *
+ * <p>[1] https://gerrit-review.googlesource.com/c/gerrit/+/334539/17
+ */
+ HttpServletRequestWrapper simulatedGetRequestForCaching =
+ new HttpServletRequestWrapper(req) {
+ @Override
+ public String getMethod() {
+ return "GET";
+ }
+ };
+
+ try (PerThreadCache unused = PerThreadCache.create(simulatedGetRequestForCaching)) {
forwardedIndexingHandler.index(id, operation, parseBody(req));
rsp.setStatus(SC_NO_CONTENT);
} catch (IOException e) {
diff --git a/src/test/docker/docker-compose.yaml b/src/test/docker/docker-compose.yaml
index 647aa6d..1365cd8 100644
--- a/src/test/docker/docker-compose.yaml
+++ b/src/test/docker/docker-compose.yaml
@@ -47,6 +47,7 @@
volumes:
- syslog-sidecar
depends_on:
+ - syslog-sidecar
- gerrit-01
- gerrit-02
diff --git a/src/test/docker/haproxy/Dockerfile b/src/test/docker/haproxy/Dockerfile
index 72cad4a..eb5920a 100644
--- a/src/test/docker/haproxy/Dockerfile
+++ b/src/test/docker/haproxy/Dockerfile
@@ -1,8 +1,5 @@
-FROM haproxy:1.8
+FROM haproxy:1.8.30-buster
+
+RUN mkdir -p /var/run/haproxy && chown -R haproxy: /var/run/haproxy
COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg
-
-RUN mkdir /var/lib/haproxy && \
- mkdir /var/run/haproxy && \
- useradd haproxy && \
- chown haproxy: /var/lib/haproxy /var/run/haproxy
