Merge branch 'stable-2.16' into stable-3.0
* stable-2.16:
Serve reindexing simulating a GET request for /meta ref caching
Auto-reload the indexTs file for auto-reindexing
Download plugins from archive-ci.gerritforge.com
Pin haproxy to 1.8.30-buster and fix associated issues
Fix issue with change indexing during the NoteDb online migration
Change-Id: I0992d42770ff1e1c546015ff97bfb9f28aed824d
diff --git a/src/main/java/com/ericsson/gerrit/plugins/highavailability/autoreindex/IndexTs.java b/src/main/java/com/ericsson/gerrit/plugins/highavailability/autoreindex/IndexTs.java
index 561bb19..b4db602 100644
--- a/src/main/java/com/ericsson/gerrit/plugins/highavailability/autoreindex/IndexTs.java
+++ b/src/main/java/com/ericsson/gerrit/plugins/highavailability/autoreindex/IndexTs.java
@@ -36,6 +36,7 @@
import java.time.format.DateTimeFormatter;
import java.util.Optional;
import java.util.concurrent.ScheduledExecutorService;
+import org.eclipse.jgit.internal.storage.file.FileSnapshot;
@Singleton
public class IndexTs
@@ -62,15 +63,17 @@
class FlusherRunner implements Runnable {
private final AbstractIndexRestApiServlet.IndexName index;
+ private final Path tsPath;
+ private FileSnapshot tsPathSnapshot;
@Override
public void run() {
+ reloadIndexTimeStampIfChanged();
LocalDateTime latestTs = getIndexTimeStamp();
Optional<LocalDateTime> currTs = getUpdateTs(index);
if (!currTs.isPresent() || latestTs.isAfter(currTs.get())) {
- Path indexTsFile = dataDir.resolve(index.name().toLowerCase());
try {
- Files.write(indexTsFile, latestTs.format(formatter).getBytes(StandardCharsets.UTF_8));
+ Files.write(tsPath, latestTs.format(formatter).getBytes(StandardCharsets.UTF_8));
} catch (IOException e) {
log.atSevere().withCause(e).log("Unable to update last timestamp for index %s", index);
}
@@ -79,6 +82,8 @@
FlusherRunner(AbstractIndexRestApiServlet.IndexName index) {
this.index = index;
+ this.tsPath = getIndexTsPath(index);
+ this.tsPathSnapshot = FileSnapshot.save(tsPath.toFile());
}
private LocalDateTime getIndexTimeStamp() {
@@ -95,6 +100,31 @@
throw new IllegalArgumentException("Unsupported index " + index);
}
}
+
+ private void reloadIndexTimeStampIfChanged() {
+ if (tsPathSnapshot.isModified(tsPath.toFile())) {
+ getUpdateTs(index).ifPresent(this::setIndexTimeStamp);
+ }
+ }
+
+ private void setIndexTimeStamp(LocalDateTime newTs) {
+ switch (index) {
+ case CHANGE:
+ changeTs = newTs;
+ break;
+ case GROUP:
+ groupTs = newTs;
+ break;
+ case ACCOUNT:
+ accountTs = newTs;
+ break;
+ case PROJECT:
+ projectTs = newTs;
+ break;
+ }
+
+ tsPathSnapshot = FileSnapshot.save(tsPath.toFile());
+ }
}
@Inject
@@ -152,7 +182,7 @@
public Optional<LocalDateTime> getUpdateTs(AbstractIndexRestApiServlet.IndexName index) {
try {
- Path indexTsFile = dataDir.resolve(index.name().toLowerCase());
+ Path indexTsFile = getIndexTsPath(index);
if (indexTsFile.toFile().exists()) {
String tsString = Files.readAllLines(indexTsFile).get(0);
return Optional.of(LocalDateTime.parse(tsString, formatter));
@@ -185,4 +215,8 @@
throw new IllegalArgumentException("Unsupported index " + index);
}
}
+
+ private Path getIndexTsPath(AbstractIndexRestApiServlet.IndexName index) {
+ return dataDir.resolve(index.name().toLowerCase());
+ }
}
diff --git a/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/ForwardedIndexChangeHandler.java b/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/ForwardedIndexChangeHandler.java
index add93d7..42f3cef 100644
--- a/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/ForwardedIndexChangeHandler.java
+++ b/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/ForwardedIndexChangeHandler.java
@@ -21,6 +21,7 @@
import com.ericsson.gerrit.plugins.highavailability.index.ForwardedIndexExecutor;
import com.google.common.base.Splitter;
import com.google.gerrit.reviewdb.client.Change;
+import com.google.gerrit.reviewdb.client.Project;
import com.google.gerrit.server.index.change.ChangeIndexer;
import com.google.gerrit.server.notedb.ChangeNotes;
import com.google.gerrit.server.project.NoSuchChangeException;
@@ -29,6 +30,7 @@
import com.google.inject.Inject;
import com.google.inject.Singleton;
import java.io.IOException;
+import java.util.List;
import java.util.Optional;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -150,7 +152,15 @@
}
private static Change.Id parseChangeId(String id) {
- return new Change.Id(Integer.parseInt(Splitter.on("~").splitToList(id).get(1)));
+ return new Change.Id(Integer.parseInt(getChangeIdParts(id).get(1)));
+ }
+
+ private static Project.NameKey parseProject(String id) {
+ return new Project.NameKey(getChangeIdParts(id).get(0));
+ }
+
+ private static List<String> getChangeIdParts(String id) {
+ return Splitter.on("~").splitToList(id);
}
private static boolean isCausedByNoSuchChangeException(Throwable throwable) {
diff --git a/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/AbstractIndexRestApiServlet.java b/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/AbstractIndexRestApiServlet.java
index 5603fc2..09d4603 100644
--- a/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/AbstractIndexRestApiServlet.java
+++ b/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/AbstractIndexRestApiServlet.java
@@ -22,12 +22,14 @@
import com.ericsson.gerrit.plugins.highavailability.forwarder.ForwardedIndexingHandler.Operation;
import com.ericsson.gerrit.plugins.highavailability.forwarder.IndexEvent;
import com.google.common.base.Charsets;
+import com.google.gerrit.server.cache.PerThreadCache;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Optional;
import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletRequestWrapper;
import javax.servlet.http.HttpServletResponse;
public abstract class AbstractIndexRestApiServlet<T> extends AbstractRestApiServlet {
@@ -85,7 +87,26 @@
setHeaders(rsp);
String path = req.getRequestURI();
T id = parse(path.substring(path.lastIndexOf('/') + 1));
- try {
+
+ /**
+ * Since [1] the change notes /meta refs are cached for all the incoming GET/HEAD REST APIs;
+ * however, the high-availability indexing API is a POST served by a regular servlet and
+ * therefore won't have any caching, which is problematic because of the high number of
+ * associated refs lookups generated.
+ *
+ * <p>Simulate an incoming GET request for allowing caching of the /meta refs lookups.
+ *
+ * <p>[1] https://gerrit-review.googlesource.com/c/gerrit/+/334539/17
+ */
+ HttpServletRequestWrapper simulatedGetRequestForCaching =
+ new HttpServletRequestWrapper(req) {
+ @Override
+ public String getMethod() {
+ return "GET";
+ }
+ };
+
+ try (PerThreadCache unused = PerThreadCache.create(simulatedGetRequestForCaching)) {
forwardedIndexingHandler.index(id, operation, parseBody(req));
rsp.setStatus(SC_NO_CONTENT);
} catch (IOException e) {
diff --git a/src/test/docker/docker-compose.yaml b/src/test/docker/docker-compose.yaml
index 647aa6d..1365cd8 100644
--- a/src/test/docker/docker-compose.yaml
+++ b/src/test/docker/docker-compose.yaml
@@ -47,6 +47,7 @@
volumes:
- syslog-sidecar
depends_on:
+ - syslog-sidecar
- gerrit-01
- gerrit-02
diff --git a/src/test/docker/gerrit/Dockerfile b/src/test/docker/gerrit/Dockerfile
index 707e5ae..ec7ebd8 100644
--- a/src/test/docker/gerrit/Dockerfile
+++ b/src/test/docker/gerrit/Dockerfile
@@ -2,7 +2,7 @@
ENV GERRIT_BRANCH=stable-3.0
-ENV GERRIT_CI_URL=https://gerrit-ci.gerritforge.com/job
+ENV GERRIT_CI_URL=https://archive-ci.gerritforge.com/job
USER root
diff --git a/src/test/docker/haproxy/Dockerfile b/src/test/docker/haproxy/Dockerfile
index 72cad4a..eb5920a 100644
--- a/src/test/docker/haproxy/Dockerfile
+++ b/src/test/docker/haproxy/Dockerfile
@@ -1,8 +1,5 @@
-FROM haproxy:1.8
+FROM haproxy:1.8.30-buster
+
+RUN mkdir -p /var/run/haproxy && chown -R haproxy: /var/run/haproxy
COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg
-
-RUN mkdir /var/lib/haproxy && \
- mkdir /var/run/haproxy && \
- useradd haproxy && \
- chown haproxy: /var/lib/haproxy /var/run/haproxy
