Support n nodes when using static strategy
So far, the plugin was able to support a maximum of two nodes when
syncing index, caches and events. This is a limitation when having more
than one standby server is wanted/needed.
Implement support for n nodes when using static strategy. The URL of the
peers in the cluster is specified as a list in the configuration. So for
example, in the high-availability configuration file of a node with the
URL http://localhost:2141, two other peers are listed as:
...
[peerInfo]
strategy = static
[peerInfo "static"]
url = http://localhost:2142
url = http://localhost:2143
...
Each of the listed nodes, in turn, will list in its configuration file
the URLs of the two other peers. In this way, all the nodes are kept in
sync no matter which is one generating "syncable" items.
Before this change, there was only one peer and the retry logic was
simple: as soon as a message was sent successfully to the other node,
the operation was considered as done. Now, the retry has to be handled
separately for each of the configured peers and the operation is only
considered successful if all the peers received the message. When some
nodes fail to receive the message, the sending is retried according to
the applicable configuration settings and if it fails after the retry,
an error is logged as before but containing also the URL of the node.
The support of n nodes in the case of dynamic strategy (using JGroups)
requires a completely different approach, thus should be tackled in a
different change.
Change-Id: I49e91c73598cee61f64e479a74504898b61c4e36
diff --git a/src/main/java/com/ericsson/gerrit/plugins/highavailability/Configuration.java b/src/main/java/com/ericsson/gerrit/plugins/highavailability/Configuration.java
index 8f1963a..d4c1272 100644
--- a/src/main/java/com/ericsson/gerrit/plugins/highavailability/Configuration.java
+++ b/src/main/java/com/ericsson/gerrit/plugins/highavailability/Configuration.java
@@ -20,6 +20,7 @@
import com.google.common.base.CharMatcher;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
import com.google.gerrit.common.Nullable;
import com.google.gerrit.extensions.annotations.PluginName;
import com.google.gerrit.server.config.ConfigUtil;
@@ -32,8 +33,11 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Objects;
import java.util.Optional;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import org.eclipse.jgit.lib.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -153,11 +157,6 @@
}
}
- @Nullable
- private static String trimTrailingSlash(@Nullable String in) {
- return in == null ? null : CharMatcher.is('/').trimTrailingFrom(in);
- }
-
public static class Main {
static final String MAIN_SECTION = "main";
static final String SHARED_DIRECTORY_KEY = "sharedDirectory";
@@ -247,17 +246,20 @@
static final String STATIC_SUBSECTION = PeerInfoStrategy.STATIC.name().toLowerCase();
static final String URL_KEY = "url";
- private final String url;
+ private final Set<String> urls;
private PeerInfoStatic(Config cfg) {
- url =
- trimTrailingSlash(
- Strings.nullToEmpty(cfg.getString(PEER_INFO_SECTION, STATIC_SUBSECTION, URL_KEY)));
- log.debug("Url: {}", url);
+ urls =
+ Arrays.stream(cfg.getStringList(PEER_INFO_SECTION, STATIC_SUBSECTION, URL_KEY))
+ .filter(Objects::nonNull)
+ .filter(s -> !s.isEmpty())
+ .map(s -> CharMatcher.is('/').trimTrailingFrom(s))
+ .collect(Collectors.toSet());
+ log.debug("Urls: {}", urls);
}
- public String url() {
- return url;
+ public Set<String> urls() {
+ return ImmutableSet.copyOf(urls);
}
}
@@ -275,6 +277,11 @@
public String myUrl() {
return myUrl;
}
+
+ @Nullable
+ private static String trimTrailingSlash(@Nullable String in) {
+ return in == null ? in : CharMatcher.is('/').trimTrailingFrom(in);
+ }
}
public static class JGroups {
diff --git a/src/main/java/com/ericsson/gerrit/plugins/highavailability/Setup.java b/src/main/java/com/ericsson/gerrit/plugins/highavailability/Setup.java
index e58f62f..86c757e 100644
--- a/src/main/java/com/ericsson/gerrit/plugins/highavailability/Setup.java
+++ b/src/main/java/com/ericsson/gerrit/plugins/highavailability/Setup.java
@@ -164,7 +164,8 @@
PeerInfoStrategy.JGROUPS, EnumSet.allOf(PeerInfoStrategy.class), "Peer info strategy");
config.setEnum(PEER_INFO_SECTION, null, STRATEGY_KEY, strategy);
if (strategy == PeerInfoStrategy.STATIC) {
- promptAndSetString("Peer URL", PEER_INFO_SECTION, STATIC_SUBSECTION, URL_KEY, null);
+ promptAndSetString(
+ titleWithNote("Peer URL", "urls"), PEER_INFO_SECTION, STATIC_SUBSECTION, URL_KEY, null);
} else {
promptAndSetString(
"JGroups cluster name",
@@ -291,7 +292,11 @@
}
private static String titleForOptionalWithNote(String prefix, String suffix) {
- return prefix + " (optional); manually repeat this line to configure more " + suffix;
+ return titleWithNote(prefix + " (optional)", suffix);
+ }
+
+ private static String titleWithNote(String prefix, String suffix) {
+ return prefix + "; manually repeat this line to configure more " + suffix;
}
private static String numberToString(int number) {
diff --git a/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/HttpSession.java b/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/HttpSession.java
index d3fce97..d9bb352 100644
--- a/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/HttpSession.java
+++ b/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/HttpSession.java
@@ -15,14 +15,11 @@
package com.ericsson.gerrit.plugins.highavailability.forwarder.rest;
import com.ericsson.gerrit.plugins.highavailability.forwarder.rest.HttpResponseHandler.HttpResult;
-import com.ericsson.gerrit.plugins.highavailability.peers.PeerInfo;
import com.google.common.base.Strings;
import com.google.common.net.MediaType;
import com.google.inject.Inject;
-import com.google.inject.Provider;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
-import java.util.Optional;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
@@ -30,20 +27,18 @@
class HttpSession {
private final CloseableHttpClient httpClient;
- private final Provider<Optional<PeerInfo>> peerInfo;
@Inject
- HttpSession(CloseableHttpClient httpClient, Provider<Optional<PeerInfo>> peerInfo) {
+ HttpSession(CloseableHttpClient httpClient) {
this.httpClient = httpClient;
- this.peerInfo = peerInfo;
}
HttpResult post(String endpoint) throws IOException {
return post(endpoint, null);
}
- HttpResult post(String endpoint, String content) throws IOException {
- HttpPost post = new HttpPost(getPeerInfo().getDirectUrl() + endpoint);
+ HttpResult post(String uri, String content) throws IOException {
+ HttpPost post = new HttpPost(uri);
if (!Strings.isNullOrEmpty(content)) {
post.addHeader("Content-Type", MediaType.JSON_UTF_8.toString());
post.setEntity(new StringEntity(content, StandardCharsets.UTF_8));
@@ -51,16 +46,7 @@
return httpClient.execute(post, new HttpResponseHandler());
}
- HttpResult delete(String endpoint) throws IOException {
- return httpClient.execute(
- new HttpDelete(getPeerInfo().getDirectUrl() + endpoint), new HttpResponseHandler());
- }
-
- private PeerInfo getPeerInfo() throws PeerInfoNotAvailableException {
- PeerInfo info = peerInfo.get().orElse(null);
- if (info == null) {
- throw new PeerInfoNotAvailableException();
- }
- return info;
+ HttpResult delete(String uri) throws IOException {
+ return httpClient.execute(new HttpDelete(uri), new HttpResponseHandler());
}
}
diff --git a/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/RestForwarder.java b/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/RestForwarder.java
index e430fea..32c56bf 100644
--- a/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/RestForwarder.java
+++ b/src/main/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/RestForwarder.java
@@ -18,6 +18,7 @@
import com.ericsson.gerrit.plugins.highavailability.cache.Constants;
import com.ericsson.gerrit.plugins.highavailability.forwarder.Forwarder;
import com.ericsson.gerrit.plugins.highavailability.forwarder.rest.HttpResponseHandler.HttpResult;
+import com.ericsson.gerrit.plugins.highavailability.peers.PeerInfo;
import com.google.common.base.Joiner;
import com.google.common.base.Supplier;
import com.google.gerrit.extensions.annotations.PluginName;
@@ -26,7 +27,12 @@
import com.google.gerrit.server.events.SupplierSerializer;
import com.google.gson.GsonBuilder;
import com.google.inject.Inject;
+import com.google.inject.Provider;
import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
import javax.net.ssl.SSLException;
import org.apache.http.HttpException;
import org.apache.http.client.ClientProtocolException;
@@ -34,150 +40,177 @@
import org.slf4j.LoggerFactory;
class RestForwarder implements Forwarder {
+ enum RequestMethod {
+ POST,
+ DELETE
+ }
+
private static final Logger log = LoggerFactory.getLogger(RestForwarder.class);
private final HttpSession httpSession;
private final String pluginRelativePath;
private final Configuration cfg;
+ private final Provider<Set<PeerInfo>> peerInfoProvider;
@Inject
- RestForwarder(HttpSession httpClient, @PluginName String pluginName, Configuration cfg) {
+ RestForwarder(
+ HttpSession httpClient,
+ @PluginName String pluginName,
+ Configuration cfg,
+ Provider<Set<PeerInfo>> peerInfoProvider) {
this.httpSession = httpClient;
- this.pluginRelativePath = Joiner.on("/").join("/plugins", pluginName);
+ this.pluginRelativePath = Joiner.on("/").join("plugins", pluginName);
this.cfg = cfg;
+ this.peerInfoProvider = peerInfoProvider;
}
@Override
public boolean indexAccount(final int accountId) {
- return new Request("index account", accountId) {
- @Override
- HttpResult send() throws IOException {
- return httpSession.post(
- Joiner.on("/").join(pluginRelativePath, "index/account", accountId));
- }
- }.execute();
+ return execute(RequestMethod.POST, "index account", "index/account", accountId);
}
@Override
public boolean indexChange(final int changeId) {
- return new Request("index change", changeId) {
- @Override
- HttpResult send() throws IOException {
- return httpSession.post(buildIndexEndpoint(changeId));
- }
- }.execute();
+ return execute(RequestMethod.POST, "index change", "index/change", changeId);
}
@Override
public boolean deleteChangeFromIndex(final int changeId) {
- return new Request("delete change", changeId) {
- @Override
- HttpResult send() throws IOException {
- return httpSession.delete(buildIndexEndpoint(changeId));
- }
- }.execute();
+ return execute(RequestMethod.DELETE, "delete change", "index/change", changeId);
}
@Override
- public boolean indexGroup(final String uuid) {
- return new Request("index group", uuid) {
- @Override
- HttpResult send() throws IOException {
- return httpSession.post(Joiner.on("/").join(pluginRelativePath, "index/group", uuid));
- }
- }.execute();
- }
-
- private String buildIndexEndpoint(int changeId) {
- return Joiner.on("/").join(pluginRelativePath, "index/change", changeId);
+ public boolean indexGroup(String uuid) {
+ return execute(RequestMethod.POST, "index group", "index/group", uuid);
}
@Override
public boolean send(final Event event) {
- return new Request("send event", event.type) {
- @Override
- HttpResult send() throws IOException {
- String serializedEvent =
- new GsonBuilder()
- .registerTypeAdapter(Supplier.class, new SupplierSerializer())
- .create()
- .toJson(event);
- return httpSession.post(Joiner.on("/").join(pluginRelativePath, "event"), serializedEvent);
- }
- }.execute();
+ String serializedEvent =
+ new GsonBuilder()
+ .registerTypeAdapter(Supplier.class, new SupplierSerializer())
+ .create()
+ .toJson(event);
+ return execute(RequestMethod.POST, "send event", "event", event.type, serializedEvent);
}
@Override
public boolean evict(final String cacheName, final Object key) {
- return new Request("invalidate cache " + cacheName, key) {
- @Override
- HttpResult send() throws IOException {
- String json = GsonParser.toJson(cacheName, key);
- return httpSession.post(Joiner.on("/").join(pluginRelativePath, "cache", cacheName), json);
- }
- }.execute();
+ String json = GsonParser.toJson(cacheName, key);
+ return execute(RequestMethod.POST, "invalidate cache " + cacheName, "cache", cacheName, json);
}
@Override
public boolean addToProjectList(String projectName) {
- return new Request("Update project_list, add ", projectName) {
- @Override
- HttpResult send() throws IOException {
- return httpSession.post(buildProjectListEndpoint(projectName));
- }
- }.execute();
+ return execute(
+ RequestMethod.POST,
+ "Update project_list, add ",
+ buildProjectListEndpoint(),
+ Url.encode(projectName));
}
@Override
public boolean removeFromProjectList(String projectName) {
- return new Request("Update project_list, remove ", projectName) {
- @Override
- HttpResult send() throws IOException {
- return httpSession.delete(buildProjectListEndpoint(projectName));
- }
- }.execute();
+ return execute(
+ RequestMethod.DELETE,
+ "Update project_list, remove ",
+ buildProjectListEndpoint(),
+ Url.encode(projectName));
}
- private String buildProjectListEndpoint(String projectName) {
- return Joiner.on("/")
- .join(pluginRelativePath, "cache", Constants.PROJECT_LIST, Url.encode(projectName));
+ private static String buildProjectListEndpoint() {
+ return Joiner.on("/").join("cache", Constants.PROJECT_LIST);
+ }
+
+ private boolean execute(RequestMethod method, String action, String endpoint, Object id) {
+ return execute(method, action, endpoint, id, null);
+ }
+
+ private boolean execute(
+ RequestMethod method, String action, String endpoint, Object id, String payload) {
+ List<CompletableFuture<Boolean>> futures =
+ peerInfoProvider
+ .get()
+ .stream()
+ .map(peer -> createRequest(method, peer, action, endpoint, id, payload))
+ .map(request -> CompletableFuture.supplyAsync(() -> request.execute()))
+ .collect(Collectors.toList());
+ CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
+ return futures.stream().allMatch(CompletableFuture::join);
+ }
+
+ private Request createRequest(
+ RequestMethod method,
+ PeerInfo peer,
+ String action,
+ String endpoint,
+ Object id,
+ String payload) {
+ String destination = peer.getDirectUrl();
+ return new Request(action, id, destination) {
+ @Override
+ HttpResult send() throws IOException {
+ String request = Joiner.on("/").join(destination, pluginRelativePath, endpoint, id);
+ switch (method) {
+ case POST:
+ return httpSession.post(request, payload);
+ case DELETE:
+ default:
+ return httpSession.delete(request);
+ }
+ }
+ };
}
private abstract class Request {
private final String action;
private final Object key;
+ private final String destination;
+
private int execCnt;
- Request(String action, Object key) {
+ Request(String action, Object key, String destination) {
this.action = action;
this.key = key;
+ this.destination = destination;
}
boolean execute() {
- log.debug("Executing {} {}", action, key);
+ log.debug("Executing {} {} towards {}", action, key, destination);
for (; ; ) {
try {
execCnt++;
tryOnce();
- log.debug("{} {} OK", action, key);
+ log.debug("{} {} towards {} OK", action, key, destination);
return true;
} catch (ForwardingException e) {
int maxTries = cfg.http().maxTries();
- log.debug("Failed to {} {} [{}/{}]", action, key, execCnt, maxTries, e);
+ log.debug(
+ "Failed to {} {} on {} [{}/{}]", action, key, destination, execCnt, maxTries, e);
if (!e.isRecoverable()) {
- log.error("{} {} failed with unrecoverable error; giving up", action, key, e);
+ log.error(
+ "{} {} towards {} failed with unrecoverable error; giving up",
+ action,
+ key,
+ destination,
+ e);
return false;
}
if (execCnt >= maxTries) {
- log.error("Failed to {} {} after {} tries; giving up", action, key, maxTries);
+ log.error(
+ "Failed to {} {} on {} after {} tries; giving up",
+ action,
+ key,
+ destination,
+ maxTries);
return false;
}
- log.debug("Retrying to {} {}", action, key);
+ log.debug("Retrying to {} {} on {}", action, key, destination);
try {
Thread.sleep(cfg.http().retryInterval());
} catch (InterruptedException ie) {
- log.error("{} {} was interrupted; giving up", action, key, ie);
+ log.error("{} {} towards {} was interrupted; giving up", action, key, destination, ie);
Thread.currentThread().interrupt();
return false;
}
diff --git a/src/main/java/com/ericsson/gerrit/plugins/highavailability/peers/PeerInfoModule.java b/src/main/java/com/ericsson/gerrit/plugins/highavailability/peers/PeerInfoModule.java
index 577d66d..68707c2 100644
--- a/src/main/java/com/ericsson/gerrit/plugins/highavailability/peers/PeerInfoModule.java
+++ b/src/main/java/com/ericsson/gerrit/plugins/highavailability/peers/PeerInfoModule.java
@@ -18,7 +18,7 @@
import com.ericsson.gerrit.plugins.highavailability.peers.jgroups.JGroupsPeerInfoProvider;
import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.inject.TypeLiteral;
-import java.util.Optional;
+import java.util.Set;
public class PeerInfoModule extends LifecycleModule {
@@ -32,11 +32,10 @@
protected void configure() {
switch (strategy) {
case STATIC:
- bind(new TypeLiteral<Optional<PeerInfo>>() {})
- .toProvider(PluginConfigPeerInfoProvider.class);
+ bind(new TypeLiteral<Set<PeerInfo>>() {}).toProvider(PluginConfigPeerInfoProvider.class);
break;
case JGROUPS:
- bind(new TypeLiteral<Optional<PeerInfo>>() {}).toProvider(JGroupsPeerInfoProvider.class);
+ bind(new TypeLiteral<Set<PeerInfo>>() {}).toProvider(JGroupsPeerInfoProvider.class);
listener().to(JGroupsPeerInfoProvider.class);
break;
default:
diff --git a/src/main/java/com/ericsson/gerrit/plugins/highavailability/peers/PluginConfigPeerInfoProvider.java b/src/main/java/com/ericsson/gerrit/plugins/highavailability/peers/PluginConfigPeerInfoProvider.java
index f39eb6b..9233c79 100644
--- a/src/main/java/com/ericsson/gerrit/plugins/highavailability/peers/PluginConfigPeerInfoProvider.java
+++ b/src/main/java/com/ericsson/gerrit/plugins/highavailability/peers/PluginConfigPeerInfoProvider.java
@@ -18,20 +18,21 @@
import com.google.inject.Inject;
import com.google.inject.Provider;
import com.google.inject.Singleton;
-import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
@Singleton
-public class PluginConfigPeerInfoProvider implements Provider<Optional<PeerInfo>> {
+public class PluginConfigPeerInfoProvider implements Provider<Set<PeerInfo>> {
- private final Optional<PeerInfo> peerInfo;
+ private final Set<PeerInfo> peers;
@Inject
PluginConfigPeerInfoProvider(Configuration cfg) {
- peerInfo = Optional.of(new PeerInfo(cfg.peerInfoStatic().url()));
+ peers = cfg.peerInfoStatic().urls().stream().map(PeerInfo::new).collect(Collectors.toSet());
}
@Override
- public Optional<PeerInfo> get() {
- return peerInfo;
+ public Set<PeerInfo> get() {
+ return peers;
}
}
diff --git a/src/main/java/com/ericsson/gerrit/plugins/highavailability/peers/jgroups/JGroupsPeerInfoProvider.java b/src/main/java/com/ericsson/gerrit/plugins/highavailability/peers/jgroups/JGroupsPeerInfoProvider.java
index cf46ac3..596f86e 100644
--- a/src/main/java/com/ericsson/gerrit/plugins/highavailability/peers/jgroups/JGroupsPeerInfoProvider.java
+++ b/src/main/java/com/ericsson/gerrit/plugins/highavailability/peers/jgroups/JGroupsPeerInfoProvider.java
@@ -16,6 +16,7 @@
import com.ericsson.gerrit.plugins.highavailability.Configuration;
import com.ericsson.gerrit.plugins.highavailability.peers.PeerInfo;
+import com.google.common.collect.ImmutableSet;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.inject.Inject;
import com.google.inject.Provider;
@@ -23,6 +24,7 @@
import java.net.InetAddress;
import java.nio.file.Path;
import java.util.Optional;
+import java.util.Set;
import org.jgroups.Address;
import org.jgroups.JChannel;
import org.jgroups.Message;
@@ -43,7 +45,7 @@
*/
@Singleton
public class JGroupsPeerInfoProvider extends ReceiverAdapter
- implements Provider<Optional<PeerInfo>>, LifecycleListener {
+ implements Provider<Set<PeerInfo>>, LifecycleListener {
private static final Logger log = LoggerFactory.getLogger(JGroupsPeerInfoProvider.class);
private static final String JGROUPS_LOG_FACTORY_PROPERTY = "jgroups.logging.log_factory_class";
@@ -85,7 +87,6 @@
@Override
public void viewAccepted(View view) {
log.info("viewAccepted(view: {}) called", view);
-
synchronized (this) {
if (view.getMembers().size() > 2) {
log.warn(
@@ -164,8 +165,8 @@
}
@Override
- public Optional<PeerInfo> get() {
- return peerInfo;
+ public Set<PeerInfo> get() {
+ return peerInfo.isPresent() ? ImmutableSet.of(peerInfo.get()) : ImmutableSet.of();
}
@Override
diff --git a/src/main/resources/Documentation/about.md b/src/main/resources/Documentation/about.md
index 331dae5..b2040d5 100644
--- a/src/main/resources/Documentation/about.md
+++ b/src/main/resources/Documentation/about.md
@@ -1,3 +1,4 @@
+
This plugin allows making Gerrit highly available by having redundant Gerrit
masters.
@@ -7,10 +8,10 @@
* sharing the git repositories using a shared file system (e.g. NFS)
* behind a load balancer (e.g. HAProxy)
-Currently, the only mode supported is one primary (active) master and one backup
-(passive) master but eventually the plan is to support `n` active masters. In
+Currently, the mode supported is one primary (active) master and multiple backup
+(passive) masters but eventually the plan is to support `n` active masters. In
the active/passive mode, the active master is handling all traffic while the
-passive is kept updated to be always ready to take over.
+passives are kept updated to be always ready to take over.
Even if database and git repositories are shared by the masters, there are a few
areas of concern in order to be able to switch traffic between masters in a
@@ -23,38 +24,38 @@
* web sessions
They need either to be shared or kept local to each master but synchronized.
-This plugin needs to be installed in both masters and will take care of sharing
+This plugin needs to be installed in all the masters and it will take care of sharing
or synchronizing them.
#### Caches
Every time a cache eviction occurs in one of the masters, the eviction will be
-forwarded the other master so its caches do not contain stale entries.
+forwarded the other masters so their caches do not contain stale entries.
#### Secondary indexes
Every time the secondary index is modified in one of the masters, e.g., a change
-is added, updated or removed from the index, the other master's index is
+is added, updated or removed from the index, the others master's index are
updated accordingly. This way, both indexes are kept synchronized.
#### Stream events
Every time a stream event occurs in one of the masters (see [more events info]
(https://gerrit-review.googlesource.com/Documentation/cmd-stream-events.html#events)),
-the event is forwarded to the other master which re-plays it. This way, the
+the event is forwarded to the other masters which re-plays it. This way, the
output of the stream-events command is the same, no matter which master a client
is connected to.
#### Web session
The built-in Gerrit H2 based web session cache is replaced with a file based
-implementation that is shared amongst both masters.
+implementation that is shared amongst the masters.
## Setup
Prerequisites:
-* Unique database server must be accessible from both masters
+* Unique database server must be accessible from all the masters
* Git repositories must be located on a shared file system
* A directory on a shared file system must be available for @PLUGIN@ to use
-For both masters:
+For the masters:
* Configure database section in gerrit.config to use the shared database
* Configure gerrit.basePath in gerrit.config to the shared repositories location
@@ -69,7 +70,7 @@
sharedDirectory = /directory/accessible/from/both/masters
[peerInfo "static"]
- url = http://backupMasterHost:8081/
+ url = http://backupMasterHost1:8081/
[http]
user = username
diff --git a/src/main/resources/Documentation/config.md b/src/main/resources/Documentation/config.md
index b59b1fc..ca5a6e5 100644
--- a/src/main/resources/Documentation/config.md
+++ b/src/main/resources/Documentation/config.md
@@ -1,7 +1,8 @@
+
@PLUGIN@ Configuration
=========================
-The @PLUGIN@ plugin must be installed on both instances and the following fields
+The @PLUGIN@ plugin must be installed on all the instances and the following fields
should be specified in `$site_path/etc/@PLUGIN@.config` file:
File '@PLUGIN@.config'
@@ -17,7 +18,8 @@
[peerInfo]
strategy = static
[peerInfo "static"]
- url = target_instance_url
+ url = first_target_instance_url
+ url = second_target_instance_url
[http]
user = username
password = password
@@ -93,7 +95,8 @@
a member joins or leaves the cluster.
```peerInfo.static.url```
-: Specify the URL for the peer instance.
+: Specify the URL for the peer instance. If more than one peer instance is to be
+ configured, add as many url entries as necessary.
```peerInfo.jgroups.myUrl```
: The URL of this instance to be broadcast to other peers. If not specified, the
diff --git a/src/test/java/com/ericsson/gerrit/plugins/highavailability/ConfigurationTest.java b/src/test/java/com/ericsson/gerrit/plugins/highavailability/ConfigurationTest.java
index efb3b75..63ab408 100644
--- a/src/test/java/com/ericsson/gerrit/plugins/highavailability/ConfigurationTest.java
+++ b/src/test/java/com/ericsson/gerrit/plugins/highavailability/ConfigurationTest.java
@@ -69,6 +69,7 @@
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.List;
import org.eclipse.jgit.lib.Config;
import org.junit.Before;
import org.junit.Test;
@@ -84,6 +85,7 @@
private static final String PASS = "fakePass";
private static final String USER = "fakeUser";
private static final String URL = "http://fakeUrl";
+ private static final List<String> URLS = ImmutableList.of(URL, "http://anotherUrl/");
private static final int TIMEOUT = 5000;
private static final int MAX_TRIES = 5;
private static final int RETRY_INTERVAL = 1000;
@@ -122,17 +124,12 @@
}
@Test
- public void testGetUrl() throws Exception {
- assertThat(getConfiguration().peerInfoStatic().url()).isEmpty();
+ public void testGetUrls() throws Exception {
+ assertThat(getConfiguration().peerInfoStatic().urls()).isEmpty();
- globalPluginConfig.setString(PEER_INFO_SECTION, STATIC_SUBSECTION, URL_KEY, URL);
- assertThat(getConfiguration().peerInfoStatic().url()).isEqualTo(URL);
- }
-
- @Test
- public void testGetUrlIsDroppingTrailingSlash() throws Exception {
- globalPluginConfig.setString(PEER_INFO_SECTION, STATIC_SUBSECTION, URL_KEY, URL + "/");
- assertThat(getConfiguration().peerInfoStatic().url()).isEqualTo(URL);
+ globalPluginConfig.setStringList(PEER_INFO_SECTION, STATIC_SUBSECTION, URL_KEY, URLS);
+ assertThat(getConfiguration().peerInfoStatic().urls())
+ .containsAllIn(ImmutableList.of(URL, "http://anotherUrl"));
}
@Test
diff --git a/src/test/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/HttpSessionTest.java b/src/test/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/HttpSessionTest.java
index b6eba3d..9b3c85c 100644
--- a/src/test/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/HttpSessionTest.java
+++ b/src/test/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/HttpSessionTest.java
@@ -15,35 +15,27 @@
package com.ericsson.gerrit.plugins.highavailability.forwarder.rest;
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
-import static com.github.tomakehurst.wiremock.client.WireMock.anyRequestedFor;
-import static com.github.tomakehurst.wiremock.client.WireMock.anyUrl;
import static com.github.tomakehurst.wiremock.client.WireMock.delete;
import static com.github.tomakehurst.wiremock.client.WireMock.equalTo;
-import static com.github.tomakehurst.wiremock.client.WireMock.exactly;
import static com.github.tomakehurst.wiremock.client.WireMock.post;
import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
-import static com.github.tomakehurst.wiremock.client.WireMock.verify;
import static com.google.common.truth.Truth.assertThat;
-import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.ericsson.gerrit.plugins.highavailability.Configuration;
import com.ericsson.gerrit.plugins.highavailability.forwarder.rest.HttpResponseHandler.HttpResult;
-import com.ericsson.gerrit.plugins.highavailability.peers.PeerInfo;
import com.github.tomakehurst.wiremock.http.Fault;
import com.github.tomakehurst.wiremock.junit.WireMockRule;
import com.github.tomakehurst.wiremock.stubbing.Scenario;
-import com.google.inject.util.Providers;
-import java.io.IOException;
import java.net.SocketTimeoutException;
-import java.util.Optional;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.mockito.Answers;
public class HttpSessionTest {
+
private static final int MAX_TRIES = 3;
private static final int RETRY_INTERVAL = 250;
private static final int TIMEOUT = 500;
@@ -65,10 +57,12 @@
@Rule public WireMockRule wireMockRule = new WireMockRule(0);
private Configuration configMock;
+ private String uri;
@Before
public void setUp() throws Exception {
String url = "http://localhost:" + wireMockRule.port();
+ uri = url + ENDPOINT;
configMock = mock(Configuration.class, Answers.RETURNS_DEEP_STUBS);
when(configMock.http().user()).thenReturn("user");
when(configMock.http().password()).thenReturn("pass");
@@ -77,11 +71,7 @@
when(configMock.http().socketTimeout()).thenReturn(TIMEOUT);
when(configMock.http().retryInterval()).thenReturn(RETRY_INTERVAL);
- PeerInfo peerInfo = mock(PeerInfo.class);
- when(peerInfo.getDirectUrl()).thenReturn(url);
- httpSession =
- new HttpSession(
- new HttpClientProvider(configMock).get(), Providers.of(Optional.of(peerInfo)));
+ httpSession = new HttpSession(new HttpClientProvider(configMock).get());
}
@Test
@@ -89,7 +79,7 @@
wireMockRule.givenThat(
post(urlEqualTo(ENDPOINT)).willReturn(aResponse().withStatus(NO_CONTENT)));
- assertThat(httpSession.post(ENDPOINT).isSuccessful()).isTrue();
+ assertThat(httpSession.post(uri).isSuccessful()).isTrue();
}
@Test
@@ -98,7 +88,7 @@
post(urlEqualTo(ENDPOINT))
.withRequestBody(equalTo(BODY))
.willReturn(aResponse().withStatus(NO_CONTENT)));
- assertThat(httpSession.post(ENDPOINT, BODY).isSuccessful()).isTrue();
+ assertThat(httpSession.post(uri, BODY).isSuccessful()).isTrue();
}
@Test
@@ -106,7 +96,7 @@
wireMockRule.givenThat(
delete(urlEqualTo(ENDPOINT)).willReturn(aResponse().withStatus(NO_CONTENT)));
- assertThat(httpSession.delete(ENDPOINT).isSuccessful()).isTrue();
+ assertThat(httpSession.delete(uri).isSuccessful()).isTrue();
}
@Test
@@ -116,7 +106,7 @@
post(urlEqualTo(ENDPOINT))
.willReturn(aResponse().withStatus(UNAUTHORIZED).withBody(expected)));
- HttpResult result = httpSession.post(ENDPOINT);
+ HttpResult result = httpSession.post(uri);
assertThat(result.isSuccessful()).isFalse();
assertThat(result.getMessage()).isEqualTo(expected);
}
@@ -128,7 +118,7 @@
post(urlEqualTo(ENDPOINT))
.willReturn(aResponse().withStatus(NOT_FOUND).withBody(expected)));
- HttpResult result = httpSession.post(ENDPOINT);
+ HttpResult result = httpSession.post(uri);
assertThat(result.isSuccessful()).isFalse();
assertThat(result.getMessage()).isEqualTo(expected);
}
@@ -139,7 +129,7 @@
post(urlEqualTo(ENDPOINT))
.willReturn(aResponse().withStatus(ERROR).withBody(ERROR_MESSAGE)));
- HttpResult result = httpSession.post(ENDPOINT);
+ HttpResult result = httpSession.post(uri);
assertThat(result.isSuccessful()).isFalse();
assertThat(result.getMessage()).isEqualTo(ERROR_MESSAGE);
}
@@ -170,7 +160,7 @@
.whenScenarioStateIs(THIRD_TRY)
.willReturn(aResponse().withFixedDelay(TIMEOUT)));
- httpSession.post(ENDPOINT);
+ httpSession.post(uri);
}
@Test
@@ -179,19 +169,6 @@
post(urlEqualTo(ENDPOINT))
.willReturn(aResponse().withFault(Fault.MALFORMED_RESPONSE_CHUNK)));
- assertThat(httpSession.post(ENDPOINT).isSuccessful()).isFalse();
- }
-
- @Test
- public void testNoRequestWhenPeerInfoUnknown() throws IOException {
- httpSession =
- new HttpSession(new HttpClientProvider(configMock).get(), Providers.of(Optional.empty()));
- try {
- httpSession.post(ENDPOINT);
- fail("Expected PeerInfoNotAvailableException");
- } catch (PeerInfoNotAvailableException e) {
- // good
- }
- verify(exactly(0), anyRequestedFor(anyUrl()));
+ assertThat(httpSession.post(uri).isSuccessful()).isFalse();
}
}
diff --git a/src/test/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/RestForwarderTest.java b/src/test/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/RestForwarderTest.java
index 6a403ed..bb40763 100644
--- a/src/test/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/RestForwarderTest.java
+++ b/src/test/java/com/ericsson/gerrit/plugins/highavailability/forwarder/rest/RestForwarderTest.java
@@ -23,22 +23,27 @@
import com.ericsson.gerrit.plugins.highavailability.Configuration;
import com.ericsson.gerrit.plugins.highavailability.cache.Constants;
import com.ericsson.gerrit.plugins.highavailability.forwarder.rest.HttpResponseHandler.HttpResult;
+import com.ericsson.gerrit.plugins.highavailability.peers.PeerInfo;
import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableSet;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.AccountGroup;
import com.google.gerrit.server.events.Event;
import com.google.gson.GsonBuilder;
+import com.google.inject.Provider;
import java.io.IOException;
+import java.util.Set;
import javax.net.ssl.SSLException;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Answers;
public class RestForwarderTest {
+ private static final String URL = "http://fake.com";
private static final String PLUGIN_NAME = "high-availability";
private static final String EMPTY_MSG = "";
private static final String ERROR = "Error";
- private static final String PLUGINS = "/plugins";
+ private static final String PLUGINS = "plugins";
private static final String PROJECT_NAME = "projectName";
private static final String PROJECT_TO_ADD = "projectToAdd";
private static final String PROJECT_TO_DELETE = "projectToDelete";
@@ -49,86 +54,94 @@
// Index
private static final int CHANGE_NUMBER = 1;
private static final String INDEX_CHANGE_ENDPOINT =
- Joiner.on("/").join(PLUGINS, PLUGIN_NAME, "index/change", CHANGE_NUMBER);
+ Joiner.on("/").join(URL, PLUGINS, PLUGIN_NAME, "index/change", CHANGE_NUMBER);
private static final int ACCOUNT_NUMBER = 2;
private static final String INDEX_ACCOUNT_ENDPOINT =
- Joiner.on("/").join(PLUGINS, PLUGIN_NAME, "index/account", ACCOUNT_NUMBER);
+ Joiner.on("/").join(URL, PLUGINS, PLUGIN_NAME, "index/account", ACCOUNT_NUMBER);
private static final String UUID = "we235jdf92nfj2351";
private static final String INDEX_GROUP_ENDPOINT =
- Joiner.on("/").join(PLUGINS, PLUGIN_NAME, "index/group", UUID);
+ Joiner.on("/").join(URL, PLUGINS, PLUGIN_NAME, "index/group", UUID);
// Event
- private static final String EVENT_ENDPOINT = Joiner.on("/").join(PLUGINS, PLUGIN_NAME, "event");
private static Event event = new Event("test-event") {};
+ private static final String EVENT_ENDPOINT =
+ Joiner.on("/").join(URL, PLUGINS, PLUGIN_NAME, "event", event.type);
private static String eventJson = new GsonBuilder().create().toJson(event);
private RestForwarder forwarder;
private HttpSession httpSessionMock;
+ @SuppressWarnings("unchecked")
@Before
public void setUp() {
httpSessionMock = mock(HttpSession.class);
Configuration configMock = mock(Configuration.class, Answers.RETURNS_DEEP_STUBS);
when(configMock.http().maxTries()).thenReturn(3);
when(configMock.http().retryInterval()).thenReturn(10);
- forwarder = new RestForwarder(httpSessionMock, PLUGIN_NAME, configMock);
+ Provider<Set<PeerInfo>> peersMock = mock(Provider.class);
+ when(peersMock.get()).thenReturn(ImmutableSet.of(new PeerInfo(URL)));
+ forwarder =
+ new RestForwarder(
+ httpSessionMock, PLUGIN_NAME, configMock, peersMock); // TODO: Create provider
}
@Test
public void testIndexAccountOK() throws Exception {
- when(httpSessionMock.post(INDEX_ACCOUNT_ENDPOINT))
+ when(httpSessionMock.post(INDEX_ACCOUNT_ENDPOINT, null))
.thenReturn(new HttpResult(SUCCESSFUL, EMPTY_MSG));
assertThat(forwarder.indexAccount(ACCOUNT_NUMBER)).isTrue();
}
@Test
public void testIndexAccountFailed() throws Exception {
- when(httpSessionMock.post(INDEX_ACCOUNT_ENDPOINT))
+ when(httpSessionMock.post(INDEX_ACCOUNT_ENDPOINT, null))
.thenReturn(new HttpResult(FAILED, EMPTY_MSG));
assertThat(forwarder.indexAccount(ACCOUNT_NUMBER)).isFalse();
}
@Test
public void testIndexAccountThrowsException() throws Exception {
- doThrow(new IOException()).when(httpSessionMock).post(INDEX_ACCOUNT_ENDPOINT);
+ doThrow(new IOException()).when(httpSessionMock).post(INDEX_ACCOUNT_ENDPOINT, null);
assertThat(forwarder.indexAccount(ACCOUNT_NUMBER)).isFalse();
}
@Test
public void testIndexGroupOK() throws Exception {
- when(httpSessionMock.post(INDEX_GROUP_ENDPOINT))
+ when(httpSessionMock.post(INDEX_GROUP_ENDPOINT, null))
.thenReturn(new HttpResult(SUCCESSFUL, EMPTY_MSG));
assertThat(forwarder.indexGroup(UUID)).isTrue();
}
@Test
public void testIndexGroupFailed() throws Exception {
- when(httpSessionMock.post(INDEX_GROUP_ENDPOINT)).thenReturn(new HttpResult(FAILED, EMPTY_MSG));
+ when(httpSessionMock.post(INDEX_GROUP_ENDPOINT, null))
+ .thenReturn(new HttpResult(FAILED, EMPTY_MSG));
assertThat(forwarder.indexGroup(UUID)).isFalse();
}
@Test
public void testIndexGroupThrowsException() throws Exception {
- doThrow(new IOException()).when(httpSessionMock).post(INDEX_GROUP_ENDPOINT);
+ doThrow(new IOException()).when(httpSessionMock).post(INDEX_GROUP_ENDPOINT, null);
assertThat(forwarder.indexGroup(UUID)).isFalse();
}
@Test
public void testIndexChangeOK() throws Exception {
- when(httpSessionMock.post(INDEX_CHANGE_ENDPOINT))
+ when(httpSessionMock.post(INDEX_CHANGE_ENDPOINT, null))
.thenReturn(new HttpResult(SUCCESSFUL, EMPTY_MSG));
assertThat(forwarder.indexChange(CHANGE_NUMBER)).isTrue();
}
@Test
public void testIndexChangeFailed() throws Exception {
- when(httpSessionMock.post(INDEX_CHANGE_ENDPOINT)).thenReturn(new HttpResult(FAILED, EMPTY_MSG));
+ when(httpSessionMock.post(INDEX_CHANGE_ENDPOINT, null))
+ .thenReturn(new HttpResult(FAILED, EMPTY_MSG));
assertThat(forwarder.indexChange(CHANGE_NUMBER)).isFalse();
}
@Test
public void testIndexChangeThrowsException() throws Exception {
- doThrow(new IOException()).when(httpSessionMock).post(INDEX_CHANGE_ENDPOINT);
+ doThrow(new IOException()).when(httpSessionMock).post(INDEX_CHANGE_ENDPOINT, null);
assertThat(forwarder.indexChange(CHANGE_NUMBER)).isFalse();
}
@@ -194,8 +207,8 @@
public void testEvictGroupsOK() throws Exception {
AccountGroup.Id key = new AccountGroup.Id(123);
String keyJson = new GsonBuilder().create().toJson(key);
- when(httpSessionMock.post(buildCacheEndpoint(Constants.GROUPS), keyJson))
- .thenReturn(new HttpResult(SUCCESSFUL, EMPTY_MSG));
+ String endpoint = buildCacheEndpoint(Constants.GROUPS);
+ when(httpSessionMock.post(endpoint, keyJson)).thenReturn(new HttpResult(SUCCESSFUL, EMPTY_MSG));
assertThat(forwarder.evict(Constants.GROUPS, key)).isTrue();
}
@@ -237,13 +250,13 @@
}
private static String buildCacheEndpoint(String name) {
- return Joiner.on("/").join(PLUGINS, PLUGIN_NAME, "cache", name);
+ return Joiner.on("/").join(URL, PLUGINS, PLUGIN_NAME, "cache", name);
}
@Test
public void testAddToProjectListOK() throws Exception {
String projectName = PROJECT_TO_ADD;
- when(httpSessionMock.post(buildProjectListCacheEndpoint(projectName)))
+ when(httpSessionMock.post(buildProjectListCacheEndpoint(projectName), null))
.thenReturn(new HttpResult(SUCCESSFUL, EMPTY_MSG));
assertThat(forwarder.addToProjectList(projectName)).isTrue();
}
@@ -251,7 +264,7 @@
@Test
public void testAddToProjectListFailed() throws Exception {
String projectName = PROJECT_TO_ADD;
- when(httpSessionMock.post(buildProjectListCacheEndpoint(projectName)))
+ when(httpSessionMock.post(buildProjectListCacheEndpoint(projectName), null))
.thenReturn(new HttpResult(FAILED, EMPTY_MSG));
assertThat(forwarder.addToProjectList(projectName)).isFalse();
}
@@ -261,7 +274,7 @@
String projectName = PROJECT_TO_ADD;
doThrow(new IOException())
.when(httpSessionMock)
- .post(buildProjectListCacheEndpoint(projectName));
+ .post(buildProjectListCacheEndpoint(projectName), null);
assertThat(forwarder.addToProjectList(projectName)).isFalse();
}
