Make JGroupsKubernetesPeerInfoProviderTest runnable in Kubernetes The JGroupsKubernetesPeerInfoProviderTest would fail when run in a Kubernetes Pod. That was due to environment variables in the pod interfering with the jgroups configuration. This change avoids that by providing a test-specific jgroups config. It also makes the test more robust by adding a wait for the first channel to be connected. Change-Id: I363de8396a67e0a08bfe5c76aa8c099c58bccb01
diff --git a/src/test/java/com/ericsson/gerrit/plugins/highavailability/peers/jgroups/JGroupsKubernetesPeerInfoProviderTest.java b/src/test/java/com/ericsson/gerrit/plugins/highavailability/peers/jgroups/JGroupsKubernetesPeerInfoProviderTest.java index 0c53959..b1cd172 100644 --- a/src/test/java/com/ericsson/gerrit/plugins/highavailability/peers/jgroups/JGroupsKubernetesPeerInfoProviderTest.java +++ b/src/test/java/com/ericsson/gerrit/plugins/highavailability/peers/jgroups/JGroupsKubernetesPeerInfoProviderTest.java
@@ -29,14 +29,19 @@ import static org.mockito.Mockito.when; import com.ericsson.gerrit.plugins.highavailability.Configuration; +import com.github.rholder.retry.RetryerBuilder; +import com.github.rholder.retry.StopStrategies; +import com.github.rholder.retry.WaitStrategies; import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.google.common.base.Predicates; import com.google.common.io.Resources; import com.google.gerrit.extensions.restapi.Url; import java.net.Inet4Address; -import java.net.NetworkInterface; import java.util.List; import java.util.Optional; +import java.util.concurrent.TimeUnit; import org.apache.http.HttpStatus; +import org.jgroups.JChannel; import org.jgroups.Message; import org.junit.After; import org.junit.Before; @@ -57,24 +62,33 @@ @Mock private InetAddressFinder finder; private JGroupsPeerInfoProvider firstJGroupsPeerInfoProvider; private JGroupsPeerInfoProvider secondJGroupsPeerInfoProvider; + private JChannel firstChannel; + private JChannel secondChannel; @Mock private MyUrlProvider myUrlProvider; @Rule public WireMockRule kubeApiMock = new WireMockRule(options().port(48443)); @Before public void setUp() throws Exception { - System.setProperty("KUBERNETES_MASTER_PROTOCOL", "http"); - System.setProperty("KUBERNETES_SERVICE_HOST", "localhost"); - System.setProperty("KUBERNETES_SERVICE_PORT", "48443"); System.setProperty("java.net.preferIPv4Stack", "true"); } @After - public void shutdown() { - System.clearProperty("KUBERNETES_MASTER_PROTOCOL"); - System.clearProperty("KUBERNETES_SERVICE_HOST"); - System.clearProperty("KUBERNETES_SERVICE_PORT"); + public void shutdown() throws Exception { System.clearProperty("java.net.preferIPv4Stack"); + + if (firstJGroupsPeerInfoProvider != null) { + firstJGroupsPeerInfoProvider.stop(); + } + if (secondJGroupsPeerInfoProvider != null) { + secondJGroupsPeerInfoProvider.stop(); + } + if (firstChannel != null && firstChannel.isConnected()) { + firstChannel.close(); + } + if (secondChannel != null && secondChannel.isConnected()) { + secondChannel.close(); + } } @Test @@ -89,21 +103,20 @@ when(myUrlProvider.get()).thenReturn("http://127.0.0.1:7800"); - NetworkInterface eth0 = NetworkInterface.getByName("eth0"); - if (eth0 != null) { - when(finder.findAddress()).thenReturn(eth0.inetAddresses().findFirst()); - } else { - when(finder.findAddress()).thenReturn(Optional.of(Inet4Address.getByName("127.0.0.1"))); - } - JChannelProvider channelProvider = new JChannelProvider(pluginConfigurationMock); + when(finder.findAddress()).thenReturn(Optional.of(Inet4Address.getByName("127.0.0.1"))); + + // Create two separate channels with different ports + firstChannel = new JChannel(getClass().getResource("test_kubernetes.xml").toString()); + secondChannel = new JChannel(getClass().getResource("test_kubernetes.xml").toString()); + firstJGroupsPeerInfoProvider = Mockito.spy( new JGroupsPeerInfoProvider( - pluginConfigurationMock, finder, myUrlProvider, channelProvider.get())); + pluginConfigurationMock, finder, myUrlProvider, firstChannel)); secondJGroupsPeerInfoProvider = Mockito.spy( new JGroupsPeerInfoProvider( - pluginConfigurationMock, finder, myUrlProvider, channelProvider.get())); + pluginConfigurationMock, finder, myUrlProvider, secondChannel)); StringBuilder kubeApiUrlBuilder = new StringBuilder(); kubeApiUrlBuilder.append("/api/v1/namespaces/"); @@ -121,11 +134,21 @@ aResponse() .withJsonBody(new ObjectMapper().readTree(respJson)) .withStatus(HttpStatus.SC_OK))); + firstJGroupsPeerInfoProvider.connect(); verify(getRequestedFor(urlEqualTo(kubeApiUrl))); + + RetryerBuilder.<Boolean>newBuilder() + .retryIfResult(Predicates.equalTo(false)) + .withStopStrategy(StopStrategies.stopAfterDelay(10, TimeUnit.SECONDS)) + .withWaitStrategy(WaitStrategies.fixedWait(1, TimeUnit.SECONDS)) + .build() + .call(() -> firstChannel.isConnected()); + secondJGroupsPeerInfoProvider.connect(); - verify(firstJGroupsPeerInfoProvider, timeout(10000)).receive(any(Message.class)); + verify(firstJGroupsPeerInfoProvider, timeout(10000).atLeastOnce()).receive(any(Message.class)); + verify(secondJGroupsPeerInfoProvider, timeout(10000).atLeastOnce()).receive(any(Message.class)); assertThat(firstJGroupsPeerInfoProvider.get()).isNotEmpty(); assertThat(firstJGroupsPeerInfoProvider.get()).hasSize(1);
diff --git a/src/test/resources/com/ericsson/gerrit/plugins/highavailability/peers/jgroups/pod-list.json b/src/test/resources/com/ericsson/gerrit/plugins/highavailability/peers/jgroups/pod-list.json index 4be7c10..e7efdd1 100644 --- a/src/test/resources/com/ericsson/gerrit/plugins/highavailability/peers/jgroups/pod-list.json +++ b/src/test/resources/com/ericsson/gerrit/plugins/highavailability/peers/jgroups/pod-list.json
@@ -17,7 +17,14 @@ "containers": [ { "image": "gerrit:test", - "name": "gerrit" + "name": "gerrit", + "ports": [ + { + "containerPort": 7800, + "name": "jgroups", + "protocol": "TCP" + } + ] } ], "hostname": "gerrit-primary-0" @@ -67,7 +74,14 @@ "containers": [ { "image": "gerrit:test", - "name": "gerrit" + "name": "gerrit", + "ports": [ + { + "containerPort": 7801, + "name": "jgroups", + "protocol": "TCP" + } + ] } ], "hostname": "gerrit-primary-1"
diff --git a/src/test/resources/com/ericsson/gerrit/plugins/highavailability/peers/jgroups/test_kubernetes.xml b/src/test/resources/com/ericsson/gerrit/plugins/highavailability/peers/jgroups/test_kubernetes.xml new file mode 100644 index 0000000..ff1bebf --- /dev/null +++ b/src/test/resources/com/ericsson/gerrit/plugins/highavailability/peers/jgroups/test_kubernetes.xml
@@ -0,0 +1,53 @@ +<!-- + Based on a configuration written by Bela Ban [1]. + + [1] https://koudingspawn.de/jgroups-on-kubernetes/ +--> +<config xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xmlns="urn:org:jgroups" + xsi:schemaLocation="urn:org:jgroups http://www.jgroups.org/schema/jgroups.xsd" +> + <TCP bind_addr="127.0.0.1" + bind_port="7800" + port_range="2" + external_addr="127.0.0.1" + thread_pool.min_threads="0" + thread_pool.max_threads="200" + thread_pool.keep_alive_time="30000" /> + <RED /> + + <org.jgroups.protocols.kubernetes.KUBE_PING + port_range="2" + namespace="gerrit" + labels="app=gerrit,mode=primary" + masterProtocol="http" + masterHost="localhost" + masterPort="48443" + /> + + <MERGE3 max_interval="30000" + min_interval="10000" /> + <FD_SOCK external_addr="${JGROUPS_EXTERNAL_ADDR}" + start_port="${FD_SOCK_PORT:9000}" /> + <FD_ALL timeout="30000" interval="5000" /> + <VERIFY_SUSPECT timeout="1500" /> + <BARRIER /> + <pbcast.NAKACK2 xmit_interval="500" + xmit_table_num_rows="100" + xmit_table_msgs_per_row="2000" + xmit_table_max_compaction_time="30000" + use_mcast_xmit="false" + discard_delivered_msgs="true" /> + <UNICAST3 + xmit_table_num_rows="100" + xmit_table_msgs_per_row="1000" + xmit_table_max_compaction_time="30000" /> + <pbcast.STABLE desired_avg_gossip="50000" max_bytes="8m" /> + <pbcast.GMS print_local_addr="true" print_physical_addrs="true" join_timeout="3000" /> + <MFC max_credits="2M" + min_threshold="0.4" /> + <FRAG2 frag_size="60K" /> + <pbcast.STATE_TRANSFER /> + <CENTRAL_LOCK /> + <COUNTER /> +</config>