[Operator] Use host IP based hashes for session stickiness in istio
So far the Gerrit Operator configured Istio to use cookies to determine
session stickiness. These cookies had a TTL of 60s. This caused some
usability issues in the Gerrit UI, when writing comments or editing
a change. In this cases, if the cookie expired and the user's request
were sent to the other instance, it could happen that the data on
this second instance had not caught up, e.g. due to NFS latency. In
that case it could happen that the user's input was reset in the UI.
Now, Istio is using the source IP for session stickiness. This is more
likely to cause imbalanced load, since CI systems who typically have
a stable IP will stick to a single instance and CI systems might thus
by chance mostly end up on a single instance and cause a high load.
This should be observed. However, the advantage over just using a
cookie with a longer TTL is that this way also supports SSH, REST
requests done from outside the browser and git, which should overall
lead to a more stable experience.
Since the NGINX ingress provider does not support this feature, it
will continue to use cookies, but with a TTL of 1h.
Inspired-By: Luca Milanesio
Inspired-By: Álvaro Vilaplana García
Change-Id: I4714e9252a4fa7aa9532d3a66e7d6c49f4017a12
diff --git a/operator/src/main/java/com/google/gerrit/k8s/operator/network/ingress/dependent/GerritClusterIngress.java b/operator/src/main/java/com/google/gerrit/k8s/operator/network/ingress/dependent/GerritClusterIngress.java
index d0bf3aa..bed92db 100644
--- a/operator/src/main/java/com/google/gerrit/k8s/operator/network/ingress/dependent/GerritClusterIngress.java
+++ b/operator/src/main/java/com/google/gerrit/k8s/operator/network/ingress/dependent/GerritClusterIngress.java
@@ -14,7 +14,6 @@
package com.google.gerrit.k8s.operator.network.ingress.dependent;
-import static com.google.gerrit.k8s.operator.api.model.network.GerritNetwork.SESSION_COOKIE_NAME;
import static com.google.gerrit.k8s.operator.network.Constants.GERRIT_FORBIDDEN_URL_PATTERN;
import static com.google.gerrit.k8s.operator.network.Constants.PROJECTS_URL_PATTERN;
import static com.google.gerrit.k8s.operator.network.Constants.RECEIVE_PACK_URL_PATTERN;
@@ -38,6 +37,7 @@
import io.fabric8.kubernetes.api.model.networking.v1.ServiceBackendPortBuilder;
import io.javaoperatorsdk.operator.api.reconciler.Context;
import io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependent;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -47,6 +47,8 @@
public class GerritClusterIngress
extends CRUDReconcileAddKubernetesDependentResource<Ingress, GerritNetwork> {
public static final String INGRESS_NAME = "gerrit-ingress";
+ public static final String SESSION_COOKIE_NAME = "Gerrit_Session";
+ public static final Duration SESSION_COOKIE_TTL = Duration.ofSeconds(3600L);
public GerritClusterIngress() {
super(Ingress.class);
@@ -117,8 +119,12 @@
annotations.put("nginx.ingress.kubernetes.io/affinity", "cookie");
annotations.put("nginx.ingress.kubernetes.io/session-cookie-name", SESSION_COOKIE_NAME);
annotations.put("nginx.ingress.kubernetes.io/session-cookie-path", "/");
- annotations.put("nginx.ingress.kubernetes.io/session-cookie-max-age", "60");
- annotations.put("nginx.ingress.kubernetes.io/session-cookie-expires", "60");
+ annotations.put(
+ "nginx.ingress.kubernetes.io/session-cookie-max-age",
+ String.valueOf(SESSION_COOKIE_TTL.getSeconds()));
+ annotations.put(
+ "nginx.ingress.kubernetes.io/session-cookie-expires",
+ String.valueOf(SESSION_COOKIE_TTL.getSeconds()));
return annotations;
}
diff --git a/operator/src/main/java/com/google/gerrit/k8s/operator/network/istio/dependent/GerritIstioDestinationRule.java b/operator/src/main/java/com/google/gerrit/k8s/operator/network/istio/dependent/GerritIstioDestinationRule.java
index 04cf05c..b3f75ef 100644
--- a/operator/src/main/java/com/google/gerrit/k8s/operator/network/istio/dependent/GerritIstioDestinationRule.java
+++ b/operator/src/main/java/com/google/gerrit/k8s/operator/network/istio/dependent/GerritIstioDestinationRule.java
@@ -14,9 +14,6 @@
package com.google.gerrit.k8s.operator.network.istio.dependent;
-import static com.google.gerrit.k8s.operator.api.model.network.GerritNetwork.SESSION_COOKIE_NAME;
-import static com.google.gerrit.k8s.operator.api.model.network.GerritNetwork.SESSION_COOKIE_TTL;
-
import com.google.gerrit.k8s.operator.api.model.cluster.GerritCluster;
import com.google.gerrit.k8s.operator.api.model.gerrit.GerritTemplate;
import com.google.gerrit.k8s.operator.api.model.network.GerritNetwork;
@@ -76,12 +73,7 @@
.withNewLoadBalancer()
.withNewLoadBalancerSettingsConsistentHashLbPolicy()
.withNewConsistentHash()
- .withNewLoadBalancerSettingsConsistentHashLBHttpCookieKey()
- .withNewHttpCookie()
- .withName(SESSION_COOKIE_NAME)
- .withTtl(SESSION_COOKIE_TTL)
- .endHttpCookie()
- .endLoadBalancerSettingsConsistentHashLBHttpCookieKey()
+ .withNewLoadBalancerSettingsConsistentHashLBUseSourceIpKey(true)
.endConsistentHash()
.endLoadBalancerSettingsConsistentHashLbPolicy()
.endLoadBalancer()
diff --git a/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_primary.yaml b/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_primary.yaml
index a45192b..09dce08 100644
--- a/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_primary.yaml
+++ b/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_primary.yaml
@@ -14,8 +14,8 @@
nginx.ingress.kubernetes.io/affinity: cookie
nginx.ingress.kubernetes.io/session-cookie-name: Gerrit_Session
nginx.ingress.kubernetes.io/session-cookie-path: /
- nginx.ingress.kubernetes.io/session-cookie-max-age: 60
- nginx.ingress.kubernetes.io/session-cookie-expires: 60
+ nginx.ingress.kubernetes.io/session-cookie-max-age: 3600
+ nginx.ingress.kubernetes.io/session-cookie-expires: 3600
spec:
rules:
- host: example.com
diff --git a/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_primary_replica.yaml b/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_primary_replica.yaml
index e0cecf1..91b9acd 100644
--- a/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_primary_replica.yaml
+++ b/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_primary_replica.yaml
@@ -19,8 +19,8 @@
nginx.ingress.kubernetes.io/affinity: cookie
nginx.ingress.kubernetes.io/session-cookie-name: Gerrit_Session
nginx.ingress.kubernetes.io/session-cookie-path: /
- nginx.ingress.kubernetes.io/session-cookie-max-age: 60
- nginx.ingress.kubernetes.io/session-cookie-expires: 60
+ nginx.ingress.kubernetes.io/session-cookie-max-age: 3600
+ nginx.ingress.kubernetes.io/session-cookie-expires: 3600
spec:
rules:
- host: example.com
diff --git a/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_primary_replica_tls.yaml b/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_primary_replica_tls.yaml
index e0aa83e..76a8de3 100644
--- a/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_primary_replica_tls.yaml
+++ b/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_primary_replica_tls.yaml
@@ -19,8 +19,8 @@
nginx.ingress.kubernetes.io/affinity: cookie
nginx.ingress.kubernetes.io/session-cookie-name: Gerrit_Session
nginx.ingress.kubernetes.io/session-cookie-path: /
- nginx.ingress.kubernetes.io/session-cookie-max-age: 60
- nginx.ingress.kubernetes.io/session-cookie-expires: 60
+ nginx.ingress.kubernetes.io/session-cookie-max-age: 3600
+ nginx.ingress.kubernetes.io/session-cookie-expires: 3600
spec:
tls:
- hosts:
diff --git a/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_receiver.yaml b/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_receiver.yaml
index 55e3bcc..ff9b4e4 100644
--- a/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_receiver.yaml
+++ b/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_receiver.yaml
@@ -9,8 +9,8 @@
nginx.ingress.kubernetes.io/affinity: cookie
nginx.ingress.kubernetes.io/session-cookie-name: Gerrit_Session
nginx.ingress.kubernetes.io/session-cookie-path: /
- nginx.ingress.kubernetes.io/session-cookie-max-age: 60
- nginx.ingress.kubernetes.io/session-cookie-expires: 60
+ nginx.ingress.kubernetes.io/session-cookie-max-age: 3600
+ nginx.ingress.kubernetes.io/session-cookie-expires: 3600
spec:
rules:
- host: example.com
diff --git a/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_receiver_replica.yaml b/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_receiver_replica.yaml
index 093dac7..0964004 100644
--- a/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_receiver_replica.yaml
+++ b/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_receiver_replica.yaml
@@ -9,8 +9,8 @@
nginx.ingress.kubernetes.io/affinity: cookie
nginx.ingress.kubernetes.io/session-cookie-name: Gerrit_Session
nginx.ingress.kubernetes.io/session-cookie-path: /
- nginx.ingress.kubernetes.io/session-cookie-max-age: 60
- nginx.ingress.kubernetes.io/session-cookie-expires: 60
+ nginx.ingress.kubernetes.io/session-cookie-max-age: 3600
+ nginx.ingress.kubernetes.io/session-cookie-expires: 3600
nginx.ingress.kubernetes.io/configuration-snippet: |-
if ($args ~ service=git-receive-pack){
set $proxy_upstream_name "gerrit-receiver-http";
diff --git a/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_receiver_replica_tls.yaml b/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_receiver_replica_tls.yaml
index 839827b..f132986 100644
--- a/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_receiver_replica_tls.yaml
+++ b/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_receiver_replica_tls.yaml
@@ -9,8 +9,8 @@
nginx.ingress.kubernetes.io/affinity: cookie
nginx.ingress.kubernetes.io/session-cookie-name: Gerrit_Session
nginx.ingress.kubernetes.io/session-cookie-path: /
- nginx.ingress.kubernetes.io/session-cookie-max-age: 60
- nginx.ingress.kubernetes.io/session-cookie-expires: 60
+ nginx.ingress.kubernetes.io/session-cookie-max-age: 3600
+ nginx.ingress.kubernetes.io/session-cookie-expires: 3600
nginx.ingress.kubernetes.io/configuration-snippet: |-
if ($args ~ service=git-receive-pack){
set $proxy_upstream_name "gerrit-receiver-http";
diff --git a/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_replica.yaml b/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_replica.yaml
index 07dfe7d..3513c84 100644
--- a/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_replica.yaml
+++ b/operator/src/test/resources/com/google/gerrit/k8s/operator/network/ingress/dependent/ingress_replica.yaml
@@ -9,8 +9,8 @@
nginx.ingress.kubernetes.io/affinity: cookie
nginx.ingress.kubernetes.io/session-cookie-name: Gerrit_Session
nginx.ingress.kubernetes.io/session-cookie-path: /
- nginx.ingress.kubernetes.io/session-cookie-max-age: 60
- nginx.ingress.kubernetes.io/session-cookie-expires: 60
+ nginx.ingress.kubernetes.io/session-cookie-max-age: 3600
+ nginx.ingress.kubernetes.io/session-cookie-expires: 3600
spec:
rules:
- host: example.com
