Create promtail config per Gerrit host
So far the install-script could only create a single promtail config.
Since the monitoring setup is able to monitor multiple Gerrit servers,
this caused manual work to create a promtail config per Gerrit server.
Now ytt will create a configuration for each Gerrit host configured
in the config.yaml. Ytt is only able to do that in a single file. Thus,
csplit is used to split the files into separate files that can then
be used to configure promtail on the respective hosts. The config-
files can then be found under
$OUTPUT/promtail/promtail-$GERRIT_HOSTNAME.yaml.
Change-Id: Ib09fba83d8a8fbd45b42e9e5388a85a37ab1a952
diff --git a/README.md b/README.md
index e5b63f0..0fee843 100644
--- a/README.md
+++ b/README.md
@@ -76,47 +76,47 @@
These options have to be configured in the `./config.yaml` before installing and
are listed here:
-| option | description |
-|-----------------------------------------|------------------------------------------------------------------------------------|
-| `gerritServers.[0].host` | Hostname (incl. port, if required) of the Gerrit server to monitor |
-| `gerritServers.[0].username` | Username of Gerrit user with 'View Metrics' capabilities |
-| `gerritServers.[0].password` | Password of Gerrit user with 'View Metrics' capabilities |
-| `namespace` | The namespace the charts are installed to |
-| `tls.skipVerify` | Whether to skip TLS certificate verification |
-| `tls.caCert` | CA certificate used for TLS certificate verification |
-| `promtail.storagePath` | Path to directory, where Promtail is allowed to save files (e.g. `positions.yaml`) |
-| `promtail.logPath` | Path to directory containing the Gerrit logs (e.g. `/var/gerrit/logs`) |
-| `prometheus.server.host` | Prometheus server ingress hostname |
-| `prometheus.server.username` | Username for Prometheus |
-| `prometheus.server.password` | Password for Prometheus |
-| `prometheus.server.tls.cert` | TLS certificate |
-| `prometheus.server.tls.key` | TLS key |
-| `prometheus.alertmanager.slack.apiUrl` | API URL of the Slack Webhook |
-| `prometheus.alertmanager.slack.channel` | Channel to which the alerts should be posted |
-| `loki.host` | Loki ingress hostname |
-| `loki.username` | Username for Loki |
-| `loki.password` | Password for Loki |
-| `loki.s3.protocol` | Protocol used for communicating with S3 |
-| `loki.s3.host` | Hostname of the S3 object store |
-| `loki.s3.accessToken` | The EC2 accessToken used for authentication with S3 |
-| `loki.s3.secret` | The secret associated with the accessToken |
-| `loki.s3.bucket` | The name of the S3 bucket |
-| `loki.s3.region` | The region in which the S3 bucket is hosted |
-| `loki.tls.cert` | TLS certificate |
-| `loki.tls.key` | TLS key |
-| `grafana.host` | Grafana ingress hostname |
-| `grafana.tls.cert` | TLS certificate |
-| `grafana.tls.key` | TLS key |
-| `grafana.admin.username` | Username for the admin user |
-| `grafana.admin.password` | Password for the admin user |
-| `grafana.ldap.enabled` | Whether to enable LDAP |
-| `grafana.ldap.host` | Hostname of LDAP server |
-| `grafana.ldap.port` | Port of LDAP server (Has to be `quoted`!) |
-| `grafana.ldap.password` | Password of LDAP server |
-| `grafana.ldap.bind_dn` | Bind DN (username) of the LDAP server |
-| `grafana.ldap.accountBases` | List of base DNs to discover accounts (Has to have the format `"['a', 'b']"`) |
-| `grafana.ldap.groupBases` | List of base DNs to discover groups (Has to have the format `"['a', 'b']"`) |
-| `grafana.dashboards.editable` | Whether dashboards can be edited manually in the UI |
+| option | description |
+|------------------------------------------|------------------------------------------------------------------------------------|
+| `gerritServers.[0].host` | Hostname (incl. port, if required) of the Gerrit server to monitor |
+| `gerritServers.[0].username` | Username of Gerrit user with 'View Metrics' capabilities |
+| `gerritServers.[0].password` | Password of Gerrit user with 'View Metrics' capabilities |
+| `gerritServers.[0].promtail.storagePath` | Path to directory, where Promtail is allowed to save files (e.g. `positions.yaml`) |
+| `gerritServers.[0].promtail.logPath` | Path to directory containing the Gerrit logs (e.g. `/var/gerrit/logs`) |
+| `namespace` | The namespace the charts are installed to |
+| `tls.skipVerify` | Whether to skip TLS certificate verification |
+| `tls.caCert` | CA certificate used for TLS certificate verification |
+| `prometheus.server.host` | Prometheus server ingress hostname |
+| `prometheus.server.username` | Username for Prometheus |
+| `prometheus.server.password` | Password for Prometheus |
+| `prometheus.server.tls.cert` | TLS certificate |
+| `prometheus.server.tls.key` | TLS key |
+| `prometheus.alertmanager.slack.apiUrl` | API URL of the Slack Webhook |
+| `prometheus.alertmanager.slack.channel` | Channel to which the alerts should be posted |
+| `loki.host` | Loki ingress hostname |
+| `loki.username` | Username for Loki |
+| `loki.password` | Password for Loki |
+| `loki.s3.protocol` | Protocol used for communicating with S3 |
+| `loki.s3.host` | Hostname of the S3 object store |
+| `loki.s3.accessToken` | The EC2 accessToken used for authentication with S3 |
+| `loki.s3.secret` | The secret associated with the accessToken |
+| `loki.s3.bucket` | The name of the S3 bucket |
+| `loki.s3.region` | The region in which the S3 bucket is hosted |
+| `loki.tls.cert` | TLS certificate |
+| `loki.tls.key` | TLS key |
+| `grafana.host` | Grafana ingress hostname |
+| `grafana.tls.cert` | TLS certificate |
+| `grafana.tls.key` | TLS key |
+| `grafana.admin.username` | Username for the admin user |
+| `grafana.admin.password` | Password for the admin user |
+| `grafana.ldap.enabled` | Whether to enable LDAP |
+| `grafana.ldap.host` | Hostname of LDAP server |
+| `grafana.ldap.port` | Port of LDAP server (Has to be `quoted`!) |
+| `grafana.ldap.password` | Password of LDAP server |
+| `grafana.ldap.bind_dn` | Bind DN (username) of the LDAP server |
+| `grafana.ldap.accountBases` | List of base DNs to discover accounts (Has to have the format `"['a', 'b']"`) |
+| `grafana.ldap.groupBases` | List of base DNs to discover groups (Has to have the format `"['a', 'b']"`) |
+| `grafana.dashboards.editable` | Whether dashboards can be edited manually in the UI |
### Encryption
@@ -165,14 +165,9 @@
```sh
$PATH_TO_PROMTAIL/promtail \
- -config.file=./dist/promtail.yaml \
- -client.external-labels=host=$(hostname)
+ -config.file=./dist/promtail.yaml
```
-The `-client.external-labels=host=$(hostname)` option will add a label to each job
-that contains the hostname. This is useful, if multiple host are scraped for logs
-and only one Grafana is used to view the logs.
-
If TLS-verification is activated, the CA-certificate used for verification
(usually the one configured for `tls.caCert`) has to be present in the
directory configured for `promtail.storagePath` in the `config.yaml` and has to
diff --git a/config.yaml b/config.yaml
index c7b9840..fffed5a 100644
--- a/config.yaml
+++ b/config.yaml
@@ -2,13 +2,13 @@
- host: gerrit.example.com
username: admin
password: secret
+ promtail:
+ storagePath: /var/promtail
+ logPath: /var/gerrit/logs
namespace: namespace
tls:
skipVerify: true
caCert:
-promtail:
- storagePath: /var/promtail
- logPath: /var/gerrit/logs
prometheus:
server:
host: prometheus.example.com
diff --git a/promtail/promtail.yaml b/promtail/promtail.yaml
index 7bd3b3c..d03e026 100644
--- a/promtail/promtail.yaml
+++ b/promtail/promtail.yaml
@@ -1,18 +1,20 @@
#@ load("@ytt:data", "data")
+#@ for i in range(len(data.values.gerritServers)):
+---
server:
http_listen_port: 9080
grpc_listen_port: 0
positions:
- filename: #@ "{}/positions.yaml".format(data.values.promtail.storagePath)
+ filename: #@ "{}/positions.yaml".format(data.values.gerritServers[i].promtail.storagePath)
clients:
- url: #@ "https://{}/loki/api/v1/push".format(data.values.loki.host)
tls_config:
insecure_skip_verify: #@ data.values.tls.skipVerify
#@ if not data.values.tls.skipVerify:
- ca_file: #@ "{}/promtail.ca.crt".format(data.values.promtail.storagePath)
+ ca_file: #@ "{}/promtail.ca.crt".format(data.values.gerritServers[i].promtail.storagePath)
#@ end
basic_auth:
username: #@ data.values.loki.username
@@ -24,7 +26,8 @@
- localhost
labels:
job: gerrit_error
- __path__: #@ "{}/error_log.json".format(data.values.promtail.logPath)
+ __path__: #@ "{}/error_log.json".format(data.values.gerritServers[i].promtail.logPath)
+ host: #@ data.values.gerritServers[i].host
entry_parser: raw
pipeline_stages:
- json:
@@ -54,7 +57,8 @@
- localhost
labels:
job: gerrit_httpd
- __path__: #@ "{}/httpd_log.json".format(data.values.promtail.logPath)
+ __path__: #@ "{}/httpd_log.json".format(data.values.gerritServers[i].promtail.logPath)
+ host: #@ data.values.gerritServers[i].host
entry_parser: raw
pipeline_stages:
- json:
@@ -72,7 +76,8 @@
- localhost
labels:
job: gerrit_sshd
- __path__: #@ "{}/sshd_log.json".format(data.values.promtail.logPath)
+ __path__: #@ "{}/sshd_log.json".format(data.values.gerritServers[i].promtail.logPath)
+ host: #@ data.values.gerritServers[i].host
entry_parser: raw
pipeline_stages:
- json:
@@ -84,3 +89,4 @@
- timestamp:
source: timestamp
format: 2006-01-02 15:04:05.999 -0700
+#@ end
diff --git a/subcommands/install.py b/subcommands/install.py
index c2db390..a03bdfb 100644
--- a/subcommands/install.py
+++ b/subcommands/install.py
@@ -71,6 +71,28 @@
yaml.dump(dashboard_cm, f)
+def _create_promtail_configs(output_dir):
+ if not os.path.exists(os.path.join(output_dir, "promtail")):
+ os.mkdir(os.path.join(output_dir, "promtail"))
+
+ with open(os.path.join(output_dir, "promtail.yaml")) as f:
+ for config in yaml.load_all(f, Loader=yaml.SafeLoader):
+ with open(
+ os.path.join(
+ output_dir,
+ "promtail",
+ "promtail-%s"
+ % config["scrape_configs"][0]["static_configs"][0]["labels"][
+ "host"
+ ],
+ ),
+ "w",
+ ) as f:
+ yaml.dump(config, f)
+
+ os.remove(os.path.join(output_dir, "promtail.yaml"))
+
+
def _run_ytt(config, output_dir):
config_string = "#@data/values\n---\n"
config_string += yaml.dump(config)
@@ -168,6 +190,8 @@
namespace = config_manager.get_config()["namespace"]
_create_dashboard_configmaps(output_dir, namespace)
+ _create_promtail_configs(output_dir)
+
if not dryrun:
if update_repo:
_update_helm_repos()
