Remove custom labels added to logs during parsing
Promtail was configured to create labels for nearly every key in the
logs. This was done to support easier label-based querying. Loki
however is not optimized to work with labels having a high cardinality.
This led to failures in Loki, if it had to handle a high number of
logs. In addition, the high number of labels led to a huge number of
chunks being created, mostly just containing a single log entry,
making querying and storage very inefficient.
This change removes all custom made labels, except for the
gerrit_version label. Logs should rather be queried using the grep-
like syntax of LogQL for which Loki is optimized.
Change-Id: I70e2a3ff4f640bc6f5d08d50212958a7bca2eae1
diff --git a/promtail/promtail.yaml b/promtail/promtail.yaml
index e0b9042..7bd3b3c 100644
--- a/promtail/promtail.yaml
+++ b/promtail/promtail.yaml
@@ -29,18 +29,8 @@
pipeline_stages:
- json:
expressions:
- level:
timestamp: '"@timestamp"'
- exception: '"exception"'
- thread: thread_name
- logger: logger_name
- class:
message:
- - json:
- source: exception
- expressions:
- exception_message:
- exception_class:
- template:
source: timestamp
template: '{{ Replace .Value "," "." 1 }}'
@@ -57,12 +47,6 @@
source: message
expression: "Gerrit Code Review (?P<gerrit_version>.*) ready"
- labels:
- level:
- exception_message:
- exception_class:
- thread:
- logger:
- class:
gerrit_version:
- job_name: gerrit_httpd
static_configs:
@@ -76,23 +60,12 @@
- json:
expressions:
timestamp:
- thread:
- user:
- method:
- status:
- protocol:
- template:
source: timestamp
template: '{{ Replace .Value "," "." 1 }}'
- timestamp:
source: timestamp
format: '02/Jan/2006:15:04:05.999 -0700'
- - labels:
- thread:
- user:
- method:
- status:
- protocol:
- job_name: gerrit_sshd
static_configs:
- targets:
@@ -105,20 +78,9 @@
- json:
expressions:
timestamp:
- session:
- thread:
- user:
- account_id:
- status:
- template:
source: timestamp
template: '{{ Replace .Value "," "." 1 }}'
- timestamp:
source: timestamp
format: 2006-01-02 15:04:05.999 -0700
- - labels:
- session:
- thread:
- user:
- account_id:
- status:
