Merge changes I2cd9c872,I26cfd395
* changes:
Scrape Loki metrics
Monitor Prometheus itself
diff --git a/dashboards/gerrit_process.json b/dashboards/gerrit_process.json
index e2e3fe0..b2851db 100644
--- a/dashboards/gerrit_process.json
+++ b/dashboards/gerrit_process.json
@@ -23,17 +23,16 @@
"dashLength": 10,
"dashes": false,
"datasource": null,
- "description": "",
"fill": 1,
"fillGradient": 0,
"gridPos": {
- "h": 13,
+ "h": 11,
"w": 12,
"x": 0,
"y": 0
},
"hiddenSeries": false,
- "id": 8,
+ "id": 14,
"legend": {
"alignAsTable": true,
"avg": true,
@@ -60,8 +59,8 @@
"steppedLine": false,
"targets": [
{
- "expr": "proc_jvm_thread_num_live{instance=\"$instance\"}",
- "legendFormat": "Java live threads",
+ "expr": "proc_cpu_system_load{instance=\"$instance\"}",
+ "legendFormat": "system load",
"refId": "A"
}
],
@@ -69,7 +68,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
- "title": "Threads",
+ "title": "System load",
"tooltip": {
"shared": true,
"sort": 0,
@@ -86,7 +85,7 @@
"yaxes": [
{
"format": "short",
- "label": "Live Threads",
+ "label": null,
"logBase": 1,
"max": null,
"min": null,
@@ -115,7 +114,7 @@
"fill": 1,
"fillGradient": 0,
"gridPos": {
- "h": 13,
+ "h": 11,
"w": 12,
"x": 12,
"y": 0
@@ -224,7 +223,7 @@
"h": 10,
"w": 12,
"x": 0,
- "y": 13
+ "y": 11
},
"hiddenSeries": false,
"id": 4,
@@ -250,7 +249,13 @@
"pointradius": 2,
"points": false,
"renderer": "flot",
- "seriesOverrides": [],
+ "seriesOverrides": [
+ {
+ "alias": "available CPUs",
+ "color": "#1F60C4",
+ "fill": 0
+ }
+ ],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
@@ -259,6 +264,11 @@
"expr": "rate(proc_cpu_usage{instance=\"$instance\"}[5m])",
"legendFormat": "used CPUs",
"refId": "A"
+ },
+ {
+ "expr": "proc_cpu_num_cores{instance=\"$instance\"}",
+ "legendFormat": "available CPUs",
+ "refId": "B"
}
],
"thresholds": [],
@@ -290,7 +300,7 @@
},
{
"format": "short",
- "label": null,
+ "label": "",
"logBase": 1,
"max": null,
"min": null,
@@ -315,7 +325,7 @@
"h": 10,
"w": 12,
"x": 12,
- "y": 13
+ "y": 11
},
"hiddenSeries": false,
"id": 6,
@@ -412,13 +422,196 @@
"dashLength": 10,
"dashes": false,
"datasource": null,
- "fill": 2,
+ "description": "",
+ "fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
- "y": 23
+ "y": 21
+ },
+ "hiddenSeries": false,
+ "id": 8,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "proc_jvm_thread_num_live{instance=\"$instance\"}",
+ "legendFormat": "Java live threads",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Threads",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "Live Threads",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 6,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 10,
+ "w": 12,
+ "x": 12,
+ "y": 21
+ },
+ "hiddenSeries": false,
+ "id": 10,
+ "interval": "",
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "max": true,
+ "min": true,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": true,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "jgit_block_cache_open_files{instance=\"$instance\"}",
+ "legendFormat": "jgit block cache",
+ "refId": "B"
+ },
+ {
+ "expr": "proc_num_open_fds{instance=\"$instance\"}-jgit_block_cache_open_files{instance=\"$instance\"}",
+ "legendFormat": "other",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Java open file descriptors",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "Open File Descriptors",
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 2,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 10,
+ "w": 12,
+ "x": 12,
+ "y": 31
},
"hiddenSeries": false,
"id": 12,
@@ -514,100 +707,6 @@
"align": false,
"alignLevel": 1
}
- },
- {
- "aliasColors": {},
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": null,
- "fill": 6,
- "fillGradient": 0,
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 12,
- "y": 23
- },
- "hiddenSeries": false,
- "id": 10,
- "interval": "",
- "legend": {
- "alignAsTable": true,
- "avg": true,
- "current": true,
- "max": true,
- "min": true,
- "show": true,
- "total": false,
- "values": true
- },
- "lines": true,
- "linewidth": 1,
- "nullPointMode": "null",
- "options": {
- "dataLinks": []
- },
- "percentage": false,
- "pointradius": 2,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [],
- "spaceLength": 10,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "jgit_block_cache_open_files{instance=\"$instance\"}",
- "legendFormat": "jgit block cache",
- "refId": "B"
- },
- {
- "expr": "proc_num_open_fds{instance=\"$instance\"}-jgit_block_cache_open_files{instance=\"$instance\"}",
- "legendFormat": "other",
- "refId": "C"
- }
- ],
- "thresholds": [],
- "timeFrom": null,
- "timeRegions": [],
- "timeShift": null,
- "title": "Java open file descriptors",
- "tooltip": {
- "shared": true,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": []
- },
- "yaxes": [
- {
- "format": "short",
- "label": "Open File Descriptors",
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ],
- "yaxis": {
- "align": false,
- "alignLevel": null
- }
}
],
"refresh": "1m",
diff --git a/promtail/promtail.yaml b/promtail/promtail.yaml
index e0b9042..7bd3b3c 100644
--- a/promtail/promtail.yaml
+++ b/promtail/promtail.yaml
@@ -29,18 +29,8 @@
pipeline_stages:
- json:
expressions:
- level:
timestamp: '"@timestamp"'
- exception: '"exception"'
- thread: thread_name
- logger: logger_name
- class:
message:
- - json:
- source: exception
- expressions:
- exception_message:
- exception_class:
- template:
source: timestamp
template: '{{ Replace .Value "," "." 1 }}'
@@ -57,12 +47,6 @@
source: message
expression: "Gerrit Code Review (?P<gerrit_version>.*) ready"
- labels:
- level:
- exception_message:
- exception_class:
- thread:
- logger:
- class:
gerrit_version:
- job_name: gerrit_httpd
static_configs:
@@ -76,23 +60,12 @@
- json:
expressions:
timestamp:
- thread:
- user:
- method:
- status:
- protocol:
- template:
source: timestamp
template: '{{ Replace .Value "," "." 1 }}'
- timestamp:
source: timestamp
format: '02/Jan/2006:15:04:05.999 -0700'
- - labels:
- thread:
- user:
- method:
- status:
- protocol:
- job_name: gerrit_sshd
static_configs:
- targets:
@@ -105,20 +78,9 @@
- json:
expressions:
timestamp:
- session:
- thread:
- user:
- account_id:
- status:
- template:
source: timestamp
template: '{{ Replace .Value "," "." 1 }}'
- timestamp:
source: timestamp
format: 2006-01-02 15:04:05.999 -0700
- - labels:
- session:
- thread:
- user:
- account_id:
- status:
