Google Git
Sign in
gerrit / gerrit-monitoring / 7d9aff0488b1ddd24129d444b21d0969adcf591b / . / dashboards / prometheus-overview.json
blob: b1566d20647e4fb4655001a44e6719ea377f0a84 [file] [log] [blame]
{
"__inputs": [
{
"name": "DS_PROMETHEUS",
"label": "Prometheus",
"description": "",
"type": "datasource",
"pluginId": "prometheus",
"pluginName": "Prometheus"
}
],
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "7.1.5"
},
{
"type": "panel",
"id": "graph",
"name": "Graph",
"version": ""
},
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "1.0.0"
}
],
"annotations": {
"list": [
{
"$$hashKey": "object:456",
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"description": "Dashboard providing insight to Prometheus monitoring system status and it's runtime info.",
"editable": true,
"gnetId": 3681,
"graphTooltip": 1,
"iteration": 1585226845807,
"links": [],
"panels": [
{
"content": "<img src=\"https://cdn.worldvectorlogo.com/logos/prometheus.svg\"/ height=\"140px\">",
"datasource": null,
"gridPos": {
"h": 5,
"w": 2,
"x": 0,
"y": 0
},
"height": "",
"id": 50,
"links": [],
"mode": "html",
"title": "",
"transparent": true,
"type": "text"
},
{
"cacheTimeout": null,
"columns": [],
"datasource":"${DS_PROMETHEUS}",
"fontSize": "100%",
"gridPos": {
"h": 5,
"w": 22,
"x": 2,
"y": 0
},
"id": 41,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 3,
"desc": false
},
"styles": [
{
"$$hashKey": "object:832",
"alias": "",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "Time",
"thresholds": [],
"type": "hidden",
"unit": "short"
},
{
"$$hashKey": "object:833",
"alias": "Prometheus instance",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": true,
"linkTargetBlank": true,
"linkTooltip": "Link to the prometheus instance",
"linkUrl": "${__cell:raw}/graph",
"mappingType": 1,
"pattern": "instance",
"thresholds": [],
"type": "string",
"unit": "short"
},
{
"$$hashKey": "object:834",
"alias": "Uptime",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "Value #A",
"thresholds": [],
"type": "number",
"unit": "s"
},
{
"$$hashKey": "object:835",
"alias": "",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": false,
"mappingType": 1,
"pattern": "__name__",
"thresholds": [],
"type": "hidden",
"unit": "short"
},
{
"$$hashKey": "object:836",
"alias": "Build git branch",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": true,
"linkTargetBlank": true,
"linkTooltip": "Link to git branch of a build",
"linkUrl": "https://github.com/prometheus/prometheus/tree/${__cell}",
"mappingType": 1,
"pattern": "branch",
"thresholds": [],
"type": "string",
"unit": "short"
},
{
"$$hashKey": "object:837",
"alias": "",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "Value #B",
"thresholds": [],
"type": "hidden",
"unit": "short"
},
{
"$$hashKey": "object:838",
"alias": "Prometheus version",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": true,
"linkTargetBlank": true,
"linkTooltip": "Prometheus release page",
"linkUrl": "https://github.com/prometheus/prometheus/releases/tag/v${__cell}",
"mappingType": 1,
"pattern": "version",
"thresholds": [],
"type": "string",
"unit": "short"
},
{
"$$hashKey": "object:839",
"alias": "Go version",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": true,
"linkTargetBlank": true,
"linkTooltip": "Go release page",
"linkUrl": "https://github.com/golang/go/releases/tag/${__cell}",
"mappingType": 1,
"pattern": "goversion",
"thresholds": [],
"type": "string",
"unit": "short"
},
{
"$$hashKey": "object:840",
"alias": "Build git revision",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": true,
"linkTargetBlank": true,
"linkTooltip": "Link to git commit of a build",
"linkUrl": "https://github.com/prometheus/prometheus/commit/${__cell}",
"mappingType": 1,
"pattern": "revision",
"thresholds": [],
"type": "string",
"unit": "short"
},
{
"$$hashKey": "object:841",
"alias": "",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "job",
"thresholds": [],
"type": "hidden",
"unit": "short"
},
{
"$$hashKey": "object:842",
"alias": "",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "",
"thresholds": [],
"type": "number",
"unit": "short"
},
{
"$$hashKey": "object:843",
"alias": "Last reload success",
"align": "auto",
"colorMode": "cell",
"colors": [
"#E02F44",
"#E02F44",
"rgba(50, 172, 45, 0)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 0,
"link": true,
"linkTargetBlank": true,
"linkTooltip": "Config page",
"linkUrl": "https://${__cell_1:raw}/config",
"mappingType": 1,
"pattern": "Value #C",
"thresholds": [
"0",
"1"
],
"type": "number",
"unit": "none"
},
{
"$$hashKey": "object:844",
"alias": "Rules",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 0,
"link": true,
"linkTargetBlank": true,
"linkTooltip": "Rules page",
"linkUrl": "https://${__cell_1:raw}/rules",
"mappingType": 1,
"pattern": "Value #D",
"thresholds": [],
"type": "number",
"unit": "short"
},
{
"$$hashKey": "object:845",
"alias": "Targets",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": null,
"link": true,
"linkTargetBlank": true,
"linkTooltip": "Targets page",
"linkUrl": "https://${__cell_1:raw}/rules",
"mappingType": 1,
"pattern": "Value #E",
"thresholds": [],
"type": "number",
"unit": "short"
},
{
"$$hashKey": "object:846",
"alias": "# series",
"align": "auto",
"colorMode": "cell",
"colors": [
"rgba(50, 172, 45, 0)",
"#FF780A",
"#C4162A"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": null,
"mappingType": 1,
"pattern": "Value #F",
"thresholds": [
"500000",
"1000000"
],
"type": "number",
"unit": "short"
},
{
"$$hashKey": "object:847",
"alias": "Time Range",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": null,
"mappingType": 1,
"pattern": "Value #G",
"thresholds": [],
"type": "number",
"unit": "s"
},
{
"$$hashKey": "object:848",
"alias": "Data",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": null,
"mappingType": 1,
"pattern": "Value #H",
"thresholds": [],
"type": "number",
"unit": "bytes"
}
],
"targets": [
{
"expr": "max(time() - process_start_time_seconds{job=\"prometheus\"}) by (instance)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "A"
},
{
"expr": "max(prometheus_build_info{job=\"prometheus\"}) by (instance, branch, goversion, version, revision)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "B"
},
{
"expr": "max(prometheus_config_last_reload_successful{job=\"prometheus\"}) by (instance)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "C"
},
{
"expr": "max(sum(prometheus_rule_group_rules{job=\"prometheus\"}) by (instance) or prometheus_build_info{job=\"prometheus\"}*0) by (instance)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "D"
},
{
"expr": "max(sum(prometheus_sd_discovered_targets{job=\"prometheus\", name=\"scrape\"}) by (instance) or prometheus_build_info{job=\"prometheus\"}*0) by (instance)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "E"
},
{
"expr": "max(prometheus_tsdb_head_series{job=\"prometheus\"}) by (instance)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "F"
},
{
"expr": "time() - max(prometheus_tsdb_lowest_timestamp_seconds{job=\"prometheus\"}) by (instance)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "G"
},
{
"expr": "min(prometheus_tsdb_storage_blocks_bytes{job=\"prometheus\"}) by (instance)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "H"
}
],
"title": "Build, uptime and runtime instance info",
"transform": "table",
"type": "table"
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"id": 56,
"panels": [],
"repeat": null,
"title": "Main info",
"type": "row"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"fill": 0,
"fillGradient": 1,
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 6
},
"hiddenSeries": false,
"id": 71,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "max(prometheus_tsdb_head_series{job=\"prometheus\"}) by (instance)",
"interval": "",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": "$long_term_interval",
"timeRegions": [],
"timeShift": null,
"title": "Number of time series",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"description": "",
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 7,
"w": 7,
"x": 8,
"y": 6
},
"hiddenSeries": false,
"id": 25,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": true,
"min": false,
"show": false,
"sort": "max",
"sortDesc": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "prometheus_target_interval_length_seconds{job=\"prometheus\",quantile=\"$quantile\"}",
"interval": "",
"legendFormat": "job {{job}} should be {{interval}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "$quantile quantile of interval length between scrapes per job",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"decimals": null,
"format": "s",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833"
},
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 6,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 9,
"x": 15,
"y": 6
},
"hiddenSeries": false,
"id": 13,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"hideEmpty": true,
"hideZero": true,
"max": true,
"min": false,
"rightSide": false,
"show": false,
"sort": "total",
"sortDesc": true,
"total": true,
"values": true
},
"lines": false,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"$$hashKey": "object:1744",
"alias": "/net_conntrack/"
}
],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"expr": "sum(increase(label_replace({__name__=~\"(:?prometheus|net_conntrack)_.*(:?fail|duplicate|out_of|error|not_found|corrupt|dropped|missed|exceeded).*_total\", job=\"prometheus\"}, \"metric_name\", \"$1\", \"__name__\", \"(.*)\")[$aggregation_interval:20s])) by (instance,metric_name,alertmanager,name,mechanism,queue) > 0",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Prometheus errors in $aggregation_interval",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:1755",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"$$hashKey": "object:1756",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 13
},
"id": 58,
"panels": [],
"repeat": null,
"title": "Requests & queries",
"type": "row"
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 14
},
"hiddenSeries": false,
"id": 18,
"legend": {
"avg": false,
"current": false,
"hideEmpty": true,
"hideZero": true,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"expr": "sum(increase(prometheus_http_requests_total{job=\"prometheus\"}[$aggregation_interval])) by (instance, handler)",
"interval": "",
"legendFormat": "{{ handler }} handler on {{ instance }}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "HTTP request count by handler in $aggregation_interval",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:1899",
"format": "none",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:1900",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 8,
"y": 14
},
"hiddenSeries": false,
"id": 16,
"legend": {
"avg": false,
"current": false,
"hideEmpty": true,
"hideZero": true,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile($quantile, sum(increase(prometheus_http_request_duration_seconds_bucket{job=\"prometheus\"}[$aggregation_interval])) by (le,instance, handler))",
"interval": "",
"legendFormat": "{{ handler }} handler on {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "$quantile quantile of HTTP request duration per handler",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 16,
"y": 14
},
"hiddenSeries": false,
"id": 19,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile($quantile, sum(increase(prometheus_http_response_size_bytes_bucket{job=\"prometheus\"}[$aggregation_interval])) by (le,instance, handler)) > 0",
"interval": "",
"legendFormat": "{{ handler }} handler on {{ instance }}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "$quantile quantile of request size by handler",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 21
},
"id": 64,
"panels": [],
"repeat": null,
"title": "Resources",
"type": "row"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"decimals": 2,
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 22
},
"hiddenSeries": false,
"id": 9,
"legend": {
"alignAsTable": false,
"avg": false,
"current": false,
"hideEmpty": false,
"max": false,
"min": false,
"rightSide": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(process_cpu_seconds_total{job=\"prometheus\"}[$aggregation_interval])) by (instance)",
"interval": "",
"legendFormat": "CPU usage of {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "CPU usage",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [
"avg"
]
},
"yaxes": [
{
"format": "none",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Allocated bytes": "#7EB26D",
"Allocated bytes - 1m max": "#BF1B00",
"Allocated bytes - 1m min": "#BF1B00",
"Allocated bytes - 5m max": "#BF1B00",
"Allocated bytes - 5m min": "#BF1B00",
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833",
"RSS": "#447EBC"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"decimals": null,
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 8,
"y": 22
},
"hiddenSeries": false,
"id": 6,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "//",
"fill": 0
},
{
"alias": "collector heap size",
"color": "#E0752D",
"linewidth": 2
},
{
"alias": "collector kubernetes memory limit",
"color": "#BF1B00",
"linewidth": 3
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(process_resident_memory_bytes{job=\"prometheus\"}) by (instance)",
"interval": "",
"legendFormat": "Resident memory of {{instance}}",
"refId": "A"
},
{
"expr": "sum(go_memstats_alloc_bytes{job=\"prometheus\"}) by (instance)",
"interval": "",
"legendFormat": "Total allocated bytes of {{instance}}",
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Memory usage",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Allocated bytes": "#F9BA8F",
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833",
"RSS": "#890F02"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 16,
"y": 22
},
"hiddenSeries": false,
"id": 7,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(go_memstats_alloc_bytes_total{job=\"prometheus\"}[$aggregation_interval])",
"interval": "",
"legendFormat": "Allocated Bytes of {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Allocations per second in $aggregation_interval",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 29
},
"id": 61,
"panels": [],
"repeat": null,
"title": "TSDB stats",
"type": "row"
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 30
},
"hiddenSeries": false,
"id": 89,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "time() - min(prometheus_tsdb_lowest_timestamp_seconds{job=\"prometheus\"}) by (instance)",
"interval": "",
"legendFormat": "Oldest data for {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": "$long_term_interval",
"timeRegions": [],
"timeShift": null,
"title": "Oldest data",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 30
},
"hiddenSeries": false,
"id": 88,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "max(prometheus_tsdb_storage_blocks_bytes{job=\"prometheus\"}) by (instance)",
"interval": "",
"legendFormat": "",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": "$long_term_interval",
"timeRegions": [],
"timeShift": null,
"title": "Size of the storage",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 37
},
"id": 75,
"panels": [],
"title": "Query engine & API",
"type": "row"
},
{
"aliasColors": {
"Allocated bytes": "#F9BA8F",
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max count collector": "#bf1b00",
"Max count harvester": "#bf1b00",
"Max to persist": "#3F6833",
"RSS": "#890F02"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 38
},
"hiddenSeries": false,
"id": 8,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"$$hashKey": "object:2178",
"alias": "/Limit/",
"color": "#E02F44",
"dashLength": 3,
"dashes": true,
"fill": 0,
"nullPointMode": "connected"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(prometheus_engine_queries{job=\"prometheus\"}) by (instance)",
"instant": false,
"interval": "",
"legendFormat": "{{instance}}",
"refId": "A"
},
{
"expr": "sum(prometheus_engine_queries_concurrent_max{job=\"prometheus\"}) by (instance)",
"interval": "",
"legendFormat": "{{instance}} Limit",
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Number of concurent queries and it's limit",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2199",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"$$hashKey": "object:2200",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 8,
"y": 38
},
"hiddenSeries": false,
"id": 15,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "prometheus_engine_query_duration_seconds{job=\"prometheus\", quantile=\"$quantile\"}",
"interval": "",
"legendFormat": "{{slice}} slice on {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "$quantile quantile of query engine evaluation duration per slice",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 16,
"y": 38
},
"hiddenSeries": false,
"id": 95,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(prometheus_api_remote_read_queries{job=\"prometheus\"}) by (instance)",
"interval": "",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Number of queries on remote read API",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 45
},
"id": 57,
"panels": [],
"repeat": null,
"title": "Rule evaluation",
"type": "row"
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 9,
"x": 0,
"y": 46
},
"hiddenSeries": false,
"id": 77,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "/LIMIT/",
"color": "#E02F44",
"dashLength": 3,
"dashes": true,
"fill": 0
},
{
"alias": "slow",
"color": "#C4162A",
"fill": 8
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "topk(20,max(prometheus_rule_group_last_duration_seconds{job=\"prometheus\"}) by (rule_group, instance) / min(prometheus_rule_group_interval_seconds{job=\"prometheus\"}) by (rule_group, instance))",
"interval": "",
"legendFormat": "Percentage of {{rule_group}} group evaluation interval on {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Percentage rule group evaluation duration from the rule group evaluation interval (top 20)",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "percentunit",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 9,
"y": 46
},
"hiddenSeries": false,
"id": 14,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "Queue length",
"yaxis": 2
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(prometheus_rule_evaluation_duration_seconds{job=\"prometheus\", quantile=\"$quantile\"}) by (instance, quantile)",
"interval": "",
"legendFormat": "evaluation duration of {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "$quantile quantile of rule evaulation duration",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 3,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 7,
"x": 17,
"y": 46
},
"hiddenSeries": false,
"id": 76,
"interval": "",
"legend": {
"avg": false,
"current": false,
"hideEmpty": true,
"hideZero": true,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "Queue length",
"yaxis": 2
}
],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"expr": "sum(topk(20,prometheus_rule_group_rules{job=\"prometheus\"})) by (instance, rule_group)",
"interval": "",
"legendFormat": "number of rules in {{rule_group}} rule group on {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Number of rules per group (top 20 groups)",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 53
},
"id": 59,
"panels": [],
"repeat": null,
"title": "Alerting",
"type": "row"
},
{
"aliasColors": {
"Alert queue capacity on o collector": "#bf1b00",
"Alert queue capacity on o harvester": "#bf1b00",
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 54
},
"hiddenSeries": false,
"id": 20,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "/capacity/",
"color": "#E02F44",
"dashLength": 3,
"dashes": true,
"fill": 0,
"nullPointMode": "connected"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(prometheus_notifications_queue_length{job=\"prometheus\"})by (instance)",
"interval": "",
"legendFormat": "Queue size of {{instance}}",
"refId": "A"
},
{
"expr": "sum(prometheus_notifications_queue_capacity{job=\"prometheus\"})by (instance)",
"interval": "",
"legendFormat": "Queue capacity of {{instance}}",
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Notification queue size and capacity",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Alert queue capacity on o collector": "#bf1b00",
"Alert queue capacity on o harvester": "#bf1b00",
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 8,
"y": 54
},
"hiddenSeries": false,
"id": 78,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"expr": "sum(increase(prometheus_notifications_sent_total{job=\"prometheus\"}[$aggregation_interval])) by (instance, alertmanager)",
"interval": "",
"legendFormat": "Sent notifications to {{alertmanager}} alertmanager from {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Number of sent notifictions per alertmanager in $aggregation_interval",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Alert queue capacity on o collector": "#bf1b00",
"Alert queue capacity on o harvester": "#bf1b00",
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 16,
"y": 54
},
"hiddenSeries": false,
"id": 79,
"legend": {
"avg": false,
"current": false,
"hideEmpty": true,
"hideZero": true,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "avg(prometheus_notifications_latency_seconds{job=\"prometheus\", quantile=\"$quantile\"}) by (instance, alertmanager)",
"interval": "",
"legendFormat": "Latency of notifications sent to {{alertmanager}} from {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "$quantile of notification latency per alertmanager",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 61
},
"id": 60,
"panels": [],
"repeat": null,
"title": "Service discovery",
"type": "row"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"decimals": 0,
"fill": 5,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 62
},
"hiddenSeries": false,
"id": 69,
"legend": {
"alignAsTable": true,
"avg": false,
"current": true,
"hideEmpty": true,
"hideZero": true,
"max": false,
"min": false,
"rightSide": true,
"show": false,
"sort": "current",
"sortDesc": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"expr": "sum(prometheus_sd_discovered_targets{job=\"prometheus\", name=\"scrape\"}) by (instance, config,name)",
"interval": "",
"intervalFactor": 5,
"legendFormat": "Number of {{name}} type targets from config {{config}} on {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Number of discovered targets per Type and config",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2495",
"decimals": 0,
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2496",
"decimals": 0,
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 8,
"y": 62
},
"hiddenSeries": false,
"id": 45,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(increase(prometheus_sd_refresh_duration_seconds_count{job=\"prometheus\"}[$aggregation_interval])) by (instance, mechanism)",
"interval": "",
"legendFormat": "Number of sync for {{mechanism}} SD mechanism on {{instance}}",
"refId": "A"
},
{
"expr": "sum(increase(prometheus_sd_consul_rpc_duration_seconds_count{job=\"prometheus\"}[$aggregation_interval])) by (instance)",
"interval": "",
"legendFormat": "Number of sync for consul SD mechanism on {{instance}}",
"refId": "B"
},
{
"expr": "sum(increase(prometheus_sd_kubernetes_http_request_duration_seconds_count{job=\"prometheus\"}[$aggregation_interval])) by (instance)",
"interval": "",
"legendFormat": "Number of sync for kubernetes SD mechanism on {{instance}}",
"refId": "C"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Service discovery sync count by mechanism in $aggregation_interval",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 16,
"y": 62
},
"hiddenSeries": false,
"id": 43,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "prometheus_sd_refresh_duration_seconds{job=\"prometheus\", quantile=\"$quantile\"}",
"interval": "",
"legendFormat": "Duration of sync for {{mechanism}} SD mechanism on {{instance}}",
"refId": "A"
},
{
"expr": "consul_rpc_duration_seconds{job=\"prometheus\", quantile=\"$quantile\"}",
"interval": "",
"legendFormat": "Number of sync for consul SD mechanism on {{instance}}",
"refId": "B"
},
{
"expr": "prometheus_sd_file_scan_duration_seconds{job=\"prometheus\", quantile=\"$quantile\"}",
"interval": "",
"legendFormat": "Duration of sync for file_scan SD mechanism on {{instance}}",
"refId": "C"
},
{
"expr": "prometheus_sd_kubernetes_http_request_duration_seconds_sum{job=\"prometheus\"} / prometheus_sd_kubernetes_http_request_duration_seconds_count{job=\"prometheus\"}",
"interval": "",
"legendFormat": "AVG Duration of sync for k8s SD mechanism on {{instance}}",
"refId": "D"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "$quantile quantile of refresh duration per SD mechanism",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 69
},
"id": 73,
"panels": [],
"title": "Compaction and retention",
"type": "row"
},
{
"aliasColors": {},
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 70
},
"hiddenSeries": false,
"id": 80,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": false,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile($quantile, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{job=\"prometheus\"}[$aggregation_interval])) by (instance, le))",
"interval": "",
"legendFormat": "Compaction duration on {{ instance }}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "$quantile quantile of compaction duration",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"decimals": null,
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"fill": 3,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 8,
"y": 70
},
"hiddenSeries": false,
"id": 81,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": true,
"targets": [
{
"expr": "sum(increase(prometheus_tsdb_compactions_total{job=\"prometheus\"}[$aggregation_interval])) by (instance) > 0",
"interval": "",
"legendFormat": "Count of horizontal compactions on {{instance}}",
"refId": "A"
},
{
"expr": "sum(increase(prometheus_tsdb_vertical_compactions_total{job=\"prometheus\"}[$aggregation_interval])) by (instance) > 0",
"interval": "",
"legendFormat": "Count of vertical compactions on {{instance}}",
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Number of vertical/horizontal compactions in last $aggregation_interval",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"decimals": null,
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"fill": 3,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 16,
"y": 70
},
"hiddenSeries": false,
"id": 82,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": true,
"targets": [
{
"expr": "sum(increase(prometheus_tsdb_size_retentions_total{job=\"prometheus\"}[$aggregation_interval])) by (instance) > 0",
"interval": "",
"legendFormat": "Number of size retention cutoffs on {{ instance }}",
"refId": "A"
},
{
"expr": "sum(increase(prometheus_tsdb_time_retentions_total{job=\"prometheus\"}[$aggregation_interval])) by (instance) > 0",
"interval": "",
"legendFormat": "Number of time retention cutoffs on {{ instance }}",
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Number of time/size retention cutoffs in $aggregation_interval",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"decimals": null,
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 77
},
"id": 62,
"panels": [],
"repeat": null,
"title": "Head block",
"type": "row"
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833",
"To persist": "#9AC48A"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 78
},
"hiddenSeries": false,
"id": 2,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "/Max.*/",
"fill": 0
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(prometheus_tsdb_head_chunks{job=\"prometheus\"}) by (instance)",
"interval": "",
"legendFormat": "Head block chunk count on {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Head chunks count",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 8,
"y": 78
},
"hiddenSeries": false,
"id": 1,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(prometheus_tsdb_head_samples_appended_total{job=\"prometheus\"}[$aggregation_interval])) by (instance)",
"interval": "",
"legendFormat": "Appended samples/s on {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Samples Appended per second in $aggregation_interval",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 16,
"y": 78
},
"hiddenSeries": false,
"id": 3,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "prometheus_tsdb_head_series{job=\"prometheus\"}",
"interval": "",
"legendFormat": "Time series count of {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Time series total count",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:262",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"$$hashKey": "object:263",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 85
},
"id": 84,
"panels": [],
"title": "WAL",
"type": "row"
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833",
"To persist": "#9AC48A"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 86
},
"hiddenSeries": false,
"id": 85,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(prometheus_tsdb_wal_fsync_duration_seconds{job=\"prometheus\",quantile=\"$quantile\"}) by (instance)",
"interval": "",
"legendFormat": "Duration of WAL fsync on {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "$quantile quantile of WAL fsync duration",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833",
"To persist": "#9AC48A"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 8,
"y": 86
},
"hiddenSeries": false,
"id": 86,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(increase(prometheus_tsdb_wal_completed_pages_total{job=\"prometheus\"}[$aggregation_interval])) by (instance)",
"interval": "",
"legendFormat": "Completed WAL pages count of {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Number of completed pages in $aggregation_interval",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833",
"To persist": "#9AC48A"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 16,
"y": 86
},
"hiddenSeries": false,
"id": 87,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(prometheus_tsdb_wal_truncate_duration_seconds_sum{job=\"prometheus\"} /prometheus_tsdb_wal_truncate_duration_seconds_count{job=\"prometheus\"}) by (instance)",
"interval": "",
"legendFormat": "Duration of WAL truncation on {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Average duration of WAL truncation",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 93
},
"id": 99,
"panels": [],
"title": "Go stats",
"type": "row"
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833",
"To persist": "#9AC48A"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 94
},
"hiddenSeries": false,
"id": 100,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(go_goroutines{job=\"prometheus\"}) by (instance)",
"interval": "",
"legendFormat": "Gorutines count of {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Number of gorutines",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833",
"To persist": "#9AC48A"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 8,
"y": 94
},
"hiddenSeries": false,
"id": 101,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "max(go_gc_duration_seconds{job=\"prometheus\"}) by (instance)",
"interval": "",
"legendFormat": "GC duration on {{instance}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Duration of Go garbage collection",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833",
"To persist": "#9AC48A"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 5,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 16,
"y": 94
},
"hiddenSeries": false,
"id": 102,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"expr": "max(go_memstats_mspan_sys_bytes{job=\"prometheus\"}) by (instance)",
"interval": "",
"legendFormat": "mspan {{instance}}",
"refId": "A"
},
{
"expr": "max(go_memstats_mcache_sys_bytes{job=\"prometheus\"}) by (instance)",
"interval": "",
"legendFormat": "mcache {{instance}}",
"refId": "B"
},
{
"expr": "max(go_memstats_buck_hash_sys_bytes{job=\"prometheus\"}) by (instance)",
"interval": "",
"legendFormat": "buck_hash {{instance}}",
"refId": "C"
},
{
"expr": "max(go_memstats_gc_sys_bytes{job=\"prometheus\"}) by (instance)",
"interval": "",
"legendFormat": "gc {{instance}}",
"refId": "D"
},
{
"expr": "max(go_memstats_other_sys_bytes{job=\"prometheus\"}) by (instance)",
"interval": "",
"legendFormat": "other {{instance}}",
"refId": "E"
},
{
"expr": "max(go_memstats_stack_sys_bytes{job=\"prometheus\"}) by (instance)",
"format": "time_series",
"hide": false,
"intervalFactor": 1,
"legendFormat": "stack {{instance}}",
"metric": "prometheus_local_storage_memory_chunks",
"refId": "F"
},
{
"expr": "max(go_memstats_heap_sys_bytes{job=\"prometheus\"}) by (instance)",
"interval": "",
"legendFormat": "heap {{instance}}",
"refId": "G"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Go system memory allocations",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2633",
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2634",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 101
},
"id": 68,
"panels": [],
"repeat": null,
"title": "Network",
"type": "row"
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 102
},
"hiddenSeries": false,
"id": 47,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "/closed/",
"transform": "negative-Y"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(increase(net_conntrack_listener_conn_accepted_total{job=\"prometheus\"}[$aggregation_interval])) by (instance, listener_name) > 0",
"interval": "",
"legendFormat": "{{listener_name}} accepted connections on {{instance}}",
"refId": "A"
},
{
"expr": "sum(increase(net_conntrack_listener_conn_closed_total{job=\"prometheus\"}[$aggregation_interval])) by (instance, listener_name) > 0",
"interval": "",
"legendFormat": "{{listener_name}} closed connections on {{instance}}",
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Accepted/closed inbound connections per listener",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Chunks": "#1F78C1",
"Chunks to persist": "#508642",
"Max chunks": "#052B51",
"Max to persist": "#3F6833"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"editable": true,
"error": false,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 102
},
"hiddenSeries": false,
"id": 90,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "/closed/",
"stack": "B",
"transform": "negative-Y"
},
{
"alias": "/established/",
"stack": "A"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(increase(net_conntrack_dialer_conn_established_total{job=\"prometheus\"}[$aggregation_interval])) by (instance, dialer_name) > 0",
"interval": "",
"legendFormat": "{{dialer_name}} established connections on {{instance}}",
"refId": "A"
},
{
"expr": "sum(increase(net_conntrack_dialer_conn_closed_total{job=\"prometheus\"}[$aggregation_interval])) by (instance, dialer_name) > 0",
"interval": "",
"legendFormat": "{{dialer_name}} closed connections on {{instance}}",
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Established/closed outbound connections per dialer ",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"refresh": "1m",
"schemaVersion": 22,
"style": "dark",
"tags": [
"prometheus",
"monitoring"
],
"templating": {
"list": [
{
"auto": true,
"auto_count": 300,
"auto_min": "1m",
"current": {
"selected": false,
"text": "1d",
"value": "1d"
},
"hide": 0,
"label": "Aggregation intarval",
"name": "aggregation_interval",
"options": [
{
"selected": false,
"text": "auto",
"value": "$__auto_interval_aggregation_interval"
},
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": false,
"text": "10m",
"value": "10m"
},
{
"selected": false,
"text": "30m",
"value": "30m"
},
{
"selected": false,
"text": "1h",
"value": "1h"
},
{
"selected": false,
"text": "6h",
"value": "6h"
},
{
"selected": false,
"text": "12h",
"value": "12h"
},
{
"selected": true,
"text": "1d",
"value": "1d"
},
{
"selected": false,
"text": "7d",
"value": "7d"
},
{
"selected": false,
"text": "14d",
"value": "14d"
},
{
"selected": false,
"text": "30d",
"value": "30d"
}
],
"query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
"refresh": 2,
"skipUrlSync": false,
"type": "interval"
},
{
"allValue": null,
"current": {
"text": "0.99",
"value": "0.99"
},
"datasource":"${DS_PROMETHEUS}",
"definition": "query_result({__name__=~\"prometheus_.*_seconds\"})",
"hide": 0,
"includeAll": false,
"index": -1,
"label": "Quantile",
"multi": false,
"name": "quantile",
"options": [],
"query": "query_result({__name__=~\"prometheus_.*_seconds\"})",
"refresh": 2,
"regex": "/quantile=\"([^\"]+)\"/",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"auto": false,
"auto_count": 30,
"auto_min": "10s",
"current": {
"selected": false,
"text": "3d",
"value": "3d"
},
"hide": 0,
"label": null,
"name": "long_term_interval",
"options": [
{
"selected": false,
"text": "10m",
"value": "10m"
},
{
"selected": false,
"text": "30m",
"value": "30m"
},
{
"selected": false,
"text": "1h",
"value": "1h"
},
{
"selected": false,
"text": "6h",
"value": "6h"
},
{
"selected": false,
"text": "12h",
"value": "12h"
},
{
"selected": false,
"text": "1d",
"value": "1d"
},
{
"selected": true,
"text": "3d",
"value": "3d"
},
{
"selected": false,
"text": "1w",
"value": "1w"
},
{
"selected": false,
"text": "4w",
"value": "4w"
},
{
"selected": false,
"text": "12w",
"value": "12w"
},
{
"selected": false,
"text": "1y",
"value": "1y"
}
],
"query": "10m,30m,1h,6h,12h,1d,3d,1w,4w,12w,1y",
"refresh": 2,
"skipUrlSync": false,
"type": "interval"
}
]
},
"time": {
"from": "now-7d",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "browser",
"title": "Prometheus",
"uid": "iLC3yp9Zz",
"variables": {
"list": []
},
"version": 3
}