blob: 5dbf501535bf8f30bef4bea1c95bc8da831d224a [file] [log] [blame]
{
"__inputs": [
{
"name": "DS_PROMETHEUS",
"label": "Prometheus",
"description": "",
"type": "datasource",
"pluginId": "prometheus",
"pluginName": "Prometheus"
}
],
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "7.1.5"
},
{
"type": "panel",
"id": "graph",
"name": "Graph",
"version": ""
},
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "1.0.0"
}
],
"annotations": {
"list": [
{
"$$hashKey": "object:386",
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"description": "Loki metrics.",
"editable": true,
"gnetId": 10880,
"graphTooltip": 0,
"links": [],
"panels": [
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 30,
"panels": [],
"title": "",
"type": "row"
},
{
"cacheTimeout": null,
"columns": [],
"datasource":"${DS_PROMETHEUS}",
"fontSize": "100%",
"gridPos": {
"h": 5,
"w": 24,
"x": 0,
"y": 1
},
"id": 10,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 3,
"desc": false
},
"styles": [
{
"$$hashKey": "object:832",
"alias": "",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "Time",
"thresholds": [],
"type": "hidden",
"unit": "short"
},
{
"$$hashKey": "object:833",
"alias": "Loki instance",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": false,
"linkTargetBlank": true,
"linkTooltip": "Link to the prometheus instance",
"linkUrl": "${__cell:raw}/graph",
"mappingType": 1,
"pattern": "instance",
"thresholds": [],
"type": "string",
"unit": "short"
},
{
"$$hashKey": "object:834",
"alias": "Uptime",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "Value #A",
"thresholds": [],
"type": "number",
"unit": "s"
},
{
"$$hashKey": "object:835",
"alias": "",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": false,
"mappingType": 1,
"pattern": "__name__",
"thresholds": [],
"type": "hidden",
"unit": "short"
},
{
"$$hashKey": "object:836",
"alias": "Build git branch",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": true,
"linkTargetBlank": true,
"linkTooltip": "Link to git branch of a build",
"linkUrl": "https://github.com/grafana/loki/tree/${__cell}",
"mappingType": 1,
"pattern": "branch",
"thresholds": [],
"type": "string",
"unit": "short"
},
{
"$$hashKey": "object:837",
"alias": "",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "Value #B",
"thresholds": [],
"type": "hidden",
"unit": "short"
},
{
"$$hashKey": "object:838",
"alias": "Loki version",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": true,
"linkTargetBlank": true,
"linkTooltip": "Loki release page",
"linkUrl": "https://github.com/grafana/loki/releases/tag/${__cell}",
"mappingType": 1,
"pattern": "version",
"thresholds": [],
"type": "string",
"unit": "short"
},
{
"$$hashKey": "object:839",
"alias": "Go version",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": true,
"linkTargetBlank": true,
"linkTooltip": "Go release page",
"linkUrl": "https://github.com/golang/go/releases/tag/${__cell}",
"mappingType": 1,
"pattern": "goversion",
"thresholds": [],
"type": "string",
"unit": "short"
},
{
"$$hashKey": "object:840",
"alias": "Build git revision",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"link": true,
"linkTargetBlank": true,
"linkTooltip": "Link to git commit of a build",
"linkUrl": "https://github.com/grafana/loki/commit/${__cell}",
"mappingType": 1,
"pattern": "revision",
"thresholds": [],
"type": "string",
"unit": "short"
},
{
"$$hashKey": "object:841",
"alias": "",
"align": "auto",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "job",
"thresholds": [],
"type": "hidden",
"unit": "short"
}
],
"targets": [
{
"expr": "max(time() - process_start_time_seconds{job=\"loki\"}) by (instance)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "A"
},
{
"expr": "max(loki_build_info{job=\"loki\"}) by (instance, branch, goversion, version, revision)",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "B"
}
],
"title": "Build, uptime and runtime instance info",
"transform": "table",
"type": "table"
},
{
"aliasColors": {},
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 6
},
"hiddenSeries": false,
"id": 12,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": false,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"$$hashKey": "object:1640",
"alias": "Number of errors",
"color": "#C4162A"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "increase(log_messages_total{level=\"error\"}[5m])",
"interval": "",
"legendFormat": "Number of errors",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Number of Loki errors per 5m",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:1612",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:1613",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 13
},
"hiddenSeries": false,
"id": 8,
"interval": "",
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "avg_over_time(loki_ingester_chunk_entries_count[5m])",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Loki's stored chunk entries",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 13
},
"hiddenSeries": false,
"id": 18,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(loki_distributor_lines_received_total{job=\"loki\"}[5m])",
"interval": "",
"legendFormat": "lines/sec",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Log lines per second",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2387",
"format": "cps",
"label": "lines/sec",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2388",
"format": "KBs",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 13
},
"hiddenSeries": false,
"id": 19,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "rate(loki_distributor_bytes_received_total{job=\"loki\"}[5m]) / 1024",
"interval": "",
"legendFormat": "kb/sec",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Log line data per second",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2387",
"format": "KBs",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2388",
"format": "KBs",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 21
},
"hiddenSeries": false,
"id": 16,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"$$hashKey": "object:2154",
"alias": "2xx",
"color": "#5794F2"
},
{
"$$hashKey": "object:2162",
"alias": "4xx",
"color": "#FA6400"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (label_replace(label_replace(rate(loki_request_duration_seconds_count{job=\"loki\", route=~\"api_prom_push|loki_api_v1_push\"}[5m]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"), \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"interval": "",
"legendFormat": "{{status}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Pushes per second",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2049",
"format": "reqps",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2050",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 21
},
"hiddenSeries": false,
"id": 14,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"$$hashKey": "object:1803",
"alias": "2xx",
"color": "#5794F2"
},
{
"$$hashKey": "object:1816",
"alias": "4xx",
"color": "#FA6400"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum by (status) (label_replace(label_replace(rate(loki_request_duration_seconds_count{job=\"loki\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[5m]), \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"interval": "",
"legendFormat": "{{status}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Queries per second",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:1775",
"format": "reqps",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:1776",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 29
},
"hiddenSeries": false,
"id": 21,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"$$hashKey": "object:2863",
"alias": "0.99",
"color": "#C4162A"
},
{
"$$hashKey": "object:2871",
"alias": "0.9",
"color": "#FADE2A"
},
{
"$$hashKey": "object:2879",
"alias": "0.5",
"color": "#1F60C4"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (loki_request_duration_seconds_bucket{job=~\"loki\", route=~\"api_prom_push|loki_api_v1_push\"}))",
"interval": "",
"legendFormat": "0.99",
"refId": "A"
},
{
"expr": "histogram_quantile(0.9, sum by (le) (loki_request_duration_seconds_bucket{job=~\"loki\", route=~\"api_prom_push|loki_api_v1_push\"}))",
"interval": "",
"legendFormat": "0.9",
"refId": "B"
},
{
"expr": "histogram_quantile(0.5, sum by (le) (loki_request_duration_seconds_bucket{job=~\"loki\", route=~\"api_prom_push|loki_api_v1_push\"}))",
"interval": "",
"legendFormat": "0.5",
"refId": "C"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Push latency",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2835",
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2836",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 29
},
"hiddenSeries": false,
"id": 25,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"$$hashKey": "object:2863",
"alias": "0.99",
"color": "#C4162A"
},
{
"$$hashKey": "object:2871",
"alias": "0.9",
"color": "#FADE2A"
},
{
"$$hashKey": "object:2879",
"alias": "0.5",
"color": "#1F60C4"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (loki_request_duration_seconds_bucket{job=\"loki\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}))",
"interval": "",
"legendFormat": "0.99",
"refId": "A"
},
{
"expr": "histogram_quantile(0.9, sum by (le) (loki_request_duration_seconds_bucket{job=\"loki\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}))",
"interval": "",
"legendFormat": "0.9",
"refId": "B"
},
{
"expr": "histogram_quantile(0.5, sum by (le) (loki_request_duration_seconds_bucket{job=\"loki\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}))",
"interval": "",
"legendFormat": "0.5",
"refId": "C"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Query latency",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2835",
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2836",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 37
},
"hiddenSeries": false,
"id": 24,
"interval": "",
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": false,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_request_duration_seconds_count{job=\"loki\", status_code=~\"success|200\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[5m])) by (route)/sum(rate(loki_request_duration_seconds_count{job=\"loki\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[5m])) by (route)",
"interval": "",
"legendFormat": "{{route}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Query Success Rate",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2980",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2981",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"collapsed": false,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 45
},
"id": 28,
"panels": [],
"title": "Ingesters",
"type": "row"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 46
},
"hiddenSeries": false,
"id": 22,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"$$hashKey": "object:2863",
"alias": "0.99",
"color": "#C4162A"
},
{
"$$hashKey": "object:2871",
"alias": "0.9",
"color": "#FADE2A"
},
{
"$$hashKey": "object:2879",
"alias": "0.5",
"color": "#1F60C4"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (loki_request_duration_seconds_bucket{job=~\"loki\", route=\"/logproto.Pusher/Push\"}))",
"interval": "",
"legendFormat": "0.99",
"refId": "A"
},
{
"expr": "histogram_quantile(0.9, sum by (le) (loki_request_duration_seconds_bucket{job=~\"loki\", route=\"/logproto.Pusher/Push\"}))",
"interval": "",
"legendFormat": "0.9",
"refId": "B"
},
{
"expr": "histogram_quantile(0.5, sum by (le) (loki_request_duration_seconds_bucket{job=~\"loki\", route=\"/logproto.Pusher/Push\"}))",
"interval": "",
"legendFormat": "0.5",
"refId": "C"
},
{
"refId": "D"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Ingester latency",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2835",
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2836",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 46
},
"hiddenSeries": false,
"id": 26,
"interval": "",
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_request_duration_seconds_count{job=\"loki\", status_code=~\"success|200\", route=\"/logproto.Pusher/Push\"}[5m])) by (route)/sum(rate(loki_request_duration_seconds_count{job=\"loki\", route=\"/logproto.Pusher/Push\"}[5m])) by (route)",
"interval": "",
"legendFormat": "{{route}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Ingester Success Rate",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2980",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2981",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 6,
"x": 0,
"y": 54
},
"hiddenSeries": false,
"id": 31,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"$$hashKey": "object:3847",
"alias": "0.99",
"color": "#C4162A"
},
{
"$$hashKey": "object:3852",
"alias": "0.9",
"color": "#FADE2A"
},
{
"$$hashKey": "object:3857",
"alias": "0.5",
"color": "#1F60C4"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (loki_ingester_chunk_age_seconds_bucket{job=\"loki\"}))",
"interval": "",
"legendFormat": "0.99",
"refId": "A"
},
{
"expr": "histogram_quantile(0.9, sum by (le) (loki_ingester_chunk_age_seconds_bucket{job=\"loki\"}))",
"interval": "",
"legendFormat": "0.9",
"refId": "B"
},
{
"expr": "histogram_quantile(0.5, sum by (le) (loki_ingester_chunk_age_seconds_bucket{job=\"loki\"}))",
"interval": "",
"legendFormat": "0.5",
"refId": "C"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Distribution of chunk ages when flushed",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:457",
"format": "s",
"label": "",
"logBase": 10,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:458",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 6,
"x": 6,
"y": 54
},
"hiddenSeries": false,
"id": 32,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"$$hashKey": "object:3847",
"alias": "0.99",
"color": "#C4162A"
},
{
"$$hashKey": "object:3852",
"alias": "0.9",
"color": "#FADE2A"
},
{
"$$hashKey": "object:3857",
"alias": "0.5",
"color": "#1F60C4"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (loki_ingester_chunk_entries_bucket{job=\"loki\"}))",
"interval": "",
"legendFormat": "0.99",
"refId": "A"
},
{
"expr": "histogram_quantile(0.9, sum by (le) (loki_ingester_chunk_entries_bucket{job=\"loki\"}))",
"interval": "",
"legendFormat": "0.9",
"refId": "B"
},
{
"expr": "histogram_quantile(0.5, sum by (le) (loki_ingester_chunk_entries_bucket{job=\"loki\"}))",
"interval": "",
"legendFormat": "0.5",
"refId": "C"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Distribution of lines per-chunk when flushed.",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:457",
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:458",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 6,
"x": 12,
"y": 54
},
"hiddenSeries": false,
"id": 33,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"$$hashKey": "object:3847",
"alias": "0.99",
"color": "#C4162A"
},
{
"$$hashKey": "object:3852",
"alias": "0.9",
"color": "#FADE2A"
},
{
"$$hashKey": "object:3857",
"alias": "0.5",
"color": "#1F60C4"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (loki_ingester_chunk_size_bytes_bucket{job=\"loki\"}))",
"interval": "",
"legendFormat": "0.99",
"refId": "A"
},
{
"expr": "histogram_quantile(0.9, sum by (le) (loki_ingester_chunk_size_bytes_bucket{job=\"loki\"}))",
"interval": "",
"legendFormat": "0.9",
"refId": "B"
},
{
"expr": "histogram_quantile(0.5, sum by (le) (loki_ingester_chunk_size_bytes_bucket{job=\"loki\"}))",
"interval": "",
"legendFormat": "0.5",
"refId": "C"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Distribution of chunk sizes when flushed.",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:457",
"format": "decbytes",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:458",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource":"${DS_PROMETHEUS}",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 6,
"x": 18,
"y": 54
},
"hiddenSeries": false,
"id": 34,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"$$hashKey": "object:3847",
"alias": "0.99",
"color": "#C4162A"
},
{
"$$hashKey": "object:3852",
"alias": "0.9",
"color": "#FADE2A"
},
{
"$$hashKey": "object:3857",
"alias": "0.5",
"color": "#1F60C4"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "histogram_quantile(0.99, sum by (le) (loki_ingester_chunk_utilization_bucket{job=\"loki\"}))",
"interval": "",
"legendFormat": "0.99",
"refId": "A"
},
{
"expr": "histogram_quantile(0.9, sum by (le) (loki_ingester_chunk_utilization_bucket{job=\"loki\"}))",
"interval": "",
"legendFormat": "0.9",
"refId": "B"
},
{
"expr": "histogram_quantile(0.5, sum by (le) (loki_ingester_chunk_utilization_bucket{job=\"loki\"}))",
"interval": "",
"legendFormat": "0.5",
"refId": "C"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Distribution of chunk utilization when flushed",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:457",
"format": "percentunit",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:458",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"schemaVersion": 22,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-2d",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
]
},
"timezone": "",
"title": "Loki",
"uid": "MQHVDmtWk",
"variables": {
"list": []
},
"version": 1
}