diff --git a/charts/incubator/mimir/.helmignore b/charts/incubator/mimir/.helmignore
deleted file mode 100644
index 77ca5567b26..00000000000
--- a/charts/incubator/mimir/.helmignore
+++ /dev/null
@@ -1,30 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/
-# OWNERS file for Kubernetes
-OWNERS
-# helm-docs templates
-*.gotmpl
-# docs folder
-/docs
-# icon
-icon.png
diff --git a/charts/incubator/mimir/CHANGELOG.md b/charts/incubator/mimir/CHANGELOG.md
deleted file mode 100644
index 6afbcc5f802..00000000000
--- a/charts/incubator/mimir/CHANGELOG.md
+++ /dev/null
@@ -1,3 +0,0 @@
----
-title: Changelog
----
diff --git a/charts/incubator/mimir/Chart.yaml b/charts/incubator/mimir/Chart.yaml
deleted file mode 100644
index ae988b11c74..00000000000
--- a/charts/incubator/mimir/Chart.yaml
+++ /dev/null
@@ -1,49 +0,0 @@
-annotations:
- max_scale_version: 24.04.0
- min_scale_version: 23.10.0
- truecharts.org/SCALE-support: "true"
- truecharts.org/category: system
- truecharts.org/max_helm_version: "3.14"
- truecharts.org/min_helm_version: "3.12"
- truecharts.org/train: incubator
-apiVersion: v2
-appVersion: 18.0.1
-dependencies:
- - name: common
- version: 17.5.0
- repository: oci://tccr.io/truecharts
- condition: ""
- alias: ""
- tags: []
- import-values: []
- - name: mimir-distributed
- version: 5.2.1
- repository: https://grafana.github.io/helm-charts
- condition: ""
- alias: "mimir"
- tags: []
- import-values: []
- - name: kube-prometheus-stack
- version: 56.9.0
- repository: oci://ghcr.io/prometheus-community/charts
- condition: ""
- alias: kps
- tags: []
- import-values: []
-deprecated: false
-description: A scalable metrics database
-home: https://truecharts.org/charts/incubator/mimir
-icon: https://truecharts.org/img/hotlink-ok/chart-icons/mimir.png
-keywords:
- - mimir
- - shell
-kubeVersion: '>=1.24.0-0'
-maintainers:
- - name: TrueCharts
- email: info@truecharts.org
- url: https://truecharts.org
-name: mimir
-sources:
- - https://grafana.com
-type: application
-version: 0.2.0
diff --git a/charts/incubator/mimir/README.md b/charts/incubator/mimir/README.md
deleted file mode 100644
index 0fc4dfd1da0..00000000000
--- a/charts/incubator/mimir/README.md
+++ /dev/null
@@ -1,28 +0,0 @@
----
-title: README
----
-
-## General Info
-
-TrueCharts can be installed as both _normal_ Helm Charts or as Apps on TrueNAS SCALE.
-However only installations using the TrueNAS SCALE Apps system are supported.
-
-For more information about this App, please check the docs on the TrueCharts [website](https://truecharts.org/charts/incubator/atuin)
-
-**This chart is not maintained by the upstream project and any issues with the chart should be raised [here](https://github.com/truecharts/charts/issues/new/choose)**
-
-## Support
-
-- Please check our [quick-start guides for TrueNAS SCALE](https://truecharts.org/manual/SCALE/guides/scale-intro).
-- See the [Website](https://truecharts.org)
-- Check our [Discord](https://discord.gg/tVsPTHWTtr)
-- Open a [issue](https://github.com/truecharts/charts/issues/new/choose)
-
----
-
-## Sponsor TrueCharts
-
-TrueCharts can only exist due to the incredible effort of our staff.
-Please consider making a [donation](https://truecharts.org/sponsor) or contributing back to the project any way you can!
-
-_All Rights Reserved - The TrueCharts Project_
diff --git a/charts/incubator/mimir/icon.png b/charts/incubator/mimir/icon.png
deleted file mode 100644
index 4673dd7feb3..00000000000
Binary files a/charts/incubator/mimir/icon.png and /dev/null differ
diff --git a/charts/incubator/mimir/output.yaml b/charts/incubator/mimir/output.yaml
deleted file mode 100644
index f61bc6d8478..00000000000
--- a/charts/incubator/mimir/output.yaml
+++ /dev/null
@@ -1,53417 +0,0 @@
----
-# Source: mimir/charts/mimir/templates/alertmanager/alertmanager-pdb.yaml
-apiVersion: policy/v1
-kind: PodDisruptionBudget
-metadata:
- name: release-name-mimir-alertmanager
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: alertmanager
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- namespace: "default"
-spec:
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: alertmanager
- maxUnavailable: 1
----
-# Source: mimir/charts/mimir/templates/compactor/compactor-pdb.yaml
-apiVersion: policy/v1
-kind: PodDisruptionBudget
-metadata:
- name: release-name-mimir-compactor
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: compactor
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- namespace: "default"
-spec:
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: compactor
- maxUnavailable: 1
----
-# Source: mimir/charts/mimir/templates/distributor/distributor-pdb.yaml
-apiVersion: policy/v1
-kind: PodDisruptionBudget
-metadata:
- name: release-name-mimir-distributor
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: distributor
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- namespace: "default"
-spec:
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: distributor
- maxUnavailable: 1
----
-# Source: mimir/charts/mimir/templates/index-cache/index-cache-pdb.yaml
-apiVersion: policy/v1
-kind: PodDisruptionBudget
-metadata:
- name: release-name-mimir-index-cache
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: index-cache
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- namespace: "default"
-spec:
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: index-cache
- maxUnavailable: 1
----
-# Source: mimir/charts/mimir/templates/ingester/ingester-pdb.yaml
-apiVersion: policy/v1
-kind: PodDisruptionBudget
-metadata:
- name: release-name-mimir-ingester
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- namespace: "default"
-spec:
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- maxUnavailable: 1
----
-# Source: mimir/charts/mimir/templates/metadata-cache/metadata-cache-pdb.yaml
-apiVersion: policy/v1
-kind: PodDisruptionBudget
-metadata:
- name: release-name-mimir-metadata-cache
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: metadata-cache
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- namespace: "default"
-spec:
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: metadata-cache
- maxUnavailable: 1
----
-# Source: mimir/charts/mimir/templates/nginx/nginx-pdb.yaml
-apiVersion: policy/v1
-kind: PodDisruptionBudget
-metadata:
- name: release-name-mimir-nginx
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: nginx
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- namespace: "default"
-spec:
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: nginx
- maxUnavailable: 1
----
-# Source: mimir/charts/mimir/templates/overrides-exporter/overrides-exporter-pdb.yaml
-apiVersion: policy/v1
-kind: PodDisruptionBudget
-metadata:
- name: release-name-mimir-overrides-exporter
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: overrides-exporter
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- namespace: "default"
-spec:
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: overrides-exporter
- maxUnavailable: 1
----
-# Source: mimir/charts/mimir/templates/querier/querier-pdb.yaml
-apiVersion: policy/v1
-kind: PodDisruptionBudget
-metadata:
- name: release-name-mimir-querier
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: querier
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- namespace: "default"
-spec:
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: querier
- maxUnavailable: 1
----
-# Source: mimir/charts/mimir/templates/query-frontend/query-frontend-pdb.yaml
-apiVersion: policy/v1
-kind: PodDisruptionBudget
-metadata:
- name: release-name-mimir-query-frontend
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-frontend
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- namespace: "default"
-spec:
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-frontend
- maxUnavailable: 1
----
-# Source: mimir/charts/mimir/templates/query-scheduler/query-scheduler-pdb.yaml
-apiVersion: policy/v1
-kind: PodDisruptionBudget
-metadata:
- name: release-name-mimir-query-scheduler
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-scheduler
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- namespace: "default"
-spec:
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-scheduler
- maxUnavailable: 1
----
-# Source: mimir/charts/mimir/templates/results-cache/results-cache-pdb.yaml
-apiVersion: policy/v1
-kind: PodDisruptionBudget
-metadata:
- name: release-name-mimir-results-cache
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: results-cache
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- namespace: "default"
-spec:
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: results-cache
- maxUnavailable: 1
----
-# Source: mimir/charts/mimir/templates/ruler/ruler-pdb.yaml
-apiVersion: policy/v1
-kind: PodDisruptionBudget
-metadata:
- name: release-name-mimir-ruler
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ruler
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- namespace: "default"
-spec:
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ruler
- maxUnavailable: 1
----
-# Source: mimir/charts/mimir/templates/store-gateway/store-gateway-pdb.yaml
-apiVersion: policy/v1
-kind: PodDisruptionBudget
-metadata:
- name: release-name-mimir-store-gateway
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- namespace: "default"
-spec:
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- maxUnavailable: 1
----
-# Source: mimir/charts/kps/charts/kube-state-metrics/templates/serviceaccount.yaml
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- labels:
- helm.sh/chart: kube-state-metrics-5.16.0
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: metrics
- app.kubernetes.io/part-of: kube-state-metrics
- app.kubernetes.io/name: kube-state-metrics
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.10.1"
- release: release-name
- name: release-name-kube-state-metrics
- namespace: default
----
-# Source: mimir/charts/kps/charts/prometheus-node-exporter/templates/serviceaccount.yaml
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: release-name-prometheus-node-exporter
- namespace: default
- labels:
- helm.sh/chart: prometheus-node-exporter-4.30.2
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: metrics
- app.kubernetes.io/part-of: prometheus-node-exporter
- app.kubernetes.io/name: prometheus-node-exporter
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "1.7.0"
- jobLabel: node-exporter
- release: release-name
----
-# Source: mimir/charts/mimir/charts/minio/templates/serviceaccount.yaml
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: "minio-sa"
----
-# Source: mimir/charts/mimir/charts/rollout_operator/templates/serviceaccount.yaml
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: release-name-rollout-operator
- labels:
- helm.sh/chart: rollout-operator-0.10.0
- app.kubernetes.io/name: rollout-operator
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "v0.9.0"
- app.kubernetes.io/managed-by: Helm
----
-# Source: mimir/charts/mimir/templates/serviceaccount.yaml
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: release-name-mimir
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
----
-# Source: mimir/charts/mimir/charts/minio/templates/secrets.yaml
-apiVersion: v1
-kind: Secret
-metadata:
- name: release-name-minio
- labels:
- app: minio
- chart: minio-5.0.14
- release: release-name
- heritage: Helm
-type: Opaque
-data:
- rootUser: "Z3JhZmFuYS1taW1pcg=="
- rootPassword: "c3VwZXJzZWNyZXQ="
----
-# Source: mimir/charts/mimir/templates/metamonitoring/logs-instance-usernames-secret.yaml
-apiVersion: v1
-kind: Secret
-metadata:
- name: release-name-mimir-logs-instance-usernames
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: meta-monitoring
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-data:
----
-# Source: mimir/charts/mimir/templates/metamonitoring/metrics-instance-usernames-secret.yaml
-apiVersion: v1
-kind: Secret
-metadata:
- name: release-name-mimir-metrics-instance-usernames
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: meta-monitoring
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-data:
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/alertmanager-overview.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-alertmanager-overview
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- alertmanager-overview.json: |-
- {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":1,"hideControls":false,"id":null,"links":[],"refresh":"30s","rows":[{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"current set of alerts stored in the Alertmanager","fill":1,"fillGradient":0,"gridPos":{},"id":2,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(alertmanager_alerts{namespace=~\"$namespace\",service=~\"$service\"}) by (namespace,service,instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Alerts","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"none","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"none","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"rate of successful and invalid alerts received by the Alertmanager","fill":1,"fillGradient":0,"gridPos":{},"id":3,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(rate(alertmanager_alerts_received_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Received","refId":"A"},{"expr":"sum(rate(alertmanager_alerts_invalid_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Invalid","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Alerts receive rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Alerts","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"rate of successful and invalid notifications sent by the Alertmanager","fill":1,"fillGradient":0,"gridPos":{},"id":4,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":"integration","seriesOverrides":[],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(rate(alertmanager_notifications_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Total","refId":"A"},{"expr":"sum(rate(alertmanager_notifications_failed_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Failed","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"$integration: Notifications Send Rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"latency of notifications sent by the Alertmanager","fill":1,"fillGradient":0,"gridPos":{},"id":5,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":"integration","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} 99th Percentile","refId":"A"},{"expr":"histogram_quantile(0.50,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Median","refId":"B"},{"expr":"sum(rate(alertmanager_notification_latency_seconds_sum{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n/\nsum(rate(alertmanager_notification_latency_seconds_count{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Average","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"$integration: Notification Duration","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Notifications","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["alertmanager-mixin"],"templating":{"list":[{"current":{"text":"Prometheus","value":"Prometheus"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":"namespace","multi":false,"name":"namespace","options":[],"query":"label_values(alertmanager_alerts, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":"service","multi":false,"name":"service","options":[],"query":"label_values(alertmanager_alerts, service)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"all","value":"$__all"},"datasource":"$datasource","hide":2,"includeAll":true,"label":null,"multi":false,"name":"integration","options":[],"query":"label_values(alertmanager_notifications_total{integration=~\".*\"}, integration)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Alertmanager / Overview","uid":"alertmanager-overview","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/apiserver.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-apiserver
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- apiserver.json: |-
- {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"content":"The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.","datasource":null,"description":"The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.","gridPos":{"h":2,"w":24,"x":0,"y":0},"id":2,"mode":"markdown","span":12,"title":"Notice","type":"text"}],"refresh":"10s","rows":[{"collapse":false,"collapsed":false,"panels":[{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"$datasource","decimals":3,"description":"How many percent of requests (both read and write) in 30 days have been answered successfully and fast enough?","format":"percentunit","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{},"id":3,"interval":"1m","legend":{"alignAsTable":true,"rightSide":true},"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":4,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"","refId":"A"}],"thresholds":"","title":"Availability (30d) > 99.000%","tooltip":{"shared":false},"type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"avg"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","decimals":3,"description":"How much error budget is left looking at our 0.990% availability guarantees?","fill":10,"fillGradient":0,"gridPos":{},"id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":8,"stack":false,"steppedLine":false,"targets":[{"expr":"100 * (apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"} - 0.990000)","format":"time_series","intervalFactor":2,"legendFormat":"errorbudget","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"ErrorBudget (30d) > 99.000%","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"decimals":3,"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"decimals":3,"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"$datasource","decimals":3,"description":"How many percent of read requests (LIST,GET) in 30 days have been answered successfully and fast enough?","format":"percentunit","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{},"id":5,"interval":"1m","legend":{"alignAsTable":true,"rightSide":true},"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":3,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"apiserver_request:availability30d{verb=\"read\", cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"","refId":"A"}],"thresholds":"","title":"Read Availability (30d)","tooltip":{"shared":false},"type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"avg"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"How many read requests (LIST,GET) per second do the apiservers get by code?","fill":10,"fillGradient":0,"gridPos":{},"id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/2../i","color":"#56A64B"},{"alias":"/3../i","color":"#F2CC0C"},{"alias":"/4../i","color":"#3274D9"},{"alias":"/5../i","color":"#E02F44"}],"spaceLength":10,"span":3,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})","format":"time_series","intervalFactor":2,"legendFormat":"{{ code }}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Read SLI - Requests","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"reqps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"reqps","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"How many percent of read requests (LIST,GET) per second are returned with errors (5xx)?","fill":1,"fillGradient":0,"gridPos":{},"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":3,"stack":false,"steppedLine":false,"targets":[{"expr":"sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})","format":"time_series","intervalFactor":2,"legendFormat":"{{ resource }}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Read SLI - Errors","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"How many seconds is the 99th percentile for reading (LIST|GET) a given resource?","fill":1,"fillGradient":0,"gridPos":{},"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":3,"stack":false,"steppedLine":false,"targets":[{"expr":"cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile{verb=\"read\", cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{ resource }}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Read SLI - Duration","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"$datasource","decimals":3,"description":"How many percent of write requests (POST|PUT|PATCH|DELETE) in 30 days have been answered successfully and fast enough?","format":"percentunit","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{},"id":9,"interval":"1m","legend":{"alignAsTable":true,"rightSide":true},"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":3,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"apiserver_request:availability30d{verb=\"write\", cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"","refId":"A"}],"thresholds":"","title":"Write Availability (30d)","tooltip":{"shared":false},"type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"avg"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"How many write requests (POST|PUT|PATCH|DELETE) per second do the apiservers get by code?","fill":10,"fillGradient":0,"gridPos":{},"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/2../i","color":"#56A64B"},{"alias":"/3../i","color":"#F2CC0C"},{"alias":"/4../i","color":"#3274D9"},{"alias":"/5../i","color":"#E02F44"}],"spaceLength":10,"span":3,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})","format":"time_series","intervalFactor":2,"legendFormat":"{{ code }}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Write SLI - Requests","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"reqps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"reqps","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"How many percent of write requests (POST|PUT|PATCH|DELETE) per second are returned with errors (5xx)?","fill":1,"fillGradient":0,"gridPos":{},"id":11,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":3,"stack":false,"steppedLine":false,"targets":[{"expr":"sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})","format":"time_series","intervalFactor":2,"legendFormat":"{{ resource }}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Write SLI - Errors","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"How many seconds is the 99th percentile for writing (POST|PUT|PATCH|DELETE) a given resource?","fill":1,"fillGradient":0,"gridPos":{},"id":12,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":3,"stack":false,"steppedLine":false,"targets":[{"expr":"cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile{verb=\"write\", cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{ resource }}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Write SLI - Duration","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":13,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{name}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Work Queue Add Rate","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":14,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{name}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Work Queue Depth","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":15,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{name}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Work Queue Latency","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":16,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":17,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU usage","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":18,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"go_goroutines{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Goroutines","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"apiserver\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":true,"label":null,"multi":false,"name":"instance","options":[],"query":"label_values(up{job=\"apiserver\", cluster=\"$cluster\"}, instance)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / API server","uid":"09ec8aa1e996d6ffcd6817bbaff4db1b","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/cluster-total.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-cluster-total
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- cluster-total.json: |-
- {"__inputs":[],"__requires":[],"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":2,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Current Bandwidth","titleSize":"h6","type":"row"},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":1},"id":3,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Received","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":1},"id":4,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Transmitted","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"columns":[{"text":"Time","value":"Time"},{"text":"Value #A","value":"Value #A"},{"text":"Value #B","value":"Value #B"},{"text":"Value #C","value":"Value #C"},{"text":"Value #D","value":"Value #D"},{"text":"Value #E","value":"Value #E"},{"text":"Value #F","value":"Value #F"},{"text":"Value #G","value":"Value #G"},{"text":"Value #H","value":"Value #H"},{"text":"namespace","value":"namespace"}],"datasource":"$datasource","fill":1,"fontSize":"90%","gridPos":{"h":9,"w":24,"x":0,"y":10},"id":5,"lines":true,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null as zero","renderer":"flot","scroll":true,"showHeader":true,"sort":{"col":0,"desc":false},"spaceLength":10,"span":24,"styles":[{"alias":"Time","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Time","thresholds":[],"type":"hidden","unit":"short"},{"alias":"Current Bandwidth Received","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Current Bandwidth Transmitted","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Average Bandwidth Received","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Average Bandwidth Transmitted","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Rate of Received Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Received Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #G","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #H","thresholds":[],"type":"number","unit":"pps"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTooltip":"Drill down","linkUrl":"d/8b7a8b326d7a6f1f04244066368c67af/kubernetes-networking-namespace-pods?orgId=1&refresh=30s&var-namespace=${__value.text}","pattern":"namespace","thresholds":[],"type":"number","unit":"short"}],"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"A","step":10},{"expr":"sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"B","step":10},{"expr":"sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"C","step":10},{"expr":"sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"D","step":10},{"expr":"sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"E","step":10},{"expr":"sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"F","step":10},{"expr":"sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"G","step":10},{"expr":"sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"H","step":10}],"timeFrom":null,"timeShift":null,"title":"Current Status","type":"table"},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":10},"id":6,"panels":[{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":11},"id":7,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Rate of Bytes Received","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":11},"id":8,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Rate of Bytes Transmitted","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Average Bandwidth","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":11},"id":9,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth History","titleSize":"h6","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":24,"x":0,"y":12},"id":10,"legend":{"alignAsTable":true,"avg":true,"current":true,"hideEmpty":true,"hideZero":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":2,"links":[],"minSpan":24,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":24,"x":0,"y":21},"id":11,"legend":{"alignAsTable":true,"avg":true,"current":true,"hideEmpty":true,"hideZero":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":2,"links":[],"minSpan":24,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":30},"id":12,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":24,"x":0,"y":31},"id":13,"legend":{"alignAsTable":true,"avg":true,"current":true,"hideEmpty":true,"hideZero":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":2,"links":[],"minSpan":24,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":24,"x":0,"y":40},"id":14,"legend":{"alignAsTable":true,"avg":true,"current":true,"hideEmpty":true,"hideZero":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":2,"links":[],"minSpan":24,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Packets","titleSize":"h6","type":"row"},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":31},"id":15,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":24,"x":0,"y":50},"id":16,"legend":{"alignAsTable":true,"avg":true,"current":true,"hideEmpty":true,"hideZero":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":2,"links":[],"minSpan":24,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":24,"x":0,"y":59},"id":17,"legend":{"alignAsTable":true,"avg":true,"current":true,"hideEmpty":true,"hideZero":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":2,"links":[],"minSpan":24,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":24,"x":0,"y":59},"id":18,"legend":{"alignAsTable":true,"avg":true,"current":true,"hideEmpty":true,"hideZero":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":2,"links":[{"targetBlank":true,"title":"What is TCP Retransmit?","url":"https://accedian.com/enterprises/blog/network-packet-loss-retransmissions-and-duplicate-acknowledgements/"}],"minSpan":24,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_OutSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))","format":"time_series","intervalFactor":1,"legendFormat":"{{instance}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of TCP Retransmits out of all sent segments","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":24,"x":0,"y":59},"id":19,"legend":{"alignAsTable":true,"avg":true,"current":true,"hideEmpty":true,"hideZero":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":2,"links":[{"targetBlank":true,"title":"Why monitor SYN retransmits?","url":"https://github.com/prometheus/node_exporter/issues/1023#issuecomment-408128365"}],"minSpan":24,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(rate(node_netstat_TcpExt_TCPSynRetrans{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))","format":"time_series","intervalFactor":1,"legendFormat":"{{instance}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of TCP SYN Retransmits out of all retransmits","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Errors","titleSize":"h6","type":"row"}],"refresh":"10s","rows":[],"schemaVersion":18,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"resolution","options":[{"selected":false,"text":"30s","value":"30s"},{"selected":true,"text":"5m","value":"5m"},{"selected":false,"text":"1h","value":"1h"}],"query":"30s,5m,1h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"interval","options":[{"selected":true,"text":"4h","value":"4h"}],"query":"4h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false},{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\"}, cluster)","refresh":2,"regex":"","sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Networking / Cluster","uid":"ff635a025bcfea7bc3dd4f508990a3e9","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/controller-manager.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-controller-manager
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- controller-manager.json: |-
- {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"10s","rows":[{"collapse":false,"collapsed":false,"panels":[{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"$datasource","format":"none","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{},"id":2,"interval":"1m","legend":{"alignAsTable":true,"rightSide":true},"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":2,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"sum(up{cluster=\"$cluster\", job=\"kube-controller-manager\"})","format":"time_series","intervalFactor":2,"legendFormat":"","refId":"A"}],"thresholds":"","title":"Up","tooltip":{"shared":false},"type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"min"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} {{name}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Work Queue Add Rate","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} {{name}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Work Queue Depth","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} {{name}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Work Queue Latency","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"2xx","refId":"A"},{"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"3xx","refId":"B"},{"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"4xx","refId":"C"},{"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"5xx","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Kube API Request Rate","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":8,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{verb}} {{url}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Post Request Latency 99th Quantile","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{verb}} {{url}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Get Request Latency 99th Quantile","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"process_resident_memory_bytes{cluster=\"$cluster\", job=\"kube-controller-manager\",instance=~\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-controller-manager\",instance=~\"$instance\"}[$__rate_interval])","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU usage","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":11,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"go_goroutines{cluster=\"$cluster\", job=\"kube-controller-manager\",instance=~\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Goroutines","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kube-controller-manager\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":true,"label":null,"multi":false,"name":"instance","options":[],"query":"label_values(up{cluster=\"$cluster\", job=\"kube-controller-manager\"}, instance)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Controller Manager","uid":"72e0e05bef5099e5f049b05fdc429ed4","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/etcd.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-etcd
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- etcd.json: |-
- {"description":"etcd sample Grafana dashboard with Prometheus","panels":[{"datasource":{"type":"datasource","uid":"-- Mixed --"},"gridPos":{"h":7,"w":6,"x":0,"y":0},"id":1,"interval":"1m","options":{"colorMode":"none","graphMode":"none","reduceOptions":{"calcs":["lastNotNull"]}},"pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(etcd_server_has_leader{job=~\".*etcd.*\", job=\"$cluster\"})","legendFormat":"{{cluster}} - {{namespace}}\n"}],"title":"Up","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"ops"}},"gridPos":{"h":7,"w":10,"x":6,"y":0},"id":2,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(rate(grpc_server_started_total{job=~\".*etcd.*\", job=\"$cluster\",grpc_type=\"unary\"}[$__rate_interval]))","legendFormat":"RPC rate"},{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(rate(grpc_server_handled_total{job=~\".*etcd.*\", job=\"$cluster\",grpc_type=\"unary\",grpc_code=~\"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[$__rate_interval]))","legendFormat":"RPC failed rate"}],"title":"RPC rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"}}},"gridPos":{"h":7,"w":8,"x":16,"y":0},"id":3,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(grpc_server_started_total{job=~\".*etcd.*\",job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})","legendFormat":"Watch streams"},{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(grpc_server_started_total{job=~\".*etcd.*\",job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})","legendFormat":"Lease streams"}],"title":"Active streams","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"bytes"}},"gridPos":{"h":7,"w":8,"x":0,"y":25},"id":4,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"etcd_mvcc_db_total_size_in_bytes{job=~\".*etcd.*\", job=\"$cluster\"}","legendFormat":"{{instance}} DB size"}],"title":"DB size","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"s"}},"gridPos":{"h":7,"w":8,"x":8,"y":25},"id":5,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])) by (instance, le))","legendFormat":"{{instance}} WAL fsync"},{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])) by (instance, le))","legendFormat":"{{instance}} DB fsync"}],"title":"Disk sync duration","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"bytes"}},"gridPos":{"h":7,"w":8,"x":16,"y":25},"id":6,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"process_resident_memory_bytes{job=~\".*etcd.*\", job=\"$cluster\"}","legendFormat":"{{instance}} resident memory"}],"title":"Memory","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"Bps"}},"gridPos":{"h":7,"w":6,"x":0,"y":50},"id":7,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"rate(etcd_network_client_grpc_received_bytes_total{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])","legendFormat":"{{instance}} client traffic in"}],"title":"Client traffic in","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"Bps"}},"gridPos":{"h":7,"w":6,"x":6,"y":50},"id":8,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"rate(etcd_network_client_grpc_sent_bytes_total{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])","legendFormat":"{{instance}} client traffic out"}],"title":"Client traffic out","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"Bps"}},"gridPos":{"h":7,"w":6,"x":12,"y":50},"id":9,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(rate(etcd_network_peer_received_bytes_total{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])) by (instance)","legendFormat":"{{instance}} peer traffic in"}],"title":"Peer traffic in","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"Bps"}},"gridPos":{"h":7,"w":6,"x":18,"y":50},"id":10,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(rate(etcd_network_peer_sent_bytes_total{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])) by (instance)","legendFormat":"{{instance}} peer traffic out"}],"title":"Peer traffic out","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"}}},"gridPos":{"h":7,"w":8,"x":0,"y":75},"id":11,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"changes(etcd_server_leader_changes_seen_total{job=~\".*etcd.*\", job=\"$cluster\"}[1d])","legendFormat":"{{instance}} total leader elections per day"}],"title":"Raft proposals","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"}}},"gridPos":{"h":7,"w":8,"x":8,"y":75},"id":12,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"changes(etcd_server_leader_changes_seen_total{job=~\".*etcd.*\", job=\"$cluster\"}[1d])","legendFormat":"{{instance}} total leader elections per day"}],"title":"Total leader elections per day","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"s"}},"gridPos":{"h":7,"w":8,"x":16,"y":75},"id":13,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])))","legendFormat":"{{instance}} peer round trip time"}],"title":"Peer round trip time","type":"timeseries"}],"refresh":"10s","schemaVersion":36,"tags":["etcd-mixin"],"templating":{"list":[{"label":"Data Source","name":"datasource","query":"prometheus","type":"datasource"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"label":"cluster","name":"cluster","query":"label_values(etcd_server_has_leader{job=~\".*etcd.*\"}, job)","refresh":2,"type":"query","hide":2}]},"time":{"from":"now-15m","to":"now"},"timezone": "utc","title":"etcd","uid":"c2f4e12cdf69feb95caa41a5a1b423d9"}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/grafana-overview.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-grafana-overview
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- grafana-overview.json: |-
- {"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"editable":true,"gnetId":null,"graphTooltip":0,"id":3085,"iteration":1631554945276,"links":[],"panels":[{"datasource":"$datasource","fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":6,"x":0,"y":0},"id":6,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"text":{},"textMode":"auto"},"pluginVersion":"8.1.3","targets":[{"expr":"grafana_alerting_result_total{job=~\"$job\", instance=~\"$instance\", state=\"alerting\"}","instant":true,"interval":"","legendFormat":"","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Firing Alerts","type":"stat"},{"datasource":"$datasource","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":6,"x":6,"y":0},"id":8,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"text":{},"textMode":"auto"},"pluginVersion":"8.1.3","targets":[{"expr":"sum(grafana_stat_totals_dashboard{job=~\"$job\", instance=~\"$instance\"})","interval":"","legendFormat":"","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Dashboards","type":"stat"},{"datasource":"$datasource","fieldConfig":{"defaults":{"custom":{"align":null,"displayMode":"auto"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":12,"x":12,"y":0},"id":10,"options":{"showHeader":true},"pluginVersion":"8.1.3","targets":[{"expr":"grafana_build_info{job=~\"$job\", instance=~\"$instance\"}","instant":true,"interval":"","legendFormat":"","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Build Info","transformations":[{"id":"labelsToFields","options":{}},{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"branch":true,"container":true,"goversion":true,"namespace":true,"pod":true,"revision":true},"indexByName":{"Time":7,"Value":11,"branch":4,"container":8,"edition":2,"goversion":6,"instance":1,"job":0,"namespace":9,"pod":10,"revision":5,"version":3},"renameByName":{}}}],"type":"table"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":5},"hiddenSeries":false,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"8.1.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by (status_code) (irate(grafana_http_request_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[1m])) ","interval":"","legendFormat":"{{status_code}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"RPS","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"$$hashKey":"object:157","format":"reqps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"$$hashKey":"object:158","format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":5},"hiddenSeries":false,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"8.1.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"exemplar":true,"expr":"histogram_quantile(0.99, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1","interval":"","legendFormat":"99th Percentile","refId":"A"},{"exemplar":true,"expr":"histogram_quantile(0.50, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1","interval":"","legendFormat":"50th Percentile","refId":"B"},{"exemplar":true,"expr":"sum(irate(grafana_http_request_duration_seconds_sum{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) * 1 / sum(irate(grafana_http_request_duration_seconds_count{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval]))","interval":"","legendFormat":"Average","refId":"C"}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Request Latency","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"$$hashKey":"object:210","format":"ms","label":null,"logBase":1,"max":null,"min":null,"show":true},{"$$hashKey":"object:211","format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}}],"schemaVersion":30,"style":"dark","tags":[],"templating":{"list":[{"current":{"selected":true,"text":"dev-cortex","value":"dev-cortex"},"description":null,"error":null,"hide":0,"includeAll":false,"label":null,"multi":false,"name":"datasource","options":[],"query":"prometheus","queryValue":"","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"allValue":".*","current":{"selected":false,"text":["default/grafana"],"value":["default/grafana"]},"datasource":"$datasource","definition":"label_values(grafana_build_info, job)","description":null,"error":null,"hide":0,"includeAll":true,"label":null,"multi":true,"name":"job","options":[],"query":{"query":"label_values(grafana_build_info, job)","refId":"Billing Admin-job-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"allValue":".*","current":{"selected":false,"text":"All","value":"$__all"},"datasource":"$datasource","definition":"label_values(grafana_build_info, instance)","description":null,"error":null,"hide":0,"includeAll":true,"label":null,"multi":true,"name":"instance","options":[],"query":{"query":"label_values(grafana_build_info, instance)","refId":"Billing Admin-instance-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["10s","30s","1m","5m","15m","30m","1h","2h","1d"]},"timezone": "utc","title":"Grafana Overview","uid":"6be0s85Mk","version":2}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/k8s-coredns.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-k8s-coredns
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- k8s-coredns.json: |-
- {"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"description":"A dashboard for the CoreDNS DNS server with updated metrics for version 1.7.0+. Based on the CoreDNS dashboard by buhay.","editable":true,"gnetId":12539,"graphTooltip":0,"iteration":1603798405693,"links":[{"icon":"external link","tags":[],"targetBlank":true,"title":"CoreDNS.io","type":"link","url":"https://coredns.io"}],"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":8,"x":0,"y":0},"hiddenSeries":false,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"total","yaxis":2}],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(rate(coredns_dns_request_count_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (proto) or\nsum(rate(coredns_dns_requests_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (proto)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{ proto }}","refId":"A","step":60}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Requests (total)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","logBase":1,"max":null,"min":0,"show":true},{"format":"pps","logBase":1,"max":null,"min":0,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":8,"x":8,"y":0},"hiddenSeries":false,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"total","yaxis":2},{"alias":"other","yaxis":2}],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(rate(coredns_dns_request_type_count_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (type) or \nsum(rate(coredns_dns_requests_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (type)","interval":"","intervalFactor":2,"legendFormat":"{{ type }}","refId":"A","step":60}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Requests (by qtype)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","logBase":1,"max":null,"min":0,"show":true},{"format":"pps","logBase":1,"max":null,"min":0,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":8,"x":16,"y":0},"hiddenSeries":false,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"total","yaxis":2}],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(rate(coredns_dns_request_count_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (zone) or\nsum(rate(coredns_dns_requests_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (zone)","interval":"","intervalFactor":2,"legendFormat":"{{ zone }}","refId":"A","step":60}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Requests (by zone)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","logBase":1,"max":null,"min":0,"show":true},{"format":"pps","logBase":1,"max":null,"min":0,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":12,"x":0,"y":7},"hiddenSeries":false,"id":8,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"total","yaxis":2}],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(coredns_dns_request_do_count_total{job=~\"$job\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_do_requests_total{job=~\"$job\",instance=~\"$instance\"}[5m]))","interval":"","intervalFactor":2,"legendFormat":"DO","refId":"A","step":40},{"expr":"sum(rate(coredns_dns_request_count_total{job=~\"$job\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_requests_total{job=~\"$job\",instance=~\"$instance\"}[5m]))","interval":"","intervalFactor":2,"legendFormat":"total","refId":"B","step":40}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Requests (DO bit)","tooltip":{"shared":true,"sort":2,"value_type":"cumulative"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","logBase":1,"max":null,"min":0,"show":true},{"format":"pps","logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":6,"x":12,"y":7},"hiddenSeries":false,"id":10,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"tcp:90","yaxis":2},{"alias":"tcp:99 ","yaxis":2},{"alias":"tcp:50","yaxis":2}],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))","interval":"","intervalFactor":2,"legendFormat":"{{ proto }}:99 ","refId":"A","step":60},{"expr":"histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))","intervalFactor":2,"legendFormat":"{{ proto }}:90","refId":"B","step":60},{"expr":"histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))","intervalFactor":2,"legendFormat":"{{ proto }}:50","refId":"C","step":60}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Requests (size, udp)","tooltip":{"shared":true,"sort":0,"value_type":"cumulative"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","logBase":1,"max":null,"min":0,"show":true},{"format":"short","logBase":1,"max":null,"min":0,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":6,"x":18,"y":7},"hiddenSeries":false,"id":12,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"tcp:90","yaxis":1},{"alias":"tcp:99 ","yaxis":1},{"alias":"tcp:50","yaxis":1}],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{ proto }}:99 ","refId":"A","step":60},{"expr":"histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{ proto }}:90","refId":"B","step":60},{"expr":"histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{ proto }}:50","refId":"C","step":60}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Requests (size,tcp)","tooltip":{"shared":true,"sort":0,"value_type":"cumulative"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","logBase":1,"max":null,"min":0,"show":true},{"format":"short","logBase":1,"max":null,"min":0,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":12,"x":0,"y":14},"hiddenSeries":false,"id":14,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(rate(coredns_dns_response_rcode_count_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (rcode) or\nsum(rate(coredns_dns_responses_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (rcode)","interval":"","intervalFactor":2,"legendFormat":"{{ rcode }}","refId":"A","step":40}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Responses (by rcode)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","logBase":1,"max":null,"min":0,"show":true},{"format":"short","logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":12,"x":12,"y":14},"hiddenSeries":false,"id":32,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(coredns_dns_request_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[5m])) by (le, job))","format":"time_series","intervalFactor":2,"legendFormat":"99%","refId":"A","step":40},{"expr":"histogram_quantile(0.90, sum(rate(coredns_dns_request_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[5m])) by (le))","format":"time_series","intervalFactor":2,"legendFormat":"90%","refId":"B","step":40},{"expr":"histogram_quantile(0.50, sum(rate(coredns_dns_request_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[5m])) by (le))","format":"time_series","intervalFactor":2,"legendFormat":"50%","refId":"C","step":40}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Responses (duration)","tooltip":{"shared":true,"sort":0,"value_type":"cumulative"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","logBase":1,"max":null,"min":0,"show":true},{"format":"short","logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":12,"x":0,"y":21},"hiddenSeries":false,"id":18,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"udp:50%","yaxis":1},{"alias":"tcp:50%","yaxis":2},{"alias":"tcp:90%","yaxis":2},{"alias":"tcp:99%","yaxis":2}],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ","interval":"","intervalFactor":2,"legendFormat":"{{ proto }}:99%","refId":"A","step":40},{"expr":"histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ","interval":"","intervalFactor":2,"legendFormat":"{{ proto }}:90%","refId":"B","step":40},{"expr":"histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ","hide":false,"intervalFactor":2,"legendFormat":"{{ proto }}:50%","metric":"","refId":"C","step":40}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Responses (size, udp)","tooltip":{"shared":true,"sort":0,"value_type":"cumulative"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","logBase":1,"max":null,"min":0,"show":true},{"format":"short","logBase":1,"max":null,"min":0,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":12,"x":12,"y":21},"hiddenSeries":false,"id":20,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"udp:50%","yaxis":1},{"alias":"tcp:50%","yaxis":1},{"alias":"tcp:90%","yaxis":1},{"alias":"tcp:99%","yaxis":1}],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ","format":"time_series","intervalFactor":2,"legendFormat":"{{ proto }}:99%","refId":"A","step":40},{"expr":"histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ","format":"time_series","intervalFactor":2,"legendFormat":"{{ proto }}:90%","refId":"B","step":40},{"expr":"histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le, proto)) ","format":"time_series","intervalFactor":2,"legendFormat":"{{ proto }}:50%","metric":"","refId":"C","step":40}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Responses (size, tcp)","tooltip":{"shared":true,"sort":0,"value_type":"cumulative"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","logBase":1,"max":null,"min":0,"show":true},{"format":"short","logBase":1,"max":null,"min":0,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":12,"x":0,"y":28},"hiddenSeries":false,"id":22,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(coredns_cache_size{job=~\"$job\",instance=~\"$instance\"}) by (type) or\nsum(coredns_cache_entries{job=~\"$job\",instance=~\"$instance\"}) by (type)","interval":"","intervalFactor":2,"legendFormat":"{{ type }}","refId":"A","step":40}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Cache (size)","tooltip":{"shared":true,"sort":2,"value_type":"cumulative"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"decbytes","logBase":1,"max":null,"min":0,"show":true},{"format":"short","logBase":1,"max":null,"min":0,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":12,"x":12,"y":28},"hiddenSeries":false,"id":24,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"misses","yaxis":2}],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(rate(coredns_cache_hits_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (type)","hide":false,"intervalFactor":2,"legendFormat":"hits:{{ type }}","refId":"A","step":40},{"expr":"sum(rate(coredns_cache_misses_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (type)","hide":false,"intervalFactor":2,"legendFormat":"misses","refId":"B","step":40}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Cache (hitrate)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","logBase":1,"max":null,"min":0,"show":true},{"format":"pps","logBase":1,"max":null,"min":0,"show":true}],"yaxis":{"align":false,"alignLevel":null}}],"refresh":"10s","schemaVersion":26,"style":"dark","tags":["dns","coredns"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"includeAll":false,"label":null,"multi":false,"name":"datasource","options":[],"query":"prometheus","queryValue":"","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"allValue":".*","current":{"selected":false,"text":"coredns","value":"coredns"},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(coredns_dns_requests_total, job)","hide":0,"includeAll":true,"label":"Job","multi":false,"name":"job","options":[],"query":{"query":"label_values(coredns_dns_requests_total, job)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"type":"query"},{"allValue":".*","current":{"selected":true,"text":"All","value":"$__all"},"datasource":"$datasource","definition":"label_values(coredns_dns_requests_total{job=~\"$job\"}, instance)","hide":0,"includeAll":true,"label":"Instance","multi":false,"name":"instance","options":[],"query":"label_values(coredns_dns_requests_total{job=~\"$job\"}, instance)","refresh":1,"regex":"","skipUrlSync":false,"sort":3,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-3h","to":"now"},"timepicker":{"refresh_intervals":["10s","30s","1m","5m","15m","30m","1h","2h","1d"]},"timezone":"utc","title":"CoreDNS","uid":"vkQ0UHxik","version":3}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-k8s-resources-cluster
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- k8s-resources-cluster.json: |-
- {"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"cluster:node_cpu:ratio_rate5m{cluster=\"$cluster\"}","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\",cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Pods","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":0,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to pods","linkUrl":"/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=${__data.fields.namespace}","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"Workloads","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":0,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to workloads","linkUrl":"/d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=${__data.fields.namespace}","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #G","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to pods","linkUrl":"/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=${__value.text}","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"F"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"G"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (w/o cache)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Pods","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":0,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to pods","linkUrl":"/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=${__data.fields.namespace}","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"Workloads","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":0,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to workloads","linkUrl":"/d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=${__data.fields.namespace}","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #G","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to pods","linkUrl":"/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=${__value.text}","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"F"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"G"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests by Namespace","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Requests","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":11,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Current Receive Bandwidth","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Current Transmit Bandwidth","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Rate of Received Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Received Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"pps"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to pods","linkUrl":"/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=${__value.text}","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(irate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(irate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(irate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum(irate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"F"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Network Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Current Network Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":12,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":13,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":14,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"avg(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Container Bandwidth by Namespace: Received","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":15,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"avg(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Container Bandwidth by Namespace: Transmitted","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Average Container Bandwidth by Namespace","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":16,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":17,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":18,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":19,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets Dropped","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","decimals":null,"fill":10,"id":20,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"ceil(sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval])))","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"IOPS(Reads+Writes)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":21,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"ThroughPut(Read+Write)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage IO","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":22,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"sort":{"col":4,"desc":true},"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"IOPS(Reads)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":3,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"iops"},{"alias":"IOPS(Writes)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":3,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"iops"},{"alias":"IOPS(Reads + Writes)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":3,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"iops"},{"alias":"Throughput(Read)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Throughput(Write)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Throughput(Read + Write)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to pods","linkUrl":"/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=${__value.text}","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum by(namespace) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum by(namespace) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"F"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Storage IO","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage IO - Distribution","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Compute Resources / Cluster","uid":"efa86fd1d0c121a26444b636a3f509a8","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-k8s-resources-multicluster
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- k8s-resources-multicluster.json: |-
- {"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"cluster:node_cpu:ratio_rate5m","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"time_series","legendFormat":"{{cluster}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=${__value.text}","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"time_series","legendFormat":"{{cluster}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (w/o cache)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=${__value.text}","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests by Cluster","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Requests","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Compute Resources / Multi-Cluster","uid":"b59e6c9f2fcbe2e16d77fc492374cc4f","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-k8s-resources-namespace
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- k8s-resources-namespace.json: |-
- {"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":3,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation (from requests)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":3,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation (from limits)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":3,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation (from requests)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":3,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation (from limits)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"quota - requests","color":"#F2495C","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false},{"alias":"quota - limits","color":"#FF9830","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false}],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null},{"expr":"scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})","format":"time_series","legendFormat":"quota - requests","legendLink":null},{"expr":"scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})","format":"time_series","legendFormat":"quota - limits","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__value.text}","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"quota - requests","color":"#F2495C","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false},{"alias":"quota - limits","color":"#FF9830","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false}],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null},{"expr":"scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})","format":"time_series","legendFormat":"quota - requests","legendLink":null},{"expr":"scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})","format":"time_series","legendFormat":"quota - limits","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (w/o cache)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Usage (RSS)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Usage (Cache)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #G","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Usage (Swap)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #H","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__value.text}","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"F"},{"expr":"sum(container_memory_cache{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"G"},{"expr":"sum(container_memory_swap{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"H"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Current Receive Bandwidth","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Current Transmit Bandwidth","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Rate of Received Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Received Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"pps"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to pods","linkUrl":"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__value.text}","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(irate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(irate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(irate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum(irate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"F"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Network Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Current Network Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":11,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":12,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":13,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":14,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":15,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets Dropped","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","decimals":null,"fill":10,"id":16,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"IOPS(Reads+Writes)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":17,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"ThroughPut(Read+Write)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage IO","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":18,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"sort":{"col":4,"desc":true},"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"IOPS(Reads)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":3,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"iops"},{"alias":"IOPS(Writes)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":3,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"iops"},{"alias":"IOPS(Reads + Writes)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":3,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"iops"},{"alias":"Throughput(Read)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Throughput(Write)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Throughput(Read + Write)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to pods","linkUrl":"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__value.text}","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"F"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Storage IO","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage IO - Distribution","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kube-state-metrics\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"namespace","options":[],"query":"label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Compute Resources / Namespace (Pods)","uid":"85a562078cdf77779eaa1add43ccec1e","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/k8s-resources-node.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-k8s-resources-node
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- k8s-resources-node.json: |-
- {"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"max capacity","color":"#F2495C","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false}],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"})","format":"time_series","legendFormat":"max capacity","legendLink":null},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"max capacity","color":"#F2495C","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false}],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"})","format":"time_series","legendFormat":"max capacity","legendLink":null},{"expr":"sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\", container!=\"\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (w/o cache)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Usage (RSS)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Usage (Cache)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #G","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Usage (Swap)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #H","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum(node_namespace_pod_container:container_memory_rss{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"F"},{"expr":"sum(node_namespace_pod_container:container_memory_cache{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"G"},{"expr":"sum(node_namespace_pod_container:container_memory_swap{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"H"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kube-state-metrics\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":true,"name":"node","options":[],"query":"label_values(kube_node_info{cluster=\"$cluster\"}, node)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Compute Resources / Node (Pods)","uid":"200ac8fdbfbb74b39aff88118e4d1c2c","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-k8s-resources-pod
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- k8s-resources-pod.json: |-
- {"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"requests","color":"#F2495C","fill":0,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false},{"alias":"limits","color":"#FF9830","fill":0,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false}],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null},{"expr":"sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n","format":"time_series","legendFormat":"requests","legendLink":null},{"expr":"sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n","format":"time_series","legendFormat":"limits","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":true,"max":true,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(increase(container_cpu_cfs_throttled_periods_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container) /sum(increase(container_cpu_cfs_periods_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[{"colorMode":"critical","fill":true,"line":true,"op":"gt","value":0.25,"yaxis":"left"}],"timeFrom":null,"timeShift":null,"title":"CPU Throttling","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Throttling","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"requests","color":"#F2495C","dashes":true,"fill":0,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false},{"alias":"limits","color":"#FF9830","dashes":true,"fill":0,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false}],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null},{"expr":"sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n","format":"time_series","legendFormat":"requests","legendLink":null},{"expr":"sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n","format":"time_series","legendFormat":"limits","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (WSS)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage (WSS)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Usage (RSS)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Usage (Cache)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #G","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Usage (Swap)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #H","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"F"},{"expr":"sum(container_memory_cache{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"G"},{"expr":"sum(container_memory_swap{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"H"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":11,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets Dropped","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","decimals":null,"fill":10,"id":12,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"ceil(sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))","format":"time_series","legendFormat":"Reads","legendLink":null},{"expr":"ceil(sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))","format":"time_series","legendFormat":"Writes","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"IOPS","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":13,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))","format":"time_series","legendFormat":"Reads","legendLink":null},{"expr":"sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))","format":"time_series","legendFormat":"Writes","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"ThroughPut","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage IO - Distribution(Pod - Read & Writes)","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","decimals":null,"fill":10,"id":14,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"ceil(sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])))","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"IOPS(Reads+Writes)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":15,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"ThroughPut(Read+Write)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage IO - Distribution(Containers)","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":16,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"sort":{"col":4,"desc":true},"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"IOPS(Reads)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":3,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"iops"},{"alias":"IOPS(Writes)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":3,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"iops"},{"alias":"IOPS(Reads + Writes)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":3,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"iops"},{"alias":"Throughput(Read)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Throughput(Write)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Throughput(Read + Write)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum by(container) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\",device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum by(container) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"F"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Storage IO","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage IO - Distribution","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kube-state-metrics\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"namespace","options":[],"query":"label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"pod","options":[],"query":"label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, pod)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Compute Resources / Pod","uid":"6581e46e4e5c7ba40a07646395ef7b23","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-k8s-resources-workload
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- k8s-resources-workload.json: |-
- {"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__value.text}","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__value.text}","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Current Receive Bandwidth","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Current Transmit Bandwidth","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Rate of Received Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Received Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"pps"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__value.text}","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"(sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"(sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"(sum(irate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"(sum(irate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"(sum(irate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"(sum(irate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"table","instant":true,"legendFormat":"","refId":"F"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Network Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Current Network Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(avg(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Container Bandwidth by Pod: Received","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(avg(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Container Bandwidth by Pod: Transmitted","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Average Container Bandwidth by Pod","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":11,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":12,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":13,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets Dropped","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kube-state-metrics\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"namespace","options":[],"query":"label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":true,"label":null,"multi":false,"name":"type","options":[],"query":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload_type)","refresh":2,"regex":"","sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"workload","options":[],"query":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}, workload)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Compute Resources / Workload","uid":"a164a7f0339f99e89cea5cb47e9be617","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-k8s-resources-workloads-namespace
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- k8s-resources-workloads-namespace.json: |-
- {"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"quota - requests","color":"#F2495C","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false},{"alias":"quota - limits","color":"#FF9830","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false}],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"time_series","legendFormat":"{{workload}} - {{workload_type}}","legendLink":null},{"expr":"scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})","format":"time_series","legendFormat":"quota - requests","legendLink":null},{"expr":"scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})","format":"time_series","legendFormat":"quota - limits","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Running Pods","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":0,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Workload","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=${__value.text}&var-type=${__data.fields.workload_type}","pattern":"workload","thresholds":[],"type":"number","unit":"short"},{"alias":"Workload Type","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"workload_type","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}) by (workload, workload_type)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"F"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"quota - requests","color":"#F2495C","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false},{"alias":"quota - limits","color":"#FF9830","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false}],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(\n container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"time_series","legendFormat":"{{workload}} - {{workload_type}}","legendLink":null},{"expr":"scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})","format":"time_series","legendFormat":"quota - requests","legendLink":null},{"expr":"scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})","format":"time_series","legendFormat":"quota - limits","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Running Pods","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":0,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Workload","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=${__value.text}&var-type=${__data.fields.workload_type}","pattern":"workload","thresholds":[],"type":"number","unit":"short"},{"alias":"Workload Type","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"workload_type","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}) by (workload, workload_type)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(\n container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(\n container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum(\n container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"F"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Current Receive Bandwidth","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Current Transmit Bandwidth","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Rate of Received Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Received Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"pps"},{"alias":"Workload","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to pods","linkUrl":"/d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=${__value.text}&var-type=$type","pattern":"workload","thresholds":[],"type":"number","unit":"short"},{"alias":"Workload Type","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"workload_type","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"(sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"(sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"(sum(irate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"(sum(irate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"(sum(irate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"(sum(irate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"legendFormat":"","refId":"F"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Network Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Current Network Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","legendFormat":"{{workload}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","legendFormat":"{{workload}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(avg(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","legendFormat":"{{workload}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Container Bandwidth by Workload: Received","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(avg(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","legendFormat":"{{workload}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Container Bandwidth by Workload: Transmitted","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Average Container Bandwidth by Workload","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","legendFormat":"{{workload}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":11,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","legendFormat":"{{workload}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":12,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","legendFormat":"{{workload}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":13,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","legendFormat":"{{workload}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets Dropped","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kube-state-metrics\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"namespace","options":[],"query":"label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"","value":""},"datasource":"$datasource","definition":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)","hide":0,"includeAll":true,"label":null,"multi":false,"name":"type","options":[],"query":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)","refresh":2,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Compute Resources / Namespace (Workloads)","uid":"a87fb0d919ec0ea5f6543124e16c42a5","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/kubelet.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-kubelet
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- kubelet.json: |-
- {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"datasource":"$datasource","fieldConfig":{"defaults":{"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"none"}},"gridPos":{"h":7,"w":4,"x":0,"y":0},"id":2,"links":[],"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"7","targets":[{"expr":"sum(kubelet_node_name{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\"})","format":"time_series","intervalFactor":2,"legendFormat":"","refId":"A"}],"title":"Running Kubelets","transparent":false,"type":"stat"},{"datasource":"$datasource","fieldConfig":{"defaults":{"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"none"}},"gridPos":{"h":7,"w":4,"x":4,"y":0},"id":3,"links":[],"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"7","targets":[{"expr":"sum(kubelet_running_pods{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"})","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"title":"Running Pods","transparent":false,"type":"stat"},{"datasource":"$datasource","fieldConfig":{"defaults":{"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"none"}},"gridPos":{"h":7,"w":4,"x":8,"y":0},"id":4,"links":[],"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"7","targets":[{"expr":"sum(kubelet_running_containers{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"})","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"title":"Running Containers","transparent":false,"type":"stat"},{"datasource":"$datasource","fieldConfig":{"defaults":{"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"none"}},"gridPos":{"h":7,"w":4,"x":12,"y":0},"id":5,"links":[],"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"7","targets":[{"expr":"sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\", state=\"actual_state_of_world\"})","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"title":"Actual Volume Count","transparent":false,"type":"stat"},{"datasource":"$datasource","fieldConfig":{"defaults":{"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"none"}},"gridPos":{"h":7,"w":4,"x":16,"y":0},"id":6,"links":[],"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"7","targets":[{"expr":"sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",state=\"desired_state_of_world\"})","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"title":"Desired Volume Count","transparent":false,"type":"stat"},{"datasource":"$datasource","fieldConfig":{"defaults":{"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"none"}},"gridPos":{"h":7,"w":4,"x":20,"y":0},"id":7,"links":[],"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"7","targets":[{"expr":"sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"title":"Config Error Count","transparent":false,"type":"stat"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":0,"y":7},"id":8,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (operation_type, instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{operation_type}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Operation Rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":12,"y":7},"id":9,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{operation_type}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Operation Error Rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":24,"x":0,"y":14},"id":10,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{operation_type}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Operation duration 99th quantile","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":0,"y":21},"id":11,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} pod","refId":"A"},{"expr":"sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} worker","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Pod Start Rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":12,"y":21},"id":12,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} pod","refId":"A"},{"expr":"histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} worker","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Pod Start Duration","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":0,"y":28},"id":13,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{operation_name}} {{volume_plugin}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Storage Operation Rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":12,"y":28},"id":14,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{operation_name}} {{volume_plugin}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Storage Operation Error Rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":24,"x":0,"y":35},"id":15,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{operation_name}} {{volume_plugin}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Storage Operation Duration 99th quantile","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":0,"y":42},"id":16,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)","format":"time_series","intervalFactor":2,"legendFormat":"{{operation_type}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Cgroup manager operation rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":12,"y":42},"id":17,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{operation_type}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Cgroup manager 99th quantile","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"Pod lifecycle event generator","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":0,"y":49},"id":18,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"PLEG relist rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":12,"y":49},"id":19,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"PLEG relist interval","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":24,"x":0,"y":56},"id":20,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"PLEG relist duration","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":24,"x":0,"y":63},"id":21,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"2xx","refId":"A"},{"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"3xx","refId":"B"},{"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"4xx","refId":"C"},{"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"5xx","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"RPC Rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":24,"x":0,"y":70},"id":22,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, verb, url, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{verb}} {{url}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Request duration 99th quantile","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":8,"x":0,"y":77},"id":23,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"process_resident_memory_bytes{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":8,"x":8,"y":77},"id":24,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":8,"x":16,"y":77},"id":25,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"go_goroutines{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Goroutines","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"refresh":"10s","rows":[],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kubelet\", metrics_path=\"/metrics\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":true,"label":"instance","multi":false,"name":"instance","options":[],"query":"label_values(up{job=\"kubelet\", metrics_path=\"/metrics\",cluster=\"$cluster\"}, instance)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Kubelet","uid":"3138fa155d5915769fbded898ac09fd9","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/namespace-by-pod.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-namespace-by-pod
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- namespace-by-pod.json: |-
- {"__inputs":[],"__requires":[],"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":2,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Current Bandwidth","titleSize":"h6","type":"row"},{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"$datasource","decimals":0,"format":"time_series","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{"h":9,"w":12,"x":0,"y":1},"height":9,"id":3,"interval":null,"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"minSpan":12,"nullPointMode":"connected","nullText":null,"options":{"fieldOptions":{"calcs":["last"],"defaults":{"max":10000000000,"min":0,"title":"$namespace","unit":"Bps"},"mappings":[],"override":{},"thresholds":[{"color":"dark-green","index":0,"value":null},{"color":"dark-yellow","index":1,"value":5000000000},{"color":"dark-red","index":2,"value":7000000000}],"values":false}},"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":12,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))","format":"time_series","instant":null,"intervalFactor":1,"legendFormat":"","refId":"A"}],"thresholds":"","timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Received","type":"gauge","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"current"},{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"$datasource","decimals":0,"format":"time_series","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{"h":9,"w":12,"x":12,"y":1},"height":9,"id":4,"interval":null,"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"minSpan":12,"nullPointMode":"connected","nullText":null,"options":{"fieldOptions":{"calcs":["last"],"defaults":{"max":10000000000,"min":0,"title":"$namespace","unit":"Bps"},"mappings":[],"override":{},"thresholds":[{"color":"dark-green","index":0,"value":null},{"color":"dark-yellow","index":1,"value":5000000000},{"color":"dark-red","index":2,"value":7000000000}],"values":false}},"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":12,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))","format":"time_series","instant":null,"intervalFactor":1,"legendFormat":"","refId":"A"}],"thresholds":"","timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Transmitted","type":"gauge","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"current"},{"columns":[{"text":"Time","value":"Time"},{"text":"Value #A","value":"Value #A"},{"text":"Value #B","value":"Value #B"},{"text":"Value #C","value":"Value #C"},{"text":"Value #D","value":"Value #D"},{"text":"Value #E","value":"Value #E"},{"text":"Value #F","value":"Value #F"},{"text":"pod","value":"pod"}],"datasource":"$datasource","fill":1,"fontSize":"100%","gridPos":{"h":9,"w":24,"x":0,"y":10},"id":5,"lines":true,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null as zero","renderer":"flot","scroll":true,"showHeader":true,"sort":{"col":0,"desc":false},"spaceLength":10,"span":24,"styles":[{"alias":"Time","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Time","thresholds":[],"type":"hidden","unit":"short"},{"alias":"Bandwidth Received","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Bandwidth Transmitted","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Rate of Received Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Received Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"pps"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTooltip":"Drill down","linkUrl":"d/7a18067ce943a40ae25454675c19ff5c/kubernetes-networking-pod?orgId=1&refresh=30s&var-namespace=$namespace&var-pod=${__value.text}","pattern":"pod","thresholds":[],"type":"number","unit":"short"}],"targets":[{"expr":"sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"A","step":10},{"expr":"sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"B","step":10},{"expr":"sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"C","step":10},{"expr":"sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"D","step":10},{"expr":"sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"E","step":10},{"expr":"sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"F","step":10}],"timeFrom":null,"timeShift":null,"title":"Current Status","type":"table"},{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":19},"id":6,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth","titleSize":"h6","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":20},"id":7,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":20},"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":29},"id":9,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":10,"w":12,"x":0,"y":30},"id":10,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":10,"w":12,"x":12,"y":30},"id":11,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Packets","titleSize":"h6","type":"row"},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":30},"id":12,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":10,"w":12,"x":0,"y":40},"id":13,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":10,"w":12,"x":12,"y":40},"id":14,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Errors","titleSize":"h6","type":"row"}],"refresh":"10s","rows":[],"schemaVersion":18,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\"}, cluster)","refresh":2,"regex":"","sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":".+","auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"kube-system","value":"kube-system"},"datasource":"$datasource","definition":"label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)","hide":0,"includeAll":true,"label":null,"multi":false,"name":"namespace","options":[],"query":"label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"resolution","options":[{"selected":false,"text":"30s","value":"30s"},{"selected":true,"text":"5m","value":"5m"},{"selected":false,"text":"1h","value":"1h"}],"query":"30s,5m,1h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"interval","options":[{"selected":true,"text":"4h","value":"4h"}],"query":"4h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Networking / Namespace (Pods)","uid":"8b7a8b326d7a6f1f04244066368c67af","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/namespace-by-workload.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-namespace-by-workload
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- namespace-by-workload.json: |-
- {"__inputs":[],"__requires":[],"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":2,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Current Bandwidth","titleSize":"h6","type":"row"},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":1},"id":3,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{ workload }}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Received","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":1},"id":4,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{ workload }}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Transmitted","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"columns":[{"text":"Time","value":"Time"},{"text":"Value #A","value":"Value #A"},{"text":"Value #B","value":"Value #B"},{"text":"Value #C","value":"Value #C"},{"text":"Value #D","value":"Value #D"},{"text":"Value #E","value":"Value #E"},{"text":"Value #F","value":"Value #F"},{"text":"Value #G","value":"Value #G"},{"text":"Value #H","value":"Value #H"},{"text":"workload","value":"workload"}],"datasource":"$datasource","fill":1,"fontSize":"90%","gridPos":{"h":9,"w":24,"x":0,"y":10},"id":5,"lines":true,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null as zero","renderer":"flot","scroll":true,"showHeader":true,"sort":{"col":0,"desc":false},"spaceLength":10,"span":24,"styles":[{"alias":"Time","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Time","thresholds":[],"type":"hidden","unit":"short"},{"alias":"Current Bandwidth Received","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Current Bandwidth Transmitted","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Average Bandwidth Received","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Average Bandwidth Transmitted","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Rate of Received Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Received Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #G","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #H","thresholds":[],"type":"number","unit":"pps"},{"alias":"Workload","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTooltip":"Drill down","linkUrl":"d/728bf77cc1166d2f3133bf25846876cc/kubernetes-networking-workload?orgId=1&refresh=30s&var-namespace=$namespace&var-type=$type&var-workload=${__value.text}","pattern":"workload","thresholds":[],"type":"number","unit":"short"}],"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"A","step":10},{"expr":"sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"B","step":10},{"expr":"sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"C","step":10},{"expr":"sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"D","step":10},{"expr":"sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"E","step":10},{"expr":"sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"F","step":10},{"expr":"sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"G","step":10},{"expr":"sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"H","step":10}],"timeFrom":null,"timeShift":null,"title":"Current Status","type":"table"},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":19},"id":6,"panels":[{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":20},"id":7,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{ workload }}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Rate of Bytes Received","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":20},"id":8,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{ workload }}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Rate of Bytes Transmitted","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Average Bandwidth","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":29},"id":9,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth HIstory","titleSize":"h6","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":38},"id":10,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{workload}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":38},"id":11,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{workload}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":39},"id":12,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":40},"id":13,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{workload}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":40},"id":14,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{workload}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Packets","titleSize":"h6","type":"row"},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":40},"id":15,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":41},"id":16,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{workload}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":41},"id":17,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{workload}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Errors","titleSize":"h6","type":"row"}],"refresh":"10s","rows":[],"schemaVersion":18,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\"}, cluster)","refresh":2,"regex":"","sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"kube-system","value":"kube-system"},"datasource":"$datasource","definition":"label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)","hide":0,"includeAll":false,"label":null,"multi":false,"name":"namespace","options":[],"query":"label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"","value":""},"datasource":"$datasource","definition":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)","hide":0,"includeAll":true,"label":null,"multi":false,"name":"type","options":[],"query":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)","refresh":2,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"resolution","options":[{"selected":false,"text":"30s","value":"30s"},{"selected":true,"text":"5m","value":"5m"},{"selected":false,"text":"1h","value":"1h"}],"query":"30s,5m,1h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"interval","options":[{"selected":true,"text":"4h","value":"4h"}],"query":"4h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Networking / Namespace (Workload)","uid":"bbb2a765a623ae38130206c7d94a160f","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-node-cluster-rsrc-use
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- node-cluster-rsrc-use.json: |-
- {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":1,"hideControls":false,"id":null,"links":[],"refresh":"30s","rows":[{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":2,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"((\n instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n *\n instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}\n) != 0 )\n/ scalar(sum(instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}))\n","format":"time_series","intervalFactor":2,"legendFormat":"{{ instance }}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":3,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Saturation (Load1 per CPU)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":4,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":5,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Major Page Faults)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"rds","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"rds","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":6,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/Receive/","stack":"A"},{"alias":"/Transmit/","stack":"B","transform":"negative-Y"}],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Receive","refId":"A"},{"expr":"instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Transmit","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Utilisation (Bytes Receive/Transmit)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":7,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/ Receive/","stack":"A"},{"alias":"/ Transmit/","stack":"B","transform":"negative-Y"}],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Receive","refId":"A"},{"expr":"instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Transmit","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Saturation (Drops Receive/Transmit)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":9,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Saturation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk IO","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":10,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum without (device) (\n max without (fstype, mountpoint) ((\n node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n -\n node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n ) != 0)\n)\n/ scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"})))\n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Space Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk Space","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["node-exporter-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(node_time_seconds, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Node Exporter / USE Method / Cluster","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/node-rsrc-use.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-node-rsrc-use
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- node-rsrc-use.json: |-
- {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":1,"hideControls":false,"id":null,"links":[],"refresh":"30s","rows":[{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":2,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Utilisation","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":3,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_load1_per_cpu:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Saturation","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Saturation (Load1 per CPU)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":4,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_memory_utilisation:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Utilisation","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":5,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Major page Faults","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Major Page Faults)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"rds","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"rds","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":6,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/Receive/","stack":"A"},{"alias":"/Transmit/","stack":"B","transform":"negative-Y"}],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Receive","refId":"A"},{"expr":"instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Transmit","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Utilisation (Bytes Receive/Transmit)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":7,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/ Receive/","stack":"A"},{"alias":"/ Transmit/","stack":"B","transform":"negative-Y"}],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Receive","refId":"A"},{"expr":"instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Transmit","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Saturation (Drops Receive/Transmit)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":9,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Saturation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk IO","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":10,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(1 -\n (\n max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n /\n max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n ) != 0\n)\n","format":"time_series","intervalFactor":2,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Space Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk Space","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["node-exporter-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(node_time_seconds, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"instance","options":[],"query":"label_values(node_exporter_build_info{job=\"node-exporter\", cluster=\"$cluster\"}, instance)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Node Exporter / USE Method / Node","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/nodes-darwin.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-nodes-darwin
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- nodes-darwin.json: |-
- {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":1,"hideControls":false,"id":null,"links":[],"refresh":"30s","rows":[{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":2,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n","format":"time_series","intervalFactor":5,"legendFormat":"{{cpu}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"fillGradient":0,"gridPos":{},"id":3,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node_load1{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"1m load average","refId":"A"},{"expr":"node_load5{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"5m load average","refId":"B"},{"expr":"node_load15{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"15m load average","refId":"C"},{"expr":"count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})","format":"time_series","intervalFactor":2,"legendFormat":"logical cores","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Load Average","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":4,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":9,"stack":false,"steppedLine":false,"targets":[{"expr":"node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"Physical Memory","refId":"A"},{"expr":"(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n","format":"time_series","intervalFactor":2,"legendFormat":"Memory Used","refId":"B"},{"expr":"(\n node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n","format":"time_series","intervalFactor":2,"legendFormat":"App Memory","refId":"C"},{"expr":"node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"Wired Memory","refId":"D"},{"expr":"node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"Compressed","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"datasource":"$datasource","fieldConfig":{"defaults":{"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)"},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"percent"}},"gridPos":{},"id":5,"span":3,"targets":[{"expr":"(\n (\n avg(node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"}) -\n avg(node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n avg(node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"})\n ) /\n avg(node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"})\n)\n*\n100\n","format":"time_series","intervalFactor":2,"legendFormat":""}],"title":"Memory Usage","transparent":false,"type":"gauge"}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"fillGradient":0,"gridPos":{},"id":6,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/ read| written/","yaxis":1},{"alias":"/ io time/","yaxis":2}],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} read","refId":"A"},{"expr":"rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} written","refId":"B"},{"expr":"rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} io time","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"datasource":"$datasource","fieldConfig":{"defaults":{"custom":{},"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"yellow","value":0.8},{"color":"red","value":0.9}]},"unit":"decbytes"},"overrides":[{"matcher":{"id":"byName","options":"Mounted on"},"properties":[{"id":"custom.width","value":260}]},{"matcher":{"id":"byName","options":"Size"},"properties":[{"id":"custom.width","value":93}]},{"matcher":{"id":"byName","options":"Used"},"properties":[{"id":"custom.width","value":72}]},{"matcher":{"id":"byName","options":"Available"},"properties":[{"id":"custom.width","value":88}]},{"matcher":{"id":"byName","options":"Used, %"},"properties":[{"id":"unit","value":"percentunit"},{"id":"custom.displayMode","value":"gradient-gauge"},{"id":"max","value":1},{"id":"min","value":0}]}]},"gridPos":{},"id":7,"span":6,"targets":[{"expr":"max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":""},{"expr":"max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":""}],"title":"Disk Space Usage","transformations":[{"id":"groupBy","options":{"fields":{"Value #A":{"aggregations":["lastNotNull"],"operation":"aggregate"},"Value #B":{"aggregations":["lastNotNull"],"operation":"aggregate"},"mountpoint":{"aggregations":[],"operation":"groupby"}}}},{"id":"merge","options":{}},{"id":"calculateField","options":{"alias":"Used","binary":{"left":"Value #A (lastNotNull)","operator":"-","reducer":"sum","right":"Value #B (lastNotNull)"},"mode":"binary","reduce":{"reducer":"sum"}}},{"id":"calculateField","options":{"alias":"Used, %","binary":{"left":"Used","operator":"/","reducer":"sum","right":"Value #A (lastNotNull)"},"mode":"binary","reduce":{"reducer":"sum"}}},{"id":"organize","options":{"excludeByName":{},"indexByName":{},"renameByName":{"Value #A (lastNotNull)":"Size","Value #B (lastNotNull)":"Available","mountpoint":"Mounted on"}}},{"id":"sortBy","options":{"fields":{},"sort":[{"field":"Mounted on"}]}}],"transparent":false,"type":"table"}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"Network received (bits/s)","fill":0,"fillGradient":0,"gridPos":{},"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8","format":"time_series","intervalFactor":1,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Received","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"Network transmitted (bits/s)","fill":0,"fillGradient":0,"gridPos":{},"id":9,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8","format":"time_series","intervalFactor":1,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Transmitted","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["node-exporter-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Instance","multi":false,"name":"instance","options":[],"query":"label_values(node_uname_info{job=\"node-exporter\", sysname=\"Darwin\"}, instance)","refresh":2,"regex":"","sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Node Exporter / MacOS","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/nodes.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-nodes
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- nodes.json: |-
- {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":1,"hideControls":false,"id":null,"links":[],"refresh":"30s","rows":[{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":2,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n","format":"time_series","intervalFactor":5,"legendFormat":"{{cpu}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"fillGradient":0,"gridPos":{},"id":3,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node_load1{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"1m load average","refId":"A"},{"expr":"node_load5{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"5m load average","refId":"B"},{"expr":"node_load15{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"15m load average","refId":"C"},{"expr":"count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})","format":"time_series","intervalFactor":2,"legendFormat":"logical cores","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Load Average","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":4,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":9,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n","format":"time_series","intervalFactor":2,"legendFormat":"memory used","refId":"A"},{"expr":"node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"memory buffers","refId":"B"},{"expr":"node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"memory cached","refId":"C"},{"expr":"node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"memory free","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"datasource":"$datasource","fieldConfig":{"defaults":{"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)"},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"percent"}},"gridPos":{},"id":5,"span":3,"targets":[{"expr":"100 -\n(\n avg(node_memory_MemAvailable_bytes{job=\"node-exporter\", instance=\"$instance\"}) /\n avg(node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"})\n* 100\n)\n","format":"time_series","intervalFactor":2,"legendFormat":""}],"title":"Memory Usage","transparent":false,"type":"gauge"}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"fillGradient":0,"gridPos":{},"id":6,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/ read| written/","yaxis":1},{"alias":"/ io time/","yaxis":2}],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} read","refId":"A"},{"expr":"rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} written","refId":"B"},{"expr":"rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} io time","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"datasource":"$datasource","fieldConfig":{"defaults":{"custom":{},"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"yellow","value":0.8},{"color":"red","value":0.9}]},"unit":"decbytes"},"overrides":[{"matcher":{"id":"byName","options":"Mounted on"},"properties":[{"id":"custom.width","value":260}]},{"matcher":{"id":"byName","options":"Size"},"properties":[{"id":"custom.width","value":93}]},{"matcher":{"id":"byName","options":"Used"},"properties":[{"id":"custom.width","value":72}]},{"matcher":{"id":"byName","options":"Available"},"properties":[{"id":"custom.width","value":88}]},{"matcher":{"id":"byName","options":"Used, %"},"properties":[{"id":"unit","value":"percentunit"},{"id":"custom.displayMode","value":"gradient-gauge"},{"id":"max","value":1},{"id":"min","value":0}]}]},"gridPos":{},"id":7,"span":6,"targets":[{"expr":"max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":""},{"expr":"max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":""}],"title":"Disk Space Usage","transformations":[{"id":"groupBy","options":{"fields":{"Value #A":{"aggregations":["lastNotNull"],"operation":"aggregate"},"Value #B":{"aggregations":["lastNotNull"],"operation":"aggregate"},"mountpoint":{"aggregations":[],"operation":"groupby"}}}},{"id":"merge","options":{}},{"id":"calculateField","options":{"alias":"Used","binary":{"left":"Value #A (lastNotNull)","operator":"-","reducer":"sum","right":"Value #B (lastNotNull)"},"mode":"binary","reduce":{"reducer":"sum"}}},{"id":"calculateField","options":{"alias":"Used, %","binary":{"left":"Used","operator":"/","reducer":"sum","right":"Value #A (lastNotNull)"},"mode":"binary","reduce":{"reducer":"sum"}}},{"id":"organize","options":{"excludeByName":{},"indexByName":{},"renameByName":{"Value #A (lastNotNull)":"Size","Value #B (lastNotNull)":"Available","mountpoint":"Mounted on"}}},{"id":"sortBy","options":{"fields":{},"sort":[{"field":"Mounted on"}]}}],"transparent":false,"type":"table"}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"Network received (bits/s)","fill":0,"fillGradient":0,"gridPos":{},"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8","format":"time_series","intervalFactor":1,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Received","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"Network transmitted (bits/s)","fill":0,"fillGradient":0,"gridPos":{},"id":9,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8","format":"time_series","intervalFactor":1,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Transmitted","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["node-exporter-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Instance","multi":false,"name":"instance","options":[],"query":"label_values(node_uname_info{job=\"node-exporter\", sysname!=\"Darwin\"}, instance)","refresh":2,"regex":"","sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Node Exporter / Nodes","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-persistentvolumesusage
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- persistentvolumesusage.json: |-
- {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"10s","rows":[{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":9,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n","format":"time_series","intervalFactor":1,"legendFormat":"Used Space","refId":"A"},{"expr":"sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n","format":"time_series","intervalFactor":1,"legendFormat":"Free Space","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Volume Space Usage","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["rgba(50, 172, 45, 0.97)","rgba(237, 129, 40, 0.89)","rgba(245, 54, 54, 0.9)"],"datasource":"$datasource","format":"percent","gauge":{"maxValue":100,"minValue":0,"show":true,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{},"id":3,"interval":"1m","legend":{"alignAsTable":true,"rightSide":true},"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":3,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"max without(instance,node) (\n(\n topk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n topk(1, kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n/\ntopk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n","format":"time_series","intervalFactor":2,"legendFormat":"","refId":"A"}],"thresholds":"80, 90","title":"Volume Space Usage","tooltip":{"shared":false},"type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"current"}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":9,"stack":true,"steppedLine":false,"targets":[{"expr":"sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n","format":"time_series","intervalFactor":1,"legendFormat":"Used inodes","refId":"A"},{"expr":"(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n","format":"time_series","intervalFactor":1,"legendFormat":" Free inodes","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Volume inodes Usage","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"none","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"none","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["rgba(50, 172, 45, 0.97)","rgba(237, 129, 40, 0.89)","rgba(245, 54, 54, 0.9)"],"datasource":"$datasource","format":"percent","gauge":{"maxValue":100,"minValue":0,"show":true,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{},"id":5,"interval":"1m","legend":{"alignAsTable":true,"rightSide":true},"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":3,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"max without(instance,node) (\ntopk(1, kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n/\ntopk(1, kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n","format":"time_series","intervalFactor":2,"legendFormat":"","refId":"A"}],"thresholds":"80, 90","title":"Volume inodes Usage","tooltip":{"shared":false},"type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"current"}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Namespace","multi":false,"name":"namespace","options":[],"query":"label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"PersistentVolumeClaim","multi":false,"name":"volume","options":[],"query":"label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\"}, persistentvolumeclaim)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-7d","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Persistent Volumes","uid":"919b92a8e8041bd567af9edab12c840c","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/pod-total.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-pod-total
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- pod-total.json: |-
- {"__inputs":[],"__requires":[],"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":2,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Current Bandwidth","titleSize":"h6","type":"row"},{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"$datasource","decimals":0,"format":"time_series","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{"h":9,"w":12,"x":0,"y":1},"height":9,"id":3,"interval":null,"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"minSpan":12,"nullPointMode":"connected","nullText":null,"options":{"fieldOptions":{"calcs":["last"],"defaults":{"max":10000000000,"min":0,"title":"$namespace: $pod","unit":"Bps"},"mappings":[],"override":{},"thresholds":[{"color":"dark-green","index":0,"value":null},{"color":"dark-yellow","index":1,"value":5000000000},{"color":"dark-red","index":2,"value":7000000000}],"values":false}},"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":12,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))","format":"time_series","instant":null,"intervalFactor":1,"legendFormat":"","refId":"A"}],"thresholds":"","timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Received","type":"gauge","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"current"},{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"$datasource","decimals":0,"format":"time_series","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{"h":9,"w":12,"x":12,"y":1},"height":9,"id":4,"interval":null,"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"minSpan":12,"nullPointMode":"connected","nullText":null,"options":{"fieldOptions":{"calcs":["last"],"defaults":{"max":10000000000,"min":0,"title":"$namespace: $pod","unit":"Bps"},"mappings":[],"override":{},"thresholds":[{"color":"dark-green","index":0,"value":null},{"color":"dark-yellow","index":1,"value":5000000000},{"color":"dark-red","index":2,"value":7000000000}],"values":false}},"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":12,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))","format":"time_series","instant":null,"intervalFactor":1,"legendFormat":"","refId":"A"}],"thresholds":"","timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Transmitted","type":"gauge","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"current"},{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":10},"id":5,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth","titleSize":"h6","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":11},"id":6,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":11},"id":7,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":20},"id":8,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":10,"w":12,"x":0,"y":21},"id":9,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":10,"w":12,"x":12,"y":21},"id":10,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Packets","titleSize":"h6","type":"row"},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":21},"id":11,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":10,"w":12,"x":0,"y":32},"id":12,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":10,"w":12,"x":12,"y":32},"id":13,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Errors","titleSize":"h6","type":"row"}],"refresh":"10s","rows":[],"schemaVersion":18,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\"}, cluster)","refresh":2,"regex":"","sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":".+","auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"kube-system","value":"kube-system"},"datasource":"$datasource","definition":"label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)","hide":0,"includeAll":true,"label":null,"multi":false,"name":"namespace","options":[],"query":"label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":".+","auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"","value":""},"datasource":"$datasource","definition":"label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)","hide":0,"includeAll":false,"label":null,"multi":false,"name":"pod","options":[],"query":"label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"resolution","options":[{"selected":false,"text":"30s","value":"30s"},{"selected":true,"text":"5m","value":"5m"},{"selected":false,"text":"1h","value":"1h"}],"query":"30s,5m,1h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"interval","options":[{"selected":true,"text":"4h","value":"4h"}],"query":"4h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Networking / Pod","uid":"7a18067ce943a40ae25454675c19ff5c","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/prometheus.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-prometheus
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- prometheus.json: |-
- {"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"60s","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":1,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Count","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"hidden","unit":"short"},{"alias":"Uptime","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"s"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"Instance","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"instance","thresholds":[],"type":"number","unit":"short"},{"alias":"Job","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"job","thresholds":[],"type":"number","unit":"short"},{"alias":"Version","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"version","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"count by (cluster, job, instance, version) (prometheus_build_info{cluster=~\"$cluster\", job=~\"$job\", instance=~\"$instance\"})","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"max by (cluster, job, instance) (time() - process_start_time_seconds{cluster=~\"$cluster\", job=~\"$job\", instance=~\"$instance\"})","format":"table","instant":true,"legendFormat":"","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Prometheus Stats","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Prometheus Stats","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(prometheus_target_sync_length_seconds_sum{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[5m])) by (cluster, job, scrape_job, instance) * 1e3","format":"time_series","legendFormat":"{{cluster}}:{{job}}:{{instance}}:{{scrape_job}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Target Sync","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ms","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by (cluster, job, instance) (prometheus_sd_discovered_targets{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"})","format":"time_series","legendFormat":"{{cluster}}:{{job}}:{{instance}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Targets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Discovery","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(prometheus_target_interval_length_seconds_sum{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3","format":"time_series","legendFormat":"{{cluster}}:{{job}}:{{instance}} {{interval}} configured","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Scrape Interval Duration","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ms","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by (cluster, job, instance) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))","format":"time_series","legendFormat":"exceeded body size limit: {{cluster}} {{job}} {{instance}}","legendLink":null},{"expr":"sum by (cluster, job, instance) (rate(prometheus_target_scrapes_exceeded_sample_limit_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))","format":"time_series","legendFormat":"exceeded sample limit: {{cluster}} {{job}} {{instance}}","legendLink":null},{"expr":"sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))","format":"time_series","legendFormat":"duplicate timestamp: {{cluster}} {{job}} {{instance}}","legendLink":null},{"expr":"sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_bounds_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))","format":"time_series","legendFormat":"out of bounds: {{cluster}} {{job}} {{instance}}","legendLink":null},{"expr":"sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_order_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))","format":"time_series","legendFormat":"out of order: {{cluster}} {{job}} {{instance}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Scrape failures","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":true,"steppedLine":false,"targets":[{"expr":"rate(prometheus_tsdb_head_samples_appended_total{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}[5m])","format":"time_series","legendFormat":"{{cluster}} {{job}} {{instance}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Appended Samples","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Retrieval","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"prometheus_tsdb_head_series{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}","format":"time_series","legendFormat":"{{cluster}} {{job}} {{instance}} head series","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Head Series","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"prometheus_tsdb_head_chunks{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}","format":"time_series","legendFormat":"{{cluster}} {{job}} {{instance}} head chunks","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Head Chunks","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":9,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"rate(prometheus_engine_query_duration_seconds_count{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])","format":"time_series","legendFormat":"{{cluster}} {{job}} {{instance}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Query Rate","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":10,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}) * 1e3","format":"time_series","legendFormat":"{{slice}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Stage Duration","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ms","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Query","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["prometheus-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":".+","current":{"selected":true,"text":"All","value":"$__all"},"datasource":"$datasource","hide":2,"includeAll":true,"label":"cluster","multi":true,"name":"cluster","options":[],"query":"label_values(prometheus_build_info{job=\"prometheus-k8s\",namespace=\"monitoring\"}, cluster)","refresh":1,"regex":"","sort":2,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":".+","current":{"selected":true,"text":"All","value":"$__all"},"datasource":"$datasource","hide":0,"includeAll":true,"label":"job","multi":true,"name":"job","options":[],"query":"label_values(prometheus_build_info{cluster=~\"$cluster\"}, job)","refresh":1,"regex":"","sort":2,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":".+","current":{"selected":true,"text":"All","value":"$__all"},"datasource":"$datasource","hide":0,"includeAll":true,"label":"instance","multi":true,"name":"instance","options":[],"query":"label_values(prometheus_build_info{cluster=~\"$cluster\", job=~\"$job\"}, instance)","refresh":1,"regex":"","sort":2,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Prometheus / Overview","uid":"","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/scheduler.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-scheduler
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- scheduler.json: |-
- {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"10s","rows":[{"collapse":false,"collapsed":false,"panels":[{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"$datasource","format":"none","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{},"id":2,"interval":"1m","legend":{"alignAsTable":true,"rightSide":true},"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":2,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"sum(up{cluster=\"$cluster\", job=\"kube-scheduler\"})","format":"time_series","intervalFactor":2,"legendFormat":"","refId":"A"}],"thresholds":"","title":"Up","tooltip":{"shared":false},"type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"min"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":5,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} e2e","refId":"A"},{"expr":"sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} binding","refId":"B"},{"expr":"sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} scheduling algorithm","refId":"C"},{"expr":"sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} volume","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Scheduling Rate","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":5,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} e2e","refId":"A"},{"expr":"histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} binding","refId":"B"},{"expr":"histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} scheduling algorithm","refId":"C"},{"expr":"histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} volume","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Scheduling latency 99th Quantile","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"2xx","refId":"A"},{"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"3xx","refId":"B"},{"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"4xx","refId":"C"},{"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"5xx","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Kube API Request Rate","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":8,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{verb}} {{url}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Post Request Latency 99th Quantile","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{verb}} {{url}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Get Request Latency 99th Quantile","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"process_resident_memory_bytes{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU usage","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"go_goroutines{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Goroutines","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kube-scheduler\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":true,"label":null,"multi":false,"name":"instance","options":[],"query":"label_values(up{job=\"kube-scheduler\", cluster=\"$cluster\"}, instance)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Scheduler","uid":"2e6b6a3b4bddf1427b3a55aa1311c656","version":0}
----
-# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/workload-total.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- namespace: default
- name: release-name-kps-workload-total
- annotations:
- {}
- labels:
- grafana_dashboard: "1"
- app: kps-grafana
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-data:
- workload-total.json: |-
- {"__inputs":[],"__requires":[],"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":2,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Current Bandwidth","titleSize":"h6","type":"row"},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":1},"id":3,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{ pod }}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Received","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":1},"id":4,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{ pod }}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Transmitted","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":10},"id":5,"panels":[{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":11},"id":6,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(avg(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{ pod }}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Rate of Bytes Received","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":11},"id":7,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(avg(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{ pod }}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Rate of Bytes Transmitted","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Average Bandwidth","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":11},"id":8,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth HIstory","titleSize":"h6","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":12},"id":9,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":12},"id":10,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":21},"id":11,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":22},"id":12,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":22},"id":13,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Packets","titleSize":"h6","type":"row"},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":22},"id":14,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":23},"id":15,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":23},"id":16,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Errors","titleSize":"h6","type":"row"}],"refresh":"10s","rows":[],"schemaVersion":18,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(kube_pod_info{job=\"kube-state-metrics\"}, cluster)","refresh":2,"regex":"","sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":".+","auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"kube-system","value":"kube-system"},"datasource":"$datasource","definition":"label_values(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)","hide":0,"includeAll":true,"label":null,"multi":false,"name":"namespace","options":[],"query":"label_values(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"","value":""},"datasource":"$datasource","definition":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)","hide":0,"includeAll":false,"label":null,"multi":false,"name":"workload","options":[],"query":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"","value":""},"datasource":"$datasource","definition":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)","hide":0,"includeAll":true,"label":null,"multi":false,"name":"type","options":[],"query":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)","refresh":2,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"resolution","options":[{"selected":false,"text":"30s","value":"30s"},{"selected":true,"text":"5m","value":"5m"},{"selected":false,"text":"1h","value":"1h"}],"query":"30s,5m,1h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"interval","options":[{"selected":true,"text":"4h","value":"4h"}],"query":"4h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Networking / Workload","uid":"728bf77cc1166d2f3133bf25846876cc","version":0}
----
-# Source: mimir/charts/mimir/charts/minio/templates/configmap.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: release-name-minio
- labels:
- app: minio
- chart: minio-5.0.14
- release: release-name
- heritage: Helm
-data:
- initialize: |-
- #!/bin/sh
- set -e ; # Have script exit in the event of a failed command.
- MC_CONFIG_DIR="/tmp/minio/mc/"
- MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}"
-
- # connectToMinio
- # Use a check-sleep-check loop to wait for MinIO service to be available
- connectToMinio() {
- SCHEME=$1
- ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts
- set -e ; # fail if we can't read the keys.
- ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ;
- set +e ; # The connections to minio are allowed to fail.
- echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ;
- MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ;
- $MC_COMMAND ;
- STATUS=$? ;
- until [ $STATUS = 0 ]
- do
- ATTEMPTS=`expr $ATTEMPTS + 1` ;
- echo \"Failed attempts: $ATTEMPTS\" ;
- if [ $ATTEMPTS -gt $LIMIT ]; then
- exit 1 ;
- fi ;
- sleep 2 ; # 1 second intervals between attempts
- $MC_COMMAND ;
- STATUS=$? ;
- done ;
- set -e ; # reset `e` as active
- return 0
- }
-
- # checkBucketExists ($bucket)
- # Check if the bucket exists, by using the exit code of `mc ls`
- checkBucketExists() {
- BUCKET=$1
- CMD=$(${MC} stat myminio/$BUCKET > /dev/null 2>&1)
- return $?
- }
-
- # createBucket ($bucket, $policy, $purge)
- # Ensure bucket exists, purging if asked to
- createBucket() {
- BUCKET=$1
- POLICY=$2
- PURGE=$3
- VERSIONING=$4
- OBJECTLOCKING=$5
-
- # Purge the bucket, if set & exists
- # Since PURGE is user input, check explicitly for `true`
- if [ $PURGE = true ]; then
- if checkBucketExists $BUCKET ; then
- echo "Purging bucket '$BUCKET'."
- set +e ; # don't exit if this fails
- ${MC} rm -r --force myminio/$BUCKET
- set -e ; # reset `e` as active
- else
- echo "Bucket '$BUCKET' does not exist, skipping purge."
- fi
- fi
-
- # Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created)
- if ! checkBucketExists $BUCKET ; then
- if [ ! -z $OBJECTLOCKING ] ; then
- if [ $OBJECTLOCKING = true ] ; then
- echo "Creating bucket with OBJECTLOCKING '$BUCKET'"
- ${MC} mb --with-lock myminio/$BUCKET
- elif [ $OBJECTLOCKING = false ] ; then
- echo "Creating bucket '$BUCKET'"
- ${MC} mb myminio/$BUCKET
- fi
- elif [ -z $OBJECTLOCKING ] ; then
- echo "Creating bucket '$BUCKET'"
- ${MC} mb myminio/$BUCKET
- else
- echo "Bucket '$BUCKET' already exists."
- fi
- fi
-
-
- # set versioning for bucket if objectlocking is disabled or not set
- if [ $OBJECTLOCKING = false ] ; then
- if [ ! -z $VERSIONING ] ; then
- if [ $VERSIONING = true ] ; then
- echo "Enabling versioning for '$BUCKET'"
- ${MC} version enable myminio/$BUCKET
- elif [ $VERSIONING = false ] ; then
- echo "Suspending versioning for '$BUCKET'"
- ${MC} version suspend myminio/$BUCKET
- fi
- fi
- else
- echo "Bucket '$BUCKET' versioning unchanged."
- fi
-
-
- # At this point, the bucket should exist, skip checking for existence
- # Set policy on the bucket
- echo "Setting policy of bucket '$BUCKET' to '$POLICY'."
- ${MC} anonymous set $POLICY myminio/$BUCKET
- }
-
- # Try connecting to MinIO instance
- scheme=http
- connectToMinio $scheme
-
-
-
- # Create the buckets
- createBucket mimir-tsdb "none" false false false
- createBucket mimir-ruler "none" false false false
- createBucket enterprise-metrics-tsdb "none" false false false
- createBucket enterprise-metrics-admin "none" false false false
- createBucket enterprise-metrics-ruler "none" false false false
-
- add-user: |-
- #!/bin/sh
- set -e ; # Have script exit in the event of a failed command.
- MC_CONFIG_DIR="/tmp/minio/mc/"
- MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}"
-
- # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters.
- # Special characters for example : ',",<,>,{,}
- MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_tmp"
-
- # connectToMinio
- # Use a check-sleep-check loop to wait for MinIO service to be available
- connectToMinio() {
- SCHEME=$1
- ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts
- set -e ; # fail if we can't read the keys.
- ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ;
- set +e ; # The connections to minio are allowed to fail.
- echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ;
- MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ;
- $MC_COMMAND ;
- STATUS=$? ;
- until [ $STATUS = 0 ]
- do
- ATTEMPTS=`expr $ATTEMPTS + 1` ;
- echo \"Failed attempts: $ATTEMPTS\" ;
- if [ $ATTEMPTS -gt $LIMIT ]; then
- exit 1 ;
- fi ;
- sleep 2 ; # 1 second intervals between attempts
- $MC_COMMAND ;
- STATUS=$? ;
- done ;
- set -e ; # reset `e` as active
- return 0
- }
-
- # checkUserExists ()
- # Check if the user exists, by using the exit code of `mc admin user info`
- checkUserExists() {
- CMD=$(${MC} admin user info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1)
- return $?
- }
-
- # createUser ($policy)
- createUser() {
- POLICY=$1
- #check accessKey_and_secretKey_tmp file
- if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then
- echo "credentials file does not exist"
- return 1
- fi
- if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then
- echo "credentials file is invalid"
- rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP
- return 1
- fi
- USER=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP)
- # Create the user if it does not exist
- if ! checkUserExists ; then
- echo "Creating user '$USER'"
- cat $MINIO_ACCESSKEY_SECRETKEY_TMP | ${MC} admin user add myminio
- else
- echo "User '$USER' already exists."
- fi
- #clean up credentials files.
- rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP
-
- # set policy for user
- if [ ! -z $POLICY -a $POLICY != " " ] ; then
- echo "Adding policy '$POLICY' for '$USER'"
- set +e ; # policy already attach errors out, allow it.
- ${MC} admin policy attach myminio $POLICY --user=$USER
- set -e
- else
- echo "User '$USER' has no policy attached."
- fi
- }
-
- # Try connecting to MinIO instance
- scheme=http
- connectToMinio $scheme
-
-
-
- # Create the users
- echo console > $MINIO_ACCESSKEY_SECRETKEY_TMP
- echo console123 >> $MINIO_ACCESSKEY_SECRETKEY_TMP
- createUser consoleAdmin
-
- add-policy: |-
- #!/bin/sh
- set -e ; # Have script exit in the event of a failed command.
- MC_CONFIG_DIR="/tmp/minio/mc/"
- MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}"
-
- # connectToMinio
- # Use a check-sleep-check loop to wait for MinIO service to be available
- connectToMinio() {
- SCHEME=$1
- ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts
- set -e ; # fail if we can't read the keys.
- ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ;
- set +e ; # The connections to minio are allowed to fail.
- echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ;
- MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ;
- $MC_COMMAND ;
- STATUS=$? ;
- until [ $STATUS = 0 ]
- do
- ATTEMPTS=`expr $ATTEMPTS + 1` ;
- echo \"Failed attempts: $ATTEMPTS\" ;
- if [ $ATTEMPTS -gt $LIMIT ]; then
- exit 1 ;
- fi ;
- sleep 2 ; # 1 second intervals between attempts
- $MC_COMMAND ;
- STATUS=$? ;
- done ;
- set -e ; # reset `e` as active
- return 0
- }
-
- # checkPolicyExists ($policy)
- # Check if the policy exists, by using the exit code of `mc admin policy info`
- checkPolicyExists() {
- POLICY=$1
- CMD=$(${MC} admin policy info myminio $POLICY > /dev/null 2>&1)
- return $?
- }
-
- # createPolicy($name, $filename)
- createPolicy () {
- NAME=$1
- FILENAME=$2
-
- # Create the name if it does not exist
- echo "Checking policy: $NAME (in /config/$FILENAME.json)"
- if ! checkPolicyExists $NAME ; then
- echo "Creating policy '$NAME'"
- else
- echo "Policy '$NAME' already exists."
- fi
- ${MC} admin policy create myminio $NAME /config/$FILENAME.json
-
- }
-
- # Try connecting to MinIO instance
- scheme=http
- connectToMinio $scheme
-
-
-
- add-svcacct: |-
- #!/bin/sh
- set -e ; # Have script exit in the event of a failed command.
- MC_CONFIG_DIR="/tmp/minio/mc/"
- MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}"
-
- # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters.
- # Special characters for example : ',",<,>,{,}
- MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_svcacct_tmp"
-
- # connectToMinio
- # Use a check-sleep-check loop to wait for MinIO service to be available
- connectToMinio() {
- SCHEME=$1
- ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts
- set -e ; # fail if we can't read the keys.
- ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ;
- set +e ; # The connections to minio are allowed to fail.
- echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ;
- MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ;
- $MC_COMMAND ;
- STATUS=$? ;
- until [ $STATUS = 0 ]
- do
- ATTEMPTS=`expr $ATTEMPTS + 1` ;
- echo \"Failed attempts: $ATTEMPTS\" ;
- if [ $ATTEMPTS -gt $LIMIT ]; then
- exit 1 ;
- fi ;
- sleep 2 ; # 2 second intervals between attempts
- $MC_COMMAND ;
- STATUS=$? ;
- done ;
- set -e ; # reset `e` as active
- return 0
- }
-
- # checkSvcacctExists ()
- # Check if the svcacct exists, by using the exit code of `mc admin user svcacct info`
- checkSvcacctExists() {
- CMD=$(${MC} admin user svcacct info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1)
- return $?
- }
-
- # createSvcacct ($user)
- createSvcacct () {
- USER=$1
- FILENAME=$2
- #check accessKey_and_secretKey_tmp file
- if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then
- echo "credentials file does not exist"
- return 1
- fi
- if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then
- echo "credentials file is invalid"
- rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP
- return 1
- fi
- SVCACCT=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP)
- # Create the svcacct if it does not exist
- if ! checkSvcacctExists ; then
- echo "Creating svcacct '$SVCACCT'"
- # Check if policy file is define
- if [ -z $FILENAME ]; then
- ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) myminio $USER
- else
- ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --policy /config/$FILENAME.json myminio $USER
- fi
- else
- echo "Svcacct '$SVCACCT' already exists."
- fi
- #clean up credentials files.
- rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP
- }
-
- # Try connecting to MinIO instance
- scheme=http
- connectToMinio $scheme
-
-
-
- custom-command: |-
- #!/bin/sh
- set -e ; # Have script exit in the event of a failed command.
- MC_CONFIG_DIR="/tmp/minio/mc/"
- MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}"
-
- # connectToMinio
- # Use a check-sleep-check loop to wait for MinIO service to be available
- connectToMinio() {
- SCHEME=$1
- ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts
- set -e ; # fail if we can't read the keys.
- ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ;
- set +e ; # The connections to minio are allowed to fail.
- echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ;
- MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ;
- $MC_COMMAND ;
- STATUS=$? ;
- until [ $STATUS = 0 ]
- do
- ATTEMPTS=`expr $ATTEMPTS + 1` ;
- echo \"Failed attempts: $ATTEMPTS\" ;
- if [ $ATTEMPTS -gt $LIMIT ]; then
- exit 1 ;
- fi ;
- sleep 2 ; # 1 second intervals between attempts
- $MC_COMMAND ;
- STATUS=$? ;
- done ;
- set -e ; # reset `e` as active
- return 0
- }
-
- # runCommand ($@)
- # Run custom mc command
- runCommand() {
- ${MC} "$@"
- return $?
- }
-
- # Try connecting to MinIO instance
- scheme=http
- connectToMinio $scheme
----
-# Source: mimir/charts/mimir/templates/alertmanager/alertmanager-config.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: release-name-mimir-alertmanager-fallback-config
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: alertmanager
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-data:
- alertmanager_fallback_config.yaml: |
- receivers:
- - name: default-receiver
- route:
- receiver: default-receiver
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-alertmanager-resources-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-alertmanager-resources.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\",resource=\"cpu\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"} > 0)",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\",resource=\"memory\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (workingset)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (go heap inuse)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Alertmanager",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?alertmanager.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Receive bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?alertmanager.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Transmit bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Network",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"alertmanager\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
- "format": "time_series",
- "legendFormat": "{{pod}} - {{device}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Disk writes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"alertmanager\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
- "format": "time_series",
- "legendFormat": "{{pod}} - {{device}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Disk reads",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Disk",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n label_name=~\"(alertmanager).*\"\n }\n)\n",
- "format": "time_series",
- "legendFormat": "{{persistentvolumeclaim}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Disk space utilization",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".*",
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": false,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": null,
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": false,
- "label": "namespace",
- "multi": false,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Alertmanager resources",
- "uid": "a6883fb22799ac74479c7db872451092",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-alertmanager-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-alertmanager.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "100px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "format": "short",
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(cluster_job_pod:cortex_alertmanager_alerts:sum{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
- "format": "time_series",
- "instant": true,
- "refId": "A"
- }
- ],
- "thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
- "title": "Total alerts",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "format": "short",
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(cluster_job_pod:cortex_alertmanager_silences:sum{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
- "format": "time_series",
- "instant": true,
- "refId": "A"
- }
- ],
- "thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
- "title": "Total silences",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "format": "short",
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max(cortex_alertmanager_tenants_discovered{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
- "format": "time_series",
- "instant": true,
- "refId": "A"
- }
- ],
- "thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
- "title": "Tenants",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": false,
- "title": "Headlines",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "QPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "99th percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "50th percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"})",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Alertmanager Distributor",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(cluster_job:cortex_alertmanager_alerts_received_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n-\nsum(cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n",
- "format": "time_series",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "APS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Alerts received",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 12,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "cortex_alertmanager_dispatcher_aggregation_groups{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "per pod Active Aggregation Groups",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Alerts grouping",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(cluster_job_integration:cortex_alertmanager_notifications_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n-\nsum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n",
- "format": "time_series",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "NPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "(\nsum(cluster_job_integration:cortex_alertmanager_notifications_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}) by(integration)\n-\nsum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}) by(integration)\n) > 0\nor on () vector(0)\n",
- "format": "time_series",
- "legendFormat": "success - {{ integration }}",
- "legendLink": null
- },
- {
- "expr": "sum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}) by(integration)",
- "format": "time_series",
- "legendFormat": "failed - {{ integration }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "NPS by integration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_alertmanager_notification_latency_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_alertmanager_notification_latency_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Alert notifications",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{operation}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Operations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "max": 1,
- "min": 0,
- "noValue": "0",
- "unit": "percentunit"
- }
- },
- "id": 12,
- "links": [ ],
- "options": {
- "legend": {
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "span": 3,
- "targets": [
- {
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\"}[$__rate_interval])) >= 0",
- "format": "time_series",
- "legendFormat": "{{operation}}",
- "legendLink": null
- }
- ],
- "title": "Error rate",
- "type": "timeseries"
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Attributes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Exists",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Alertmanager Configuration Object Store (Alertmanager accesses)",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Get",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: GetRange",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Upload",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Delete",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (cortex_alertmanager_tenants_owned{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Per pod tenants",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 20,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (cluster_job_pod:cortex_alertmanager_alerts:sum{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Per pod alerts",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (cluster_job_pod:cortex_alertmanager_silences:sum{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Per pod silences",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Replication",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 22,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_alertmanager_sync_configs_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_sync_configs_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_alertmanager_sync_configs_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Syncs/sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 23,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(reason) (rate(cortex_alertmanager_sync_configs_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{reason}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Syncs/sec (by reason)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 24,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum (rate(cortex_alertmanager_ring_check_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "errors",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Ring check errors/sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Tenant configuration sync",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 25,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(outcome) (rate(cortex_alertmanager_state_initial_sync_completed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "interval": "1m",
- "legendFormat": "{{outcome}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Initial syncs /sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 26,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "interval": "1m",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "interval": "1m",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "interval": "1m",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Initial sync duration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 27,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_alertmanager_state_fetch_replica_state_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_state_fetch_replica_state_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "interval": "1m",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_alertmanager_state_fetch_replica_state_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "interval": "1m",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Fetch state from other alertmanagers /sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Sharding initial state sync",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 28,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(cluster_job:cortex_alertmanager_state_replication_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n-\nsum(cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n",
- "format": "time_series",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Replicate state to other alertmanagers /sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 29,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(cluster_job:cortex_alertmanager_partial_state_merges_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n-\nsum(cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n",
- "format": "time_series",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Merge state from other alertmanagers /sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 30,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_alertmanager_state_persist_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_state_persist_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_alertmanager_state_persist_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Persist state to remote storage /sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Sharding runtime state sync",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": true,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "namespace",
- "multi": true,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Alertmanager",
- "uid": "b0d38d318bbddd80476246d4930f9e55",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-compactor-resources-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-compactor-resources.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\",resource=\"cpu\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (go heap inuse)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "CPU and memory",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_rss{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"} > 0)",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\",resource=\"memory\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (RSS)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"} > 0)",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\",resource=\"memory\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (workingset)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?compactor.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Receive bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?compactor.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Transmit bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Network",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"compactor\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
- "format": "time_series",
- "legendFormat": "{{pod}} - {{device}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Disk writes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"compactor\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
- "format": "time_series",
- "legendFormat": "{{pod}} - {{device}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Disk reads",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n label_name=~\"(compactor).*\"\n }\n)\n",
- "format": "time_series",
- "legendFormat": "{{persistentvolumeclaim}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Disk space utilization",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Disk",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": true,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": false,
- "label": "namespace",
- "multi": true,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Compactor resources",
- "uid": "09a5c49e9cdb2f2b24c6d184574a07fd",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-compactor-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-compactor.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "completed": "#7EB26D",
- "failed": "#E24D42",
- "started": "#34CCEB"
- },
- "bars": true,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Per-instance runs\nNumber of times a compactor instance triggers a compaction across all tenants that it manages.\n\n",
- "fill": 10,
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": false,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_compactor_runs_started_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "started",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_compactor_runs_completed_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "completed",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_compactor_runs_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Per-instance runs / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "description": "### Tenants compaction progress\nIn a multi-tenant cluster, display the progress of tenants that are compacted while compaction is running.\n\n",
- "fieldConfig": {
- "defaults": {
- "max": 1,
- "noValue": 1,
- "unit": "percentunit"
- }
- },
- "id": 2,
- "links": [ ],
- "options": {
- "legend": {
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "span": 3,
- "targets": [
- {
- "expr": "(\n cortex_compactor_tenants_processing_succeeded{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"} +\n cortex_compactor_tenants_processing_failed{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"} +\n cortex_compactor_tenants_skipped{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}\n)\n/\ncortex_compactor_tenants_discovered{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"} > 0\n",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "title": "Tenants compaction progress",
- "type": "timeseries"
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Longest time since last successful run\nDisplays the amount of time since the most recent successful execution\nof the compactor.\nThe value shown will be for the compactor replica that has the longest time since its\nlast successful run.\nThe table to the right shows a summary for all compactor replicas.\n\nIf there is no time value, one of the following messages might appear:\n\n- If you see \"No compactor data\" in this panel, that means that no compactors are active yet.\n\n- If you see \"No successful runs\" in this panel, that means that compactors are active, but none\n of them were successfully executed yet.\n\nThese might be expected - for example, if you just recently restarted your compactors,\nthey might not have had a chance to complete their first compaction run.\nHowever, if these messages persist, you should check the health of your compactors.\n\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "decimals": 1,
- "noValue": "No compactor data",
- "thresholds": {
- "mode": "absolute",
- "steps": [ ]
- },
- "unit": "s"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "Last run"
- },
- "properties": [
- {
- "id": "custom.width",
- "value": 74
- },
- {
- "id": "mappings",
- "value": [
- {
- "options": {
- "from": "-Infinity",
- "result": {
- "color": "text",
- "text": "No successful runs since startup yet"
- },
- "to": 0
- },
- "type": "range"
- }
- ]
- },
- {
- "id": "color",
- "value": {
- "mode": "thresholds"
- }
- },
- {
- "id": "thresholds",
- "value": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": 0
- },
- {
- "color": "yellow",
- "value": 7200
- },
- {
- "color": "orange",
- "value": 21600
- },
- {
- "color": "red",
- "value": 43200
- }
- ]
- }
- }
- ]
- }
- ]
- },
- "fill": 1,
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "options": {
- "reduceOptions": {
- "calcs": [
- "first"
- ],
- "fields": "/^Last run$/",
- "values": false
- },
- "textMode": "value"
- },
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod)\n(\n (time() * (max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[1h]) !=bool 0))\n -\n max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[1h])\n)\n",
- "format": "table",
- "instant": true,
- "interval": "",
- "intervalFactor": null,
- "legendFormat": "",
- "legendLink": null,
- "step": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Longest time since last successful run",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "transformations": [
- {
- "id": "organize",
- "options": {
- "renameByName": {
- "Value": "Last run",
- "pod": "Compactor"
- }
- }
- },
- {
- "id": "sortBy",
- "options": {
- "sort": [
- {
- "desc": true,
- "field": "Last run"
- }
- ]
- }
- }
- ],
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Last successful run per-compactor replica\nDisplays the compactor replicas, and for each, shows how long it has been since\nits last successful compaction run.\n\nThe value in the status column is based on how long it has been since the last successful compaction.\n\n- Okay: less than 2 hours\n- Delayed: more than 2 hours\n- Late: more than 6 hours\n- Very late: more than 12 hours\n\nIf the status of any compactor replicas are *Late* or *Very late*, check their health.\n\n",
- "fieldConfig": {
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "Status"
- },
- "properties": [
- {
- "id": "custom.displayMode",
- "value": "color-background"
- },
- {
- "id": "mappings",
- "value": [
- {
- "options": {
- "from": "-Infinity",
- "result": {
- "color": "transparent",
- "text": "N/A"
- },
- "to": 0
- },
- "type": "range"
- },
- {
- "options": {
- "from": 0,
- "result": {
- "color": "green",
- "text": "Ok"
- },
- "to": 7200
- },
- "type": "range"
- },
- {
- "options": {
- "from": 7200,
- "result": {
- "color": "yellow",
- "text": "Delayed"
- },
- "to": 21600
- },
- "type": "range"
- },
- {
- "options": {
- "from": 21600,
- "result": {
- "color": "orange",
- "text": "Late"
- },
- "to": 43200
- },
- "type": "range"
- },
- {
- "options": {
- "from": 43200,
- "result": {
- "color": "red",
- "text": "Very late"
- },
- "to": "Infinity"
- },
- "type": "range"
- },
- {
- "options": {
- "match": "null+nan",
- "result": {
- "color": "transparent",
- "text": "Unknown"
- }
- },
- "type": "special"
- }
- ]
- },
- {
- "id": "custom.width",
- "value": 86
- },
- {
- "id": "custom.align",
- "value": "center"
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "Last run"
- },
- "properties": [
- {
- "id": "unit",
- "value": "s"
- },
- {
- "id": "custom.width",
- "value": 74
- },
- {
- "id": "mappings",
- "value": [
- {
- "options": {
- "from": "-Infinity",
- "result": {
- "text": "Never"
- },
- "to": 0
- },
- "type": "range"
- }
- ]
- }
- ]
- }
- ]
- },
- "fill": 1,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod)\n(\n (time() * (max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[1h]) !=bool 0))\n -\n max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[1h])\n)\n",
- "format": "table",
- "instant": true,
- "legendFormat": "Last run",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Last successful run per-compactor replica",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "transformations": [
- {
- "id": "organize",
- "options": {
- "renameByName": {
- "Value": "Last run",
- "pod": "Compactor"
- }
- }
- },
- {
- "id": "sortBy",
- "options": {
- "sort": [
- {
- "desc": true,
- "field": "Last run"
- }
- ]
- }
- },
- {
- "id": "calculateField",
- "options": {
- "alias": "One",
- "binary": {
- "left": "Last run",
- "operator": "/",
- "right": "Last run"
- },
- "mode": "binary",
- "replaceFields": false
- }
- },
- {
- "id": "calculateField",
- "options": {
- "alias": "Status",
- "binary": {
- "left": "Last run",
- "operator": "*",
- "right": "One"
- },
- "mode": "binary",
- "replaceFields": false
- }
- },
- {
- "id": "filterFieldsByName",
- "options": {
- "include": {
- "names": [
- "Compactor",
- "Last run",
- "Status"
- ]
- }
- }
- }
- ],
- "type": "table",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Summary",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### TSDB compactions / sec\nRate of TSDB compactions. Single TSDB compaction takes one or more input blocks and produces one or more (during \"split\" phase) output blocks.\n\n",
- "fill": 1,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(prometheus_tsdb_compactions_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "compactions",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "TSDB compactions / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### TSDB compaction duration\nDisplay the amount of time that it has taken to run a single TSDB compaction.\n\n",
- "fill": 1,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(prometheus_tsdb_compaction_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(prometheus_tsdb_compaction_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "TSDB compaction duration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(max by(user) (cortex_bucket_blocks_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}))",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Average blocks / tenant",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Tenants with largest number of blocks\nThe 10 tenants with the largest number of blocks.\n\n",
- "fill": 1,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "topk(10, max by(user) (cortex_bucket_blocks_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}))",
- "format": "time_series",
- "legendFormat": "{{user}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Tenants with largest number of blocks",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_compactor_blocks_marked_for_deletion_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "blocks",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Blocks marked for deletion / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_compactor_blocks_cleaned_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_compactor_block_cleanup_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Blocks deletions / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Garbage collector",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_compactor_meta_syncs_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_compactor_meta_sync_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_compactor_meta_sync_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Metadata syncs / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_compactor_meta_sync_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_compactor_meta_sync_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Metadata sync duration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Metadata sync",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{operation}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Operations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "max": 1,
- "min": 0,
- "noValue": "0",
- "unit": "percentunit"
- }
- },
- "id": 14,
- "links": [ ],
- "options": {
- "legend": {
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "span": 3,
- "targets": [
- {
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval])) >= 0",
- "format": "time_series",
- "legendFormat": "{{operation}}",
- "legendLink": null
- }
- ],
- "title": "Error rate",
- "type": "timeseries"
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Attributes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Exists",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Object Store",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Get",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: GetRange",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Upload",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 20,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Delete",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 22,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Key-value store for compactors ring",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": true,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "namespace",
- "multi": true,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Compactor",
- "uid": "1b3443aea86db629e6efdb7d05c53823",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-config-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-config.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 12,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "count(cortex_config_hash{cluster=~\"$cluster\", namespace=~\"$namespace\"}) by (sha256)",
- "format": "time_series",
- "legendFormat": "sha256:{{sha256}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Startup config file hashes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "instances",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Startup config file",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 12,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "count(cortex_runtime_config_hash{cluster=~\"$cluster\", namespace=~\"$namespace\"}) by (sha256)",
- "format": "time_series",
- "legendFormat": "sha256:{{sha256}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Runtime config file hashes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "instances",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Runtime config file",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": true,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "namespace",
- "multi": true,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Config",
- "uid": "5d9d0b4724c0f80d68467088ec61e003",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-object-store-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-object-store.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(component) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{component}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "RPS / component",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "max": 1,
- "min": 0,
- "noValue": "0",
- "unit": "percentunit"
- }
- },
- "id": 2,
- "links": [ ],
- "options": {
- "legend": {
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "span": 6,
- "targets": [
- {
- "expr": "sum by(component) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) / sum by(component) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) >= 0",
- "format": "time_series",
- "legendFormat": "{{component}}",
- "legendLink": null
- }
- ],
- "title": "Error rate / component",
- "type": "timeseries"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Components",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{operation}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "RPS / operation",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "max": 1,
- "min": 0,
- "noValue": "0",
- "unit": "percentunit"
- }
- },
- "id": 4,
- "links": [ ],
- "options": {
- "legend": {
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "span": 6,
- "targets": [
- {
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) >= 0",
- "format": "time_series",
- "legendFormat": "{{operation}}",
- "legendLink": null
- }
- ],
- "title": "Error rate / operation",
- "type": "timeseries"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Operations",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Op: Get",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Op: GetRange",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Op: Exists",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Op: Attributes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Op: Upload",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Op: Delete",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": true,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "namespace",
- "multi": true,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Object Store",
- "uid": "e1324ee2a434f4158c00a9ee279d3292",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-overrides-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-overrides.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "",
- "rows": [
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "datasource": "${datasource}",
- "id": 1,
- "span": 12,
- "targets": [
- {
- "expr": "max by(limit_name) (cortex_limits_defaults{cluster=~\"$cluster\",namespace=~\"$namespace\"})",
- "instant": true,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Defaults",
- "transformations": [
- {
- "id": "labelsToFields",
- "options": { }
- },
- {
- "id": "merge",
- "options": { }
- },
- {
- "id": "organize",
- "options": {
- "excludeByName": {
- "Time": true
- },
- "indexByName": {
- "Value": 1,
- "limit_name": 0
- }
- }
- },
- {
- "id": "sortBy",
- "options": {
- "fields": { },
- "sort": [
- {
- "field": "limit_name"
- }
- ]
- }
- }
- ],
- "type": "table"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "datasource": "${datasource}",
- "id": 2,
- "span": 12,
- "targets": [
- {
- "expr": "max by(user, limit_name) (cortex_limits_overrides{cluster=~\"$cluster\",namespace=~\"$namespace\",user=~\"${tenant_id}\"})",
- "instant": true,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Per-tenant overrides",
- "transformations": [
- {
- "id": "labelsToFields",
- "options": {
- "mode": "columns",
- "valueLabel": "limit_name"
- }
- },
- {
- "id": "merge",
- "options": { }
- },
- {
- "id": "organize",
- "options": {
- "excludeByName": {
- "Time": true
- },
- "indexByName": {
- "user": 0
- }
- }
- }
- ],
- "type": "table"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".*",
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": false,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": null,
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": false,
- "label": "namespace",
- "multi": false,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "current": {
- "selected": true,
- "text": ".*",
- "value": ".*"
- },
- "hide": 0,
- "label": "Tenant ID",
- "name": "tenant_id",
- "options": [
- {
- "selected": true,
- "text": ".*",
- "value": ".*"
- }
- ],
- "query": ".*",
- "type": "textbox"
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Overrides",
- "uid": "1e2c358600ac53f09faea133f811b5bb",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-overview-networking-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-overview-networking.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Receive bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Transmit bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"})",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"})",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Inflight requests (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}))",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}))",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- },
- {
- "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "TCP connections (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Writes",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Receive bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Transmit bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"})",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"})",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Inflight requests (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}))",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}))",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- },
- {
- "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "TCP connections (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Reads",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Receive bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Transmit bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"})",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"})",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Inflight requests (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"}))",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"}))",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- },
- {
- "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "TCP connections (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Backend",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".*",
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": false,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": null,
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": false,
- "label": "namespace",
- "multi": false,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Overview networking",
- "uid": "e15c71d372cc541367a088f10d9fcd92",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-overview-resources-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-overview-resources.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (workingset)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (go heap inuse)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Writes",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"distributor|ingester|mimir-write\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
- "format": "time_series",
- "legendFormat": "{{pod}} - {{device}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Disk writes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"distributor|ingester|mimir-write\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
- "format": "time_series",
- "legendFormat": "{{pod}} - {{device}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Disk reads",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n label_name=~\"(distributor|ingester|mimir-write).*\"\n }\n)\n",
- "format": "time_series",
- "legendFormat": "{{persistentvolumeclaim}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Disk space utilization",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (workingset)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (go heap inuse)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Reads",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (workingset)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (go heap inuse)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Backend",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
- "format": "time_series",
- "legendFormat": "{{pod}} - {{device}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Disk writes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
- "format": "time_series",
- "legendFormat": "{{pod}} - {{device}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Disk reads",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n label_name=~\"(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"\n }\n)\n",
- "format": "time_series",
- "legendFormat": "{{persistentvolumeclaim}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Disk space utilization",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".*",
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": false,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": null,
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": false,
- "label": "namespace",
- "multi": false,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Overview resources",
- "uid": "a9b92d3c4d1af325d872a9e9a7083d71",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-overview-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-overview.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "content": "The 'Status' panel shows an overview on the cluster health over the time.\nTo investigate failures, see a specific dashboard:\n\n- Writes\n- Reads\n- Rule evaluations\n- Alerting notifications\n- Object storage\n",
- "datasource": null,
- "description": "",
- "id": 1,
- "mode": "markdown",
- "span": 3,
- "title": "",
- "transparent": true,
- "type": "text"
- },
- {
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "#7EB26D",
- "value": null
- },
- {
- "color": "#EAB839",
- "value": 0.01
- },
- {
- "color": "#E24D42",
- "value": 0.050000000000000003
- }
- ]
- }
- }
- },
- "id": 2,
- "options": {
- "showValue": "never"
- },
- "span": 6,
- "targets": [
- {
- "datasource": {
- "uid": "$datasource"
- },
- "exemplar": false,
- "expr": "(\n # gRPC errors are not tracked as 5xx but \"error\".\n sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\",status_code=~\"5.*|error\"}[$__rate_interval]))\n or\n # Handle the case no failure has been tracked yet.\n vector(0)\n)\n/\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))\n",
- "instant": false,
- "legendFormat": "Writes",
- "range": true
- },
- {
- "datasource": {
- "uid": "$datasource"
- },
- "exemplar": false,
- "expr": "(\n sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"5.*\"}[$__rate_interval]))\n or\n # Handle the case no failure has been tracked yet.\n vector(0)\n)\n/\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n",
- "instant": false,
- "legendFormat": "Reads",
- "range": true
- },
- {
- "datasource": {
- "uid": "$datasource"
- },
- "exemplar": false,
- "expr": "(\n (\n sum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n +\n # Consider missed evaluations as failures.\n sum(rate(cortex_prometheus_rule_group_iterations_missed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n )\n or\n # Handle the case no failure has been tracked yet.\n vector(0)\n)\n/\nsum(rate(cortex_prometheus_rule_evaluations_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "instant": false,
- "legendFormat": "Rule evaluations",
- "range": true
- },
- {
- "datasource": {
- "uid": "$datasource"
- },
- "exemplar": false,
- "expr": "(\n # Failed notifications from ruler to Alertmanager (handling the case the ruler metrics are missing).\n ((sum(rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n) or vector(0))\n +\n # Failed notifications from Alertmanager to receivers (handling the case the alertmanager metrics are missing).\n ((sum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n) or vector(0))\n)\n/\n(\n # Total notifications from ruler to Alertmanager (handling the case the ruler metrics are missing).\n ((sum(rate(cortex_prometheus_notifications_sent_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n) or vector(0))\n +\n # Total notifications from Alertmanager to receivers (handling the case the alertmanager metrics are missing).\n ((sum(cluster_job_integration:cortex_alertmanager_notifications_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n) or vector(0))\n)\n",
- "instant": false,
- "legendFormat": "Alerting notifications",
- "range": true
- },
- {
- "datasource": {
- "uid": "$datasource"
- },
- "exemplar": false,
- "expr": "sum(rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))\n/\nsum(rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))\n",
- "instant": false,
- "legendFormat": "Object storage",
- "range": true
- }
- ],
- "title": "Status",
- "type": "state-timeline"
- },
- {
- "id": 3,
- "options": {
- "alertInstanceLabelFilter": "cluster=~\"$cluster\", namespace=~\"$namespace\"",
- "alertName": "Mimir",
- "dashboardAlerts": false,
- "maxItems": 100,
- "sortOrder": 3,
- "stateFilter": {
- "error": true,
- "firing": true,
- "noData": false,
- "normal": false,
- "pending": false
- }
- },
- "span": 3,
- "title": "Firing alerts",
- "type": "alertlist"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Mimir cluster health",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "content": "These panels show an overview on the write path. \nTo examine the write path in detail, see a specific dashboard:\n\n- Writes\n- Writes resources\n- Writes networking\n- Overview resources\n- Overview networking\n",
- "datasource": null,
- "description": "",
- "id": 4,
- "mode": "markdown",
- "span": 3,
- "title": "",
- "transparent": true,
- "type": "text"
- },
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Write requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "99th percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "50th percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Write latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(cluster_namespace_job:cortex_distributor_received_samples:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})",
- "format": "time_series",
- "legendFormat": "samples / sec",
- "legendLink": null
- },
- {
- "expr": "sum(cluster_namespace_job:cortex_distributor_received_exemplars:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})",
- "format": "time_series",
- "legendFormat": "exemplars / sec",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Ingestion / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "cps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Writes",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "content": "These panels show an overview on the read path. \nTo examine the read path in detail, see a specific dashboard:\n\n- Reads\n- Reads resources\n- Reads networking\n- Overview resources\n- Overview networking\n- Queries\n- Compactor\n",
- "datasource": null,
- "description": "",
- "id": 8,
- "mode": "markdown",
- "span": 3,
- "title": "",
- "transparent": true,
- "type": "text"
- },
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Read requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "99th percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "50th percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Read latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "instant queries",
- "color": "#429D48"
- },
- {
- "alias": "range queries",
- "color": "#F1C731"
- },
- {
- "alias": "\"label names\" queries",
- "color": "#2A66CF"
- },
- {
- "alias": "\"label values\" queries",
- "color": "#9E44C1"
- },
- {
- "alias": "series queries",
- "color": "#FFAB57"
- },
- {
- "alias": "remote read queries",
- "color": "#C79424"
- },
- {
- "alias": "metadata queries",
- "color": "#84D586"
- },
- {
- "alias": "exemplar queries",
- "color": "#A1C4FC"
- },
- {
- "alias": "other",
- "color": "#C788DE"
- }
- ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_query\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "instant queries",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_query_range\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "range queries",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_labels\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "\"label names\" queries",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_label_name_values\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "\"label values\" queries",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_series\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "series queries",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_read\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "remote read queries",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_metadata\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "metadata queries",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_query_exemplars\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "exemplar queries",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_.*\",route!~\".*(query|query_range|label.*|series|read|metadata|query_exemplars)\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "other",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Queries / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Reads",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "content": "These panels show an overview on the recording and alerting rules evaluation.\nTo examine the rules evaluation and alerts notifications in detail, see a specific dashboard:\n\n- Ruler\n- Alertmanager\n- Alertmanager resources\n- Overview resources\n- Overview networking\n",
- "datasource": null,
- "description": "",
- "id": 12,
- "mode": "markdown",
- "span": 3,
- "title": "",
- "transparent": true,
- "type": "text"
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_prometheus_rule_evaluations_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "success",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_prometheus_rule_group_iterations_missed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "missed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Rule evaluations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n /\nsum (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "average",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Rule evaluations latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_prometheus_notifications_sent_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n -\nsum(rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Alerting notifications sent to Alertmanager / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Recording and alerting rules",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "content": "These panels show an overview on the long-term storage (object storage).\nTo examine the storage in detail, see a specific dashboard:\n\n- Object store\n- Compactor\n",
- "datasource": null,
- "description": "",
- "id": 16,
- "mode": "markdown",
- "span": 3,
- "title": "",
- "transparent": true,
- "type": "text"
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))\n-\nsum(rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "attributes",
- "color": "#429D48"
- },
- {
- "alias": "delete",
- "color": "#F1C731"
- },
- {
- "alias": "exists",
- "color": "#2A66CF"
- },
- {
- "alias": "get",
- "color": "#9E44C1"
- },
- {
- "alias": "get_range",
- "color": "#FFAB57"
- },
- {
- "alias": "iter",
- "color": "#C79424"
- },
- {
- "alias": "upload",
- "color": "#84D586"
- }
- ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{operation}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Operations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(max by(user) (max_over_time(cortex_bucket_blocks_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[15m])))",
- "format": "time_series",
- "legendFormat": "blocks",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Total number of blocks in the storage",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Long-term storage (object storage)",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": true,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "namespace",
- "multi": true,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Overview",
- "uid": "ffcd83628d7d4b5a03d1cafd159e6c9c",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-queries-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-queries.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_query_frontend_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_frontend_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Queue duration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_query_frontend_retries_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) * 1 / sum(rate(cortex_query_frontend_retries_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Retries",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (cortex_query_frontend_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Queue length (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "noValue": "0",
- "unit": "short"
- }
- },
- "id": 4,
- "links": [ ],
- "options": {
- "legend": {
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "span": 3,
- "targets": [
- {
- "expr": "sum by(user) (cortex_query_frontend_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}) > 0",
- "format": "time_series",
- "legendFormat": "{{user}}",
- "legendLink": null
- }
- ],
- "title": "Queue length (per user)",
- "type": "timeseries"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Query-frontend",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Queue duration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (cortex_query_scheduler_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Queue length (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "noValue": "0",
- "unit": "short"
- }
- },
- "id": 7,
- "links": [ ],
- "options": {
- "legend": {
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "span": 4,
- "targets": [
- {
- "expr": "sum by(user) (cortex_query_scheduler_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}) > 0",
- "format": "time_series",
- "legendFormat": "{{user}}",
- "legendLink": null
- }
- ],
- "title": "Queue length (per user)",
- "type": "timeseries"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Query-scheduler",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Intervals per query\nThe average number of split queries (partitioned by time) executed a single input query.\n\n",
- "fill": 1,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_frontend_split_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) / sum(rate(cortex_frontend_query_range_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", method=\"split_by_interval_and_results_cache\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "splitting rate",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Intervals per query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "# Query the new metric introduced in Mimir 2.10.\n(\n sum by(request_type) (rate(cortex_frontend_query_result_cache_hits_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))\n /\n sum by(request_type) (rate(cortex_frontend_query_result_cache_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))\n)\n# Otherwise fallback to the previous general-purpose metrics.\nor\n(\n label_replace(\n # Query metrics before and after migration to new memcached backend.\n sum (\n rate(cortex_cache_hits{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n or\n rate(thanos_cache_memcached_hits_total{name=\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n )\n /\n sum (\n rate(cortex_cache_fetched_keys{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n or\n rate(thanos_cache_memcached_requests_total{name=~\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n ),\n \"request_type\", \"query_range\", \"\", \"\")\n)\n",
- "format": "time_series",
- "legendFormat": "{{request_type}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Query results cache hit ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Query results cache skipped\nThe % of queries whose results could not be cached.\nIt is tracked for each split query when the splitting by interval is enabled.\n\n",
- "fill": 10,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_frontend_query_result_cache_skipped_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (reason) /\nignoring (reason) group_left sum(rate(cortex_frontend_query_result_cache_attempted_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "{{reason}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Query results cache skipped",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Query-frontend - query splitting and results cache",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Sharded queries ratio\nThe % of queries that have been successfully rewritten and executed in a shardable way.\nThis panel only takes into account the type of queries that are supported by query sharding (eg. range queries).\n\n",
- "fill": 1,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_frontend_query_sharding_rewrites_succeeded_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) /\nsum(rate(cortex_frontend_query_sharding_rewrites_attempted_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "sharded queries ratio",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Sharded queries ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Number of sharded queries per query\nThe number of sharded queries that have been executed for a single input query. It only tracks queries that\nhave been successfully rewritten in a shardable way.\n\n",
- "fill": 1,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_frontend_sharded_queries_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_frontend_sharded_queries_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_frontend_sharded_queries_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) * 1 / sum(rate(cortex_frontend_sharded_queries_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Number of sharded queries per query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Query-frontend - query sharding",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_series_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})) * 1",
- "format": "time_series",
- "legendFormat": "99th percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_series_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})) * 1",
- "format": "time_series",
- "legendFormat": "50th percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "1 * sum(cluster_job:cortex_ingester_queried_series_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}) / sum(cluster_job:cortex_ingester_queried_series_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Series per query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_samples_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})) * 1",
- "format": "time_series",
- "legendFormat": "99th percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_samples_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})) * 1",
- "format": "time_series",
- "legendFormat": "50th percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "1 * sum(cluster_job:cortex_ingester_queried_samples_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}) / sum(cluster_job:cortex_ingester_queried_samples_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Samples per query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_exemplars_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})) * 1",
- "format": "time_series",
- "legendFormat": "99th percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_exemplars_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})) * 1",
- "format": "time_series",
- "legendFormat": "50th percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "1 * sum(cluster_job:cortex_ingester_queried_exemplars_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}) / sum(cluster_job:cortex_ingester_queried_exemplars_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Exemplars per query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Ingester",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Number of store-gateways hit per query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Refetches of missing blocks per query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {
- "Failure Rate": "#E24D42"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Consistency checks failed\nRate of queries that had to run with consistency checks and those checks failed. A failed consistency check means that some of at least one block which had to be queried wasn't present in any of the store-gateways.\n\n",
- "fill": 1,
- "id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) / sum(rate(cortex_querier_blocks_consistency_checks_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Failure Rate",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Consistency checks failed",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Rejected queries\nThe proportion of all queries received by queriers that were rejected for some reason.\n\n",
- "fill": 1,
- "id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (reason) (rate(cortex_querier_queries_rejected_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) / ignoring (reason) group_left sum(rate(cortex_querier_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_query(_range)?\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{reason}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Rejected queries",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Querier",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 20,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max(cortex_bucket_index_loaded{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"})",
- "format": "time_series",
- "legendFormat": "Max",
- "legendLink": null
- },
- {
- "expr": "min(cortex_bucket_index_loaded{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"})",
- "format": "time_series",
- "legendFormat": "Min",
- "legendLink": null
- },
- {
- "expr": "avg(cortex_bucket_index_loaded{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"})",
- "format": "time_series",
- "legendFormat": "Average",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Bucket indexes loaded (per querier)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_bucket_index_loads_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) - sum(rate(cortex_bucket_index_load_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_bucket_index_load_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Bucket indexes load / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 22,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_bucket_index_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_index_load_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Bucket indexes load latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 23,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_bucket_store_series_blocks_queried_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "blocks",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Blocks queried / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 24,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(data_type) (\n # Exclude \"chunks refetched\".\n rate(cortex_bucket_store_series_data_size_fetched_bytes_sum{component=\"store-gateway\", stage!=\"refetched\", cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])\n)\n",
- "format": "time_series",
- "legendFormat": "{{data_type}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Data fetched / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "binBps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 25,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(data_type) (\n # Exclude \"chunks processed\" to only count \"chunks returned\", other than postings and series.\n rate(cortex_bucket_store_series_data_size_touched_bytes_sum{component=\"store-gateway\", stage!=\"processed\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])\n)\n",
- "format": "time_series",
- "legendFormat": "{{data_type}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Data touched / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "binBps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Store-gateway",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 26,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(stage) (rate(cortex_bucket_store_series_request_stage_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n/\nsum by(stage) (rate(cortex_bucket_store_series_request_stage_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "{{stage}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Series request average latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 27,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum by(stage, le) (rate(cortex_bucket_store_series_request_stage_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])))\n",
- "format": "time_series",
- "legendFormat": "{{stage}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Series request 99th percentile latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Series batch preloading efficiency\nThis panel shows the % of time reduced by preloading, for Series() requests which have been\nsplit to 2+ batches. If a Series() request is served within a single batch, then preloading\nis not triggered, and thus not counted in this measurement.\n\n",
- "fill": 1,
- "id": 28,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "# Clamping min to 0 because if preloading not useful at all, then the actual value we get is\n# slightly negative because of the small overhead introduced by preloading.\nclamp_min(1 - (\n sum(rate(cortex_bucket_store_series_batch_preloading_wait_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n /\n sum(rate(cortex_bucket_store_series_batch_preloading_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n), 0)\n",
- "format": "time_series",
- "legendFormat": "% of time reduced by preloading",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Series batch preloading efficiency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Blocks currently owned\nThis panel shows the number of blocks owned by each store-gateway replica.\nFor each owned block, the store-gateway keeps its index-header on disk, and\neventually loaded in memory (if index-header lazy loading is disabled, or lazy loading\nis enabled and the index-header was loaded).\n\n",
- "fill": 0,
- "id": 29,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "cortex_bucket_store_blocks_loaded{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Blocks currently owned",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 30,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_bucket_store_block_loads_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Blocks loaded / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 31,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_bucket_store_block_drops_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Blocks dropped / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 32,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "cortex_bucket_store_indexheader_lazy_load_total{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"} - cortex_bucket_store_indexheader_lazy_unload_total{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Lazy loaded index-headers",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 33,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Index-header lazy load duration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Index-header lazy load gate latency\nTime spent waiting for a turn to load an index header. This time is not included in \"Index-header lazy load duration.\"\n\n",
- "fill": 1,
- "id": 34,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_stores_gate_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",gate=\"index_header\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_stores_gate_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",gate=\"index_header\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_bucket_stores_gate_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",gate=\"index_header\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_stores_gate_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",gate=\"index_header\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Index-header lazy load gate latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 35,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_bucket_store_series_hash_cache_hits_total{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n/\nsum(rate(cortex_bucket_store_series_hash_cache_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "hit ratio",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Series hash cache hit ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 36,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(thanos_store_index_cache_hits_total{item_type=\"ExpandedPostings\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n/\nsum(rate(thanos_store_index_cache_requests_total{item_type=\"ExpandedPostings\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "hit ratio",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "ExpandedPostings cache hit ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 37,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_cache_memory_hits_total{name=\"chunks-attributes-cache\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n/\nsum(rate(cortex_cache_memory_requests_total{name=\"chunks-attributes-cache\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "hit ratio",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Chunks attributes in-memory cache hit ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": true,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "namespace",
- "multi": true,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Queries",
- "uid": "b3abe8d5c040395cc36615cb4334c92d",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-reads-networking-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-reads-networking.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Receive bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Transmit bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"})",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"})",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Inflight requests (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}))",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}))",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- },
- {
- "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "TCP connections (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Summary",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Receive bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Transmit bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"})",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"})",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Inflight requests (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"}))",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"}))",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- },
- {
- "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "TCP connections (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Query-frontend",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Receive bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Transmit bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"})",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"})",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Inflight requests (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"}))",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"}))",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- },
- {
- "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "TCP connections (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Query-scheduler",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Receive bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Transmit bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"})",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"})",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Inflight requests (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"}))",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"}))",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- },
- {
- "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "TCP connections (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Querier",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Receive bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Transmit bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"})",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"})",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Inflight requests (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 20,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"}))",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"}))",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- },
- {
- "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "TCP connections (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Store-gateway",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Receive bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 22,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Transmit bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 23,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"})",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"})",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Inflight requests (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 24,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"}))",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"}))",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- },
- {
- "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "TCP connections (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Ruler",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".*",
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": false,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": null,
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": false,
- "label": "namespace",
- "multi": false,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Reads networking",
- "uid": "54b2a0a4748b3bd1aefa92ce5559a1c2",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-reads-resources-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-reads-resources.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (workingset)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (go heap inuse)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Summary",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\",resource=\"cpu\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"} > 0)",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\",resource=\"memory\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (workingset)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (go heap inuse)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Query-frontend",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\",resource=\"cpu\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\"} > 0)",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\",resource=\"memory\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (workingset)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (go heap inuse)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Query-scheduler",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\",resource=\"cpu\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"} > 0)",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\",resource=\"memory\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (workingset)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (go heap inuse)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Querier",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\",resource=\"cpu\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (go heap inuse)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Ingester",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_rss{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} > 0)",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\",resource=\"memory\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (RSS)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} > 0)",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\",resource=\"memory\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (workingset)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Rules",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\",resource=\"cpu\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Ruler",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"} > 0)",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\",resource=\"memory\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (workingset)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 20,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (go heap inuse)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\",resource=\"cpu\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 22,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (go heap inuse)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Store-gateway",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 23,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_rss{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"} > 0)",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\",resource=\"memory\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (RSS)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 24,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"} > 0)",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\",resource=\"memory\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (workingset)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 25,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"store-gateway\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
- "format": "time_series",
- "legendFormat": "{{pod}} - {{device}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Disk writes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 26,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"store-gateway\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
- "format": "time_series",
- "legendFormat": "{{pod}} - {{device}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Disk reads",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 27,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n label_name=~\"(store-gateway).*\"\n }\n)\n",
- "format": "time_series",
- "legendFormat": "{{persistentvolumeclaim}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Disk space utilization",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".*",
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": false,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": null,
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": false,
- "label": "namespace",
- "multi": false,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Reads resources",
- "uid": "cc86fd5aa9301c6528986572ad974db9",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-reads-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-reads.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "175px",
- "panels": [
- {
- "content": "\n This dashboard shows health metrics for the read path.\n It is broken into sections for each service on the read path, and organized by the order in which the read request flows.\n
\n Incoming queries travel from the gateway → query frontend → query scheduler → querier → ingester and/or store-gateway (depending on the time range of the query).\n
\n For each service, there are 3 panels showing (1) requests per second to that service, (2) average, median, and p99 latency of requests to that service, and (3) p99 latency of requests to each instance of that service.\n
\n\n The dashboard also shows metrics for the 4 optional caches that can be deployed:\n the query results cache, the metadata cache, the chunks cache, and the index cache.\n
\n These panels will show “no data” if the caches are not deployed.\n
\n\n Lastly, it also includes metrics for how the ingester and store-gateway interact with object storage.\n
\n",
- "datasource": null,
- "description": "",
- "id": 1,
- "mode": "markdown",
- "span": 12,
- "title": "",
- "transparent": true,
- "type": "text"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": false,
- "title": "Reads dashboard description",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "100px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Instant queries per second\nRate of instant queries per second being made to the system.\nIncludes both queries made to the /prometheus API as\nwell as queries from the ruler.\n\n",
- "fill": 1,
- "format": "reqps",
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(\n rate(\n cortex_request_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",\n route=~\"(prometheus|api_prom)_api_v1_query\"\n }[$__rate_interval]\n )\n or\n rate(\n cortex_prometheus_rule_evaluations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"\n }[$__rate_interval]\n )\n)\n",
- "format": "time_series",
- "instant": true,
- "refId": "A"
- }
- ],
- "thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
- "title": "Instant queries / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Range queries per second\nRate of range queries per second being made to\nMimir via the /prometheus API.\n\n",
- "fill": 1,
- "format": "reqps",
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_query_range\"}[$__rate_interval]))",
- "format": "time_series",
- "instant": true,
- "refId": "A"
- }
- ],
- "thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
- "title": "Range queries / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### \"Label names\" queries per second\nRate of \"label names\" endpoint queries per second being made to\nMimir via the /prometheus API.\n\n",
- "fill": 1,
- "format": "reqps",
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_labels\"}[$__rate_interval]))",
- "format": "time_series",
- "instant": true,
- "refId": "A"
- }
- ],
- "thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
- "title": "Label names queries / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### \"Label values\" queries per second\nRate of specific \"label values\" endpoint queries per second being made to\nMimir via the /prometheus API.\n\n",
- "fill": 1,
- "format": "reqps",
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_label_name_values\"}[$__rate_interval]))",
- "format": "time_series",
- "instant": true,
- "refId": "A"
- }
- ],
- "thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
- "title": "Label values queries / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Series queries per second\nRate of series queries per second being made to\nMimir via the /prometheus API.\n\n",
- "fill": 1,
- "format": "reqps",
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_series\"}[$__rate_interval]))",
- "format": "time_series",
- "instant": true,
- "refId": "A"
- }
- ],
- "thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
- "title": "Series queries / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": false,
- "title": "Headlines",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "99th percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "50th percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "drawStyle": "line",
- "fillOpacity": 0,
- "lineWidth": 1,
- "pointSize": 5,
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [ ]
- },
- "unit": "s"
- },
- "overrides": [ ]
- },
- "id": 9,
- "links": [ ],
- "options": {
- "legend": {
- "displayMode": "hidden",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "span": 4,
- "targets": [
- {
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))",
- "format": "time_series",
- "legendFormat": "",
- "legendLink": null
- }
- ],
- "title": "Per pod p99 latency",
- "type": "timeseries"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Query-frontend",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Requests / sec\n\n The query scheduler is an optional service that moves\n the internal queue from the query-frontend into a\n separate component.\n If this service is not deployed,\n these panels will show \"No data.\"\n
\n\n",
- "fill": 10,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Latency (Time in Queue)\n\n The query scheduler is an optional service that moves\n the internal queue from the query-frontend into a\n separate component.\n If this service is not deployed,\n these panels will show \"No data.\"\n
\n\n",
- "fill": 1,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency (Time in Queue)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Latency (Time in Queue) by Queue Dimension\n\n The query scheduler is an optional service that moves\n the internal queue from the query-frontend into a\n separate component.\n If this service is not deployed,\n these panels will show \"No data.\"\n
\n\n",
- "fill": 1,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "label_replace(histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le, additional_queue_dimensions)) * 1e3, \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
- "format": "time_series",
- "legendFormat": "99th Percentile: {{ additional_queue_dimensions }}",
- "refId": "A"
- },
- {
- "expr": "label_replace(histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le, additional_queue_dimensions)) * 1e3, \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
- "format": "time_series",
- "legendFormat": "50th Percentile: {{ additional_queue_dimensions }}",
- "refId": "B"
- },
- {
- "expr": "label_replace(sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (additional_queue_dimensions) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (additional_queue_dimensions), \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
- "format": "time_series",
- "legendFormat": "Average: {{ additional_queue_dimensions }}",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency (Time in Queue) by Queue Dimension",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Query-scheduler",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum (\n rate(thanos_memcached_operations_total{name=\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_operations_total{name=\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n)\n",
- "format": "time_series",
- "legendFormat": "Requests/s",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n) by (le)) * 1e3\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n) by (le)) * 1e3\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n)\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Cache – query results",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_querier_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "99th percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "50th percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "1e3 * sum(cluster_job_route:cortex_querier_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_querier_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "drawStyle": "line",
- "fillOpacity": 0,
- "lineWidth": 1,
- "pointSize": 5,
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [ ]
- },
- "unit": "s"
- },
- "overrides": [ ]
- },
- "id": 17,
- "links": [ ],
- "options": {
- "legend": {
- "displayMode": "hidden",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "span": 4,
- "targets": [
- {
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_querier_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))",
- "format": "time_series",
- "legendFormat": "",
- "legendLink": null
- }
- ],
- "title": "Per pod p99 latency",
- "type": "timeseries"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Querier",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "99th percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "50th percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"})",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "drawStyle": "line",
- "fillOpacity": 0,
- "lineWidth": 1,
- "pointSize": 5,
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [ ]
- },
- "unit": "s"
- },
- "overrides": [ ]
- },
- "id": 20,
- "links": [ ],
- "options": {
- "legend": {
- "displayMode": "hidden",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "span": 4,
- "targets": [
- {
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__rate_interval])))",
- "format": "time_series",
- "legendFormat": "",
- "legendLink": null
- }
- ],
- "title": "Per pod p99 latency",
- "type": "timeseries"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Ingester",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 22,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "99th percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "50th percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"})",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "drawStyle": "line",
- "fillOpacity": 0,
- "lineWidth": 1,
- "pointSize": 5,
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [ ]
- },
- "unit": "s"
- },
- "overrides": [ ]
- },
- "id": 23,
- "links": [ ],
- "options": {
- "legend": {
- "displayMode": "hidden",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "span": 4,
- "targets": [
- {
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval])))",
- "format": "time_series",
- "legendFormat": "",
- "legendLink": null
- }
- ],
- "title": "Per pod p99 latency",
- "type": "timeseries"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Store-gateway",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Replicas\nThe maximum, and current number of querier replicas.\nPlease note that the current number of replicas can still show 1 replica even when scaled to 0.\nSince HPA never reports 0 replicas, the query will report 0 only if the HPA is not active.\n\n",
- "fill": 1,
- "id": 24,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "/Max .+/",
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "/Current .+/",
- "fill": 0
- },
- {
- "alias": "/Min .+/",
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_max_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n # Add the scaletargetref_name label which is more readable than \"kube-hpa-...\"\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n)\n",
- "format": "time_series",
- "legendFormat": "Max {{ scaletargetref_name }}",
- "legendLink": null
- },
- {
- "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_status_current_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n # HPA doesn't go to 0 replicas, so we multiply by 0 if the HPA is not active.\n * on (cluster, namespace, horizontalpodautoscaler)\n kube_horizontalpodautoscaler_status_condition{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\", condition=\"ScalingActive\", status=\"true\"}\n # Add the scaletargetref_name label which is more readable than \"kube-hpa-...\"\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n)\n",
- "format": "time_series",
- "legendFormat": "Current {{ scaletargetref_name }}",
- "legendLink": null
- },
- {
- "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_min_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n # Add the scaletargetref_name label which is more readable than \"kube-hpa-...\"\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n)\n",
- "format": "time_series",
- "legendFormat": "Min {{ scaletargetref_name }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Replicas",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Scaling metric (desired replicas)\nThis panel shows the result scaling metric exposed by KEDA divided by the target/threshold used.\nIt should represent the desired number of replicas, ignoring the min/max constraints which are applied later.\n\n",
- "fill": 1,
- "id": 25,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "Target per replica",
- "yaxis": 2
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left\n label_replace(label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ), \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\")\n)\n",
- "format": "time_series",
- "legendFormat": "{{ scaler }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Scaling metric (desired replicas)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": true
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Autoscaler failures rate\nThe rate of failures in the KEDA custom metrics API server. Whenever an error occurs, the KEDA custom\nmetrics server is unable to query the scaling metric from Prometheus so the autoscaler woudln't work properly.\n\n",
- "fill": 1,
- "id": 26,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(cluster, namespace, scaler, metric, scaledObject) (\n label_replace(\n rate(keda_scaler_errors[$__rate_interval]),\n \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n )\n) +\non(cluster, namespace, metric, scaledObject) group_left\nlabel_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"} * 0,\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n ),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n",
- "format": "time_series",
- "legendFormat": "{{scaler}} failures",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Autoscaler failures rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Querier - autoscaling",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 27,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 28,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Store-gateway – key-value store for store-gateways ring",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 29,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(operation) (\n # Backwards compatibility\n rate(\n thanos_memcached_operations_total{\n component=\"store-gateway\",\n name=\"index-cache\",\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n }[$__rate_interval]\n )\n or ignoring(backend)\n rate(\n thanos_cache_operations_total{\n component=\"store-gateway\",\n name=\"index-cache\",\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n }[$__rate_interval]\n )\n)\n",
- "format": "time_series",
- "legendFormat": "{{operation}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 30,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n)\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency (getmulti)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Hit ratio\nEven if you do not set up memcached for the blocks index cache, you will still see data in this panel because the store-gateway by default has an\nin-memory blocks index cache.\n\n",
- "fill": 1,
- "id": 31,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(item_type) (\n rate(\n thanos_store_index_cache_hits_total{\n component=\"store-gateway\",\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n }[$__rate_interval]\n )\n)\n/\nsum by(item_type) (\n rate(\n thanos_store_index_cache_requests_total{\n component=\"store-gateway\",\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n }[$__rate_interval]\n )\n)\n",
- "format": "time_series",
- "legendFormat": "{{item_type}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Hit ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Memcached – block index cache (store-gateway accesses)",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 32,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(operation) (\n # Backwards compatibility\n rate(thanos_memcached_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n)\n",
- "format": "time_series",
- "legendFormat": "{{operation}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 33,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n)\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency (getmulti)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 34,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(\n # Backwards compatibility\n rate(thanos_cache_memcached_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n)\n/\nsum(\n # Backwards compatibility\n rate(thanos_cache_memcached_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n)\n",
- "format": "time_series",
- "legendFormat": "items",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Hit ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Memcached – chunks cache (store-gateway accesses)",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 35,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(operation) (\n # Backwards compatibility\n rate(thanos_memcached_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n",
- "format": "time_series",
- "legendFormat": "{{operation}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 36,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n)\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency (getmulti)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 37,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(\n # Backwards compatibility\n rate(thanos_cache_memcached_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n/\nsum(\n # Backwards compatibility\n rate(thanos_cache_memcached_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n",
- "format": "time_series",
- "legendFormat": "items",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Hit ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Memcached – metadata cache (store-gateway accesses)",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 38,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(operation) (\n # Backwards compatibility\n rate(thanos_memcached_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_operations_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n",
- "format": "time_series",
- "legendFormat": "{{operation}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 39,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "99th Percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "50th Percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n)\n",
- "format": "time_series",
- "intervalFactor": 2,
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency (getmulti)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 40,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(\n # Backwards compatibility\n rate(thanos_cache_memcached_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_hits_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n/\nsum(\n # Backwards compatibility\n rate(thanos_cache_memcached_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_requests_total{\n cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n",
- "format": "time_series",
- "legendFormat": "items",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Hit ratio",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Memcached – metadata cache (querier accesses)",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 41,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{operation}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Operations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "max": 1,
- "min": 0,
- "noValue": "0",
- "unit": "percentunit"
- }
- },
- "id": 42,
- "links": [ ],
- "options": {
- "legend": {
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "span": 3,
- "targets": [
- {
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval])) >= 0",
- "format": "time_series",
- "legendFormat": "{{operation}}",
- "legendLink": null
- }
- ],
- "title": "Error rate",
- "type": "timeseries"
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 43,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Attributes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 44,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Exists",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Blocks object store (store-gateway accesses)",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 45,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Get",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 46,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: GetRange",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 47,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Upload",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 48,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Delete",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 49,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{operation}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Operations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "max": 1,
- "min": 0,
- "noValue": "0",
- "unit": "percentunit"
- }
- },
- "id": 50,
- "links": [ ],
- "options": {
- "legend": {
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "span": 3,
- "targets": [
- {
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval])) >= 0",
- "format": "time_series",
- "legendFormat": "{{operation}}",
- "legendLink": null
- }
- ],
- "title": "Error rate",
- "type": "timeseries"
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 51,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Attributes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 52,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Exists",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Blocks object store (querier accesses)",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 53,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Get",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 54,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: GetRange",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 55,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Upload",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 56,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Delete",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": true,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "namespace",
- "multi": true,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Reads",
- "uid": "e327503188913dc38ad571c647eef643",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-remote-ruler-reads-resources-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-remote-ruler-reads-resources.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\",resource=\"cpu\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"} > 0)",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\",resource=\"memory\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (workingset)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (go heap inuse)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Query-frontend (dedicated to ruler)",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\",resource=\"cpu\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"} > 0)",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\",resource=\"memory\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (workingset)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (go heap inuse)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Query-scheduler (dedicated to ruler)",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\",resource=\"cpu\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"} > 0)",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\",resource=\"memory\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (workingset)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (go heap inuse)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Querier (dedicated to ruler)",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".*",
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": false,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": null,
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": false,
- "label": "namespace",
- "multi": false,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Remote ruler reads resources",
- "uid": "1940f6ef765a506a171faa2056c956c3",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-remote-ruler-reads-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-remote-ruler-reads.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "175px",
- "panels": [
- {
- "content": "\n This dashboard shows health metrics for the ruler read path when remote operational mode is enabled.\n It is broken into sections for each service on the ruler read path, and organized by the order in which the read request flows.\n
\n For each service, there are three panels showing (1) requests per second to that service, (2) average, median, and p99 latency of requests to that service, and (3) p99 latency of requests to each instance of that service.\n
\n",
- "datasource": null,
- "description": "",
- "id": 1,
- "mode": "markdown",
- "span": 12,
- "title": "",
- "transparent": true,
- "type": "text"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": false,
- "title": "Remote ruler reads dashboard description",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "100px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Evaluations per second\nRate of rule expressions evaluated per second.\n\n",
- "fill": 1,
- "format": "reqps",
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(\n rate(\n cortex_request_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\",\n route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"\n }[$__rate_interval]\n )\n)\n",
- "format": "time_series",
- "instant": true,
- "refId": "A"
- }
- ],
- "thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
- "title": "Evaluations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": false,
- "title": "Headlines",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "99th percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "50th percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"})",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "drawStyle": "line",
- "fillOpacity": 0,
- "lineWidth": 1,
- "pointSize": 5,
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [ ]
- },
- "unit": "s"
- },
- "overrides": [ ]
- },
- "id": 5,
- "links": [ ],
- "options": {
- "legend": {
- "displayMode": "hidden",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "span": 4,
- "targets": [
- {
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"}[$__rate_interval])))",
- "format": "time_series",
- "legendFormat": "",
- "legendLink": null
- }
- ],
- "title": "Per pod p99 latency",
- "type": "timeseries"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Query-frontend (dedicated to ruler)",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Requests / sec\n\n The query scheduler is an optional service that moves\n the internal queue from the query-frontend into a\n separate component.\n If this service is not deployed,\n these panels will show \"No data.\"\n
\n\n",
- "fill": 10,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Latency (Time in Queue)\n\n The query scheduler is an optional service that moves\n the internal queue from the query-frontend into a\n separate component.\n If this service is not deployed,\n these panels will show \"No data.\"\n
\n\n",
- "fill": 1,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency (Time in Queue)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Latency (Time in Queue) by Queue Dimension\n\n The query scheduler is an optional service that moves\n the internal queue from the query-frontend into a\n separate component.\n If this service is not deployed,\n these panels will show \"No data.\"\n
\n\n",
- "fill": 1,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "label_replace(histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) by (le, additional_queue_dimensions)) * 1e3, \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
- "format": "time_series",
- "legendFormat": "99th Percentile: {{ additional_queue_dimensions }}",
- "refId": "A"
- },
- {
- "expr": "label_replace(histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) by (le, additional_queue_dimensions)) * 1e3, \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
- "format": "time_series",
- "legendFormat": "50th Percentile: {{ additional_queue_dimensions }}",
- "refId": "B"
- },
- {
- "expr": "label_replace(sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) by (additional_queue_dimensions) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) by (additional_queue_dimensions), \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
- "format": "time_series",
- "legendFormat": "Average: {{ additional_queue_dimensions }}",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency (Time in Queue) by Queue Dimension",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Query-scheduler (dedicated to ruler)",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_querier_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "99th percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "50th percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "1e3 * sum(cluster_job_route:cortex_querier_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_querier_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "drawStyle": "line",
- "fillOpacity": 0,
- "lineWidth": 1,
- "pointSize": 5,
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [ ]
- },
- "unit": "s"
- },
- "overrides": [ ]
- },
- "id": 11,
- "links": [ ],
- "options": {
- "legend": {
- "displayMode": "hidden",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "span": 4,
- "targets": [
- {
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_querier_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))",
- "format": "time_series",
- "legendFormat": "",
- "legendLink": null
- }
- ],
- "title": "Per pod p99 latency",
- "type": "timeseries"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Querier (dedicated to ruler)",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Replicas\nThe maximum and current number of ruler-querier replicas.\nNote: The current number of replicas can still show 1 replica even when scaled to 0.\nBecause HPA never reports 0 replicas, the query will report 0 only if the HPA is not active.\n\n",
- "fill": 1,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "/Max .+/",
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "/Current .+/",
- "fill": 0
- },
- {
- "alias": "/Min .+/",
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_max_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}\n # Add the scaletargetref_name label for readability\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}\n)\n",
- "format": "time_series",
- "legendFormat": "Max {{ scaletargetref_name }}",
- "legendLink": null
- },
- {
- "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_status_current_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}\n # HPA doesn't go to 0 replicas, so we multiply by 0 if the HPA is not active\n * on (cluster, namespace, horizontalpodautoscaler)\n kube_horizontalpodautoscaler_status_condition{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\", condition=\"ScalingActive\", status=\"true\"}\n # Add the scaletargetref_name label for readability\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}\n)\n",
- "format": "time_series",
- "legendFormat": "Current {{ scaletargetref_name }}",
- "legendLink": null
- },
- {
- "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_min_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}\n # Add the scaletargetref_name label for readability\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}\n)\n",
- "format": "time_series",
- "legendFormat": "Min {{ scaletargetref_name }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Replicas",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Scaling metric (CPU): Desired replicas\nThis panel shows the scaling metric exposed by KEDA divided by the target/threshold used.\nIt should represent the desired number of replicas, ignoring the min/max constraints applied later.\n\n",
- "fill": 1,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*cpu.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
- "format": "time_series",
- "legendFormat": "{{ scaler }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Scaling metric (CPU): Desired replicas",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Scaling metric (memory): Desired replicas\nThis panel shows the scaling metric exposed by KEDA divided by the target/threshold used.\nIt should represent the desired number of replicas, ignoring the min/max constraints applied later.\n\n",
- "fill": 1,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*memory.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
- "format": "time_series",
- "legendFormat": "{{ scaler }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Scaling metric (memory): Desired replicas",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Autoscaler failures rate\nThe rate of failures in the KEDA custom metrics API server. Whenever an error occurs, the KEDA custom\nmetrics server is unable to query the scaling metric from Prometheus so the autoscaler woudln't work properly.\n\n",
- "fill": 1,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(cluster, namespace, scaler, metric, scaledObject) (\n label_replace(\n rate(keda_scaler_errors[$__rate_interval]),\n \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n )\n) +\non(cluster, namespace, metric, scaledObject) group_left\nlabel_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"} * 0,\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n ),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n",
- "format": "time_series",
- "legendFormat": "{{scaler}} failures",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Autoscaler failures rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Ruler-Querier - autoscaling",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": true,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "namespace",
- "multi": true,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Remote ruler reads",
- "uid": "f103238f7f5ab2f1345ce650cbfbfe2f",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-rollout-progress-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-rollout-progress.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "panels": [
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "custom": {
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "fillOpacity": 80,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineWidth": 1,
- "scaleDistribution": {
- "type": "linear"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [ ],
- "max": 1,
- "min": 0,
- "thresholds": {
- "mode": "absolute",
- "steps": [ ]
- },
- "unit": "percentunit"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "Ready"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "green",
- "mode": "fixed"
- }
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "Updated"
- },
- "properties": [
- {
- "id": "color",
- "value": {
- "fixedColor": "blue",
- "mode": "fixed"
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 13,
- "w": 10,
- "x": 0,
- "y": 0
- },
- "id": 1,
- "links": [ ],
- "options": {
- "barRadius": 0,
- "barWidth": 0.96999999999999997,
- "fullHighlight": false,
- "groupWidth": 0.69999999999999996,
- "legend": {
- "calcs": [ ],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "orientation": "horizontal",
- "showValue": "auto",
- "stacking": "none",
- "tooltip": {
- "mode": "multi",
- "sort": "none"
- },
- "xField": "Workload",
- "xTickLabelRotation": 0,
- "xTickLabelSpacing": 0
- },
- "targets": [
- {
- "expr": "(\n sum by(cortex_service) (\n label_replace(\n label_replace(\n label_replace(\n {__name__=~\"kube_(deployment|statefulset)_status_replicas_updated\", cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"cortex_service\", \"$1\", \"deployment\", \"(.+)\"\n ),\n \"cortex_service\", \"$1\", \"statefulset\", \"(.+)\"\n ),\n # Strip the -zone-X suffix, if there is one\n \"cortex_service\", \"$1\", \"cortex_service\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n /\n sum by(cortex_service) (\n label_replace(\n label_replace(\n label_replace(\n {__name__=~\"kube_(deployment|statefulset)_status_replicas\", cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"cortex_service\", \"$1\", \"deployment\", \"(.+)\"\n ),\n \"cortex_service\", \"$1\", \"statefulset\", \"(.+)\"\n ),\n # Strip the -zone-X suffix, if there is one\n \"cortex_service\", \"$1\", \"cortex_service\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n) and (\n sum by(cortex_service) (\n label_replace(\n label_replace(\n label_replace(\n {__name__=~\"kube_(deployment|statefulset)_status_replicas\", cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"cortex_service\", \"$1\", \"deployment\", \"(.+)\"\n ),\n \"cortex_service\", \"$1\", \"statefulset\", \"(.+)\"\n ),\n # Strip the -zone-X suffix, if there is one\n \"cortex_service\", \"$1\", \"cortex_service\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n > 0\n)\n",
- "format": "table",
- "instant": true,
- "intervalFactor": null,
- "legendFormat": "__auto",
- "legendLink": null,
- "step": null
- },
- {
- "expr": "(\n sum by(cortex_service) (\n label_replace(\n label_replace(\n label_replace(\n {__name__=~\"kube_(deployment|statefulset)_status_replicas_ready\", cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"cortex_service\", \"$1\", \"deployment\", \"(.+)\"\n ),\n \"cortex_service\", \"$1\", \"statefulset\", \"(.+)\"\n ),\n # Strip the -zone-X suffix, if there is one\n \"cortex_service\", \"$1\", \"cortex_service\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n /\n sum by(cortex_service) (\n label_replace(\n label_replace(\n label_replace(\n {__name__=~\"kube_(deployment|statefulset)_status_replicas\", cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"cortex_service\", \"$1\", \"deployment\", \"(.+)\"\n ),\n \"cortex_service\", \"$1\", \"statefulset\", \"(.+)\"\n ),\n # Strip the -zone-X suffix, if there is one\n \"cortex_service\", \"$1\", \"cortex_service\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n) and (\n sum by(cortex_service) (\n label_replace(\n label_replace(\n label_replace(\n {__name__=~\"kube_(deployment|statefulset)_status_replicas\", cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"cortex_service\", \"$1\", \"deployment\", \"(.+)\"\n ),\n \"cortex_service\", \"$1\", \"statefulset\", \"(.+)\"\n ),\n # Strip the -zone-X suffix, if there is one\n \"cortex_service\", \"$1\", \"cortex_service\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n > 0\n)\n",
- "format": "table",
- "instant": true,
- "intervalFactor": null,
- "legendFormat": "__auto",
- "legendLink": null,
- "step": null
- }
- ],
- "title": "Rollout progress",
- "transformations": [
- {
- "id": "joinByField",
- "options": {
- "byField": "cortex_service",
- "mode": "outer"
- }
- },
- {
- "id": "organize",
- "options": {
- "excludeByName": {
- "Time 1": true,
- "Time 2": true
- },
- "renameByName": {
- "Value #A": "Updated",
- "Value #B": "Ready",
- "cortex_service": "Workload"
- }
- }
- },
- {
- "id": "sortBy",
- "options": {
- "sort": [
- {
- "field": "Workload"
- }
- ]
- }
- }
- ],
- "type": "barchart"
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "decimals": 1,
- "noValue": "",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "percentunit"
- },
- "overrides": [ ]
- },
- "fill": 1,
- "gridPos": {
- "h": 4,
- "w": 2,
- "x": 10,
- "y": 0
- },
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))\n",
- "format": null,
- "instant": false,
- "interval": "",
- "intervalFactor": null,
- "legendFormat": "",
- "legendLink": null,
- "step": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Writes - 2xx",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "decimals": 1,
- "noValue": "",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "orange",
- "value": 0.20000000000000001
- },
- {
- "color": "red",
- "value": 0.5
- }
- ]
- },
- "unit": "percentunit"
- },
- "overrides": [ ]
- },
- "fill": 1,
- "gridPos": {
- "h": 4,
- "w": 2,
- "x": 12,
- "y": 0
- },
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))\n",
- "format": null,
- "instant": false,
- "interval": "",
- "intervalFactor": null,
- "legendFormat": "",
- "legendLink": null,
- "step": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Writes - 4xx",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "decimals": 1,
- "noValue": "",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 0.01
- }
- ]
- },
- "unit": "percentunit"
- },
- "overrides": [ ]
- },
- "fill": 1,
- "gridPos": {
- "h": 4,
- "w": 2,
- "x": 14,
- "y": 0
- },
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))\n",
- "format": null,
- "instant": false,
- "interval": "",
- "intervalFactor": null,
- "legendFormat": "",
- "legendLink": null,
- "step": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Writes - 5xx",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "decimals": 1,
- "noValue": "",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "orange",
- "value": 0.20000000000000001
- },
- {
- "color": "red",
- "value": 0.5
- }
- ]
- },
- "unit": "s"
- },
- "overrides": [ ]
- },
- "fill": 1,
- "gridPos": {
- "h": 4,
- "w": 8,
- "x": 16,
- "y": 0
- },
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\"}))\n",
- "format": null,
- "instant": false,
- "interval": "",
- "intervalFactor": null,
- "legendFormat": "",
- "legendLink": null,
- "step": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Writes 99th latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "decimals": 1,
- "noValue": "",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- }
- ]
- },
- "unit": "percentunit"
- },
- "overrides": [ ]
- },
- "fill": 1,
- "gridPos": {
- "h": 4,
- "w": 2,
- "x": 10,
- "y": 4
- },
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n",
- "format": null,
- "instant": false,
- "interval": "",
- "intervalFactor": null,
- "legendFormat": "",
- "legendLink": null,
- "step": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Reads - 2xx",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "decimals": 1,
- "noValue": "",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "orange",
- "value": 0.01
- },
- {
- "color": "red",
- "value": 0.050000000000000003
- }
- ]
- },
- "unit": "percentunit"
- },
- "overrides": [ ]
- },
- "fill": 1,
- "gridPos": {
- "h": 4,
- "w": 2,
- "x": 12,
- "y": 4
- },
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n",
- "format": null,
- "instant": false,
- "interval": "",
- "intervalFactor": null,
- "legendFormat": "",
- "legendLink": null,
- "step": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Reads - 4xx",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "decimals": 1,
- "noValue": "",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 0.01
- }
- ]
- },
- "unit": "percentunit"
- },
- "overrides": [ ]
- },
- "fill": 1,
- "gridPos": {
- "h": 4,
- "w": 2,
- "x": 14,
- "y": 4
- },
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n",
- "format": null,
- "instant": false,
- "interval": "",
- "intervalFactor": null,
- "legendFormat": "",
- "legendLink": null,
- "step": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Reads - 5xx",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "decimals": 1,
- "noValue": "",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "orange",
- "value": 1
- },
- {
- "color": "red",
- "value": 2.5
- }
- ]
- },
- "unit": "s"
- },
- "overrides": [ ]
- },
- "fill": 1,
- "gridPos": {
- "h": 4,
- "w": 8,
- "x": 16,
- "y": 4
- },
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}))\n",
- "format": null,
- "instant": false,
- "interval": "",
- "intervalFactor": null,
- "legendFormat": "",
- "legendLink": null,
- "step": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Reads 99th latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "decimals": 0,
- "noValue": "All healthy",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "orange",
- "value": 1
- },
- {
- "color": "red",
- "value": 2
- }
- ]
- },
- "unit": "short"
- },
- "overrides": [ ]
- },
- "fill": 1,
- "gridPos": {
- "h": 3,
- "w": 10,
- "x": 0,
- "y": 13
- },
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "options": {
- "text": {
- "titleSize": 14,
- "valueSize": 14
- },
- "textMode": "value_and_name"
- },
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "kube_deployment_status_replicas_unavailable{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n> 0\n",
- "format": null,
- "instant": true,
- "interval": "",
- "intervalFactor": null,
- "legendFormat": "{{deployment}}",
- "legendLink": null,
- "step": null
- },
- {
- "expr": "kube_statefulset_status_replicas_current{cluster=~\"$cluster\", namespace=~\"$namespace\"} -\nkube_statefulset_status_replicas_ready {cluster=~\"$cluster\", namespace=~\"$namespace\"}\n> 0\n",
- "format": null,
- "instant": true,
- "interval": "",
- "intervalFactor": null,
- "legendFormat": "{{statefulset}}",
- "legendLink": null,
- "step": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Unhealthy pods",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "stat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "overrides": [
- {
- "matcher": {
- "id": "byRegexp",
- "options": "r.*"
- },
- "properties": [
- {
- "id": "custom.align",
- "value": "center"
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 6,
- "x": 10,
- "y": 8
- },
- "id": 11,
- "targets": [
- {
- "expr": "count by(container, version) (\n label_replace(\n kube_pod_container_info{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"version\", \"$1\", \"image\", \".*:(.*)\"\n )\n)\n",
- "instant": true,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "title": "Pods count per version",
- "transformations": [
- {
- "id": "labelsToFields",
- "options": {
- "valueLabel": "version"
- }
- },
- {
- "id": "organize",
- "options": {
- "excludeByName": {
- "Time": true
- },
- "indexByName": {
- "Time": 0,
- "container": 1
- }
- }
- },
- {
- "id": "sortBy",
- "options": {
- "fields": { },
- "sort": [
- {
- "field": "container"
- }
- ]
- }
- }
- ],
- "type": "table"
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "fillOpacity": 10
- },
- "unit": "percentunit"
- }
- },
- "gridPos": {
- "h": 8,
- "w": 8,
- "x": 16,
- "y": 8
- },
- "id": 12,
- "links": [ ],
- "options": {
- "legend": {
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\"}))[1h:])\n)\n",
- "format": "time_series",
- "legendFormat": "writes",
- "legendLink": null
- },
- {
- "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}))[1h:])\n)\n",
- "format": "time_series",
- "legendFormat": "reads",
- "legendLink": null
- }
- ],
- "title": "Latency vs 24h ago",
- "type": "timeseries"
- }
- ],
- "refresh": "10s",
- "rows": null,
- "schemaVersion": 27,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".*",
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": false,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": null,
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": false,
- "label": "namespace",
- "multi": false,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Rollout progress",
- "uid": "7f0b5567d543a1698e695b530eb7f5de",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-ruler-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-ruler.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "100px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "format": "short",
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(cortex_ruler_managers_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"})",
- "format": "time_series",
- "instant": true,
- "refId": "A"
- }
- ],
- "thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
- "title": "Active configurations",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "format": "short",
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"})",
- "format": "time_series",
- "instant": true,
- "refId": "A"
- }
- ],
- "thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
- "title": "Total rules",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Read from ingesters - QPS\nNote: Even while operating in Remote ruler mode you will still see values for this panel.\n\nThis is because the metrics are inclusive of intermediate services and are showing the requests that ultimately reach the ingesters.\n\nFor a more detailed view of the read path when using remote ruler mode, see the Remote ruler reads dashboard.\n\n",
- "fill": 1,
- "format": "reqps",
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*|ruler-querier.*))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))",
- "format": "time_series",
- "instant": true,
- "refId": "A"
- }
- ],
- "thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
- "title": "Read from ingesters - QPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "format": "reqps",
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))",
- "format": "time_series",
- "instant": true,
- "refId": "A"
- }
- ],
- "thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
- "title": "Write to ingesters - QPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": false,
- "title": "Headlines",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "failed": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_prometheus_rule_evaluations_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "success",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_prometheus_rule_group_iterations_missed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "missed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Evaluations per second",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n /\nsum (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "average",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Rule evaluations global",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "QPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Writes (ingesters)",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*|ruler-querier.*))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "QPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*|ruler-querier.*))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*|ruler-querier.*))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*|ruler-querier.*))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*|ruler-querier.*))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Reads (ingesters)",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", kv_name=~\"ruler\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", kv_name=~\"ruler\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", kv_name=~\"ruler\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", kv_name=~\"ruler\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", kv_name=~\"ruler\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Ruler - key-value store for rulers ring",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Number of store-gateways hit per query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Refetches of missing blocks per query",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {
- "Failures / sec": "#E24D42"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Consistency checks failed\nRate of queries that had to run with consistency checks and those checks failed. A failed consistency check means that some of at least one block which had to be queried wasn't present in any of the store-gateways.\n\n",
- "fill": 1,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) / sum(rate(cortex_querier_blocks_consistency_checks_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Failures / sec",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Consistency checks failed",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": 1,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Ruler - blocks storage",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "noValue": 0,
- "unit": "short"
- }
- },
- "id": 16,
- "links": [ ],
- "options": {
- "legend": {
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "span": 4,
- "targets": [
- {
- "expr": "sum by(user) (rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_notifications_sent_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]) > 0)\n> 0\n",
- "format": "time_series",
- "legendFormat": "{{ user }}",
- "legendLink": null
- }
- ],
- "title": "Delivery errors",
- "type": "timeseries"
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "noValue": 0,
- "unit": "percentunit"
- }
- },
- "fill": 1,
- "id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(user) (rate(cortex_prometheus_notifications_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_notifications_queue_capacity{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) > 0\n",
- "format": "time_series",
- "legendFormat": "{{ user }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Queue length",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "noValue": 0,
- "unit": "short"
- }
- },
- "id": 18,
- "links": [ ],
- "options": {
- "legend": {
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "span": 4,
- "targets": [
- {
- "expr": "sum by (user) (increase(cortex_prometheus_notifications_dropped_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) > 0\n",
- "format": "time_series",
- "legendFormat": "{{ user }}",
- "legendLink": null
- }
- ],
- "title": "Dropped",
- "type": "timeseries"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Notifications",
- "titleSize": "h6"
- },
- {
- "collapse": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(user) (rate(cortex_prometheus_rule_group_iterations_missed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) > 0",
- "format": "time_series",
- "legendFormat": "{{ user }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Missed iterations",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 20,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "rate(cortex_prometheus_rule_group_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])\n /\nrate(cortex_prometheus_rule_group_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])\n",
- "format": "time_series",
- "legendFormat": "{{ user }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(rule_group) (rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) > 0",
- "format": "time_series",
- "legendFormat": "{{ rule_group }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Failures",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Group evaluations",
- "titleSize": "h6"
- },
- {
- "collapse": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 22,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
- "format": "time_series",
- "legendFormat": "{{ user }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Rule evaluation per user",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 23,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{operation}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Operations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "max": 1,
- "min": 0,
- "noValue": "0",
- "unit": "percentunit"
- }
- },
- "id": 24,
- "links": [ ],
- "options": {
- "legend": {
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "span": 3,
- "targets": [
- {
- "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\"}[$__rate_interval])) >= 0",
- "format": "time_series",
- "legendFormat": "{{operation}}",
- "legendLink": null
- }
- ],
- "title": "Error rate",
- "type": "timeseries"
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 25,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Attributes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 26,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Exists",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Ruler configuration object store (ruler accesses)",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 27,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Get",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 28,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: GetRange",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 29,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Upload",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 30,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency of op: Delete",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": true,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "namespace",
- "multi": true,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Ruler",
- "uid": "631e15d5d85afb2ca8e35d62984eeaa0",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-scaling-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-scaling.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "200px",
- "panels": [
- {
- "id": 1,
- "options": {
- "content": "This dashboard identifies scaling-related issues by suggesting services that you might want to scale up.\nThe table that follows contains a suggested number of replicas and the reason why.\nIf the system is failing and depending on the reason, try scaling up to the specified number.\nThe specified numbers are intended as helpful guidelines when things go wrong, rather than prescriptive guidelines.\n\nReasons:\n- **sample_rate**: There are not enough replicas to handle the\n sample rate. Applies to distributor and ingesters.\n- **active_series**: There are not enough replicas\n to handle the number of active series. Applies to ingesters.\n- **cpu_usage**: There are not enough replicas\n based on the CPU usage of the jobs vs the resource requests.\n Applies to all jobs.\n- **memory_usage**: There are not enough replicas based on the memory\n usage vs the resource requests. Applies to all jobs.\n- **active_series_limits**: There are not enough replicas to hold 60% of the\n sum of all the per tenant series limits.\n- **sample_rate_limits**: There are not enough replicas to handle 60% of the\n sum of all the per tenant rate limits.\n",
- "mode": "markdown"
- },
- "span": 12,
- "title": "",
- "type": "text"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Service scaling",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "400px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "sort": {
- "col": 0,
- "desc": false
- },
- "spaceLength": 10,
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "styles": [
- {
- "alias": "Time",
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "pattern": "Time",
- "type": "hidden"
- },
- {
- "alias": "Required Replicas",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 0,
- "link": false,
- "linkTargetBlank": false,
- "linkTooltip": "Drill down",
- "linkUrl": "",
- "pattern": "Value",
- "thresholds": [ ],
- "type": "number",
- "unit": "short"
- },
- {
- "alias": "Cluster",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "link": false,
- "linkTargetBlank": false,
- "linkTooltip": "Drill down",
- "linkUrl": "",
- "pattern": "__name__",
- "thresholds": [ ],
- "type": "hidden",
- "unit": "short"
- },
- {
- "alias": "Cluster",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "link": false,
- "linkTargetBlank": false,
- "linkTooltip": "Drill down",
- "linkUrl": "",
- "pattern": "cluster",
- "thresholds": [ ],
- "type": "number",
- "unit": "short"
- },
- {
- "alias": "Service",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "link": false,
- "linkTargetBlank": false,
- "linkTooltip": "Drill down",
- "linkUrl": "",
- "pattern": "deployment",
- "thresholds": [ ],
- "type": "number",
- "unit": "short"
- },
- {
- "alias": "Namespace",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "link": false,
- "linkTargetBlank": false,
- "linkTooltip": "Drill down",
- "linkUrl": "",
- "pattern": "namespace",
- "thresholds": [ ],
- "type": "number",
- "unit": "short"
- },
- {
- "alias": "Reason",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "link": false,
- "linkTargetBlank": false,
- "linkTooltip": "Drill down",
- "linkUrl": "",
- "pattern": "reason",
- "thresholds": [ ],
- "type": "number",
- "unit": "short"
- },
- {
- "alias": "",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "pattern": "/.*/",
- "thresholds": [ ],
- "type": "string",
- "unit": "short"
- }
- ],
- "targets": [
- {
- "expr": "sort_desc(\n cluster_namespace_deployment_reason:required_replicas:count{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n > ignoring(reason) group_left\n cluster_namespace_deployment:actual_replicas:count{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\n",
- "format": "table",
- "instant": true,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Workload-based scaling",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "transform": "table",
- "type": "table",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Scaling",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": true,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "namespace",
- "multi": true,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Scaling",
- "uid": "64bbad83507b7289b514725658e10352",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-slow-queries-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-slow-queries.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "",
- "rows": [
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${loki_datasource}",
- "fill": 1,
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(response_time) [$__auto]) by ()",
- "format": "time_series",
- "legendFormat": "p99",
- "legendLink": null
- },
- {
- "expr": "quantile_over_time(0.5, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(response_time) [$__auto]) by ()",
- "format": "time_series",
- "legendFormat": "p50",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Response time",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${loki_datasource}",
- "fill": 1,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_series_count[$__auto]) by ()",
- "format": "time_series",
- "legendFormat": "p99",
- "legendLink": null
- },
- {
- "expr": "quantile_over_time(0.5, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_series_count[$__auto]) by ()",
- "format": "time_series",
- "legendFormat": "p50",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Fetched series",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${loki_datasource}",
- "fill": 1,
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_chunk_bytes[$__auto]) by ()",
- "format": "time_series",
- "legendFormat": "p99",
- "legendLink": null
- },
- {
- "expr": "quantile_over_time(0.5, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_chunk_bytes[$__auto]) by ()",
- "format": "time_series",
- "legendFormat": "p50",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Fetched chunks",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${loki_datasource}",
- "fill": 1,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap response_size_bytes[$__auto]) by ()",
- "format": "time_series",
- "legendFormat": "p99",
- "legendLink": null
- },
- {
- "expr": "quantile_over_time(0.5, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap response_size_bytes[$__auto]) by ()",
- "format": "time_series",
- "legendFormat": "p50",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Response size",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${loki_datasource}",
- "fill": 1,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(length) [$__auto]) by ()",
- "format": "time_series",
- "legendFormat": "p99",
- "legendLink": null
- },
- {
- "expr": "quantile_over_time(0.5, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(length) [$__auto]) by ()",
- "format": "time_series",
- "legendFormat": "p50",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Time span",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Accross tenants",
- "titleSize": "h6"
- },
- {
- "collapse": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${loki_datasource}",
- "fill": 1,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(response_time) [$__auto]) by (user))",
- "format": "time_series",
- "legendFormat": "{{user}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "P99 response time",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${loki_datasource}",
- "fill": 1,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_series_count[$__auto]) by (user))",
- "format": "time_series",
- "legendFormat": "{{user}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "P99 fetched series",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${loki_datasource}",
- "fill": 1,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_chunk_bytes[$__auto]) by (user))",
- "format": "time_series",
- "legendFormat": "{{user}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "P99 fetched chunks",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${loki_datasource}",
- "fill": 1,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap response_size_bytes[$__auto]) by (user))",
- "format": "time_series",
- "legendFormat": "{{user}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "P99 response size",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${loki_datasource}",
- "fill": 1,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(length) [$__auto]) by (user))",
- "format": "time_series",
- "legendFormat": "{{user}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "P99 time span",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Top 10 tenants",
- "titleSize": "h6"
- },
- {
- "collapse": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${loki_datasource}",
- "fill": 1,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(response_time) [$__auto]) by (user_agent))",
- "format": "time_series",
- "legendFormat": "{{user_agent}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "P99 response time",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${loki_datasource}",
- "fill": 1,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_series_count[$__auto]) by (user_agent))",
- "format": "time_series",
- "legendFormat": "{{user_agent}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "P99 fetched series",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${loki_datasource}",
- "fill": 1,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_chunk_bytes[$__auto]) by (user_agent))",
- "format": "time_series",
- "legendFormat": "{{user_agent}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "P99 fetched chunks",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${loki_datasource}",
- "fill": 1,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap response_size_bytes[$__auto]) by (user_agent))",
- "format": "time_series",
- "legendFormat": "{{user_agent}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "P99 response size",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "${loki_datasource}",
- "fill": 1,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(length) [$__auto]) by (user_agent))",
- "format": "time_series",
- "legendFormat": "{{user_agent}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "P99 time span",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Top 10 User-Agents",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "datasource": "${loki_datasource}",
- "fieldConfig": {
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "fetched_chunk_bytes"
- },
- "properties": [
- {
- "id": "unit",
- "value": "bytes"
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "fetched_index_bytes"
- },
- "properties": [
- {
- "id": "unit",
- "value": "bytes"
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "response_size_bytes"
- },
- "properties": [
- {
- "id": "unit",
- "value": "bytes"
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "results_cache_hit_bytes"
- },
- "properties": [
- {
- "id": "unit",
- "value": "bytes"
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "results_cache_miss_bytes"
- },
- "properties": [
- {
- "id": "unit",
- "value": "bytes"
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "estimated_series_count"
- },
- "properties": [
- {
- "id": "unit",
- "value": "short"
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "fetched_chunks_count"
- },
- "properties": [
- {
- "id": "unit",
- "value": "short"
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "fetched_series_count"
- },
- "properties": [
- {
- "id": "unit",
- "value": "short"
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "Time span"
- },
- "properties": [
- {
- "id": "unit",
- "value": "s"
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "Duration"
- },
- "properties": [
- {
- "id": "unit",
- "value": "s"
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "Step"
- },
- "properties": [
- {
- "id": "unit",
- "value": "s"
- }
- ]
- },
- {
- "matcher": {
- "id": "byName",
- "options": "queue_time_seconds"
- },
- "properties": [
- {
- "id": "unit",
- "value": "s"
- }
- ]
- }
- ]
- },
- "height": "500px",
- "id": 16,
- "span": 12,
- "targets": [
- {
- "expr": "{cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | label_format response_time_seconds=\"{{ duration .response_time }}\",param_step_seconds=\"{{ div .param_step 1000 }}\",length_seconds=\"{{ duration .length }}\"",
- "instant": false,
- "legendFormat": "",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Slow queries",
- "transformations": [
- {
- "id": "extractFields",
- "options": {
- "source": "labels"
- }
- },
- {
- "id": "organize",
- "options": {
- "excludeByName": {
- "Line": true,
- "Time": true,
- "caller": true,
- "cluster": true,
- "component": true,
- "container": true,
- "gossip_ring_member": true,
- "host": true,
- "id": true,
- "job": true,
- "labels": true,
- "length": true,
- "level": true,
- "line": true,
- "method": true,
- "msg": true,
- "name": true,
- "namespace": true,
- "param_step": true,
- "path": true,
- "pod": true,
- "pod_template_hash": true,
- "query_wall_time_seconds": true,
- "response_time": true,
- "stream": true,
- "traceID": true,
- "tsNs": true
- },
- "indexByName": {
- "err": 10,
- "length_seconds": 3,
- "param_end": 5,
- "param_query": 8,
- "param_start": 4,
- "param_step_seconds": 7,
- "param_time": 6,
- "response_time_seconds": 9,
- "status": 1,
- "ts": 0,
- "user": 2
- },
- "renameByName": {
- "err": "Error",
- "length_seconds": "Time span",
- "param_end": "End",
- "param_query": "Query",
- "param_start": "Start",
- "param_step_seconds": "Step",
- "param_time": "Time (instant query)",
- "response_time_seconds": "Duration",
- "ts": "Completion date",
- "user": "Tenant ID"
- }
- }
- },
- {
- "id": "convertFieldType",
- "options": {
- "conversions": [
- {
- "destinationType": "number",
- "targetField": "estimated_series_count"
- },
- {
- "destinationType": "number",
- "targetField": "fetched_chunk_bytes"
- },
- {
- "destinationType": "number",
- "targetField": "fetched_chunks_count"
- },
- {
- "destinationType": "number",
- "targetField": "fetched_index_bytes"
- },
- {
- "destinationType": "number",
- "targetField": "fetched_series_count"
- },
- {
- "destinationType": "number",
- "targetField": "queue_time_seconds"
- },
- {
- "destinationType": "number",
- "targetField": "response_size_bytes"
- },
- {
- "destinationType": "number",
- "targetField": "results_cache_hit_bytes"
- },
- {
- "destinationType": "number",
- "targetField": "results_cache_miss_bytes"
- },
- {
- "destinationType": "number",
- "targetField": "sharded_queries"
- },
- {
- "destinationType": "number",
- "targetField": "split_queries"
- },
- {
- "destinationType": "number",
- "targetField": "Time span"
- },
- {
- "destinationType": "number",
- "targetField": "Duration"
- },
- {
- "destinationType": "number",
- "targetField": "Step"
- },
- {
- "destinationType": "number",
- "targetField": "queue_time_seconds"
- }
- ]
- }
- }
- ],
- "type": "table"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".*",
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": false,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": null,
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": false,
- "label": "namespace",
- "multi": false,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "hide": 0,
- "includeAll": false,
- "label": "Loki data source",
- "multi": false,
- "name": "loki_datasource",
- "query": "loki",
- "type": "datasource"
- },
- {
- "current": {
- "selected": true,
- "text": "5s",
- "value": "5s"
- },
- "hide": 0,
- "label": "Min duration",
- "name": "min_duration",
- "options": [
- {
- "selected": true,
- "text": "5s",
- "value": "5s"
- }
- ],
- "query": "5s",
- "type": "textbox"
- },
- {
- "current": {
- "selected": true,
- "text": ".*",
- "value": ".*"
- },
- "hide": 0,
- "label": "Tenant ID",
- "name": "tenant_id",
- "options": [
- {
- "selected": true,
- "text": ".*",
- "value": ".*"
- }
- ],
- "query": ".*",
- "type": "textbox"
- },
- {
- "current": {
- "selected": true,
- "text": ".*",
- "value": ".*"
- },
- "hide": 0,
- "label": "User-Agent HTTP Header",
- "name": "user_agent",
- "options": [
- {
- "selected": true,
- "text": ".*",
- "value": ".*"
- }
- ],
- "query": ".*",
- "type": "textbox"
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Slow queries",
- "uid": "6089e1ce1e678788f46312a0a1e647e6",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-tenants-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-tenants.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "25px",
- "panels": [
- {
- "content": "\n This dashboard shows various metrics detailed by tenant (user) selected above.\n
\n",
- "datasource": null,
- "description": "",
- "id": 1,
- "mode": "markdown",
- "span": 12,
- "title": "",
- "transparent": true,
- "type": "text"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": false,
- "title": "Tenants dashboard description",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### All series\nNumber of active and in-memory series per user, and active series matching custom trackers (in parenthesis).\nNote that these counts include all series regardless of the type of data (counter, gauge, native histogram, etc.).\nNote that active series matching custom trackers are included in the total active series count.\n\n",
- "fill": 1,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "limit",
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(\n (\n cortex_ingester_memory_series_created_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n - cortex_ingester_memory_series_removed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n )\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
- "format": "time_series",
- "legendFormat": "in-memory",
- "legendLink": null
- },
- {
- "expr": "max(cortex_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"max_global_series_per_user\", user=\"$user\"})\nor\nmax(cortex_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"max_global_series_per_user\"})\n",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "sum(\n cortex_ingester_active_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
- "format": "time_series",
- "legendFormat": "active",
- "legendLink": null
- },
- {
- "expr": "sum by (name) (\n cortex_ingester_active_series_custom_tracker{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n) > 0\n",
- "format": "time_series",
- "legendFormat": "active ({{ name }})",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "All series",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Native histogram series\nNumber of active native histogram series per user, and active native histogram series matching custom trackers (in parenthesis).\nNote that active series matching custom trackers are included in the total active series count.\n\n",
- "fill": 1,
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "limit",
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(\n cortex_ingester_active_native_histogram_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
- "format": "time_series",
- "legendFormat": "active",
- "legendLink": null
- },
- {
- "expr": "sum by (name) (\n cortex_ingester_active_native_histogram_series_custom_tracker{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n) > 0\n",
- "format": "time_series",
- "legendFormat": "active ({{ name }})",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Native histogram series",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Total number of buckets used by native histogram series\nTotal number of buckets in active native histogram series per user, and total active native histogram buckets matching custom trackers (in parenthesis).\n\n",
- "fill": 1,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "limit",
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(\n cortex_ingester_active_native_histogram_buckets{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
- "format": "time_series",
- "legendFormat": "buckets",
- "legendLink": null
- },
- {
- "expr": "sum by (name) (\n cortex_ingester_active_native_histogram_buckets_custom_tracker{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n) > 0\n",
- "format": "time_series",
- "legendFormat": "buckets ({{ name }})",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Total number of buckets used by native histogram series",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Active series and native histograms",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Series with exemplars\nNumber of series with exemplars currently in storage.\n\n",
- "fill": 1,
- "id": 5,
- "legend": {
- "show": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(\n cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
- "format": "time_series",
- "legendFormat": "series",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Series with exemplars",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Newest seen sample age\nThe age of the newest received sample seen in the distributors.\n\n",
- "fill": 1,
- "id": 6,
- "legend": {
- "show": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "time() - max(cortex_distributor_latest_seen_sample_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"} > 0)",
- "format": "time_series",
- "legendFormat": "age",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Newest seen sample age",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Oldest exemplar age\nThe age of the oldest exemplar stored in circular storage.\nUseful to check for what time range the current exemplar buffer limit allows.\nThis usually means the max age for all exemplars for a typical setup.\nThis is not true though if one of the series timestamp is in future compared to rest series.\n\n",
- "fill": 1,
- "id": 7,
- "legend": {
- "show": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "time() - min(cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"} > 0)",
- "format": "time_series",
- "legendFormat": "age",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Oldest exemplar age",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Samples and exemplars",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Distributor requests incoming rate\nThe rate of requests that have come in to the distributor, including rejected requests.\n\n",
- "fill": 1,
- "id": 8,
- "legend": {
- "show": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_distributor_requests_in_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "rate",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Distributor requests incoming rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Distributor requests received (accepted) rate\nThe rate of received requests, excluding rejected requests.\n\n",
- "fill": 1,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "limit",
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_distributor_received_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "rate",
- "legendLink": null
- },
- {
- "expr": "max(cortex_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"request_rate\", user=\"$user\"})\nor\nmax(cortex_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"request_rate\"})\n",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Distributor requests received (accepted) rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Distributor discarded requests rate\nThe rate of each request's discarding reason.\n\n",
- "fill": 1,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (reason) (rate(cortex_discarded_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{ reason }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Distributor discarded requests rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Distributor ingestion requests",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Distributor samples incoming rate\nThe rate of samples that have come in to the distributor, including rejected or deduped exemplars.\n\n",
- "fill": 1,
- "id": 11,
- "legend": {
- "show": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_distributor_samples_in_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "rate",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Distributor samples incoming rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Distributor samples received (accepted) rate\nThe rate of received samples, excluding rejected and deduped samples.\n\n",
- "fill": 1,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "limit",
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_distributor_received_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "rate",
- "legendLink": null
- },
- {
- "expr": "max(cortex_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"ingestion_rate\", user=\"$user\"})\nor\nmax(cortex_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"ingestion_rate\"})\n",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Distributor samples received (accepted) rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Distributor deduplicated/non-HA\nThe rate of deduplicated samples and the rate of received samples for a user that has HA tracking turned on, but the sample didn't contain both HA labels.\n\n",
- "fill": 1,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_distributor_deduped_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "deduplicated",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_distributor_non_ha_samples_received_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "non-HA",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Distributor deduplicated/non-HA",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Distributor and ingester discarded samples rate\nThe rate of each sample's discarding reason.\n\n",
- "fill": 1,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (reason) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{ reason }} (distributor)",
- "legendLink": null
- },
- {
- "expr": "sum by (reason) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{ reason }} (ingester)",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Distributor and ingester discarded samples rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Samples ingestion funnel",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Distributor exemplars incoming rate\nThe rate of exemplars that have come in to the distributor, including rejected or deduped exemplars.\n\n",
- "fill": 1,
- "id": 15,
- "legend": {
- "show": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_distributor_exemplars_in_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "rate",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Distributor exemplars incoming rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Distributor exemplars received (accepted) rate\nThe rate of received exemplars, excluding rejected and deduped exemplars.\nThis number can be sensibly lower than incoming rate because we dedupe the HA sent exemplars, and then reject based on time.\nSee discarded rate for reasons why exemplars are being discarded.\n\n",
- "fill": 1,
- "id": 16,
- "legend": {
- "show": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_distributor_received_exemplars_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "rate",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Distributor exemplars received (accepted) rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Distributor discarded exemplars rate\nThe rate of each exmplars' discarding reason.\n\n",
- "fill": 1,
- "id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (reason) (rate(cortex_discarded_exemplars_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{ reason }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Distributor discarded exemplars rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Ingester appended exemplars rate\nTotal number of exemplars appended in the ingesters.\nThis can be lower than ingested exemplars rate since TSDB does not append the same exemplar twice, and those can be frequent.\n\n",
- "fill": 1,
- "id": 18,
- "legend": {
- "show": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(\n rate(cortex_ingester_tsdb_exemplar_exemplars_appended_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval])\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
- "format": "time_series",
- "legendFormat": "rate",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Ingester appended exemplars rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Exemplars ingestion funnel",
- "titleSize": "h6"
- },
- {
- "collapse": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Symbol table size for loaded blocks\nSize of symbol table in memory for loaded blocks, averaged by ingester.\n\n",
- "fill": 1,
- "id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (job) (cortex_ingester_tsdb_symbol_table_size_bytes{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"})",
- "format": "time_series",
- "legendFormat": "{{ job }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Symbol table size for loaded blocks",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Space used by local blocks\nThe number of bytes that are currently used for local storage by all blocks.\n\n",
- "fill": 1,
- "id": 20,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (job) (cortex_ingester_tsdb_storage_blocks_bytes{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"})",
- "format": "time_series",
- "legendFormat": "{{ job }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Space used by local blocks",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Ingesters' storage",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Number of groups\nTotal number of rule groups for a tenant.\n\n",
- "fill": 1,
- "id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "limit",
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "count(sum by (rule_group) (cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}))",
- "format": "time_series",
- "legendFormat": "groups",
- "legendLink": null
- },
- {
- "expr": "max(cortex_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"ruler_max_rule_groups_per_tenant\", user=\"$user\"})\nor\nmax(cortex_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"ruler_max_rule_groups_per_tenant\"})\n",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Number of groups",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Number of rules\nTotal number of rules for a tenant.\n\n",
- "fill": 1,
- "id": 22,
- "legend": {
- "show": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"})",
- "format": "time_series",
- "legendFormat": "rules",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Number of rules",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 23,
- "legend": {
- "show": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_prometheus_rule_evaluations_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "rate",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Total evaluations rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 24,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (rule_group) (rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval])) > 0",
- "format": "time_series",
- "legendFormat": "{{ rule_group }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Failed evaluations rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Rules",
- "titleSize": "h6"
- },
- {
- "collapse": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 25,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "sort": {
- "col": 2,
- "desc": true
- },
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "styles": [
- {
- "alias": "Time",
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "pattern": "Time",
- "type": "hidden"
- },
- {
- "alias": "rules",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "link": false,
- "linkTargetBlank": false,
- "linkTooltip": "Drill down",
- "linkUrl": "",
- "pattern": "Value #A",
- "thresholds": [ ],
- "type": "number",
- "unit": "short"
- },
- {
- "alias": "",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "pattern": "/.*/",
- "thresholds": [ ],
- "type": "string",
- "unit": "short"
- }
- ],
- "targets": [
- {
- "expr": "topk($limit, sum by (rule_group) (cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}))",
- "format": "table",
- "instant": true,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Top $limit biggest groups",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "transform": "table",
- "type": "table",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 26,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "sort": {
- "col": 2,
- "desc": true
- },
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "styles": [
- {
- "alias": "Time",
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "pattern": "Time",
- "type": "hidden"
- },
- {
- "alias": "seconds",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "link": false,
- "linkTargetBlank": false,
- "linkTooltip": "Drill down",
- "linkUrl": "",
- "pattern": "Value #A",
- "thresholds": [ ],
- "type": "number",
- "unit": "short"
- },
- {
- "alias": "",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "pattern": "/.*/",
- "thresholds": [ ],
- "type": "string",
- "unit": "short"
- }
- ],
- "targets": [
- {
- "expr": "topk($limit, sum by (rule_group) (cortex_prometheus_rule_group_last_duration_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}))",
- "format": "table",
- "instant": true,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Top $limit slowest groups (last evaluation)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "transform": "table",
- "type": "table",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Top rules",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 27,
- "legend": {
- "show": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_prometheus_notifications_sent_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "rate",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Sent notifications rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {
- "rate": "#E24D42"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 28,
- "legend": {
- "show": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "rate",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Failed notifications rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Notifications",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 29,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (user) (cortex_alertmanager_alerts{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"})",
- "format": "time_series",
- "legendFormat": "alerts",
- "legendLink": null
- },
- {
- "expr": "sum by (user) (cortex_alertmanager_silences{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"})",
- "format": "time_series",
- "legendFormat": "silences",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Alerts",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 30,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "(\nsum(rate(cortex_alertmanager_notifications_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval]))\n-\non() (sum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval])) or on () vector(0))\n) > 0\n",
- "format": "time_series",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "NPS",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 31,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "(\nsum(rate(cortex_alertmanager_notifications_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval])) by(integration)\n-\n(sum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval])) by(integration) or\n (sum(rate(cortex_alertmanager_notifications_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval])) by(integration) * 0)\n)) > 0\n",
- "format": "time_series",
- "legendFormat": "success - {{ integration }}",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval])) by(integration)",
- "format": "time_series",
- "legendFormat": "failed - {{ integration }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "NPS by integration",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Alertmanager",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 32,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_query_frontend_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", user=\"$user\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Queries / Sec",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Rate of Read Requests - query-frontend",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 33,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(cortex_query_scheduler_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\", user=\"$user\"})",
- "format": "time_series",
- "legendFormat": "Queue Length",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Number of Queries Queued - query-scheduler",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Read Path - Queries (User)",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 34,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_query_frontend_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", user=\"$user\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Queries / Sec",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Rate of Read Requests - ruler-query-frontend",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 35,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(cortex_query_scheduler_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\", user=\"$user\"})",
- "format": "time_series",
- "legendFormat": "Queue Length",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Number of Queries Queued - ruler-query-scheduler",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Read Path - Queries (Ruler)",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": true,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "namespace",
- "multi": true,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": null,
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": false,
- "label": "user",
- "multi": false,
- "name": "user",
- "options": [ ],
- "query": "label_values(cortex_ingester_active_series{cluster=~\"$cluster\", namespace=~\"$namespace\"}, user)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "current": {
- "selected": true,
- "text": "10",
- "value": "10"
- },
- "hide": 0,
- "includeAll": false,
- "multi": false,
- "name": "limit",
- "options": [
- {
- "selected": true,
- "text": "10",
- "value": "10"
- },
- {
- "selected": false,
- "text": "50",
- "value": "50"
- },
- {
- "selected": false,
- "text": "100",
- "value": "100"
- },
- {
- "selected": false,
- "text": "500",
- "value": "500"
- },
- {
- "selected": false,
- "text": "1000",
- "value": "1000"
- }
- ],
- "type": "custom"
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Tenants",
- "uid": "35fa247ce651ba189debf33d7ae41611",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-top-tenants-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-top-tenants.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "25px",
- "panels": [
- {
- "content": "\n This dashboard shows the top tenants based on multiple selection criterias.\n Rows are collapsed by default to avoid querying all of them.\n Use the templating variable \"limit\" above to select the amount of users to be shown.\n
\n",
- "datasource": null,
- "description": "",
- "id": 1,
- "mode": "markdown",
- "span": 12,
- "title": "",
- "transparent": true,
- "type": "text"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": false,
- "title": "Top tenants dashboard description",
- "titleSize": "h6"
- },
- {
- "collapse": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "sort": {
- "col": 2,
- "desc": true
- },
- "spaceLength": 10,
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "styles": [
- {
- "alias": "Time",
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "pattern": "Time",
- "type": "hidden"
- },
- {
- "alias": "series",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "link": false,
- "linkTargetBlank": false,
- "linkTooltip": "Drill down",
- "linkUrl": "",
- "pattern": "Value #A",
- "thresholds": [ ],
- "type": "number",
- "unit": "short"
- },
- {
- "alias": "",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "pattern": "/.*/",
- "thresholds": [ ],
- "type": "string",
- "unit": "short"
- }
- ],
- "targets": [
- {
- "expr": "topk($limit,\n sum by (user) (\n cortex_ingester_active_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n )\n)\n",
- "format": "table",
- "instant": true,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Top $limit users by active series",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "transform": "table",
- "type": "table",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "By active series",
- "titleSize": "h6"
- },
- {
- "collapse": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "sort": {
- "col": 2,
- "desc": true
- },
- "spaceLength": 10,
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "styles": [
- {
- "alias": "Time",
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "pattern": "Time",
- "type": "hidden"
- },
- {
- "alias": "series",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "link": false,
- "linkTargetBlank": false,
- "linkTooltip": "Drill down",
- "linkUrl": "",
- "pattern": "Value #A",
- "thresholds": [ ],
- "type": "number",
- "unit": "short"
- },
- {
- "alias": "",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "pattern": "/.*/",
- "thresholds": [ ],
- "type": "string",
- "unit": "short"
- }
- ],
- "targets": [
- {
- "expr": "topk($limit, sum by (user) (\n (\n sum by (user, cluster, namespace) (cortex_ingester_memory_series_created_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} )\n -\n sum by (user, cluster, namespace) (cortex_ingester_memory_series_removed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} )\n )\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"} )\n)\n)",
- "format": "table",
- "instant": true,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Top $limit users by in-memory series (series created - series removed)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "transform": "table",
- "type": "table",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "By in-memory series",
- "titleSize": "h6"
- },
- {
- "collapse": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (user) (\n (\n sum by (user, cluster, namespace) (cortex_ingester_memory_series_created_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} )\n -\n sum by (user, cluster, namespace) (cortex_ingester_memory_series_removed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} )\n )\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"} )\n)\n\nand\ntopk($limit, sum by (user) (\n (\n sum by (user, cluster, namespace) (cortex_ingester_memory_series_created_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} @ end())\n -\n sum by (user, cluster, namespace) (cortex_ingester_memory_series_removed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} @ end())\n )\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"} @ end())\n)\n - sum by (user) (\n (\n sum by (user, cluster, namespace) (cortex_ingester_memory_series_created_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} @ start())\n -\n sum by (user, cluster, namespace) (cortex_ingester_memory_series_removed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} @ start())\n )\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"} @ start())\n)\n)\n",
- "format": "time_series",
- "legendFormat": "{{ user }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Top $limit users by in-memory series (series created - series removed) that grew the most between query range start and query range end",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "By in-memory series growth",
- "titleSize": "h6"
- },
- {
- "collapse": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "sort": {
- "col": 2,
- "desc": true
- },
- "spaceLength": 10,
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "styles": [
- {
- "alias": "Time",
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "pattern": "Time",
- "type": "hidden"
- },
- {
- "alias": "samples/s",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "link": false,
- "linkTargetBlank": false,
- "linkTooltip": "Drill down",
- "linkUrl": "",
- "pattern": "Value #A",
- "thresholds": [ ],
- "type": "number",
- "unit": "short"
- },
- {
- "alias": "",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "pattern": "/.*/",
- "thresholds": [ ],
- "type": "string",
- "unit": "short"
- }
- ],
- "targets": [
- {
- "expr": "topk($limit, sum by (user) (rate(cortex_distributor_received_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}[5m])))",
- "format": "table",
- "instant": true,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Top $limit users by received samples rate in last 5m",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "transform": "table",
- "type": "table",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "By samples rate",
- "titleSize": "h6"
- },
- {
- "collapse": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (user) (rate(cortex_distributor_received_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))\nand\ntopk($limit,\n sum by (user) (rate(cortex_distributor_received_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval] @ end()))\n -\n sum by (user) (rate(cortex_distributor_received_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval] @ start()))\n)\n",
- "format": "time_series",
- "legendFormat": "{{ user }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Top $limit users by received samples rate that grew the most between query range start and query range end",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "By samples rate growth",
- "titleSize": "h6"
- },
- {
- "collapse": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "sort": {
- "col": 2,
- "desc": true
- },
- "spaceLength": 10,
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "styles": [
- {
- "alias": "Time",
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "pattern": "Time",
- "type": "hidden"
- },
- {
- "alias": "samples/s",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "link": false,
- "linkTargetBlank": false,
- "linkTooltip": "Drill down",
- "linkUrl": "",
- "pattern": "Value #A",
- "thresholds": [ ],
- "type": "number",
- "unit": "short"
- },
- {
- "alias": "",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "pattern": "/.*/",
- "thresholds": [ ],
- "type": "string",
- "unit": "short"
- }
- ],
- "targets": [
- {
- "expr": "topk($limit, sum by (user) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*|distributor.*|cortex|mimir|mimir-write.*))\"}[5m])))",
- "format": "table",
- "instant": true,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Top $limit users by discarded samples rate in last 5m",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "transform": "table",
- "type": "table",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "By discarded samples rate",
- "titleSize": "h6"
- },
- {
- "collapse": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (user) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*|distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))\nand\ntopk($limit,\n sum by (user) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*|distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval] @ end()))\n -\n sum by (user) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*|distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval] @ start()))\n)\n",
- "format": "time_series",
- "legendFormat": "{{ user }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Top $limit users by discarded samples rate that grew the most between query range start and query range end",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "By discarded samples rate growth",
- "titleSize": "h6"
- },
- {
- "collapse": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "sort": {
- "col": 2,
- "desc": true
- },
- "spaceLength": 10,
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "styles": [
- {
- "alias": "Time",
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "pattern": "Time",
- "type": "hidden"
- },
- {
- "alias": "series",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "link": false,
- "linkTargetBlank": false,
- "linkTooltip": "Drill down",
- "linkUrl": "",
- "pattern": "Value #A",
- "thresholds": [ ],
- "type": "number",
- "unit": "short"
- },
- {
- "alias": "",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "pattern": "/.*/",
- "thresholds": [ ],
- "type": "string",
- "unit": "short"
- }
- ],
- "targets": [
- {
- "expr": "topk($limit,\n sum by (user) (\n cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n )\n)\n",
- "format": "table",
- "instant": true,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Top $limit users by series with exemplars",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "transform": "table",
- "type": "table",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "By series with exemplars",
- "titleSize": "h6"
- },
- {
- "collapse": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "sort": {
- "col": 2,
- "desc": true
- },
- "spaceLength": 10,
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "styles": [
- {
- "alias": "Time",
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "pattern": "Time",
- "type": "hidden"
- },
- {
- "alias": "exemplars/s",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "link": false,
- "linkTargetBlank": false,
- "linkTooltip": "Drill down",
- "linkUrl": "",
- "pattern": "Value #A",
- "thresholds": [ ],
- "type": "number",
- "unit": "short"
- },
- {
- "alias": "",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "pattern": "/.*/",
- "thresholds": [ ],
- "type": "string",
- "unit": "short"
- }
- ],
- "targets": [
- {
- "expr": "topk($limit, sum by (user) (rate(cortex_distributor_received_exemplars_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}[5m])))",
- "format": "table",
- "instant": true,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Top $limit users by received exemplars rate in last 5m",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "transform": "table",
- "type": "table",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "By exemplars rate",
- "titleSize": "h6"
- },
- {
- "collapse": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "sort": {
- "col": 3,
- "desc": true
- },
- "spaceLength": 10,
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "styles": [
- {
- "alias": "Time",
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "pattern": "Time",
- "type": "hidden"
- },
- {
- "alias": "rules",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "link": false,
- "linkTargetBlank": false,
- "linkTooltip": "Drill down",
- "linkUrl": "",
- "pattern": "Value #A",
- "thresholds": [ ],
- "type": "number",
- "unit": "short"
- },
- {
- "alias": "",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "pattern": "/.*/",
- "thresholds": [ ],
- "type": "string",
- "unit": "short"
- }
- ],
- "targets": [
- {
- "expr": "topk($limit, sum by (rule_group, user) (cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}))",
- "format": "table",
- "instant": true,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Top $limit biggest groups",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "transform": "table",
- "type": "table",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "By rule group size",
- "titleSize": "h6"
- },
- {
- "collapse": true,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "sort": {
- "col": 3,
- "desc": true
- },
- "spaceLength": 10,
- "span": 12,
- "stack": false,
- "steppedLine": false,
- "styles": [
- {
- "alias": "Time",
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "pattern": "Time",
- "type": "hidden"
- },
- {
- "alias": "seconds",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "link": false,
- "linkTargetBlank": false,
- "linkTooltip": "Drill down",
- "linkUrl": "",
- "pattern": "Value #A",
- "thresholds": [ ],
- "type": "number",
- "unit": "short"
- },
- {
- "alias": "",
- "colorMode": null,
- "colors": [ ],
- "dateFormat": "YYYY-MM-DD HH:mm:ss",
- "decimals": 2,
- "pattern": "/.*/",
- "thresholds": [ ],
- "type": "string",
- "unit": "short"
- }
- ],
- "targets": [
- {
- "expr": "topk($limit, sum by (rule_group, user) (cortex_prometheus_rule_group_last_duration_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}))",
- "format": "table",
- "instant": true,
- "legendFormat": "",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Top $limit slowest groups (last evaluation)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "transform": "table",
- "type": "table",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "By rule group evaluation time",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": true,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "namespace",
- "multi": true,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "current": {
- "selected": true,
- "text": "10",
- "value": "10"
- },
- "hide": 0,
- "includeAll": false,
- "multi": false,
- "name": "limit",
- "options": [
- {
- "selected": true,
- "text": "10",
- "value": "10"
- },
- {
- "selected": false,
- "text": "50",
- "value": "50"
- },
- {
- "selected": false,
- "text": "100",
- "value": "100"
- }
- ],
- "type": "custom"
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Top tenants",
- "uid": "bc6e12d4fe540e4a1785b9d3ca0ffdd9",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-writes-networking-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-writes-networking.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Receive bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Transmit bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"})",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"})",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Inflight requests (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}))",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}))",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- },
- {
- "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "TCP connections (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Summary",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Receive bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Transmit bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"})",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"})",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Inflight requests (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"}))",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"}))",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- },
- {
- "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "TCP connections (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Distributor",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Receive bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Transmit bandwidth",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"})",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"})",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Inflight requests (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"}))",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- },
- {
- "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"}))",
- "format": "time_series",
- "legendFormat": "highest",
- "legendLink": null
- },
- {
- "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "TCP connections (per pod)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Ingester",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".*",
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": false,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": null,
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": false,
- "label": "namespace",
- "multi": false,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Writes networking",
- "uid": "978c1cb452585c96697a238eaac7fe2d",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-writes-resources-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-writes-resources.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 1,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (workingset)",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (go heap inuse)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Summary",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\",resource=\"cpu\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"} > 0)",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\",resource=\"memory\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (workingset)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (go heap inuse)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Distributor",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (cortex_ingester_memory_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "In-memory series",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\",resource=\"cpu\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "CPU",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Ingester",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 9,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_rss{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} > 0)",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\",resource=\"memory\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (RSS)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "request",
- "color": "#FFC000",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "limit",
- "color": "#E02F44",
- "dashLength": 5,
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- },
- {
- "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} > 0)",
- "format": "time_series",
- "legendFormat": "limit",
- "legendLink": null
- },
- {
- "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\",resource=\"memory\"})",
- "format": "time_series",
- "legendFormat": "request",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (workingset)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
- "format": "time_series",
- "legendFormat": "{{pod}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Memory (go heap inuse)",
- "tooltip": {
- "sort": 2
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "bytes",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 12,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"ingester\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
- "format": "time_series",
- "legendFormat": "{{pod}} - {{device}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Disk writes",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n container=~\"ingester\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n",
- "format": "time_series",
- "legendFormat": "{{pod}} - {{device}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Disk reads",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "Bps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 0,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{\n cluster=~\"$cluster\", namespace=~\"$namespace\",\n label_name=~\"(ingester).*\"\n }\n)\n",
- "format": "time_series",
- "legendFormat": "{{persistentvolumeclaim}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Disk space utilization",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "percentunit",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".*",
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": false,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": null,
- "current": {
- "text": "prod",
- "value": "prod"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": false,
- "label": "namespace",
- "multi": false,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Writes resources",
- "uid": "bc9160e50b52e89e0e49c840fea3d379",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: mimir-writes-dashboard
- namespace: default
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- grafana_dashboard: "1"
- annotations:
- k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
-data:
- mimir-writes.json: |-
- {
- "__requires": [
- {
- "id": "grafana",
- "name": "Grafana",
- "type": "grafana",
- "version": "8.0.0"
- }
- ],
- "annotations": {
- "list": [ ]
- },
- "editable": true,
- "gnetId": null,
- "graphTooltip": 1,
- "hideControls": false,
- "links": [
- {
- "asDropdown": true,
- "icon": "external link",
- "includeVars": true,
- "keepTime": true,
- "tags": [
- "mimir"
- ],
- "targetBlank": false,
- "title": "Mimir dashboards",
- "type": "dashboards"
- }
- ],
- "refresh": "10s",
- "rows": [
- {
- "collapse": false,
- "height": "125px",
- "panels": [
- {
- "content": "\n This dashboard shows various health metrics for the write path.\n It is broken into sections for each service on the write path,\n and organized by the order in which the write request flows.\n
\n Incoming metrics data travels from the gateway → distributor → ingester.\n
\n For each service, there are 3 panels showing\n (1) requests per second to that service,\n (2) average, median, and p99 latency of requests to that service, and\n (3) p99 latency of requests to each instance of that service.\n
\n\n It also includes metrics for the key-value (KV) stores used to manage\n the high-availability tracker and the ingesters.\n
\n",
- "datasource": null,
- "description": "",
- "id": 1,
- "mode": "markdown",
- "span": 12,
- "title": "",
- "transparent": true,
- "type": "text"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": false,
- "title": "Writes dashboard description",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "100px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "format": "short",
- "id": 2,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(cluster_namespace_job:cortex_distributor_received_samples:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})",
- "format": "time_series",
- "instant": true,
- "refId": "A"
- }
- ],
- "thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
- "title": "Samples / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Exemplars / sec\nThe total number of received exemplars by the distributors, excluding rejected and deduped exemplars, but not necessarily ingested by the ingesters.\n\n",
- "fill": 1,
- "format": "short",
- "id": 3,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(cluster_namespace_job:cortex_distributor_received_exemplars:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})",
- "format": "time_series",
- "instant": true,
- "refId": "A"
- }
- ],
- "thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
- "title": "Exemplars / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### In-memory series\nThe number of series not yet flushed to object storage that are held in ingester memory.\n\n",
- "fill": 1,
- "format": "short",
- "id": 4,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(cortex_ingester_memory_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}\n/ on(cluster, namespace) group_left\nmax by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}))\n",
- "format": "time_series",
- "instant": true,
- "refId": "A"
- }
- ],
- "thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
- "title": "In-memory series",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Exemplars in ingesters\nNumber of TSDB exemplars currently in ingesters' storage.\n\n",
- "fill": 1,
- "format": "short",
- "id": 5,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(cortex_ingester_tsdb_exemplar_exemplars_in_storage{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}\n/ on(cluster, namespace) group_left\nmax by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}))\n",
- "format": "time_series",
- "instant": true,
- "refId": "A"
- }
- ],
- "thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
- "title": "Exemplars in ingesters",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "format": "short",
- "id": 6,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 2,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "count(count by(user) (cortex_ingester_active_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}))",
- "format": "time_series",
- "instant": true,
- "refId": "A"
- }
- ],
- "thresholds": "70,80",
- "timeFrom": null,
- "timeShift": null,
- "title": "Tenants",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "singlestat",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": false,
- "title": "Headlines",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Requests / sec\nThe rate of successful, failed and rejected requests to distributor.\nRejected requests are requests that distributor fails to handle because of distributor instance limits.\nWhen distributor is configured to use \"early\" request rejection, then rejected requests are NOT included in other metrics.\nWhen distributor is not configured to use \"early\" request rejection, then rejected requests are also counted as \"errors\".\n\n",
- "fill": 10,
- "id": 7,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 8,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "99th percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "50th percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "drawStyle": "line",
- "fillOpacity": 0,
- "lineWidth": 1,
- "pointSize": 5,
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [ ]
- },
- "unit": "s"
- },
- "overrides": [ ]
- },
- "id": 9,
- "links": [ ],
- "options": {
- "legend": {
- "displayMode": "hidden",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "span": 4,
- "targets": [
- {
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval])))",
- "format": "time_series",
- "legendFormat": "",
- "legendLink": null
- }
- ],
- "title": "Per pod p99 latency",
- "type": "timeseries"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Distributor",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Requests / sec\nThe rate of successful, failed and rejected requests to ingester.\nRejected requests are requests that ingester fails to handle because of ingester instance limits (ingester-max-inflight-push-requests, ingester-max-inflight-push-requests-bytes, ingester-max-ingestion-rate).\nWhen ingester is configured to use \"early\" request rejection, then rejected requests are NOT included in other metrics.\nWhen ingester is not configured to use \"early\" request rejection, then rejected requests are also counted as \"errors\".\n\n",
- "fill": 10,
- "id": 10,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",route=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 11,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 4,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "99th percentile",
- "refId": "A",
- "step": 10
- },
- {
- "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"})) * 1e3",
- "format": "time_series",
- "legendFormat": "50th percentile",
- "refId": "B",
- "step": 10
- },
- {
- "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"})",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C",
- "step": 10
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "fieldConfig": {
- "defaults": {
- "custom": {
- "drawStyle": "line",
- "fillOpacity": 0,
- "lineWidth": 1,
- "pointSize": 5,
- "showPoints": "never",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- }
- },
- "thresholds": {
- "mode": "absolute",
- "steps": [ ]
- },
- "unit": "s"
- },
- "overrides": [ ]
- },
- "id": 12,
- "links": [ ],
- "options": {
- "legend": {
- "displayMode": "hidden",
- "showLegend": false
- },
- "tooltip": {
- "mode": "multi",
- "sort": "desc"
- }
- },
- "span": 4,
- "targets": [
- {
- "exemplar": true,
- "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"}[$__rate_interval])))",
- "format": "time_series",
- "legendFormat": "",
- "legendLink": null
- }
- ],
- "title": "Per pod p99 latency",
- "type": "timeseries"
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Ingester",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Replicas\nThe maximum and current number of distributor replicas.\nNote: The current number of replicas can still show 1 replica even when scaled to 0.\nBecause HPA never reports 0 replicas, the query will report 0 only if the HPA is not active.\n\n",
- "fill": 1,
- "id": 13,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [
- {
- "alias": "/Max .+/",
- "dashes": true,
- "fill": 0
- },
- {
- "alias": "/Current .+/",
- "fill": 0
- },
- {
- "alias": "/Min .+/",
- "dashes": true,
- "fill": 0
- }
- ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_max_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}\n # Add the scaletargetref_name label for readability\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}\n)\n",
- "format": "time_series",
- "legendFormat": "Max {{ scaletargetref_name }}",
- "legendLink": null
- },
- {
- "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_status_current_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}\n # HPA doesn't go to 0 replicas, so we multiply by 0 if the HPA is not active\n * on (cluster, namespace, horizontalpodautoscaler)\n kube_horizontalpodautoscaler_status_condition{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\", condition=\"ScalingActive\", status=\"true\"}\n # Add the scaletargetref_name label for readability\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}\n)\n",
- "format": "time_series",
- "legendFormat": "Current {{ scaletargetref_name }}",
- "legendLink": null
- },
- {
- "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_min_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}\n # Add the scaletargetref_name label for readability\n + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}\n)\n",
- "format": "time_series",
- "legendFormat": "Min {{ scaletargetref_name }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Replicas",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Scaling metric (CPU): Desired replicas\nThis panel shows the scaling metric exposed by KEDA divided by the target/threshold used.\nIt should represent the desired number of replicas, ignoring the min/max constraints applied later.\n\n",
- "fill": 1,
- "id": 14,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*cpu.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
- "format": "time_series",
- "legendFormat": "{{ scaler }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Scaling metric (CPU): Desired replicas",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Scaling metric (memory): Desired replicas\nThis panel shows the scaling metric exposed by KEDA divided by the target/threshold used.\nIt should represent the desired number of replicas, ignoring the min/max constraints applied later.\n\n",
- "fill": 1,
- "id": 15,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*memory.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n )\n)\n",
- "format": "time_series",
- "legendFormat": "{{ scaler }}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Scaling metric (memory): Desired replicas",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Autoscaler failures rate\nThe rate of failures in the KEDA custom metrics API server. Whenever an error occurs, the KEDA custom\nmetrics server is unable to query the scaling metric from Prometheus so the autoscaler woudln't work properly.\n\n",
- "fill": 1,
- "id": 16,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by(cluster, namespace, scaler, metric, scaledObject) (\n label_replace(\n rate(keda_scaler_errors[$__rate_interval]),\n \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n )\n) +\non(cluster, namespace, metric, scaledObject) group_left\nlabel_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"} * 0,\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n ),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n",
- "format": "time_series",
- "legendFormat": "{{scaler}} failures",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Autoscaler failures rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Distributor - autoscaling",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 17,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-hatracker\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 18,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-hatracker\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-hatracker\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-hatracker\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-hatracker\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Distributor - key-value store for high-availability (HA) deduplication",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 19,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-(lifecycler|ring)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 20,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-(lifecycler|ring)\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-(lifecycler|ring)\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-(lifecycler|ring)\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-(lifecycler|ring)\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Distributor - key-value store for distributors ring",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "1xx": "#EAB839",
- "2xx": "#7EB26D",
- "3xx": "#6ED0E0",
- "4xx": "#EF843C",
- "5xx": "#E24D42",
- "OK": "#7EB26D",
- "cancel": "#A9A9A9",
- "error": "#E24D42",
- "success": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 21,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", kv_name=~\"ingester-.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
- "format": "time_series",
- "legendFormat": "{{status}}",
- "refId": "A"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Requests / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "reqps",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 22,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", kv_name=~\"ingester-.*\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", kv_name=~\"ingester-.*\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", kv_name=~\"ingester-.*\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", kv_name=~\"ingester-.*\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Ingester - key-value store for the ingesters ring",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Uploaded blocks / sec\nThe rate of blocks being uploaded from the ingesters\nto object storage.\n\n",
- "fill": 10,
- "id": 23,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_ingester_shipper_uploads_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) - sum(rate(cortex_ingester_shipper_upload_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_ingester_shipper_upload_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Uploaded blocks / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Upload latency\nThe average, median (50th percentile), and 99th percentile time\nthe ingesters take to upload blocks to object storage.\n\n",
- "fill": 1,
- "id": 24,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",component=\"ingester\",operation=\"upload\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Upload latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Ingester - shipper",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Compactions per second\nIngesters maintain a local TSDB per-tenant on disk. Each TSDB maintains a head block for each\nactive time series; these blocks get periodically compacted (by default, every 2h).\nThis panel shows the rate of compaction operations across all TSDBs on all ingesters.\n\n",
- "fill": 10,
- "id": 25,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_ingester_tsdb_compactions_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_ingester_tsdb_compactions_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Compactions / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Compaction latency\nThe average, median (50th percentile), and 99th percentile time ingesters take to compact TSDB head blocks\non the local filesystem.\n\n",
- "fill": 1,
- "id": 26,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "99th Percentile",
- "refId": "A"
- },
- {
- "expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) by (le)) * 1e3",
- "format": "time_series",
- "legendFormat": "50th Percentile",
- "refId": "B"
- },
- {
- "expr": "sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "Average",
- "refId": "C"
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Compactions latency",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ms",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Ingester - TSDB head",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### WAL truncations per second\nThe WAL is truncated each time a new TSDB block is written. This panel measures the rate of\ntruncations.\n\n",
- "fill": 10,
- "id": 27,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_ingester_tsdb_wal_truncations_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) - sum(rate(cortex_ingester_tsdb_wal_truncations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_ingester_tsdb_wal_truncations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "WAL truncations / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": {
- "failed": "#E24D42",
- "successful": "#7EB26D"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Checkpoints created per second\nCheckpoints are created as part of the WAL truncation process.\nThis metric measures the rate of checkpoint creation.\n\n",
- "fill": 10,
- "id": 28,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_ingester_tsdb_checkpoint_creations_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) - sum(rate(cortex_ingester_tsdb_checkpoint_creations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "successful",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_ingester_tsdb_checkpoint_creations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "failed",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Checkpoints created / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "datasource": "$datasource",
- "description": "### WAL truncations latency (including checkpointing)\nAverage time taken to perform a full WAL truncation,\nincluding the time taken for the checkpointing to complete.\n\n",
- "fieldConfig": {
- "defaults": {
- "noValue": "0",
- "unit": "s"
- }
- },
- "id": 29,
- "links": [ ],
- "options": {
- "legend": {
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "span": 3,
- "targets": [
- {
- "expr": "sum(rate(cortex_ingester_tsdb_wal_truncate_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))\n/\nsum(rate(cortex_ingester_tsdb_wal_truncate_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) >= 0\n",
- "format": "time_series",
- "legendFormat": "avg",
- "legendLink": null
- }
- ],
- "title": "WAL truncations latency (includes checkpointing)",
- "type": "timeseries"
- },
- {
- "aliasColors": {
- "WAL": "#E24D42",
- "mmap-ed chunks": "#E28A42"
- },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 10,
- "id": 30,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 0,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": true,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(rate(cortex_ingester_tsdb_wal_corruptions_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "WAL",
- "legendLink": null
- },
- {
- "expr": "sum(rate(cortex_ingester_tsdb_mmap_chunk_corruptions_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "mmap-ed chunks",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Corruptions / sec",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ops",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Ingester - TSDB write ahead log (WAL)",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Distributor exemplars incoming rate\nThe rate of exemplars that have come in to the distributor, including rejected or deduped exemplars.\n\n",
- "fill": 1,
- "id": 31,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(cluster_namespace_job:cortex_distributor_exemplars_in:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})",
- "format": "time_series",
- "legendFormat": "incoming exemplars",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Distributor exemplars incoming rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ex/s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Distributor exemplars received rate\nThe rate of received exemplars, excluding rejected and deduped exemplars.\nThis number can be sensibly lower than incoming rate because we dedupe the HA sent exemplars, and then reject based on time, see `cortex_discarded_exemplars_total` for specific reasons rates.\n\n",
- "fill": 1,
- "id": 32,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(cluster_namespace_job:cortex_distributor_received_exemplars:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})",
- "format": "time_series",
- "legendFormat": "received exemplars",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Distributor exemplars received rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ex/s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Ingester ingested exemplars rate\nThe rate of exemplars ingested in the ingesters.\nEvery exemplar is sent to the replication factor number of ingesters, so the sum of rates from all ingesters is divided by the replication factor.\nThis ingested exemplars rate should match the distributor's received exemplars rate.\n\n",
- "fill": 1,
- "id": 33,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(\n cluster_namespace_job:cortex_ingester_ingested_exemplars:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
- "format": "time_series",
- "legendFormat": "ingested exemplars",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Ingester ingested exemplars rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ex/s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "description": "### Ingester appended exemplars rate\nThe rate of exemplars appended in the ingesters.\nThis can be lower than ingested exemplars rate since TSDB does not append the same exemplar twice, and those can be frequent.\n\n",
- "fill": 1,
- "id": 34,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 3,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum(\n cluster_namespace_job:cortex_ingester_tsdb_exemplar_exemplars_appended:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}\n / on(cluster, namespace) group_left\n max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
- "format": "time_series",
- "legendFormat": "appended exemplars",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Ingester appended exemplars rate",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "ex/s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Exemplars",
- "titleSize": "h6"
- },
- {
- "collapse": false,
- "height": "250px",
- "panels": [
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 35,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (reason) (rate(cortex_distributor_instance_rejected_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{reason}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Rejected distributor requests",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "req/s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- },
- {
- "aliasColors": { },
- "bars": false,
- "dashLength": 10,
- "dashes": false,
- "datasource": "$datasource",
- "fill": 1,
- "id": 36,
- "legend": {
- "avg": false,
- "current": false,
- "max": false,
- "min": false,
- "show": true,
- "total": false,
- "values": false
- },
- "lines": true,
- "linewidth": 1,
- "links": [ ],
- "nullPointMode": "null as zero",
- "percentage": false,
- "pointradius": 5,
- "points": false,
- "renderer": "flot",
- "seriesOverrides": [ ],
- "spaceLength": 10,
- "span": 6,
- "stack": false,
- "steppedLine": false,
- "targets": [
- {
- "expr": "sum by (reason) (rate(cortex_ingester_instance_rejected_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
- "format": "time_series",
- "legendFormat": "{{reason}}",
- "legendLink": null
- }
- ],
- "thresholds": [ ],
- "timeFrom": null,
- "timeShift": null,
- "title": "Rejected ingester requests",
- "tooltip": {
- "shared": false,
- "sort": 0,
- "value_type": "individual"
- },
- "type": "graph",
- "xaxis": {
- "buckets": null,
- "mode": "time",
- "name": null,
- "show": true,
- "values": [ ]
- },
- "yaxes": [
- {
- "format": "req/s",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": 0,
- "show": true
- },
- {
- "format": "short",
- "label": null,
- "logBase": 1,
- "max": null,
- "min": null,
- "show": false
- }
- ]
- }
- ],
- "repeat": null,
- "repeatIteration": null,
- "repeatRowId": null,
- "showTitle": true,
- "title": "Instance Limits",
- "titleSize": "h6"
- }
- ],
- "schemaVersion": 14,
- "style": "dark",
- "tags": [
- "mimir"
- ],
- "templating": {
- "list": [
- {
- "current": {
- "text": "default",
- "value": "default"
- },
- "hide": 0,
- "label": "Data source",
- "name": "datasource",
- "options": [ ],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "cluster",
- "multi": true,
- "name": "cluster",
- "options": [ ],
- "query": "label_values(cortex_build_info, cluster)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- },
- {
- "allValue": ".+",
- "current": {
- "selected": true,
- "text": "All",
- "value": "$__all"
- },
- "datasource": "$datasource",
- "hide": 0,
- "includeAll": true,
- "label": "namespace",
- "multi": true,
- "name": "namespace",
- "options": [ ],
- "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
- "refresh": 1,
- "regex": "",
- "sort": 1,
- "tagValuesQuery": "",
- "tags": [ ],
- "tagsQuery": "",
- "type": "query",
- "useTags": false
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {
- "refresh_intervals": [
- "5s",
- "10s",
- "30s",
- "1m",
- "5m",
- "15m",
- "30m",
- "1h",
- "2h",
- "1d"
- ],
- "time_options": [
- "5m",
- "15m",
- "1h",
- "6h",
- "12h",
- "24h",
- "2d",
- "7d",
- "30d"
- ]
- },
- "timezone": "utc",
- "title": "Mimir / Writes",
- "uid": "8280707b8f16e7b87b840fc1cc92d4c5",
- "version": 0
- }
----
-# Source: mimir/charts/mimir/templates/mimir-config.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: release-name-mimir-config
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- namespace: "default"
-data:
- mimir.yaml: |
-
- activity_tracker:
- filepath: /active-query-tracker/activity.log
- alertmanager:
- data_dir: /data
- enable_api: true
- external_url: /alertmanager
- fallback_config_file: /configs/alertmanager_fallback_config.yaml
- alertmanager_storage:
- backend: s3
- s3:
- access_key_id: grafana-mimir
- bucket_name: mimir-ruler
- endpoint: release-name-minio.default.svc:9000
- insecure: true
- secret_access_key: supersecret
- blocks_storage:
- backend: s3
- bucket_store:
- index_cache:
- backend: memcached
- memcached:
- addresses: dns+release-name-mimir-index-cache.default.svc:11211
- max_idle_connections: 150
- max_item_size: 5242880
- timeout: 450ms
- metadata_cache:
- backend: memcached
- memcached:
- addresses: dns+release-name-mimir-metadata-cache.default.svc:11211
- max_idle_connections: 150
- max_item_size: 1048576
- sync_dir: /data/tsdb-sync
- s3:
- access_key_id: grafana-mimir
- bucket_name: mimir-tsdb
- endpoint: release-name-minio.default.svc:9000
- insecure: true
- secret_access_key: supersecret
- tsdb:
- dir: /data/tsdb
- head_compaction_interval: 15m
- wal_replay_concurrency: 3
- compactor:
- compaction_interval: 30m
- data_dir: /data
- deletion_delay: 2h
- first_level_compaction_wait_period: 25m
- max_closing_blocks_concurrency: 2
- max_opening_blocks_concurrency: 4
- sharding_ring:
- wait_stability_min_duration: 1m
- symbols_flushers_concurrency: 4
- frontend:
- cache_results: true
- parallelize_shardable_queries: true
- query_sharding_target_series_per_shard: 2500
- results_cache:
- backend: memcached
- memcached:
- addresses: dns+release-name-mimir-results-cache.default.svc:11211
- max_item_size: 5242880
- timeout: 500ms
- scheduler_address: release-name-mimir-query-scheduler-headless.default.svc:9095
- frontend_worker:
- grpc_client_config:
- max_send_msg_size: 419430400
- scheduler_address: release-name-mimir-query-scheduler-headless.default.svc:9095
- ingester:
- ring:
- final_sleep: 0s
- num_tokens: 512
- tokens_file_path: /data/tokens
- unregister_on_shutdown: false
- zone_awareness_enabled: true
- ingester_client:
- grpc_client_config:
- max_recv_msg_size: 104857600
- max_send_msg_size: 104857600
- limits:
- max_cache_freshness: 10m
- max_query_parallelism: 240
- max_total_query_length: 12000h
- memberlist:
- abort_if_cluster_join_fails: false
- compression_enabled: false
- join_members:
- - dns+release-name-mimir-gossip-ring.default.svc.cluster.local.:7946
- querier:
- max_concurrent: 16
- query_scheduler:
- max_outstanding_requests_per_tenant: 800
- ruler:
- alertmanager_url: dnssrvnoa+http://_http-metrics._tcp.release-name-mimir-alertmanager-headless.default.svc.cluster.local./alertmanager
- enable_api: true
- rule_path: /data
- ruler_storage:
- backend: s3
- cache:
- backend: memcached
- memcached:
- addresses: dns+release-name-mimir-metadata-cache.default.svc:11211
- max_item_size: 1048576
- s3:
- access_key_id: grafana-mimir
- bucket_name: mimir-ruler
- endpoint: release-name-minio.default.svc:9000
- insecure: true
- secret_access_key: supersecret
- runtime_config:
- file: /var/mimir/runtime.yaml
- server:
- grpc_server_max_connection_age: 2m
- grpc_server_max_connection_age_grace: 5m
- grpc_server_max_connection_idle: 1m
- store_gateway:
- sharding_ring:
- kvstore:
- prefix: multi-zone/
- tokens_file_path: /data/tokens
- unregister_on_shutdown: false
- wait_stability_min_duration: 1m
- zone_awareness_enabled: true
- usage_stats:
- installation_mode: helm
----
-# Source: mimir/charts/mimir/templates/nginx/nginx-configmap.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: release-name-mimir-nginx
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: nginx
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- namespace: "default"
-data:
- nginx.conf: |
- worker_processes 5; ## Default: 1
- error_log /dev/stderr error;
- pid /tmp/nginx.pid;
- worker_rlimit_nofile 8192;
-
- events {
- worker_connections 4096; ## Default: 1024
- }
-
- http {
- client_body_temp_path /tmp/client_temp;
- proxy_temp_path /tmp/proxy_temp_path;
- fastcgi_temp_path /tmp/fastcgi_temp;
- uwsgi_temp_path /tmp/uwsgi_temp;
- scgi_temp_path /tmp/scgi_temp;
-
- default_type application/octet-stream;
- log_format main '$remote_addr - $remote_user [$time_local] $status '
- '"$request" $body_bytes_sent "$http_referer" '
- '"$http_user_agent" "$http_x_forwarded_for"';
- access_log /dev/stderr main;
-
- sendfile on;
- tcp_nopush on;
- resolver kube-dns.kube-system.svc.cluster.local.;
-
- # Ensure that X-Scope-OrgID is always present, default to the no_auth_tenant for backwards compatibility when multi-tenancy was turned off.
- map $http_x_scope_orgid $ensured_x_scope_orgid {
- default $http_x_scope_orgid;
- "" "anonymous";
- }
-
- proxy_read_timeout 300;
- server {
- listen 8080;
- listen [::]:8080;
-
- location = / {
- return 200 'OK';
- auth_basic off;
- }
-
- proxy_set_header X-Scope-OrgID $ensured_x_scope_orgid;
-
- # Distributor endpoints
- location /distributor {
- set $distributor release-name-mimir-distributor-headless.default.svc.cluster.local.;
- proxy_pass http://$distributor:8080$request_uri;
- }
- location = /api/v1/push {
- set $distributor release-name-mimir-distributor-headless.default.svc.cluster.local.;
- proxy_pass http://$distributor:8080$request_uri;
- }
- location /otlp/v1/metrics {
- set $distributor release-name-mimir-distributor-headless.default.svc.cluster.local.;
- proxy_pass http://$distributor:8080$request_uri;
- }
-
- # Alertmanager endpoints
- location /alertmanager {
- set $alertmanager release-name-mimir-alertmanager-headless.default.svc.cluster.local.;
- proxy_pass http://$alertmanager:8080$request_uri;
- }
- location = /multitenant_alertmanager/status {
- set $alertmanager release-name-mimir-alertmanager-headless.default.svc.cluster.local.;
- proxy_pass http://$alertmanager:8080$request_uri;
- }
- location = /api/v1/alerts {
- set $alertmanager release-name-mimir-alertmanager-headless.default.svc.cluster.local.;
- proxy_pass http://$alertmanager:8080$request_uri;
- }
-
- # Ruler endpoints
- location /prometheus/config/v1/rules {
- set $ruler release-name-mimir-ruler.default.svc.cluster.local.;
- proxy_pass http://$ruler:8080$request_uri;
- }
- location /prometheus/api/v1/rules {
- set $ruler release-name-mimir-ruler.default.svc.cluster.local.;
- proxy_pass http://$ruler:8080$request_uri;
- }
-
- location /prometheus/api/v1/alerts {
- set $ruler release-name-mimir-ruler.default.svc.cluster.local.;
- proxy_pass http://$ruler:8080$request_uri;
- }
- location = /ruler/ring {
- set $ruler release-name-mimir-ruler.default.svc.cluster.local.;
- proxy_pass http://$ruler:8080$request_uri;
- }
-
- # Rest of /prometheus goes to the query frontend
- location /prometheus {
- set $query_frontend release-name-mimir-query-frontend.default.svc.cluster.local.;
- proxy_pass http://$query_frontend:8080$request_uri;
- }
-
- # Buildinfo endpoint can go to any component
- location = /api/v1/status/buildinfo {
- set $query_frontend release-name-mimir-query-frontend.default.svc.cluster.local.;
- proxy_pass http://$query_frontend:8080$request_uri;
- }
-
- # Compactor endpoint for uploading blocks
- location /api/v1/upload/block/ {
- set $compactor release-name-mimir-compactor.default.svc.cluster.local.;
- proxy_pass http://$compactor:8080$request_uri;
- }
- }
- }
----
-# Source: mimir/charts/mimir/templates/runtime-configmap.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: release-name-mimir-runtime
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- namespace: "default"
-data:
- runtime.yaml: |
-
- {}
----
-# Source: mimir/templates/common.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: release-name-mimir-grafana-datasource
- namespace: default
- labels:
- app: "mimir-0.1.8"
- app.kubernetes.io/instance: "release-name"
- app.kubernetes.io/managed-by: "Helm"
- app.kubernetes.io/name: "mimir"
- app.kubernetes.io/version: "18.0.1"
- grafana_datasources: "1"
- helm-revision: "1"
- helm.sh/chart: "mimir-0.1.8"
- release: "release-name"
-data:
- datasource.yaml: |-
- apiVersion: 1
- datasources:
- - name: Mimir-Metamonitoring
- type: prometheus
- uid: meta
- url: http://release-name-mimir-nginx.default/prometheus
- access: proxy
- isDefault: false
- jsonData:
- httpMethod: POST
- timeInterval: 30s
- httpHeaderName1: 'X-Scope-OrgID'
- secureJsonData:
- httpHeaderValue1: 'metamonitoring'
- - name: Mimir
- type: prometheus
- uid: meta
- url: http://release-name-mimir-nginx.default/prometheus
- access: proxy
- isDefault: true
- jsonData:
- httpMethod: POST
- timeInterval: 30s
----
-# Source: mimir/charts/mimir/charts/minio/templates/pvc.yaml
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
- name: release-name-minio
- labels:
- app: minio
- chart: minio-5.0.14
- release: release-name
- heritage: Helm
-spec:
- accessModes:
- - "ReadWriteOnce"
- resources:
- requests:
- storage: "5Gi"
----
-# Source: mimir/charts/kps/charts/kube-state-metrics/templates/role.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
- labels:
- helm.sh/chart: kube-state-metrics-5.16.0
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: metrics
- app.kubernetes.io/part-of: kube-state-metrics
- app.kubernetes.io/name: kube-state-metrics
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.10.1"
- release: release-name
- name: release-name-kube-state-metrics
-rules:
-
-- apiGroups: ["certificates.k8s.io"]
- resources:
- - certificatesigningrequests
- verbs: ["list", "watch"]
-
-- apiGroups: [""]
- resources:
- - configmaps
- verbs: ["list", "watch"]
-
-- apiGroups: ["batch"]
- resources:
- - cronjobs
- verbs: ["list", "watch"]
-
-- apiGroups: ["extensions", "apps"]
- resources:
- - daemonsets
- verbs: ["list", "watch"]
-
-- apiGroups: ["extensions", "apps"]
- resources:
- - deployments
- verbs: ["list", "watch"]
-
-- apiGroups: [""]
- resources:
- - endpoints
- verbs: ["list", "watch"]
-
-- apiGroups: ["autoscaling"]
- resources:
- - horizontalpodautoscalers
- verbs: ["list", "watch"]
-
-- apiGroups: ["extensions", "networking.k8s.io"]
- resources:
- - ingresses
- verbs: ["list", "watch"]
-
-- apiGroups: ["batch"]
- resources:
- - jobs
- verbs: ["list", "watch"]
-
-- apiGroups: ["coordination.k8s.io"]
- resources:
- - leases
- verbs: ["list", "watch"]
-
-- apiGroups: [""]
- resources:
- - limitranges
- verbs: ["list", "watch"]
-
-- apiGroups: ["admissionregistration.k8s.io"]
- resources:
- - mutatingwebhookconfigurations
- verbs: ["list", "watch"]
-
-- apiGroups: [""]
- resources:
- - namespaces
- verbs: ["list", "watch"]
-
-- apiGroups: ["networking.k8s.io"]
- resources:
- - networkpolicies
- verbs: ["list", "watch"]
-
-- apiGroups: [""]
- resources:
- - nodes
- verbs: ["list", "watch"]
-
-- apiGroups: [""]
- resources:
- - persistentvolumeclaims
- verbs: ["list", "watch"]
-
-- apiGroups: [""]
- resources:
- - persistentvolumes
- verbs: ["list", "watch"]
-
-- apiGroups: ["policy"]
- resources:
- - poddisruptionbudgets
- verbs: ["list", "watch"]
-
-- apiGroups: [""]
- resources:
- - pods
- verbs: ["list", "watch"]
-
-- apiGroups: ["extensions", "apps"]
- resources:
- - replicasets
- verbs: ["list", "watch"]
-
-- apiGroups: [""]
- resources:
- - replicationcontrollers
- verbs: ["list", "watch"]
-
-- apiGroups: [""]
- resources:
- - resourcequotas
- verbs: ["list", "watch"]
-
-- apiGroups: [""]
- resources:
- - secrets
- verbs: ["list", "watch"]
-
-- apiGroups: [""]
- resources:
- - services
- verbs: ["list", "watch"]
-
-- apiGroups: ["apps"]
- resources:
- - statefulsets
- verbs: ["list", "watch"]
-
-- apiGroups: ["storage.k8s.io"]
- resources:
- - storageclasses
- verbs: ["list", "watch"]
-
-- apiGroups: ["admissionregistration.k8s.io"]
- resources:
- - validatingwebhookconfigurations
- verbs: ["list", "watch"]
-
-- apiGroups: ["storage.k8s.io"]
- resources:
- - volumeattachments
- verbs: ["list", "watch"]
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-agent-cluster-role.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
- name: release-name-mimir-grafana-agent
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: meta-monitoring
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-rules:
- - apiGroups:
- - ""
- resources:
- - nodes
- - nodes/proxy
- - nodes/metrics
- - services
- - endpoints
- - pods
- - events
- verbs:
- - get
- - list
- - watch
- - apiGroups:
- - networking.k8s.io
- resources:
- - ingresses
- verbs:
- - get
- - list
- - watch
- - nonResourceURLs:
- - /metrics
- - /metrics/cadvisor
- verbs:
- - get
----
-# Source: mimir/charts/kps/charts/kube-state-metrics/templates/clusterrolebinding.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
- labels:
- helm.sh/chart: kube-state-metrics-5.16.0
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: metrics
- app.kubernetes.io/part-of: kube-state-metrics
- app.kubernetes.io/name: kube-state-metrics
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.10.1"
- release: release-name
- name: release-name-kube-state-metrics
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: release-name-kube-state-metrics
-subjects:
-- kind: ServiceAccount
- name: release-name-kube-state-metrics
- namespace: default
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-agent-cluster-role-binding.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
- name: release-name-mimir-grafana-agent
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: meta-monitoring
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: release-name-mimir-grafana-agent
-subjects:
- - kind: ServiceAccount
- name: release-name-mimir
- namespace: default
----
-# Source: mimir/charts/mimir/charts/rollout_operator/templates/role.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
- name: release-name-rollout-operator
-rules:
-- apiGroups:
- - ""
- resources:
- - pods
- verbs:
- - list
- - get
- - watch
- - delete
-- apiGroups:
- - apps
- resources:
- - statefulsets
- verbs:
- - list
- - get
- - watch
-- apiGroups:
- - apps
- resources:
- - statefulsets/status
- verbs:
- - update
----
-# Source: mimir/charts/mimir/charts/rollout_operator/templates/rolebinding.yaml
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
- name: release-name-rollout-operator
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: Role
- name: release-name-rollout-operator
-subjects:
-- kind: ServiceAccount
- name: release-name-rollout-operator
----
-# Source: mimir/charts/kps/charts/kube-state-metrics/templates/service.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-kube-state-metrics
- namespace: default
- labels:
- helm.sh/chart: kube-state-metrics-5.16.0
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: metrics
- app.kubernetes.io/part-of: kube-state-metrics
- app.kubernetes.io/name: kube-state-metrics
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.10.1"
- release: release-name
- annotations:
- prometheus.io/scrape: 'true'
-spec:
- type: "ClusterIP"
- ports:
- - name: "http"
- protocol: TCP
- port: 8080
- targetPort: 8080
-
- selector:
- app.kubernetes.io/name: kube-state-metrics
- app.kubernetes.io/instance: release-name
----
-# Source: mimir/charts/kps/charts/prometheus-node-exporter/templates/service.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-prometheus-node-exporter
- namespace: default
- labels:
- helm.sh/chart: prometheus-node-exporter-4.30.2
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: metrics
- app.kubernetes.io/part-of: prometheus-node-exporter
- app.kubernetes.io/name: prometheus-node-exporter
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "1.7.0"
- jobLabel: node-exporter
- release: release-name
- annotations:
- prometheus.io/scrape: "true"
-spec:
- type: ClusterIP
- ports:
- - port: 9100
- targetPort: 9100
- protocol: TCP
- name: http-metrics
- selector:
- app.kubernetes.io/name: prometheus-node-exporter
- app.kubernetes.io/instance: release-name
----
-# Source: mimir/charts/kps/templates/exporters/core-dns/service.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-kps-coredns
- labels:
- app: kps-coredns
- jobLabel: coredns
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
- namespace: kube-system
-spec:
- clusterIP: None
- ports:
- - name: http-metrics
- port: 9153
- protocol: TCP
- targetPort: 9153
- selector:
- k8s-app: kube-dns
----
-# Source: mimir/charts/kps/templates/exporters/kube-controller-manager/service.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-kps-kube-controller-manager
- labels:
- app: kps-kube-controller-manager
- jobLabel: kube-controller-manager
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
- namespace: kube-system
-spec:
- clusterIP: None
- ports:
- - name: http-metrics
- port: 10257
- protocol: TCP
- targetPort: 10257
- selector:
- component: kube-controller-manager
- type: ClusterIP
----
-# Source: mimir/charts/kps/templates/exporters/kube-etcd/service.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-kps-kube-etcd
- labels:
- app: kps-kube-etcd
- jobLabel: kube-etcd
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
- namespace: kube-system
-spec:
- clusterIP: None
- ports:
- - name: http-metrics
- port: 2381
- protocol: TCP
- targetPort: 2381
- selector:
- component: etcd
- type: ClusterIP
----
-# Source: mimir/charts/kps/templates/exporters/kube-scheduler/service.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-kps-kube-scheduler
- labels:
- app: kps-kube-scheduler
- jobLabel: kube-scheduler
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
- namespace: kube-system
-spec:
- clusterIP: None
- ports:
- - name: http-metrics
- port: 10259
- protocol: TCP
- targetPort: 10259
- selector:
- component: kube-scheduler
- type: ClusterIP
----
-# Source: mimir/charts/mimir/charts/minio/templates/console-service.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-minio-console
- labels:
- app: minio
- chart: minio-5.0.14
- release: release-name
- heritage: Helm
-spec:
- type: ClusterIP
- ports:
- - name: http
- port: 9001
- protocol: TCP
- targetPort: 9001
- selector:
- app: minio
- release: release-name
----
-# Source: mimir/charts/mimir/charts/minio/templates/service.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-minio
- labels:
- app: minio
- chart: minio-5.0.14
- release: release-name
- heritage: Helm
- monitoring: "true"
-spec:
- type: ClusterIP
- ports:
- - name: http
- port: 9000
- protocol: TCP
- targetPort: 9000
- selector:
- app: minio
- release: release-name
----
-# Source: mimir/charts/mimir/templates/alertmanager/alertmanager-svc-headless.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-alertmanager-headless
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: alertmanager
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- prometheus.io/service-monitor: "false"
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- clusterIP: None
- publishNotReadyAddresses: true
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- - port: 9095
- protocol: TCP
- name: grpc
- targetPort: grpc
- - port: 9094
- protocol: TCP
- name: cluster
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: alertmanager
----
-# Source: mimir/charts/mimir/templates/alertmanager/alertmanager-svc.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-alertmanager
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: alertmanager
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- - port: 9095
- protocol: TCP
- name: grpc
- targetPort: grpc
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: alertmanager
----
-# Source: mimir/charts/mimir/templates/compactor/compactor-svc.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-compactor
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: compactor
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- - port: 9095
- protocol: TCP
- name: grpc
- targetPort: grpc
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: compactor
----
-# Source: mimir/charts/mimir/templates/distributor/distributor-svc-headless.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-distributor-headless
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: distributor
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- prometheus.io/service-monitor: "false"
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- clusterIP: None
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- - port: 9095
- protocol: TCP
- name: grpc
- targetPort: grpc
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: distributor
----
-# Source: mimir/charts/mimir/templates/distributor/distributor-svc.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-distributor
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: distributor
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- - port: 9095
- protocol: TCP
- name: grpc
- targetPort: grpc
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: distributor
----
-# Source: mimir/charts/mimir/templates/gossip-ring/gossip-ring-svc.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-gossip-ring
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: gossip-ring
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- namespace: "default"
-spec:
- type: ClusterIP
- clusterIP: None
- ports:
- - name: gossip-ring
- port: 7946
- appProtocol: tcp
- protocol: TCP
- targetPort: 7946
- publishNotReadyAddresses: true
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/part-of: memberlist
----
-# Source: mimir/charts/mimir/templates/index-cache/index-cache-svc-headless.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-index-cache
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: index-cache
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- clusterIP: None
- ports:
- - name: memcached-client
- port: 11211
- targetPort: 11211
-
- - name: http-metrics
- port: 9150
- targetPort: 9150
-
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: index-cache
----
-# Source: mimir/charts/mimir/templates/ingester/ingester-svc-headless.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-ingester-headless
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- prometheus.io/service-monitor: "false"
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- clusterIP: None
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- - port: 9095
- protocol: TCP
- name: grpc
- targetPort: grpc
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
----
-# Source: mimir/charts/mimir/templates/ingester/ingester-svc.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-ingester-zone-a
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- name: "ingester-zone-a"
- rollout-group: ingester
- zone: zone-a
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- - port: 9095
- protocol: TCP
- name: grpc
- targetPort: grpc
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- rollout-group: ingester
- zone: zone-a
----
-# Source: mimir/charts/mimir/templates/ingester/ingester-svc.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-ingester-zone-b
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- name: "ingester-zone-b"
- rollout-group: ingester
- zone: zone-b
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- - port: 9095
- protocol: TCP
- name: grpc
- targetPort: grpc
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- rollout-group: ingester
- zone: zone-b
----
-# Source: mimir/charts/mimir/templates/ingester/ingester-svc.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-ingester-zone-c
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- name: "ingester-zone-c"
- rollout-group: ingester
- zone: zone-c
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- - port: 9095
- protocol: TCP
- name: grpc
- targetPort: grpc
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- rollout-group: ingester
- zone: zone-c
----
-# Source: mimir/charts/mimir/templates/metadata-cache/metadata-cache-svc-headless.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-metadata-cache
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: metadata-cache
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- clusterIP: None
- ports:
- - name: memcached-client
- port: 11211
- targetPort: 11211
-
- - name: http-metrics
- port: 9150
- targetPort: 9150
-
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: metadata-cache
----
-# Source: mimir/charts/mimir/templates/nginx/nginx-svc.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-nginx
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: nginx
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- ports:
- - name: http-metric
- port: 80
- targetPort: http-metric
- protocol: TCP
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: nginx
----
-# Source: mimir/charts/mimir/templates/overrides-exporter/overrides-exporter-svc.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-overrides-exporter
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: overrides-exporter
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- - port: 9095
- protocol: TCP
- name: grpc
- targetPort: grpc
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: overrides-exporter
----
-# Source: mimir/charts/mimir/templates/querier/querier-svc.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-querier
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: querier
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- - port: 9095
- protocol: TCP
- name: grpc
- targetPort: grpc
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: querier
----
-# Source: mimir/charts/mimir/templates/query-frontend/query-frontend-svc.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-query-frontend
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-frontend
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- - port: 9095
- protocol: TCP
- name: grpc
- targetPort: grpc
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-frontend
----
-# Source: mimir/charts/mimir/templates/query-scheduler/query-scheduler-svc-headless.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-query-scheduler-headless
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-scheduler
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- prometheus.io/service-monitor: "false"
- annotations:
- {}
-spec:
- type: ClusterIP
- clusterIP: None
- publishNotReadyAddresses: true
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- - port: 9095
- protocol: TCP
- name: grpc
- targetPort: grpc
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-scheduler
----
-# Source: mimir/charts/mimir/templates/query-scheduler/query-scheduler-svc.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-query-scheduler
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-scheduler
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
-spec:
- type: ClusterIP
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- - port: 9095
- protocol: TCP
- name: grpc
- targetPort: grpc
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-scheduler
----
-# Source: mimir/charts/mimir/templates/results-cache/results-cache-svc-headless.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-results-cache
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: results-cache
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- clusterIP: None
- ports:
- - name: memcached-client
- port: 11211
- targetPort: 11211
-
- - name: http-metrics
- port: 9150
- targetPort: 9150
-
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: results-cache
----
-# Source: mimir/charts/mimir/templates/ruler/ruler-svc.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-ruler
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ruler
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ruler
----
-# Source: mimir/charts/mimir/templates/store-gateway/store-gateway-svc-headless.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-store-gateway-headless
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- prometheus.io/service-monitor: "false"
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- clusterIP: None
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- - port: 9095
- protocol: TCP
- name: grpc
- targetPort: grpc
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
----
-# Source: mimir/charts/mimir/templates/store-gateway/store-gateway-svc.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-store-gateway-zone-a
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- name: "store-gateway-zone-a"
- rollout-group: store-gateway
- zone: zone-a
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- - port: 9095
- protocol: TCP
- name: grpc
- targetPort: grpc
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- rollout-group: store-gateway
- zone: zone-a
----
-# Source: mimir/charts/mimir/templates/store-gateway/store-gateway-svc.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-store-gateway-zone-b
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- name: "store-gateway-zone-b"
- rollout-group: store-gateway
- zone: zone-b
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- - port: 9095
- protocol: TCP
- name: grpc
- targetPort: grpc
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- rollout-group: store-gateway
- zone: zone-b
----
-# Source: mimir/charts/mimir/templates/store-gateway/store-gateway-svc.yaml
-apiVersion: v1
-kind: Service
-metadata:
- name: release-name-mimir-store-gateway-zone-c
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- name: "store-gateway-zone-c"
- rollout-group: store-gateway
- zone: zone-c
- annotations:
- {}
- namespace: "default"
-spec:
- type: ClusterIP
- ports:
- - port: 8080
- protocol: TCP
- name: http-metrics
- targetPort: http-metrics
- - port: 9095
- protocol: TCP
- name: grpc
- targetPort: grpc
- selector:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- rollout-group: store-gateway
- zone: zone-c
----
-# Source: mimir/charts/kps/charts/prometheus-node-exporter/templates/daemonset.yaml
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
- name: release-name-prometheus-node-exporter
- namespace: default
- labels:
- helm.sh/chart: prometheus-node-exporter-4.30.2
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: metrics
- app.kubernetes.io/part-of: prometheus-node-exporter
- app.kubernetes.io/name: prometheus-node-exporter
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "1.7.0"
- jobLabel: node-exporter
- release: release-name
-spec:
- selector:
- matchLabels:
- app.kubernetes.io/name: prometheus-node-exporter
- app.kubernetes.io/instance: release-name
- revisionHistoryLimit: 10
- updateStrategy:
- rollingUpdate:
- maxUnavailable: 1
- type: RollingUpdate
- template:
- metadata:
- annotations:
- cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
- labels:
- helm.sh/chart: prometheus-node-exporter-4.30.2
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: metrics
- app.kubernetes.io/part-of: prometheus-node-exporter
- app.kubernetes.io/name: prometheus-node-exporter
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "1.7.0"
- jobLabel: node-exporter
- release: release-name
- spec:
- automountServiceAccountToken: false
- securityContext:
- fsGroup: 65534
- runAsGroup: 65534
- runAsNonRoot: true
- runAsUser: 65534
- serviceAccountName: release-name-prometheus-node-exporter
- containers:
- - name: node-exporter
- image: quay.io/prometheus/node-exporter:v1.7.0
- imagePullPolicy: IfNotPresent
- args:
- - --path.procfs=/host/proc
- - --path.sysfs=/host/sys
- - --path.rootfs=/host/root
- - --path.udev.data=/host/root/run/udev/data
- - --web.listen-address=[$(HOST_IP)]:9100
- - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/)
- - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$
- securityContext:
- readOnlyRootFilesystem: true
- env:
- - name: HOST_IP
- value: 0.0.0.0
- ports:
- - name: http-metrics
- containerPort: 9100
- protocol: TCP
- livenessProbe:
- failureThreshold: 3
- httpGet:
- httpHeaders:
- path: /
- port: 9100
- scheme: HTTP
- initialDelaySeconds: 0
- periodSeconds: 10
- successThreshold: 1
- timeoutSeconds: 1
- readinessProbe:
- failureThreshold: 3
- httpGet:
- httpHeaders:
- path: /
- port: 9100
- scheme: HTTP
- initialDelaySeconds: 0
- periodSeconds: 10
- successThreshold: 1
- timeoutSeconds: 1
- volumeMounts:
- - name: proc
- mountPath: /host/proc
- readOnly: true
- - name: sys
- mountPath: /host/sys
- readOnly: true
- - name: root
- mountPath: /host/root
- mountPropagation: HostToContainer
- readOnly: true
- hostNetwork: true
- hostPID: true
- nodeSelector:
- kubernetes.io/os: linux
- tolerations:
- - effect: NoSchedule
- operator: Exists
- volumes:
- - name: proc
- hostPath:
- path: /proc
- - name: sys
- hostPath:
- path: /sys
- - name: root
- hostPath:
- path: /
----
-# Source: mimir/charts/kps/charts/kube-state-metrics/templates/deployment.yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: release-name-kube-state-metrics
- namespace: default
- labels:
- helm.sh/chart: kube-state-metrics-5.16.0
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: metrics
- app.kubernetes.io/part-of: kube-state-metrics
- app.kubernetes.io/name: kube-state-metrics
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.10.1"
- release: release-name
-spec:
- selector:
- matchLabels:
- app.kubernetes.io/name: kube-state-metrics
- app.kubernetes.io/instance: release-name
- replicas: 1
- strategy:
- type: RollingUpdate
- revisionHistoryLimit: 10
- template:
- metadata:
- labels:
- helm.sh/chart: kube-state-metrics-5.16.0
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: metrics
- app.kubernetes.io/part-of: kube-state-metrics
- app.kubernetes.io/name: kube-state-metrics
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.10.1"
- release: release-name
- spec:
- hostNetwork: false
- serviceAccountName: release-name-kube-state-metrics
- securityContext:
- fsGroup: 65534
- runAsGroup: 65534
- runAsNonRoot: true
- runAsUser: 65534
- seccompProfile:
- type: RuntimeDefault
- containers:
- - name: kube-state-metrics
- args:
- - --port=8080
- - --resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,horizontalpodautoscalers,ingresses,jobs,leases,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,volumeattachments
- imagePullPolicy: IfNotPresent
- image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.10.1
- ports:
- - containerPort: 8080
- name: "http"
- livenessProbe:
- failureThreshold: 3
- httpGet:
- httpHeaders:
- path: /healthz
- port: 8080
- scheme: HTTP
- initialDelaySeconds: 5
- periodSeconds: 10
- successThreshold: 1
- timeoutSeconds: 5
- readinessProbe:
- failureThreshold: 3
- httpGet:
- httpHeaders:
- path: /
- port: 8080
- scheme: HTTP
- initialDelaySeconds: 5
- periodSeconds: 10
- successThreshold: 1
- timeoutSeconds: 5
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
----
-# Source: mimir/charts/mimir/charts/minio/templates/deployment.yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: release-name-minio
- labels:
- app: minio
- chart: minio-5.0.14
- release: release-name
- heritage: Helm
-spec:
- strategy:
- type: RollingUpdate
- rollingUpdate:
- maxSurge: 100%
- maxUnavailable: 0
- replicas: 1
- selector:
- matchLabels:
- app: minio
- release: release-name
- template:
- metadata:
- name: release-name-minio
- labels:
- app: minio
- release: release-name
- annotations:
- checksum/secrets: c8339c63fe17e834336f905a9b41c8c19ae107390219fc9b9ad71d3783e47601
- checksum/config: 1cd78d08302cd4659fe486040b24a41a7b9528e97c02fe2446ae70478df68cfd
- spec:
- securityContext:
- runAsUser: 1000
- runAsGroup: 1000
- fsGroup: 1000
- fsGroupChangePolicy: OnRootMismatch
-
- serviceAccountName: minio-sa
- containers:
- - name: minio
- image: "quay.io/minio/minio:RELEASE.2023-09-30T07-02-29Z"
- imagePullPolicy: IfNotPresent
- command:
- - "/bin/sh"
- - "-ce"
- - "/usr/bin/docker-entrypoint.sh minio server /export -S /etc/minio/certs/ --address :9000 --console-address :9001"
- volumeMounts:
- - name: minio-user
- mountPath: "/tmp/credentials"
- readOnly: true
- - name: export
- mountPath: /export
- ports:
- - name: http
- containerPort: 9000
- - name: http-console
- containerPort: 9001
- env:
- - name: MINIO_ROOT_USER
- valueFrom:
- secretKeyRef:
- name: release-name-minio
- key: rootUser
- - name: MINIO_ROOT_PASSWORD
- valueFrom:
- secretKeyRef:
- name: release-name-minio
- key: rootPassword
- - name: MINIO_PROMETHEUS_AUTH_TYPE
- value: "public"
- resources:
- requests:
- cpu: 100m
- memory: 128Mi
- volumes:
- - name: export
- persistentVolumeClaim:
- claimName: release-name-minio
- - name: minio-user
- secret:
- secretName: release-name-minio
----
-# Source: mimir/charts/mimir/charts/rollout_operator/templates/deployment.yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: release-name-rollout-operator
- labels:
- helm.sh/chart: rollout-operator-0.10.0
- app.kubernetes.io/name: rollout-operator
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "v0.9.0"
- app.kubernetes.io/managed-by: Helm
-spec:
- replicas: 1
- minReadySeconds: 10
- selector:
- matchLabels:
- app.kubernetes.io/name: rollout-operator
- app.kubernetes.io/instance: release-name
- strategy:
- rollingUpdate:
- maxSurge: 0
- maxUnavailable: 1
- template:
- metadata:
- labels:
- app.kubernetes.io/name: rollout-operator
- app.kubernetes.io/instance: release-name
- spec:
- serviceAccountName: release-name-rollout-operator
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- containers:
- - name: rollout-operator
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- image: "grafana/rollout-operator:v0.9.0"
- imagePullPolicy: IfNotPresent
- args:
- - -kubernetes.namespace=default
- ports:
- - name: http-metrics
- containerPort: 8001
- protocol: TCP
- readinessProbe:
- httpGet:
- path: /ready
- port: http-metrics
- initialDelaySeconds: 5
- timeoutSeconds: 1
- resources:
- limits:
- cpu: "1"
- memory: 200Mi
- requests:
- cpu: 100m
- memory: 100Mi
----
-# Source: mimir/charts/mimir/templates/distributor/distributor-dep.yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: release-name-mimir-distributor
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: distributor
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- # If replicas is not number (when using values file it's float64, when using --set arg it's int64) and is false (i.e. null) don't set it
- replicas: 2
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: distributor
- strategy:
- rollingUpdate:
- maxSurge: 15%
- maxUnavailable: 0
- type: RollingUpdate
- template:
- metadata:
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: distributor
- app.kubernetes.io/part-of: memberlist
- annotations:
- checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
- namespace: "default"
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- initContainers:
- []
- containers:
- - name: distributor
- image: "grafana/mimir:2.11.0"
- imagePullPolicy: IfNotPresent
- args:
- - "-target=distributor"
- - "-config.expand-env=true"
- - "-config.file=/etc/mimir/mimir.yaml"
- volumeMounts:
- - name: config
- mountPath: /etc/mimir
- - name: runtime-config
- mountPath: /var/mimir
- - name: storage
- mountPath: "/data"
- subPath:
- - name: active-queries
- mountPath: /active-query-tracker
- ports:
- - name: http-metrics
- containerPort: 8080
- protocol: TCP
- - name: grpc
- containerPort: 9095
- protocol: TCP
- - name: memberlist
- containerPort: 7946
- protocol: TCP
- livenessProbe:
- null
- readinessProbe:
- httpGet:
- path: /ready
- port: http-metrics
- initialDelaySeconds: 45
- resources:
- limits:
- cpu: 3.5
- memory: 5.7Gi
- requests:
- cpu: 20m
- memory: 10Mi
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- env:
- - name: "GOMAXPROCS"
- value: "8"
- envFrom:
- nodeSelector:
- {}
- affinity:
- {}
- topologySpreadConstraints:
- - maxSkew: 1
- topologyKey: kubernetes.io/hostname
- whenUnsatisfiable: ScheduleAnyway
- labelSelector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: distributor
- tolerations:
- []
- terminationGracePeriodSeconds: 60
- volumes:
- - name: config
- configMap:
- name: release-name-mimir-config
- items:
- - key: "mimir.yaml"
- path: "mimir.yaml"
- - name: runtime-config
- configMap:
- name: release-name-mimir-runtime
- - name: storage
- emptyDir: {}
- - name: active-queries
- emptyDir: {}
----
-# Source: mimir/charts/mimir/templates/nginx/nginx-dep.yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: release-name-mimir-nginx
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: nginx
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- replicas: 1
- strategy:
- rollingUpdate:
- maxSurge: 15%
- maxUnavailable: 0
- type: RollingUpdate
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: nginx
- template:
- metadata:
- annotations:
- checksum/config: 3207e30c8838f0e7fa2003ecc19522fc61ad4af5452ee2f1bf7bfe7312d9c21e
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: nginx
- namespace: "default"
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- terminationGracePeriodSeconds: 30
- containers:
- - name: nginx
- image: docker.io/nginxinc/nginx-unprivileged:1.25-alpine
- imagePullPolicy: IfNotPresent
- ports:
- - name: http-metric
- containerPort: 8080
- protocol: TCP
- env:
- envFrom:
- readinessProbe:
- httpGet:
- path: /
- port: http-metric
- initialDelaySeconds: 15
- timeoutSeconds: 1
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- volumeMounts:
- - name: config
- mountPath: /etc/nginx
- - name: tmp
- mountPath: /tmp
- - name: docker-entrypoint-d-override
- mountPath: /docker-entrypoint.d
- resources:
- limits:
- cpu: 1.4
- memory: 731Mi
- requests:
- cpu: 20m
- memory: 10Mi
- topologySpreadConstraints:
- - maxSkew: 1
- topologyKey: kubernetes.io/hostname
- whenUnsatisfiable: ScheduleAnyway
- labelSelector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: nginx
- volumes:
- - name: config
- configMap:
- name: release-name-mimir-nginx
- - name: tmp
- emptyDir: {}
- - name: docker-entrypoint-d-override
- emptyDir: {}
----
-# Source: mimir/charts/mimir/templates/overrides-exporter/overrides-exporter-dep.yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- annotations:
- {}
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: overrides-exporter
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- name: release-name-mimir-overrides-exporter
- namespace: "default"
-spec:
- replicas: 1
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: overrides-exporter
- strategy:
- rollingUpdate:
- maxSurge: 15%
- maxUnavailable: 0
- type: RollingUpdate
- template:
- metadata:
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: overrides-exporter
- annotations:
- checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
- namespace: "default"
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- initContainers:
- []
- containers:
- - name: overrides-exporter
- image: "grafana/mimir:2.11.0"
- imagePullPolicy: IfNotPresent
- args:
- - "-target=overrides-exporter"
- - "-config.expand-env=true"
- - "-config.file=/etc/mimir/mimir.yaml"
- volumeMounts:
- - name: config
- mountPath: /etc/mimir
- - name: runtime-config
- mountPath: /var/mimir
- - name: storage
- mountPath: "/data"
- subPath:
- - name: active-queries
- mountPath: /active-query-tracker
- ports:
- - name: http-metrics
- containerPort: 8080
- protocol: TCP
- - name: grpc
- containerPort: 9095
- protocol: TCP
- livenessProbe:
- httpGet:
- path: /ready
- port: http-metrics
- initialDelaySeconds: 45
- readinessProbe:
- httpGet:
- path: /ready
- port: http-metrics
- initialDelaySeconds: 45
- resources:
- limits:
- cpu: 100m
- memory: 128Mi
- requests:
- cpu: 20m
- memory: 10Mi
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- env:
- envFrom:
- nodeSelector:
- {}
- affinity:
- {}
-
- tolerations:
- []
- terminationGracePeriodSeconds: 60
- volumes:
- - name: config
- configMap:
- name: release-name-mimir-config
- items:
- - key: "mimir.yaml"
- path: "mimir.yaml"
- - name: runtime-config
- configMap:
- name: release-name-mimir-runtime
- - name: storage
- emptyDir: {}
- - name: active-queries
- emptyDir: {}
----
-# Source: mimir/charts/mimir/templates/querier/querier-dep.yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: release-name-mimir-querier
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: querier
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- # If replicas is not number (when using values file it's float64, when using --set arg it's int64) and is false (i.e. null) don't set it
- replicas: 1
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: querier
- strategy:
- rollingUpdate:
- maxSurge: 15%
- maxUnavailable: 0
- type: RollingUpdate
- template:
- metadata:
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: querier
- app.kubernetes.io/part-of: memberlist
- annotations:
- checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- initContainers:
- []
- containers:
- - name: querier
- image: "grafana/mimir:2.11.0"
- imagePullPolicy: IfNotPresent
- args:
- - "-target=querier"
- - "-config.expand-env=true"
- - "-config.file=/etc/mimir/mimir.yaml"
- volumeMounts:
- - name: config
- mountPath: /etc/mimir
- - name: runtime-config
- mountPath: /var/mimir
- - name: storage
- mountPath: "/data"
- subPath:
- - name: active-queries
- mountPath: /active-query-tracker
- ports:
- - name: http-metrics
- containerPort: 8080
- protocol: TCP
- - name: grpc
- containerPort: 9095
- protocol: TCP
- - name: memberlist
- containerPort: 7946
- protocol: TCP
- livenessProbe:
- null
- readinessProbe:
- httpGet:
- path: /ready
- port: http-metrics
- initialDelaySeconds: 45
- resources:
- limits:
- cpu: 2.8
- memory: 5.6Gi
- requests:
- cpu: 20m
- memory: 10Mi
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- env:
- - name: "GOMAXPROCS"
- value: "5"
- envFrom:
- nodeSelector:
- {}
- affinity:
- {}
- topologySpreadConstraints:
- - maxSkew: 1
- topologyKey: kubernetes.io/hostname
- whenUnsatisfiable: ScheduleAnyway
- labelSelector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: querier
- tolerations:
- []
- terminationGracePeriodSeconds: 180
- volumes:
- - name: config
- configMap:
- name: release-name-mimir-config
- items:
- - key: "mimir.yaml"
- path: "mimir.yaml"
- - name: runtime-config
- configMap:
- name: release-name-mimir-runtime
- - name: storage
- emptyDir: {}
- - name: active-queries
- emptyDir: {}
----
-# Source: mimir/charts/mimir/templates/query-frontend/query-frontend-dep.yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: release-name-mimir-query-frontend
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-frontend
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- # If replicas is not number (when using values file it's float64, when using --set arg it's int64) and is false (i.e. null) don't set it
- replicas: 1
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-frontend
- strategy:
- rollingUpdate:
- maxSurge: 15%
- maxUnavailable: 0
- type: RollingUpdate
- template:
- metadata:
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: query-frontend
- annotations:
- checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
- namespace: "default"
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- initContainers:
- []
- containers:
- - name: query-frontend
- image: "grafana/mimir:2.11.0"
- imagePullPolicy: IfNotPresent
- args:
- - "-target=query-frontend"
- - "-config.expand-env=true"
- - "-config.file=/etc/mimir/mimir.yaml"
- volumeMounts:
- - name: runtime-config
- mountPath: /var/mimir
- - name: config
- mountPath: /etc/mimir
- - name: storage
- mountPath: /data
- - name: active-queries
- mountPath: /active-query-tracker
- ports:
- - name: http-metrics
- containerPort: 8080
- protocol: TCP
- - name: grpc
- containerPort: 9095
- protocol: TCP
- livenessProbe:
- null
- readinessProbe:
- httpGet:
- path: /ready
- port: http-metrics
- initialDelaySeconds: 45
- resources:
- limits:
- cpu: 2.8
- memory: 2.8Gi
- requests:
- cpu: 20m
- memory: 10Mi
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- env:
- envFrom:
- nodeSelector:
- {}
- affinity:
- {}
- topologySpreadConstraints:
- - maxSkew: 1
- topologyKey: kubernetes.io/hostname
- whenUnsatisfiable: ScheduleAnyway
- labelSelector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-frontend
- tolerations:
- []
- terminationGracePeriodSeconds: 180
- volumes:
- - name: config
- configMap:
- name: release-name-mimir-config
- items:
- - key: "mimir.yaml"
- path: "mimir.yaml"
- - name: runtime-config
- configMap:
- name: release-name-mimir-runtime
- - name: storage
- emptyDir: {}
- - name: active-queries
- emptyDir: {}
----
-# Source: mimir/charts/mimir/templates/query-scheduler/query-scheduler-dep.yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: release-name-mimir-query-scheduler
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-scheduler
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
-spec:
- replicas: 2
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-scheduler
- strategy:
- rollingUpdate:
- maxSurge: 1
- maxUnavailable: 0
- type: RollingUpdate
- template:
- metadata:
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: query-scheduler
- annotations:
- checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- initContainers:
- []
- containers:
- - name: query-scheduler
- image: "grafana/mimir:2.11.0"
- imagePullPolicy: IfNotPresent
- args:
- - "-target=query-scheduler"
- - "-config.expand-env=true"
- - "-config.file=/etc/mimir/mimir.yaml"
- - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity
- - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity
- volumeMounts:
- - name: runtime-config
- mountPath: /var/mimir
- - name: config
- mountPath: /etc/mimir
- - name: storage
- mountPath: /data
- - name: active-queries
- mountPath: /active-query-tracker
- ports:
- - name: http-metrics
- containerPort: 8080
- protocol: TCP
- - name: grpc
- containerPort: 9095
- protocol: TCP
- livenessProbe:
- null
- readinessProbe:
- httpGet:
- path: /ready
- port: http-metrics
- initialDelaySeconds: 45
- resources:
- requests:
- cpu: 100m
- memory: 128Mi
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- env:
- envFrom:
- nodeSelector:
- {}
- affinity:
- {}
- topologySpreadConstraints:
- - maxSkew: 1
- topologyKey: kubernetes.io/hostname
- whenUnsatisfiable: ScheduleAnyway
- labelSelector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-scheduler
- tolerations:
- []
- terminationGracePeriodSeconds: 180
- volumes:
- - name: config
- configMap:
- name: release-name-mimir-config
- items:
- - key: "mimir.yaml"
- path: "mimir.yaml"
- - name: runtime-config
- configMap:
- name: release-name-mimir-runtime
- - name: storage
- emptyDir: {}
- - name: active-queries
- emptyDir: {}
----
-# Source: mimir/charts/mimir/templates/ruler/ruler-dep.yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: release-name-mimir-ruler
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ruler
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- replicas: 1
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ruler
- strategy:
- rollingUpdate:
- maxSurge: 50%
- maxUnavailable: 0
- type: RollingUpdate
- template:
- metadata:
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: ruler
- app.kubernetes.io/part-of: memberlist
- annotations:
- checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
- namespace: "default"
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- initContainers:
- []
- containers:
- - name: ruler
- image: "grafana/mimir:2.11.0"
- imagePullPolicy: IfNotPresent
- args:
- - "-target=ruler"
- - "-config.expand-env=true"
- - "-config.file=/etc/mimir/mimir.yaml"
- volumeMounts:
- - name: config
- mountPath: /etc/mimir
- - name: runtime-config
- mountPath: /var/mimir
- - name: storage
- mountPath: "/data"
- subPath:
- - name: active-queries
- mountPath: /active-query-tracker
- ports:
- - name: http-metrics
- containerPort: 8080
- protocol: TCP
- - name: grpc
- containerPort: 9095
- protocol: TCP
- - name: memberlist
- containerPort: 7946
- protocol: TCP
- livenessProbe:
- null
- readinessProbe:
- httpGet:
- path: /ready
- port: http-metrics
- initialDelaySeconds: 45
- resources:
- limits:
- cpu: 1.4
- memory: 2.8Gi
- requests:
- cpu: 20m
- memory: 10Mi
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- env:
- envFrom:
- nodeSelector:
- {}
- affinity:
- {}
- topologySpreadConstraints:
- - maxSkew: 1
- topologyKey: kubernetes.io/hostname
- whenUnsatisfiable: ScheduleAnyway
- labelSelector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ruler
- tolerations:
- []
- terminationGracePeriodSeconds: 180
- volumes:
- - name: config
- configMap:
- name: release-name-mimir-config
- items:
- - key: "mimir.yaml"
- path: "mimir.yaml"
- - name: runtime-config
- configMap:
- name: release-name-mimir-runtime
- - name: storage
- emptyDir: {}
- - name: active-queries
- emptyDir: {}
----
-# Source: mimir/charts/mimir/templates/alertmanager/alertmanager-statefulset.yaml
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
- name: release-name-mimir-alertmanager
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: alertmanager
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- replicas: 2
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: alertmanager
- updateStrategy:
- type: RollingUpdate
- serviceName: release-name-mimir-alertmanager
- volumeClaimTemplates:
- - metadata:
- name: storage
- spec:
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: "1Gi"
- template:
- metadata:
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: alertmanager
- app.kubernetes.io/part-of: memberlist
- annotations:
- checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
- checksum/alertmanager-fallback-config: d2d10e73bd2feee8e9f9250f8ef6b276de6d4b2eeb9127a16e77eb0f5db7718f
- namespace: "default"
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- initContainers:
- []
- nodeSelector:
- {}
- affinity:
- {}
- topologySpreadConstraints:
- - maxSkew: 1
- topologyKey: kubernetes.io/hostname
- whenUnsatisfiable: ScheduleAnyway
- labelSelector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: alertmanager
- tolerations:
- []
- terminationGracePeriodSeconds: 60
- volumes:
- - name: config
- configMap:
- name: release-name-mimir-config
- items:
- - key: "mimir.yaml"
- path: "mimir.yaml"
- - name: runtime-config
- configMap:
- name: release-name-mimir-runtime
- - name: tmp
- emptyDir: {}
- - name: active-queries
- emptyDir: {}
- - name: alertmanager-fallback-config
- configMap:
- name: release-name-mimir-alertmanager-fallback-config
- containers:
- - name: alertmanager
- image: "grafana/mimir:2.11.0"
- imagePullPolicy: IfNotPresent
- args:
- - "-target=alertmanager"
- - "-config.expand-env=true"
- - "-config.file=/etc/mimir/mimir.yaml"
- volumeMounts:
- - name: config
- mountPath: /etc/mimir
- - name: runtime-config
- mountPath: /var/mimir
- - name: storage
- mountPath: "/data"
- - name: alertmanager-fallback-config
- mountPath: /configs/
- - name: tmp
- mountPath: /tmp
- - name: active-queries
- mountPath: /active-query-tracker
- ports:
- - name: http-metrics
- containerPort: 8080
- protocol: TCP
- - name: grpc
- containerPort: 9095
- protocol: TCP
- - name: memberlist
- containerPort: 7946
- protocol: TCP
- livenessProbe:
- null
- readinessProbe:
- httpGet:
- path: /ready
- port: http-metrics
- initialDelaySeconds: 45
- resources:
- limits:
- cpu: 1.4
- memory: 1.4Gi
- requests:
- cpu: 20m
- memory: 10Mi
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- env:
- envFrom:
----
-# Source: mimir/charts/mimir/templates/compactor/compactor-statefulset.yaml
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
- name: release-name-mimir-compactor
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: compactor
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- podManagementPolicy: OrderedReady
- replicas: 1
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: compactor
- updateStrategy:
- type: RollingUpdate
- serviceName: release-name-mimir-compactor
- volumeClaimTemplates:
- - metadata:
- name: storage
- spec:
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: "20Gi"
- template:
- metadata:
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: compactor
- app.kubernetes.io/part-of: memberlist
- annotations:
- checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
- namespace: "default"
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- initContainers:
- []
- nodeSelector:
- {}
- affinity:
- {}
- topologySpreadConstraints:
- - maxSkew: 1
- topologyKey: kubernetes.io/hostname
- whenUnsatisfiable: ScheduleAnyway
- labelSelector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: compactor
- tolerations:
- []
- terminationGracePeriodSeconds: 240
- volumes:
- - name: config
- configMap:
- name: release-name-mimir-config
- items:
- - key: "mimir.yaml"
- path: "mimir.yaml"
- - name: runtime-config
- configMap:
- name: release-name-mimir-runtime
- - name: active-queries
- emptyDir: {}
- containers:
- - name: compactor
- image: "grafana/mimir:2.11.0"
- imagePullPolicy: IfNotPresent
- args:
- - "-target=compactor"
- - "-config.expand-env=true"
- - "-config.file=/etc/mimir/mimir.yaml"
- volumeMounts:
- - name: config
- mountPath: /etc/mimir
- - name: runtime-config
- mountPath: /var/mimir
- - name: storage
- mountPath: "/data"
- - name: active-queries
- mountPath: /active-query-tracker
- ports:
- - name: http-metrics
- containerPort: 8080
- protocol: TCP
- - name: grpc
- containerPort: 9095
- protocol: TCP
- - name: memberlist
- containerPort: 7946
- protocol: TCP
- livenessProbe:
- null
- readinessProbe:
- httpGet:
- path: /ready
- port: http-metrics
- initialDelaySeconds: 60
- resources:
- limits:
- cpu: 1.4
- memory: 2.1Gi
- requests:
- cpu: 20m
- memory: 10Mi
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- env:
- envFrom:
----
-# Source: mimir/charts/mimir/templates/index-cache/index-cache-statefulset.yaml
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
- name: release-name-mimir-index-cache
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: memcached
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- podManagementPolicy: Parallel
- replicas: 3
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: index-cache
- updateStrategy:
- type: RollingUpdate
- serviceName: release-name-mimir-index-cache
-
- template:
- metadata:
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: index-cache
- annotations:
-
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- initContainers:
- []
- nodeSelector:
- {}
- affinity:
- {}
-
- tolerations:
- []
- terminationGracePeriodSeconds: 60
- containers:
- - name: memcached
- image: memcached:1.6.22-alpine
- imagePullPolicy: IfNotPresent
- resources:
- limits:
- memory: 2458Mi
- requests:
- cpu: 500m
- memory: 2458Mi
- ports:
- - containerPort: 11211
- name: client
- args:
- - -m 2048
- - --extended=modern,track_sizes
- - -I 5m
- - -c 16384
- - -v
- - -u 11211
- env:
- envFrom:
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- - name: exporter
- image: prom/memcached-exporter:v0.14.2
- imagePullPolicy: IfNotPresent
- ports:
- - containerPort: 9150
- name: http-metrics
- args:
- - "--memcached.address=localhost:11211"
- - "--web.listen-address=0.0.0.0:9150"
- resources:
- limits: {}
- requests: {}
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
----
-# Source: mimir/charts/mimir/templates/ingester/ingester-statefulset.yaml
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
- name: release-name-mimir-ingester-zone-a
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- name: "ingester-zone-a"
- rollout-group: ingester
- zone: zone-a
- annotations:
- rollout-max-unavailable: "50"
- namespace: "default"
-spec:
- podManagementPolicy: Parallel
- replicas: 1
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- rollout-group: ingester
- zone: zone-a
- updateStrategy:
- type: OnDelete
- serviceName: release-name-mimir-ingester-headless
- volumeClaimTemplates:
- - metadata:
- name: storage
- spec:
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: "50Gi"
- template:
- metadata:
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: ingester
- app.kubernetes.io/part-of: memberlist
- name: "ingester-zone-a"
- rollout-group: ingester
- zone: zone-a
- annotations:
- checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
- namespace: "default"
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- initContainers:
- []
- nodeSelector:
- {}
- affinity:
- podAntiAffinity:
- requiredDuringSchedulingIgnoredDuringExecution:
- - labelSelector:
- matchExpressions:
- - key: rollout-group
- operator: In
- values:
- - ingester
- - key: zone
- operator: NotIn
- values:
- - zone-a
- topologyKey: kubernetes.io/hostname
- topologySpreadConstraints:
- - maxSkew: 1
- topologyKey: kubernetes.io/hostname
- whenUnsatisfiable: ScheduleAnyway
- labelSelector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- tolerations:
- []
- terminationGracePeriodSeconds: 240
- volumes:
- - name: config
- configMap:
- name: release-name-mimir-config
- items:
- - key: "mimir.yaml"
- path: "mimir.yaml"
- - name: runtime-config
- configMap:
- name: release-name-mimir-runtime
- - name: active-queries
- emptyDir: {}
- containers:
- - name: ingester
- image: "grafana/mimir:2.11.0"
- imagePullPolicy: IfNotPresent
- args:
- - "-target=ingester"
- - "-config.expand-env=true"
- - "-config.file=/etc/mimir/mimir.yaml"
- - "-ingester.ring.instance-availability-zone=zone-a"
- volumeMounts:
- - name: config
- mountPath: /etc/mimir
- - name: runtime-config
- mountPath: /var/mimir
- - name: storage
- mountPath: "/data"
- - name: active-queries
- mountPath: /active-query-tracker
- ports:
- - name: http-metrics
- containerPort: 8080
- protocol: TCP
- - name: grpc
- containerPort: 9095
- protocol: TCP
- - name: memberlist
- containerPort: 7946
- protocol: TCP
- livenessProbe:
- null
- readinessProbe:
- httpGet:
- path: /ready
- port: http-metrics
- initialDelaySeconds: 60
- resources:
- limits:
- cpu: 5
- memory: 12Gi
- requests:
- cpu: 20m
- memory: 10Mi
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- env:
- envFrom:
----
-# Source: mimir/charts/mimir/templates/ingester/ingester-statefulset.yaml
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
- name: release-name-mimir-ingester-zone-b
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- name: "ingester-zone-b"
- rollout-group: ingester
- zone: zone-b
- annotations:
- rollout-max-unavailable: "50"
- namespace: "default"
-spec:
- podManagementPolicy: Parallel
- replicas: 1
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- rollout-group: ingester
- zone: zone-b
- updateStrategy:
- type: OnDelete
- serviceName: release-name-mimir-ingester-headless
- volumeClaimTemplates:
- - metadata:
- name: storage
- spec:
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: "50Gi"
- template:
- metadata:
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: ingester
- app.kubernetes.io/part-of: memberlist
- name: "ingester-zone-b"
- rollout-group: ingester
- zone: zone-b
- annotations:
- checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
- namespace: "default"
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- initContainers:
- []
- nodeSelector:
- {}
- affinity:
- podAntiAffinity:
- requiredDuringSchedulingIgnoredDuringExecution:
- - labelSelector:
- matchExpressions:
- - key: rollout-group
- operator: In
- values:
- - ingester
- - key: zone
- operator: NotIn
- values:
- - zone-b
- topologyKey: kubernetes.io/hostname
- topologySpreadConstraints:
- - maxSkew: 1
- topologyKey: kubernetes.io/hostname
- whenUnsatisfiable: ScheduleAnyway
- labelSelector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- tolerations:
- []
- terminationGracePeriodSeconds: 240
- volumes:
- - name: config
- configMap:
- name: release-name-mimir-config
- items:
- - key: "mimir.yaml"
- path: "mimir.yaml"
- - name: runtime-config
- configMap:
- name: release-name-mimir-runtime
- - name: active-queries
- emptyDir: {}
- containers:
- - name: ingester
- image: "grafana/mimir:2.11.0"
- imagePullPolicy: IfNotPresent
- args:
- - "-target=ingester"
- - "-config.expand-env=true"
- - "-config.file=/etc/mimir/mimir.yaml"
- - "-ingester.ring.instance-availability-zone=zone-b"
- volumeMounts:
- - name: config
- mountPath: /etc/mimir
- - name: runtime-config
- mountPath: /var/mimir
- - name: storage
- mountPath: "/data"
- - name: active-queries
- mountPath: /active-query-tracker
- ports:
- - name: http-metrics
- containerPort: 8080
- protocol: TCP
- - name: grpc
- containerPort: 9095
- protocol: TCP
- - name: memberlist
- containerPort: 7946
- protocol: TCP
- livenessProbe:
- null
- readinessProbe:
- httpGet:
- path: /ready
- port: http-metrics
- initialDelaySeconds: 60
- resources:
- limits:
- cpu: 5
- memory: 12Gi
- requests:
- cpu: 20m
- memory: 10Mi
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- env:
- envFrom:
----
-# Source: mimir/charts/mimir/templates/ingester/ingester-statefulset.yaml
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
- name: release-name-mimir-ingester-zone-c
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- name: "ingester-zone-c"
- rollout-group: ingester
- zone: zone-c
- annotations:
- rollout-max-unavailable: "50"
- namespace: "default"
-spec:
- podManagementPolicy: Parallel
- replicas: 1
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- rollout-group: ingester
- zone: zone-c
- updateStrategy:
- type: OnDelete
- serviceName: release-name-mimir-ingester-headless
- volumeClaimTemplates:
- - metadata:
- name: storage
- spec:
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: "50Gi"
- template:
- metadata:
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: ingester
- app.kubernetes.io/part-of: memberlist
- name: "ingester-zone-c"
- rollout-group: ingester
- zone: zone-c
- annotations:
- checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
- namespace: "default"
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- initContainers:
- []
- nodeSelector:
- {}
- affinity:
- podAntiAffinity:
- requiredDuringSchedulingIgnoredDuringExecution:
- - labelSelector:
- matchExpressions:
- - key: rollout-group
- operator: In
- values:
- - ingester
- - key: zone
- operator: NotIn
- values:
- - zone-c
- topologyKey: kubernetes.io/hostname
- topologySpreadConstraints:
- - maxSkew: 1
- topologyKey: kubernetes.io/hostname
- whenUnsatisfiable: ScheduleAnyway
- labelSelector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- tolerations:
- []
- terminationGracePeriodSeconds: 240
- volumes:
- - name: config
- configMap:
- name: release-name-mimir-config
- items:
- - key: "mimir.yaml"
- path: "mimir.yaml"
- - name: runtime-config
- configMap:
- name: release-name-mimir-runtime
- - name: active-queries
- emptyDir: {}
- containers:
- - name: ingester
- image: "grafana/mimir:2.11.0"
- imagePullPolicy: IfNotPresent
- args:
- - "-target=ingester"
- - "-config.expand-env=true"
- - "-config.file=/etc/mimir/mimir.yaml"
- - "-ingester.ring.instance-availability-zone=zone-c"
- volumeMounts:
- - name: config
- mountPath: /etc/mimir
- - name: runtime-config
- mountPath: /var/mimir
- - name: storage
- mountPath: "/data"
- - name: active-queries
- mountPath: /active-query-tracker
- ports:
- - name: http-metrics
- containerPort: 8080
- protocol: TCP
- - name: grpc
- containerPort: 9095
- protocol: TCP
- - name: memberlist
- containerPort: 7946
- protocol: TCP
- livenessProbe:
- null
- readinessProbe:
- httpGet:
- path: /ready
- port: http-metrics
- initialDelaySeconds: 60
- resources:
- limits:
- cpu: 5
- memory: 12Gi
- requests:
- cpu: 20m
- memory: 10Mi
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- env:
- envFrom:
----
-# Source: mimir/charts/mimir/templates/metadata-cache/metadata-cache-statefulset.yaml
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
- name: release-name-mimir-metadata-cache
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: memcached
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- podManagementPolicy: Parallel
- replicas: 1
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: metadata-cache
- updateStrategy:
- type: RollingUpdate
- serviceName: release-name-mimir-metadata-cache
-
- template:
- metadata:
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: metadata-cache
- annotations:
-
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- initContainers:
- []
- nodeSelector:
- {}
- affinity:
- {}
-
- tolerations:
- []
- terminationGracePeriodSeconds: 60
- containers:
- - name: memcached
- image: memcached:1.6.22-alpine
- imagePullPolicy: IfNotPresent
- resources:
- limits:
- memory: 614Mi
- requests:
- cpu: 500m
- memory: 614Mi
- ports:
- - containerPort: 11211
- name: client
- args:
- - -m 512
- - --extended=modern,track_sizes
- - -I 1m
- - -c 16384
- - -v
- - -u 11211
- env:
- envFrom:
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- - name: exporter
- image: prom/memcached-exporter:v0.14.2
- imagePullPolicy: IfNotPresent
- ports:
- - containerPort: 9150
- name: http-metrics
- args:
- - "--memcached.address=localhost:11211"
- - "--web.listen-address=0.0.0.0:9150"
- resources:
- limits: {}
- requests: {}
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
----
-# Source: mimir/charts/mimir/templates/results-cache/results-cache-statefulset.yaml
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
- name: release-name-mimir-results-cache
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: memcached
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- {}
- namespace: "default"
-spec:
- podManagementPolicy: Parallel
- replicas: 1
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: results-cache
- updateStrategy:
- type: RollingUpdate
- serviceName: release-name-mimir-results-cache
-
- template:
- metadata:
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: results-cache
- annotations:
-
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- initContainers:
- []
- nodeSelector:
- {}
- affinity:
- {}
-
- tolerations:
- []
- terminationGracePeriodSeconds: 60
- containers:
- - name: memcached
- image: memcached:1.6.22-alpine
- imagePullPolicy: IfNotPresent
- resources:
- limits:
- memory: 614Mi
- requests:
- cpu: 500m
- memory: 614Mi
- ports:
- - containerPort: 11211
- name: client
- args:
- - -m 512
- - --extended=modern,track_sizes
- - -I 5m
- - -c 16384
- - -v
- - -u 11211
- env:
- envFrom:
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- - name: exporter
- image: prom/memcached-exporter:v0.14.2
- imagePullPolicy: IfNotPresent
- ports:
- - containerPort: 9150
- name: http-metrics
- args:
- - "--memcached.address=localhost:11211"
- - "--web.listen-address=0.0.0.0:9150"
- resources:
- limits: {}
- requests: {}
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
----
-# Source: mimir/charts/mimir/templates/store-gateway/store-gateway-statefulset.yaml
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
- name: release-name-mimir-store-gateway-zone-a
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- name: "store-gateway-zone-a"
- rollout-group: store-gateway
- zone: zone-a
- annotations:
- rollout-max-unavailable: "50"
- namespace: "default"
-spec:
- podManagementPolicy: OrderedReady
- replicas: 1
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- rollout-group: store-gateway
- zone: zone-a
- updateStrategy:
- type: OnDelete
- serviceName: release-name-mimir-store-gateway-headless
- volumeClaimTemplates:
- - metadata:
- name: storage
- spec:
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: "10Gi"
- template:
- metadata:
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: store-gateway
- app.kubernetes.io/part-of: memberlist
- name: "store-gateway-zone-a"
- rollout-group: store-gateway
- zone: zone-a
- annotations:
- checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
- namespace: "default"
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- initContainers:
- []
- nodeSelector:
- {}
- affinity:
- podAntiAffinity:
- requiredDuringSchedulingIgnoredDuringExecution:
- - labelSelector:
- matchExpressions:
- - key: rollout-group
- operator: In
- values:
- - store-gateway
- - key: zone
- operator: NotIn
- values:
- - zone-a
- topologyKey: kubernetes.io/hostname
- topologySpreadConstraints:
- - maxSkew: 1
- topologyKey: kubernetes.io/hostname
- whenUnsatisfiable: ScheduleAnyway
- labelSelector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- tolerations:
- []
- terminationGracePeriodSeconds: 240
- volumes:
- - name: config
- configMap:
- name: release-name-mimir-config
- items:
- - key: "mimir.yaml"
- path: "mimir.yaml"
- - name: runtime-config
- configMap:
- name: release-name-mimir-runtime
- - name: active-queries
- emptyDir: {}
- containers:
- - name: store-gateway
- image: "grafana/mimir:2.11.0"
- imagePullPolicy: IfNotPresent
- args:
- - "-target=store-gateway"
- - "-config.expand-env=true"
- - "-config.file=/etc/mimir/mimir.yaml"
- - "-store-gateway.sharding-ring.instance-availability-zone=zone-a"
- volumeMounts:
- - name: config
- mountPath: /etc/mimir
- - name: runtime-config
- mountPath: /var/mimir
- - name: storage
- mountPath: "/data"
- - name: active-queries
- mountPath: /active-query-tracker
- ports:
- - name: http-metrics
- containerPort: 8080
- protocol: TCP
- - name: grpc
- containerPort: 9095
- protocol: TCP
- - name: memberlist
- containerPort: 7946
- protocol: TCP
- livenessProbe:
- null
- readinessProbe:
- httpGet:
- path: /ready
- port: http-metrics
- initialDelaySeconds: 60
- resources:
- limits:
- cpu: 1.4
- memory: 2.1Gi
- requests:
- cpu: 20m
- memory: 10Mi
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- env:
- - name: "GOMAXPROCS"
- value: "5"
- - name: "GOMEMLIMIT"
- value: "10485760"
- envFrom:
----
-# Source: mimir/charts/mimir/templates/store-gateway/store-gateway-statefulset.yaml
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
- name: release-name-mimir-store-gateway-zone-b
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- name: "store-gateway-zone-b"
- rollout-group: store-gateway
- zone: zone-b
- annotations:
- rollout-max-unavailable: "50"
- namespace: "default"
-spec:
- podManagementPolicy: OrderedReady
- replicas: 1
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- rollout-group: store-gateway
- zone: zone-b
- updateStrategy:
- type: OnDelete
- serviceName: release-name-mimir-store-gateway-headless
- volumeClaimTemplates:
- - metadata:
- name: storage
- spec:
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: "10Gi"
- template:
- metadata:
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: store-gateway
- app.kubernetes.io/part-of: memberlist
- name: "store-gateway-zone-b"
- rollout-group: store-gateway
- zone: zone-b
- annotations:
- checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
- namespace: "default"
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- initContainers:
- []
- nodeSelector:
- {}
- affinity:
- podAntiAffinity:
- requiredDuringSchedulingIgnoredDuringExecution:
- - labelSelector:
- matchExpressions:
- - key: rollout-group
- operator: In
- values:
- - store-gateway
- - key: zone
- operator: NotIn
- values:
- - zone-b
- topologyKey: kubernetes.io/hostname
- topologySpreadConstraints:
- - maxSkew: 1
- topologyKey: kubernetes.io/hostname
- whenUnsatisfiable: ScheduleAnyway
- labelSelector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- tolerations:
- []
- terminationGracePeriodSeconds: 240
- volumes:
- - name: config
- configMap:
- name: release-name-mimir-config
- items:
- - key: "mimir.yaml"
- path: "mimir.yaml"
- - name: runtime-config
- configMap:
- name: release-name-mimir-runtime
- - name: active-queries
- emptyDir: {}
- containers:
- - name: store-gateway
- image: "grafana/mimir:2.11.0"
- imagePullPolicy: IfNotPresent
- args:
- - "-target=store-gateway"
- - "-config.expand-env=true"
- - "-config.file=/etc/mimir/mimir.yaml"
- - "-store-gateway.sharding-ring.instance-availability-zone=zone-b"
- volumeMounts:
- - name: config
- mountPath: /etc/mimir
- - name: runtime-config
- mountPath: /var/mimir
- - name: storage
- mountPath: "/data"
- - name: active-queries
- mountPath: /active-query-tracker
- ports:
- - name: http-metrics
- containerPort: 8080
- protocol: TCP
- - name: grpc
- containerPort: 9095
- protocol: TCP
- - name: memberlist
- containerPort: 7946
- protocol: TCP
- livenessProbe:
- null
- readinessProbe:
- httpGet:
- path: /ready
- port: http-metrics
- initialDelaySeconds: 60
- resources:
- limits:
- cpu: 1.4
- memory: 2.1Gi
- requests:
- cpu: 20m
- memory: 10Mi
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- env:
- - name: "GOMAXPROCS"
- value: "5"
- - name: "GOMEMLIMIT"
- value: "10485760"
- envFrom:
----
-# Source: mimir/charts/mimir/templates/store-gateway/store-gateway-statefulset.yaml
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
- name: release-name-mimir-store-gateway-zone-c
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- name: "store-gateway-zone-c"
- rollout-group: store-gateway
- zone: zone-c
- annotations:
- rollout-max-unavailable: "50"
- namespace: "default"
-spec:
- podManagementPolicy: OrderedReady
- replicas: 1
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- rollout-group: store-gateway
- zone: zone-c
- updateStrategy:
- type: OnDelete
- serviceName: release-name-mimir-store-gateway-headless
- volumeClaimTemplates:
- - metadata:
- name: storage
- spec:
- accessModes:
- - ReadWriteOnce
- resources:
- requests:
- storage: "10Gi"
- template:
- metadata:
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: store-gateway
- app.kubernetes.io/part-of: memberlist
- name: "store-gateway-zone-c"
- rollout-group: store-gateway
- zone: zone-c
- annotations:
- checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
- namespace: "default"
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- initContainers:
- []
- nodeSelector:
- {}
- affinity:
- podAntiAffinity:
- requiredDuringSchedulingIgnoredDuringExecution:
- - labelSelector:
- matchExpressions:
- - key: rollout-group
- operator: In
- values:
- - store-gateway
- - key: zone
- operator: NotIn
- values:
- - zone-c
- topologyKey: kubernetes.io/hostname
- topologySpreadConstraints:
- - maxSkew: 1
- topologyKey: kubernetes.io/hostname
- whenUnsatisfiable: ScheduleAnyway
- labelSelector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- tolerations:
- []
- terminationGracePeriodSeconds: 240
- volumes:
- - name: config
- configMap:
- name: release-name-mimir-config
- items:
- - key: "mimir.yaml"
- path: "mimir.yaml"
- - name: runtime-config
- configMap:
- name: release-name-mimir-runtime
- - name: active-queries
- emptyDir: {}
- containers:
- - name: store-gateway
- image: "grafana/mimir:2.11.0"
- imagePullPolicy: IfNotPresent
- args:
- - "-target=store-gateway"
- - "-config.expand-env=true"
- - "-config.file=/etc/mimir/mimir.yaml"
- - "-store-gateway.sharding-ring.instance-availability-zone=zone-c"
- volumeMounts:
- - name: config
- mountPath: /etc/mimir
- - name: runtime-config
- mountPath: /var/mimir
- - name: storage
- mountPath: "/data"
- - name: active-queries
- mountPath: /active-query-tracker
- ports:
- - name: http-metrics
- containerPort: 8080
- protocol: TCP
- - name: grpc
- containerPort: 9095
- protocol: TCP
- - name: memberlist
- containerPort: 7946
- protocol: TCP
- livenessProbe:
- null
- readinessProbe:
- httpGet:
- path: /ready
- port: http-metrics
- initialDelaySeconds: 60
- resources:
- limits:
- cpu: 1.4
- memory: 2.1Gi
- requests:
- cpu: 20m
- memory: 10Mi
- securityContext:
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- readOnlyRootFilesystem: true
- env:
- - name: "GOMAXPROCS"
- value: "5"
- - name: "GOMEMLIMIT"
- value: "10485760"
- envFrom:
----
-# Source: mimir/charts/mimir/templates/minio/create-bucket-job.yaml
-# Minio provides post-install hook to create bucket
-# however the hook won't be executed if helm install is run
-# with --wait flag. Hence this job is a workaround for that.
-# See https://github.com/grafana/mimir/issues/2464
-apiVersion: batch/v1
-kind: Job
-metadata:
- name: release-name-mimir-make-minio-buckets-5.0.14
- namespace: "default"
- labels:
- app: mimir-make-bucket-job
- chart: mimir-5.2.1
- release: release-name
- heritage: Helm
-spec:
- template:
- metadata:
- labels:
- app: mimir-job
- release: release-name
- spec:
- restartPolicy: OnFailure
- volumes:
- - name: minio-configuration
- projected:
- sources:
- - configMap:
- name: release-name-minio
- - secret:
- name: release-name-minio
- containers:
- - name: minio-mc
- image: "quay.io/minio/mc:RELEASE.2023-09-29T16-41-22Z"
- imagePullPolicy: IfNotPresent
- command: ["/bin/sh", "/config/initialize"]
- env:
- - name: MINIO_ENDPOINT
- value: release-name-minio
- - name: MINIO_PORT
- value: "9000"
- volumeMounts:
- - name: minio-configuration
- mountPath: /config
- resources:
- requests:
- memory: 128Mi
----
-# Source: mimir/charts/mimir/templates/metamonitoring/grafana-agent.yaml
-apiVersion: monitoring.grafana.com/v1alpha1
-kind: GrafanaAgent
-metadata:
- name: release-name-mimir-meta-monitoring
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: meta-monitoring
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-spec:
- serviceAccountName: release-name-mimir
- containers:
- # The container specs here are merged with the ones in the operator using a strategic merge patch.
- - name: config-reloader
- - name: grafana-agent
-
- logs:
- instanceSelector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: meta-monitoring
- # cluster label for logs is added in the LogsInstance
- metrics:
- scrapeInterval: 60s
- instanceSelector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: meta-monitoring
- externalLabels:
- cluster: release-name
----
-# Source: mimir/charts/mimir/templates/metamonitoring/logs-instance.yaml
-apiVersion: monitoring.grafana.com/v1alpha1
-kind: LogsInstance
-metadata:
- name: release-name-mimir-meta-monitoring
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: meta-monitoring
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-spec:
- clients:
-
-
- # Supply an empty namespace selector to look in all namespaces. Remove
- # this to only look in the same namespace as the LogsInstance CR
- podLogsNamespaceSelector: {}
-
- podLogsSelector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
----
-# Source: mimir/charts/mimir/templates/metamonitoring/metrics-instance.yaml
-apiVersion: monitoring.grafana.com/v1alpha1
-kind: MetricsInstance
-metadata:
- name: release-name-mimir-meta-monitoring
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: meta-monitoring
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-spec:
- remoteWrite:
- - url: http://{{ if ne .Release.Name "mimir" }}{{ .Release.Name }}-{{ end }}mimir-nginx.{{ .Release.Namespace }}/prometheus
- basicAuth:
-
- # Supply an empty namespace selector to look in all namespaces. Remove
- # this to only look in the same namespace as the MetricsInstance CR
- serviceMonitorNamespaceSelector: {}
-
- serviceMonitorSelector:
- # Scrape ServiceMonitors from all components
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
----
-# Source: mimir/charts/mimir/templates/metamonitoring/pod-logs.yaml
-apiVersion: monitoring.grafana.com/v1alpha1
-kind: PodLogs
-metadata:
- name: release-name-mimir-meta-monitoring
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: meta-monitoring
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-spec:
- pipelineStages:
- - cri: { }
- relabelings:
- - action: replace # For consistency with metrics
- replacement: $1
- separator: /
- sourceLabels:
- - __meta_kubernetes_namespace
- - __meta_kubernetes_pod_container_name
- targetLabel: job
- - action: replace # Necessary for slow queries dashboard
- sourceLabels:
- - __meta_kubernetes_pod_container_name
- targetLabel: name
- - targetLabel: cluster
- replacement: release-name
-
- namespaceSelector:
- matchNames:
- - "default"
-
- selector:
- matchLabels:
- # Scrape logs from all components
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/alertmanager.rules.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-alertmanager.rules
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: alertmanager.rules
- rules:
- - alert: AlertmanagerFailedReload
- annotations:
- description: Configuration has failed to load for {{ $labels.namespace }}/{{ $labels.pod}}.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerfailedreload
- summary: Reloading an Alertmanager configuration has failed.
- expr: |-
- # Without max_over_time, failed scrapes could create false negatives, see
- # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
- max_over_time(alertmanager_config_last_reload_successful{job="release-name-kps-alertmanager",namespace="default"}[5m]) == 0
- for: 10m
- labels:
- severity: critical
- - alert: AlertmanagerMembersInconsistent
- annotations:
- description: Alertmanager {{ $labels.namespace }}/{{ $labels.pod}} has only found {{ $value }} members of the {{$labels.job}} cluster.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagermembersinconsistent
- summary: A member of an Alertmanager cluster has not found all other cluster members.
- expr: |-
- # Without max_over_time, failed scrapes could create false negatives, see
- # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
- max_over_time(alertmanager_cluster_members{job="release-name-kps-alertmanager",namespace="default"}[5m])
- < on (namespace,service,cluster) group_left
- count by (namespace,service,cluster) (max_over_time(alertmanager_cluster_members{job="release-name-kps-alertmanager",namespace="default"}[5m]))
- for: 15m
- labels:
- severity: critical
- - alert: AlertmanagerFailedToSendAlerts
- annotations:
- description: Alertmanager {{ $labels.namespace }}/{{ $labels.pod}} failed to send {{ $value | humanizePercentage }} of notifications to {{ $labels.integration }}.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerfailedtosendalerts
- summary: An Alertmanager instance failed to send notifications.
- expr: |-
- (
- rate(alertmanager_notifications_failed_total{job="release-name-kps-alertmanager",namespace="default"}[5m])
- /
- ignoring (reason) group_left rate(alertmanager_notifications_total{job="release-name-kps-alertmanager",namespace="default"}[5m])
- )
- > 0.01
- for: 5m
- labels:
- severity: warning
- - alert: AlertmanagerClusterFailedToSendAlerts
- annotations:
- description: The minimum notification failure rate to {{ $labels.integration }} sent from any instance in the {{$labels.job}} cluster is {{ $value | humanizePercentage }}.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerclusterfailedtosendalerts
- summary: All Alertmanager instances in a cluster failed to send notifications to a critical integration.
- expr: |-
- min by (namespace,service, integration) (
- rate(alertmanager_notifications_failed_total{job="release-name-kps-alertmanager",namespace="default", integration=~`.*`}[5m])
- /
- ignoring (reason) group_left rate(alertmanager_notifications_total{job="release-name-kps-alertmanager",namespace="default", integration=~`.*`}[5m])
- )
- > 0.01
- for: 5m
- labels:
- severity: critical
- - alert: AlertmanagerClusterFailedToSendAlerts
- annotations:
- description: The minimum notification failure rate to {{ $labels.integration }} sent from any instance in the {{$labels.job}} cluster is {{ $value | humanizePercentage }}.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerclusterfailedtosendalerts
- summary: All Alertmanager instances in a cluster failed to send notifications to a non-critical integration.
- expr: |-
- min by (namespace,service, integration) (
- rate(alertmanager_notifications_failed_total{job="release-name-kps-alertmanager",namespace="default", integration!~`.*`}[5m])
- /
- ignoring (reason) group_left rate(alertmanager_notifications_total{job="release-name-kps-alertmanager",namespace="default", integration!~`.*`}[5m])
- )
- > 0.01
- for: 5m
- labels:
- severity: warning
- - alert: AlertmanagerConfigInconsistent
- annotations:
- description: Alertmanager instances within the {{$labels.job}} cluster have different configurations.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerconfiginconsistent
- summary: Alertmanager instances within the same cluster have different configurations.
- expr: |-
- count by (namespace,service,cluster) (
- count_values by (namespace,service,cluster) ("config_hash", alertmanager_config_hash{job="release-name-kps-alertmanager",namespace="default"})
- )
- != 1
- for: 20m
- labels:
- severity: critical
- - alert: AlertmanagerClusterDown
- annotations:
- description: '{{ $value | humanizePercentage }} of Alertmanager instances within the {{$labels.job}} cluster have been up for less than half of the last 5m.'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerclusterdown
- summary: Half or more of the Alertmanager instances within the same cluster are down.
- expr: |-
- (
- count by (namespace,service,cluster) (
- avg_over_time(up{job="release-name-kps-alertmanager",namespace="default"}[5m]) < 0.5
- )
- /
- count by (namespace,service,cluster) (
- up{job="release-name-kps-alertmanager",namespace="default"}
- )
- )
- >= 0.5
- for: 5m
- labels:
- severity: critical
- - alert: AlertmanagerClusterCrashlooping
- annotations:
- description: '{{ $value | humanizePercentage }} of Alertmanager instances within the {{$labels.job}} cluster have restarted at least 5 times in the last 10m.'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerclustercrashlooping
- summary: Half or more of the Alertmanager instances within the same cluster are crashlooping.
- expr: |-
- (
- count by (namespace,service,cluster) (
- changes(process_start_time_seconds{job="release-name-kps-alertmanager",namespace="default"}[10m]) > 4
- )
- /
- count by (namespace,service,cluster) (
- up{job="release-name-kps-alertmanager",namespace="default"}
- )
- )
- >= 0.5
- for: 5m
- labels:
- severity: critical
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/config-reloaders.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-config-reloaders
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: config-reloaders
- rules:
- - alert: ConfigReloaderSidecarErrors
- annotations:
- description: 'Errors encountered while the {{$labels.pod}} config-reloader sidecar attempts to sync config in {{$labels.namespace}} namespace.
-
- As a result, configuration for service running in {{$labels.pod}} may be stale and cannot be updated anymore.'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/configreloadersidecarerrors
- summary: config-reloader sidecar has not had a successful reload for 10m
- expr: max_over_time(reloader_last_reload_successful{namespace=~".+"}[5m]) == 0
- for: 10m
- labels:
- severity: warning
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/etcd.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-etcd
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: etcd
- rules:
- - alert: etcdMembersDown
- annotations:
- description: 'etcd cluster "{{ $labels.job }}": members are down ({{ $value }}).'
- summary: etcd cluster members are down.
- expr: |-
- max without (endpoint) (
- sum without (instance) (up{job=~".*etcd.*"} == bool 0)
- or
- count without (To) (
- sum without (instance) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01
- )
- )
- > 0
- for: 10m
- labels:
- severity: critical
- - alert: etcdInsufficientMembers
- annotations:
- description: 'etcd cluster "{{ $labels.job }}": insufficient members ({{ $value }}).'
- summary: etcd cluster has insufficient number of members.
- expr: sum(up{job=~".*etcd.*"} == bool 1) without (instance) < ((count(up{job=~".*etcd.*"}) without (instance) + 1) / 2)
- for: 3m
- labels:
- severity: critical
- - alert: etcdNoLeader
- annotations:
- description: 'etcd cluster "{{ $labels.job }}": member {{ $labels.instance }} has no leader.'
- summary: etcd cluster has no leader.
- expr: etcd_server_has_leader{job=~".*etcd.*"} == 0
- for: 1m
- labels:
- severity: critical
- - alert: etcdHighNumberOfLeaderChanges
- annotations:
- description: 'etcd cluster "{{ $labels.job }}": {{ $value }} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.'
- summary: etcd cluster has high number of leader changes.
- expr: increase((max without (instance) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 4
- for: 5m
- labels:
- severity: warning
- - alert: etcdHighNumberOfFailedGRPCRequests
- annotations:
- description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.'
- summary: etcd cluster has high number of failed grpc requests.
- expr: |-
- 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code)
- /
- sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code)
- > 1
- for: 10m
- labels:
- severity: warning
- - alert: etcdHighNumberOfFailedGRPCRequests
- annotations:
- description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.'
- summary: etcd cluster has high number of failed grpc requests.
- expr: |-
- 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code)
- /
- sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code)
- > 5
- for: 5m
- labels:
- severity: critical
- - alert: etcdGRPCRequestsSlow
- annotations:
- description: 'etcd cluster "{{ $labels.job }}": 99th percentile of gRPC requests is {{ $value }}s on etcd instance {{ $labels.instance }} for {{ $labels.grpc_method }} method.'
- summary: etcd grpc requests are slow
- expr: |-
- histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type))
- > 0.15
- for: 10m
- labels:
- severity: critical
- - alert: etcdMemberCommunicationSlow
- annotations:
- description: 'etcd cluster "{{ $labels.job }}": member communication with {{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance }}.'
- summary: etcd cluster member communication is slow.
- expr: |-
- histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
- > 0.15
- for: 10m
- labels:
- severity: warning
- - alert: etcdHighNumberOfFailedProposals
- annotations:
- description: 'etcd cluster "{{ $labels.job }}": {{ $value }} proposal failures within the last 30 minutes on etcd instance {{ $labels.instance }}.'
- summary: etcd cluster has high number of proposal failures.
- expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
- for: 15m
- labels:
- severity: warning
- - alert: etcdHighFsyncDurations
- annotations:
- description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.'
- summary: etcd cluster 99th percentile fsync durations are too high.
- expr: |-
- histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
- > 0.5
- for: 10m
- labels:
- severity: warning
- - alert: etcdHighFsyncDurations
- annotations:
- description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.'
- summary: etcd cluster 99th percentile fsync durations are too high.
- expr: |-
- histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
- > 1
- for: 10m
- labels:
- severity: critical
- - alert: etcdHighCommitDurations
- annotations:
- description: 'etcd cluster "{{ $labels.job }}": 99th percentile commit durations {{ $value }}s on etcd instance {{ $labels.instance }}.'
- summary: etcd cluster 99th percentile commit durations are too high.
- expr: |-
- histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
- > 0.25
- for: 10m
- labels:
- severity: warning
- - alert: etcdDatabaseQuotaLowSpace
- annotations:
- description: 'etcd cluster "{{ $labels.job }}": database size exceeds the defined quota on etcd instance {{ $labels.instance }}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.'
- summary: etcd cluster database is running full.
- expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100 > 95
- for: 10m
- labels:
- severity: critical
- - alert: etcdExcessiveDatabaseGrowth
- annotations:
- description: 'etcd cluster "{{ $labels.job }}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{ $labels.instance }}, please check as it might be disruptive.'
- summary: etcd cluster database growing very fast.
- expr: predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 4*60*60) > etcd_server_quota_backend_bytes{job=~".*etcd.*"}
- for: 10m
- labels:
- severity: warning
- - alert: etcdDatabaseHighFragmentationRatio
- annotations:
- description: 'etcd cluster "{{ $labels.job }}": database size in use on instance {{ $labels.instance }} is {{ $value | humanizePercentage }} of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.'
- runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation
- summary: etcd database size in use is less than 50% of the actual allocated storage.
- expr: (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600
- for: 10m
- labels:
- severity: warning
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/general.rules.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-general.rules
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: general.rules
- rules:
- - alert: TargetDown
- annotations:
- description: '{{ printf "%.4g" $value }}% of the {{ $labels.job }}/{{ $labels.service }} targets in {{ $labels.namespace }} namespace are down.'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/targetdown
- summary: One or more targets are unreachable.
- expr: 100 * (count(up == 0) BY (cluster, job, namespace, service) / count(up) BY (cluster, job, namespace, service)) > 10
- for: 10m
- labels:
- severity: warning
- - alert: Watchdog
- annotations:
- description: 'This is an alert meant to ensure that the entire alerting pipeline is functional.
-
- This alert is always firing, therefore it should always be firing in Alertmanager
-
- and always fire against a receiver. There are integrations with various notification
-
- mechanisms that send a notification when this alert is not firing. For example the
-
- "DeadMansSnitch" integration in PagerDuty.
-
- '
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/watchdog
- summary: An alert that should always be firing to certify that Alertmanager is working properly.
- expr: vector(1)
- labels:
- severity: none
- - alert: InfoInhibitor
- annotations:
- description: 'This is an alert that is used to inhibit info alerts.
-
- By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with
-
- other alerts.
-
- This alert fires whenever there''s a severity="info" alert, and stops firing when another alert with a
-
- severity of ''warning'' or ''critical'' starts firing on the same namespace.
-
- This alert should be routed to a null receiver and configured to inhibit alerts with severity="info".
-
- '
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/infoinhibitor
- summary: Info-level alert inhibition.
- expr: ALERTS{severity = "info"} == 1 unless on (namespace) ALERTS{alertname != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1
- labels:
- severity: none
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-k8s.rules.container-cpu-usage-seconds-total
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: k8s.rules.container_cpu_usage_seconds_total
- rules:
- - expr: |-
- sum by (cluster, namespace, pod, container) (
- irate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m])
- ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (
- 1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
- )
- record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-k8s.rules.container-memory-cache
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: k8s.rules.container_memory_cache
- rules:
- - expr: |-
- container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
- * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (1,
- max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
- )
- record: node_namespace_pod_container:container_memory_cache
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-k8s.rules.container-memory-rss
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: k8s.rules.container_memory_rss
- rules:
- - expr: |-
- container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
- * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (1,
- max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
- )
- record: node_namespace_pod_container:container_memory_rss
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-k8s.rules.container-memory-swap
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: k8s.rules.container_memory_swap
- rules:
- - expr: |-
- container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
- * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (1,
- max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
- )
- record: node_namespace_pod_container:container_memory_swap
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-k8s.rules.container-memory-working-set-bytes
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: k8s.rules.container_memory_working_set_bytes
- rules:
- - expr: |-
- container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
- * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (1,
- max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
- )
- record: node_namespace_pod_container:container_memory_working_set_bytes
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-k8s.rules.container-resource
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: k8s.rules.container_resource
- rules:
- - expr: |-
- kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster)
- group_left() max by (namespace, pod, cluster) (
- (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
- )
- record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests
- - expr: |-
- sum by (namespace, cluster) (
- sum by (namespace, pod, cluster) (
- max by (namespace, pod, container, cluster) (
- kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"}
- ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
- kube_pod_status_phase{phase=~"Pending|Running"} == 1
- )
- )
- )
- record: namespace_memory:kube_pod_container_resource_requests:sum
- - expr: |-
- kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster)
- group_left() max by (namespace, pod, cluster) (
- (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
- )
- record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests
- - expr: |-
- sum by (namespace, cluster) (
- sum by (namespace, pod, cluster) (
- max by (namespace, pod, container, cluster) (
- kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"}
- ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
- kube_pod_status_phase{phase=~"Pending|Running"} == 1
- )
- )
- )
- record: namespace_cpu:kube_pod_container_resource_requests:sum
- - expr: |-
- kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster)
- group_left() max by (namespace, pod, cluster) (
- (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
- )
- record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits
- - expr: |-
- sum by (namespace, cluster) (
- sum by (namespace, pod, cluster) (
- max by (namespace, pod, container, cluster) (
- kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"}
- ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
- kube_pod_status_phase{phase=~"Pending|Running"} == 1
- )
- )
- )
- record: namespace_memory:kube_pod_container_resource_limits:sum
- - expr: |-
- kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster)
- group_left() max by (namespace, pod, cluster) (
- (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
- )
- record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits
- - expr: |-
- sum by (namespace, cluster) (
- sum by (namespace, pod, cluster) (
- max by (namespace, pod, container, cluster) (
- kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"}
- ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
- kube_pod_status_phase{phase=~"Pending|Running"} == 1
- )
- )
- )
- record: namespace_cpu:kube_pod_container_resource_limits:sum
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-k8s.rules.pod-owner
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: k8s.rules.pod_owner
- rules:
- - expr: |-
- max by (cluster, namespace, workload, pod) (
- label_replace(
- label_replace(
- kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"},
- "replicaset", "$1", "owner_name", "(.*)"
- ) * on (replicaset, namespace) group_left(owner_name) topk by (replicaset, namespace) (
- 1, max by (replicaset, namespace, owner_name) (
- kube_replicaset_owner{job="kube-state-metrics"}
- )
- ),
- "workload", "$1", "owner_name", "(.*)"
- )
- )
- labels:
- workload_type: deployment
- record: namespace_workload_pod:kube_pod_owner:relabel
- - expr: |-
- max by (cluster, namespace, workload, pod) (
- label_replace(
- kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"},
- "workload", "$1", "owner_name", "(.*)"
- )
- )
- labels:
- workload_type: daemonset
- record: namespace_workload_pod:kube_pod_owner:relabel
- - expr: |-
- max by (cluster, namespace, workload, pod) (
- label_replace(
- kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"},
- "workload", "$1", "owner_name", "(.*)"
- )
- )
- labels:
- workload_type: statefulset
- record: namespace_workload_pod:kube_pod_owner:relabel
- - expr: |-
- max by (cluster, namespace, workload, pod) (
- label_replace(
- kube_pod_owner{job="kube-state-metrics", owner_kind="Job"},
- "workload", "$1", "owner_name", "(.*)"
- )
- )
- labels:
- workload_type: job
- record: namespace_workload_pod:kube_pod_owner:relabel
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-kube-apiserver-availability.rules
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - interval: 3m
- name: kube-apiserver-availability.rules
- rules:
- - expr: avg_over_time(code_verb:apiserver_request_total:increase1h[30d]) * 24 * 30
- record: code_verb:apiserver_request_total:increase30d
- - expr: sum by (cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"})
- labels:
- verb: read
- record: code:apiserver_request_total:increase30d
- - expr: sum by (cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"})
- labels:
- verb: write
- record: code:apiserver_request_total:increase30d
- - expr: sum by (cluster, verb, scope) (increase(apiserver_request_sli_duration_seconds_count{job="apiserver"}[1h]))
- record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h
- - expr: sum by (cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h[30d]) * 24 * 30)
- record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d
- - expr: sum by (cluster, verb, scope, le) (increase(apiserver_request_sli_duration_seconds_bucket[1h]))
- record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h
- - expr: sum by (cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h[30d]) * 24 * 30)
- record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d
- - expr: |-
- 1 - (
- (
- # write too slow
- sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"})
- -
- sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"})
- ) +
- (
- # read too slow
- sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"LIST|GET"})
- -
- (
- (
- sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"})
- or
- vector(0)
- )
- +
- sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"})
- +
- sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"})
- )
- ) +
- # errors
- sum by (cluster) (code:apiserver_request_total:increase30d{code=~"5.."} or vector(0))
- )
- /
- sum by (cluster) (code:apiserver_request_total:increase30d)
- labels:
- verb: all
- record: apiserver_request:availability30d
- - expr: |-
- 1 - (
- sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"LIST|GET"})
- -
- (
- # too slow
- (
- sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"})
- or
- vector(0)
- )
- +
- sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"})
- +
- sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"})
- )
- +
- # errors
- sum by (cluster) (code:apiserver_request_total:increase30d{verb="read",code=~"5.."} or vector(0))
- )
- /
- sum by (cluster) (code:apiserver_request_total:increase30d{verb="read"})
- labels:
- verb: read
- record: apiserver_request:availability30d
- - expr: |-
- 1 - (
- (
- # too slow
- sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"})
- -
- sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"})
- )
- +
- # errors
- sum by (cluster) (code:apiserver_request_total:increase30d{verb="write",code=~"5.."} or vector(0))
- )
- /
- sum by (cluster) (code:apiserver_request_total:increase30d{verb="write"})
- labels:
- verb: write
- record: apiserver_request:availability30d
- - expr: sum by (cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m]))
- labels:
- verb: read
- record: code_resource:apiserver_request_total:rate5m
- - expr: sum by (cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))
- labels:
- verb: write
- record: code_resource:apiserver_request_total:rate5m
- - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"2.."}[1h]))
- record: code_verb:apiserver_request_total:increase1h
- - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"3.."}[1h]))
- record: code_verb:apiserver_request_total:increase1h
- - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"4.."}[1h]))
- record: code_verb:apiserver_request_total:increase1h
- - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"5.."}[1h]))
- record: code_verb:apiserver_request_total:increase1h
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-kube-apiserver-burnrate.rules
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: kube-apiserver-burnrate.rules
- rules:
- - expr: |-
- (
- (
- # too slow
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1d]))
- -
- (
- (
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1d]))
- or
- vector(0)
- )
- +
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1d]))
- +
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1d]))
- )
- )
- +
- # errors
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1d]))
- )
- /
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d]))
- labels:
- verb: read
- record: apiserver_request:burnrate1d
- - expr: |-
- (
- (
- # too slow
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1h]))
- -
- (
- (
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1h]))
- or
- vector(0)
- )
- +
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1h]))
- +
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1h]))
- )
- )
- +
- # errors
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1h]))
- )
- /
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h]))
- labels:
- verb: read
- record: apiserver_request:burnrate1h
- - expr: |-
- (
- (
- # too slow
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[2h]))
- -
- (
- (
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[2h]))
- or
- vector(0)
- )
- +
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[2h]))
- +
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[2h]))
- )
- )
- +
- # errors
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[2h]))
- )
- /
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h]))
- labels:
- verb: read
- record: apiserver_request:burnrate2h
- - expr: |-
- (
- (
- # too slow
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[30m]))
- -
- (
- (
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[30m]))
- or
- vector(0)
- )
- +
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[30m]))
- +
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[30m]))
- )
- )
- +
- # errors
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[30m]))
- )
- /
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m]))
- labels:
- verb: read
- record: apiserver_request:burnrate30m
- - expr: |-
- (
- (
- # too slow
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[3d]))
- -
- (
- (
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[3d]))
- or
- vector(0)
- )
- +
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[3d]))
- +
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[3d]))
- )
- )
- +
- # errors
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[3d]))
- )
- /
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d]))
- labels:
- verb: read
- record: apiserver_request:burnrate3d
- - expr: |-
- (
- (
- # too slow
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))
- -
- (
- (
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[5m]))
- or
- vector(0)
- )
- +
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[5m]))
- +
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[5m]))
- )
- )
- +
- # errors
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[5m]))
- )
- /
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m]))
- labels:
- verb: read
- record: apiserver_request:burnrate5m
- - expr: |-
- (
- (
- # too slow
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[6h]))
- -
- (
- (
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[6h]))
- or
- vector(0)
- )
- +
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[6h]))
- +
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[6h]))
- )
- )
- +
- # errors
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[6h]))
- )
- /
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h]))
- labels:
- verb: read
- record: apiserver_request:burnrate6h
- - expr: |-
- (
- (
- # too slow
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1d]))
- -
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1d]))
- )
- +
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d]))
- )
- /
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d]))
- labels:
- verb: write
- record: apiserver_request:burnrate1d
- - expr: |-
- (
- (
- # too slow
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1h]))
- -
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1h]))
- )
- +
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h]))
- )
- /
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h]))
- labels:
- verb: write
- record: apiserver_request:burnrate1h
- - expr: |-
- (
- (
- # too slow
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[2h]))
- -
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[2h]))
- )
- +
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h]))
- )
- /
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h]))
- labels:
- verb: write
- record: apiserver_request:burnrate2h
- - expr: |-
- (
- (
- # too slow
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[30m]))
- -
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[30m]))
- )
- +
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m]))
- )
- /
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m]))
- labels:
- verb: write
- record: apiserver_request:burnrate30m
- - expr: |-
- (
- (
- # too slow
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[3d]))
- -
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[3d]))
- )
- +
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d]))
- )
- /
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d]))
- labels:
- verb: write
- record: apiserver_request:burnrate3d
- - expr: |-
- (
- (
- # too slow
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))
- -
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[5m]))
- )
- +
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m]))
- )
- /
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))
- labels:
- verb: write
- record: apiserver_request:burnrate5m
- - expr: |-
- (
- (
- # too slow
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[6h]))
- -
- sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[6h]))
- )
- +
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h]))
- )
- /
- sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h]))
- labels:
- verb: write
- record: apiserver_request:burnrate6h
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-kube-apiserver-histogram.rules
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: kube-apiserver-histogram.rules
- rules:
- - expr: histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0
- labels:
- quantile: '0.99'
- verb: read
- record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile
- - expr: histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0
- labels:
- quantile: '0.99'
- verb: write
- record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-kube-apiserver-slos
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: kube-apiserver-slos
- rules:
- - alert: KubeAPIErrorBudgetBurn
- annotations:
- description: The API server is burning too much error budget.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
- summary: The API server is burning too much error budget.
- expr: |-
- sum(apiserver_request:burnrate1h) > (14.40 * 0.01000)
- and
- sum(apiserver_request:burnrate5m) > (14.40 * 0.01000)
- for: 2m
- labels:
- long: 1h
- severity: critical
- short: 5m
- - alert: KubeAPIErrorBudgetBurn
- annotations:
- description: The API server is burning too much error budget.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
- summary: The API server is burning too much error budget.
- expr: |-
- sum(apiserver_request:burnrate6h) > (6.00 * 0.01000)
- and
- sum(apiserver_request:burnrate30m) > (6.00 * 0.01000)
- for: 15m
- labels:
- long: 6h
- severity: critical
- short: 30m
- - alert: KubeAPIErrorBudgetBurn
- annotations:
- description: The API server is burning too much error budget.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
- summary: The API server is burning too much error budget.
- expr: |-
- sum(apiserver_request:burnrate1d) > (3.00 * 0.01000)
- and
- sum(apiserver_request:burnrate2h) > (3.00 * 0.01000)
- for: 1h
- labels:
- long: 1d
- severity: warning
- short: 2h
- - alert: KubeAPIErrorBudgetBurn
- annotations:
- description: The API server is burning too much error budget.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
- summary: The API server is burning too much error budget.
- expr: |-
- sum(apiserver_request:burnrate3d) > (1.00 * 0.01000)
- and
- sum(apiserver_request:burnrate6h) > (1.00 * 0.01000)
- for: 3h
- labels:
- long: 3d
- severity: warning
- short: 6h
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-kube-prometheus-general.rules
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: kube-prometheus-general.rules
- rules:
- - expr: count without(instance, pod, node) (up == 1)
- record: count:up1
- - expr: count without(instance, pod, node) (up == 0)
- record: count:up0
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-kube-prometheus-node-recording.rules
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: kube-prometheus-node-recording.rules
- rules:
- - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m])) BY (instance)
- record: instance:node_cpu:rate:sum
- - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance)
- record: instance:node_network_receive_bytes:rate:sum
- - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance)
- record: instance:node_network_transmit_bytes:rate:sum
- - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance)
- record: instance:node_cpu:ratio
- - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m]))
- record: cluster:node_cpu:sum_rate5m
- - expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu))
- record: cluster:node_cpu:ratio
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-kube-scheduler.rules
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: kube-scheduler.rules
- rules:
- - expr: histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
- labels:
- quantile: '0.99'
- record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile
- - expr: histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
- labels:
- quantile: '0.99'
- record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile
- - expr: histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
- labels:
- quantile: '0.99'
- record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile
- - expr: histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
- labels:
- quantile: '0.9'
- record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile
- - expr: histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
- labels:
- quantile: '0.9'
- record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile
- - expr: histogram_quantile(0.9, sum(rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
- labels:
- quantile: '0.9'
- record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile
- - expr: histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
- labels:
- quantile: '0.5'
- record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile
- - expr: histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
- labels:
- quantile: '0.5'
- record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile
- - expr: histogram_quantile(0.5, sum(rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
- labels:
- quantile: '0.5'
- record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kube-state-metrics.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-kube-state-metrics
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: kube-state-metrics
- rules:
- - alert: KubeStateMetricsListErrors
- annotations:
- description: kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricslisterrors
- summary: kube-state-metrics is experiencing errors in list operations.
- expr: |-
- (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m])) by (cluster)
- /
- sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m])) by (cluster))
- > 0.01
- for: 15m
- labels:
- severity: critical
- - alert: KubeStateMetricsWatchErrors
- annotations:
- description: kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricswatcherrors
- summary: kube-state-metrics is experiencing errors in watch operations.
- expr: |-
- (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m])) by (cluster)
- /
- sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics"}[5m])) by (cluster))
- > 0.01
- for: 15m
- labels:
- severity: critical
- - alert: KubeStateMetricsShardingMismatch
- annotations:
- description: kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricsshardingmismatch
- summary: kube-state-metrics sharding is misconfigured.
- expr: stdvar (kube_state_metrics_total_shards{job="kube-state-metrics"}) by (cluster) != 0
- for: 15m
- labels:
- severity: critical
- - alert: KubeStateMetricsShardsMissing
- annotations:
- description: kube-state-metrics shards are missing, some Kubernetes objects are not being exposed.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricsshardsmissing
- summary: kube-state-metrics shards are missing.
- expr: |-
- 2^max(kube_state_metrics_total_shards{job="kube-state-metrics"}) by (cluster) - 1
- -
- sum( 2 ^ max by (cluster, shard_ordinal) (kube_state_metrics_shard_ordinal{job="kube-state-metrics"}) ) by (cluster)
- != 0
- for: 15m
- labels:
- severity: critical
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kubelet.rules.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-kubelet.rules
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: kubelet.rules
- rules:
- - expr: histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (cluster, instance, le) * on (cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"})
- labels:
- quantile: '0.99'
- record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
- - expr: histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (cluster, instance, le) * on (cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"})
- labels:
- quantile: '0.9'
- record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
- - expr: histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (cluster, instance, le) * on (cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"})
- labels:
- quantile: '0.5'
- record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kubernetes-apps.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-kubernetes-apps
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: kubernetes-apps
- rules:
- - alert: KubePodCrashLooping
- annotations:
- description: 'Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is in waiting state (reason: "CrashLoopBackOff").'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodcrashlooping
- summary: Pod is crash looping.
- expr: max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", job="kube-state-metrics", namespace=~".*"}[5m]) >= 1
- for: 15m
- labels:
- severity: warning
- - alert: KubePodNotReady
- annotations:
- description: Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
- summary: Pod has been in a non-ready state for more than 15 minutes.
- expr: |-
- sum by (namespace, pod, cluster) (
- max by (namespace, pod, cluster) (
- kube_pod_status_phase{job="kube-state-metrics", namespace=~".*", phase=~"Pending|Unknown|Failed"}
- ) * on (namespace, pod, cluster) group_left(owner_kind) topk by (namespace, pod, cluster) (
- 1, max by (namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"})
- )
- ) > 0
- for: 15m
- labels:
- severity: warning
- - alert: KubeDeploymentGenerationMismatch
- annotations:
- description: Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentgenerationmismatch
- summary: Deployment generation mismatch due to possible roll-back
- expr: |-
- kube_deployment_status_observed_generation{job="kube-state-metrics", namespace=~".*"}
- !=
- kube_deployment_metadata_generation{job="kube-state-metrics", namespace=~".*"}
- for: 15m
- labels:
- severity: warning
- - alert: KubeDeploymentReplicasMismatch
- annotations:
- description: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 15 minutes.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch
- summary: Deployment has not matched the expected number of replicas.
- expr: |-
- (
- kube_deployment_spec_replicas{job="kube-state-metrics", namespace=~".*"}
- >
- kube_deployment_status_replicas_available{job="kube-state-metrics", namespace=~".*"}
- ) and (
- changes(kube_deployment_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}[10m])
- ==
- 0
- )
- for: 15m
- labels:
- severity: warning
- - alert: KubeDeploymentRolloutStuck
- annotations:
- description: Rollout of deployment {{ $labels.namespace }}/{{ $labels.deployment }} is not progressing for longer than 15 minutes.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentrolloutstuck
- summary: Deployment rollout is not progressing.
- expr: |-
- kube_deployment_status_condition{condition="Progressing", status="false",job="kube-state-metrics", namespace=~".*"}
- != 0
- for: 15m
- labels:
- severity: warning
- - alert: KubeStatefulSetReplicasMismatch
- annotations:
- description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetreplicasmismatch
- summary: StatefulSet has not matched the expected number of replicas.
- expr: |-
- (
- kube_statefulset_status_replicas_ready{job="kube-state-metrics", namespace=~".*"}
- !=
- kube_statefulset_status_replicas{job="kube-state-metrics", namespace=~".*"}
- ) and (
- changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}[10m])
- ==
- 0
- )
- for: 15m
- labels:
- severity: warning
- - alert: KubeStatefulSetGenerationMismatch
- annotations:
- description: StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetgenerationmismatch
- summary: StatefulSet generation mismatch due to possible roll-back
- expr: |-
- kube_statefulset_status_observed_generation{job="kube-state-metrics", namespace=~".*"}
- !=
- kube_statefulset_metadata_generation{job="kube-state-metrics", namespace=~".*"}
- for: 15m
- labels:
- severity: warning
- - alert: KubeStatefulSetUpdateNotRolledOut
- annotations:
- description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetupdatenotrolledout
- summary: StatefulSet update has not been rolled out.
- expr: |-
- (
- max without (revision) (
- kube_statefulset_status_current_revision{job="kube-state-metrics", namespace=~".*"}
- unless
- kube_statefulset_status_update_revision{job="kube-state-metrics", namespace=~".*"}
- )
- *
- (
- kube_statefulset_replicas{job="kube-state-metrics", namespace=~".*"}
- !=
- kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}
- )
- ) and (
- changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}[5m])
- ==
- 0
- )
- for: 15m
- labels:
- severity: warning
- - alert: KubeDaemonSetRolloutStuck
- annotations:
- description: DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished or progressed for at least 15 minutes.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetrolloutstuck
- summary: DaemonSet rollout is stuck.
- expr: |-
- (
- (
- kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~".*"}
- !=
- kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
- ) or (
- kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~".*"}
- !=
- 0
- ) or (
- kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics", namespace=~".*"}
- !=
- kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
- ) or (
- kube_daemonset_status_number_available{job="kube-state-metrics", namespace=~".*"}
- !=
- kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
- )
- ) and (
- changes(kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics", namespace=~".*"}[5m])
- ==
- 0
- )
- for: 15m
- labels:
- severity: warning
- - alert: KubeContainerWaiting
- annotations:
- description: pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on container {{ $labels.container}} has been in waiting state for longer than 1 hour.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontainerwaiting
- summary: Pod container waiting longer than 1 hour
- expr: sum by (namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="kube-state-metrics", namespace=~".*"}) > 0
- for: 1h
- labels:
- severity: warning
- - alert: KubeDaemonSetNotScheduled
- annotations:
- description: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled.'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetnotscheduled
- summary: DaemonSet pods are not scheduled.
- expr: |-
- kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
- -
- kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~".*"} > 0
- for: 10m
- labels:
- severity: warning
- - alert: KubeDaemonSetMisScheduled
- annotations:
- description: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run.'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetmisscheduled
- summary: DaemonSet pods are misscheduled.
- expr: kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~".*"} > 0
- for: 15m
- labels:
- severity: warning
- - alert: KubeJobNotCompleted
- annotations:
- description: Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than {{ "43200" | humanizeDuration }} to complete.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobnotcompleted
- summary: Job did not complete in time
- expr: |-
- time() - max by (namespace, job_name, cluster) (kube_job_status_start_time{job="kube-state-metrics", namespace=~".*"}
- and
- kube_job_status_active{job="kube-state-metrics", namespace=~".*"} > 0) > 43200
- labels:
- severity: warning
- - alert: KubeJobFailed
- annotations:
- description: Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. Removing failed job after investigation should clear this alert.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobfailed
- summary: Job failed to complete.
- expr: kube_job_failed{job="kube-state-metrics", namespace=~".*"} > 0
- for: 15m
- labels:
- severity: warning
- - alert: KubeHpaReplicasMismatch
- annotations:
- description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has not matched the desired number of replicas for longer than 15 minutes.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpareplicasmismatch
- summary: HPA has not matched desired number of replicas.
- expr: |-
- (kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics", namespace=~".*"}
- !=
- kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"})
- and
- (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}
- >
- kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics", namespace=~".*"})
- and
- (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}
- <
- kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics", namespace=~".*"})
- and
- changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}[15m]) == 0
- for: 15m
- labels:
- severity: warning
- - alert: KubeHpaMaxedOut
- annotations:
- description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has been running at max replicas for longer than 15 minutes.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpamaxedout
- summary: HPA is running at max replicas
- expr: |-
- kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}
- ==
- kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics", namespace=~".*"}
- for: 15m
- labels:
- severity: warning
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kubernetes-resources.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-kubernetes-resources
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: kubernetes-resources
- rules:
- - alert: KubeCPUOvercommit
- annotations:
- description: Cluster {{ $labels.cluster }} has overcommitted CPU resource requests for Pods by {{ $value }} CPU shares and cannot tolerate node failure.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuovercommit
- summary: Cluster has overcommitted CPU resource requests.
- expr: |-
- sum(namespace_cpu:kube_pod_container_resource_requests:sum{job="kube-state-metrics",}) by (cluster) - (sum(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster) - max(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster)) > 0
- and
- (sum(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster) - max(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster)) > 0
- for: 10m
- labels:
- severity: warning
- - alert: KubeMemoryOvercommit
- annotations:
- description: Cluster {{ $labels.cluster }} has overcommitted memory resource requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node failure.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryovercommit
- summary: Cluster has overcommitted memory resource requests.
- expr: |-
- sum(namespace_memory:kube_pod_container_resource_requests:sum{}) by (cluster) - (sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster) - max(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster)) > 0
- and
- (sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster) - max(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster)) > 0
- for: 10m
- labels:
- severity: warning
- - alert: KubeCPUQuotaOvercommit
- annotations:
- description: Cluster {{ $labels.cluster }} has overcommitted CPU resource requests for Namespaces.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuquotaovercommit
- summary: Cluster has overcommitted CPU resource requests.
- expr: |-
- sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(cpu|requests.cpu)"})) by (cluster)
- /
- sum(kube_node_status_allocatable{resource="cpu", job="kube-state-metrics"}) by (cluster)
- > 1.5
- for: 5m
- labels:
- severity: warning
- - alert: KubeMemoryQuotaOvercommit
- annotations:
- description: Cluster {{ $labels.cluster }} has overcommitted memory resource requests for Namespaces.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryquotaovercommit
- summary: Cluster has overcommitted memory resource requests.
- expr: |-
- sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(memory|requests.memory)"})) by (cluster)
- /
- sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster)
- > 1.5
- for: 5m
- labels:
- severity: warning
- - alert: KubeQuotaAlmostFull
- annotations:
- description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaalmostfull
- summary: Namespace quota is going to be full.
- expr: |-
- kube_resourcequota{job="kube-state-metrics", type="used"}
- / ignoring(instance, job, type)
- (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
- > 0.9 < 1
- for: 15m
- labels:
- severity: info
- - alert: KubeQuotaFullyUsed
- annotations:
- description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotafullyused
- summary: Namespace quota is fully used.
- expr: |-
- kube_resourcequota{job="kube-state-metrics", type="used"}
- / ignoring(instance, job, type)
- (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
- == 1
- for: 15m
- labels:
- severity: info
- - alert: KubeQuotaExceeded
- annotations:
- description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaexceeded
- summary: Namespace quota has exceeded the limits.
- expr: |-
- kube_resourcequota{job="kube-state-metrics", type="used"}
- / ignoring(instance, job, type)
- (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
- > 1
- for: 15m
- labels:
- severity: warning
- - alert: CPUThrottlingHigh
- annotations:
- description: '{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}.'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/cputhrottlinghigh
- summary: Processes experience elevated CPU throttling.
- expr: |-
- sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (cluster, container, pod, namespace)
- /
- sum(increase(container_cpu_cfs_periods_total{}[5m])) by (cluster, container, pod, namespace)
- > ( 25 / 100 )
- for: 15m
- labels:
- severity: info
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kubernetes-storage.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-kubernetes-storage
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: kubernetes-storage
- rules:
- - alert: KubePersistentVolumeFillingUp
- annotations:
- description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is only {{ $value | humanizePercentage }} free.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup
- summary: PersistentVolume is filling up.
- expr: |-
- (
- kubelet_volume_stats_available_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
- /
- kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
- ) < 0.03
- and
- kubelet_volume_stats_used_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0
- unless on (cluster, namespace, persistentvolumeclaim)
- kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
- unless on (cluster, namespace, persistentvolumeclaim)
- kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
- for: 1m
- labels:
- severity: critical
- - alert: KubePersistentVolumeFillingUp
- annotations:
- description: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup
- summary: PersistentVolume is filling up.
- expr: |-
- (
- kubelet_volume_stats_available_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
- /
- kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
- ) < 0.15
- and
- kubelet_volume_stats_used_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0
- and
- predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0
- unless on (cluster, namespace, persistentvolumeclaim)
- kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
- unless on (cluster, namespace, persistentvolumeclaim)
- kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
- for: 1h
- labels:
- severity: warning
- - alert: KubePersistentVolumeInodesFillingUp
- annotations:
- description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} only has {{ $value | humanizePercentage }} free inodes.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup
- summary: PersistentVolumeInodes are filling up.
- expr: |-
- (
- kubelet_volume_stats_inodes_free{job="kubelet", namespace=~".*", metrics_path="/metrics"}
- /
- kubelet_volume_stats_inodes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
- ) < 0.03
- and
- kubelet_volume_stats_inodes_used{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0
- unless on (cluster, namespace, persistentvolumeclaim)
- kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
- unless on (cluster, namespace, persistentvolumeclaim)
- kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
- for: 1m
- labels:
- severity: critical
- - alert: KubePersistentVolumeInodesFillingUp
- annotations:
- description: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup
- summary: PersistentVolumeInodes are filling up.
- expr: |-
- (
- kubelet_volume_stats_inodes_free{job="kubelet", namespace=~".*", metrics_path="/metrics"}
- /
- kubelet_volume_stats_inodes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
- ) < 0.15
- and
- kubelet_volume_stats_inodes_used{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0
- and
- predict_linear(kubelet_volume_stats_inodes_free{job="kubelet", namespace=~".*", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0
- unless on (cluster, namespace, persistentvolumeclaim)
- kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
- unless on (cluster, namespace, persistentvolumeclaim)
- kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
- for: 1h
- labels:
- severity: warning
- - alert: KubePersistentVolumeErrors
- annotations:
- description: The persistent volume {{ $labels.persistentvolume }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} has status {{ $labels.phase }}.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeerrors
- summary: PersistentVolume is having issues with provisioning.
- expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
- for: 5m
- labels:
- severity: critical
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-kubernetes-system-apiserver
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: kubernetes-system-apiserver
- rules:
- - alert: KubeClientCertificateExpiration
- annotations:
- description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration
- summary: Client certificate is about to expire.
- expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on (job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800
- for: 5m
- labels:
- severity: warning
- - alert: KubeClientCertificateExpiration
- annotations:
- description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration
- summary: Client certificate is about to expire.
- expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on (job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400
- for: 5m
- labels:
- severity: critical
- - alert: KubeAggregatedAPIErrors
- annotations:
- description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapierrors
- summary: Kubernetes aggregated API has reported errors.
- expr: sum by (name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4
- labels:
- severity: warning
- - alert: KubeAggregatedAPIDown
- annotations:
- description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapidown
- summary: Kubernetes aggregated API is down.
- expr: (1 - max by (name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice{job="apiserver"}[10m]))) * 100 < 85
- for: 5m
- labels:
- severity: warning
- - alert: KubeAPIDown
- annotations:
- description: KubeAPI has disappeared from Prometheus target discovery.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapidown
- summary: Target disappeared from Prometheus target discovery.
- expr: absent(up{job="apiserver"} == 1)
- for: 15m
- labels:
- severity: critical
- - alert: KubeAPITerminatedRequests
- annotations:
- description: The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapiterminatedrequests
- summary: The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.
- expr: sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20
- for: 5m
- labels:
- severity: warning
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-kubernetes-system-controller-manager
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: kubernetes-system-controller-manager
- rules:
- - alert: KubeControllerManagerDown
- annotations:
- description: KubeControllerManager has disappeared from Prometheus target discovery.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontrollermanagerdown
- summary: Target disappeared from Prometheus target discovery.
- expr: absent(up{job="kube-controller-manager"} == 1)
- for: 15m
- labels:
- severity: critical
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-kubernetes-system-kubelet
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: kubernetes-system-kubelet
- rules:
- - alert: KubeNodeNotReady
- annotations:
- description: '{{ $labels.node }} has been unready for more than 15 minutes.'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodenotready
- summary: Node is not ready.
- expr: kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0
- for: 15m
- labels:
- severity: warning
- - alert: KubeNodeUnreachable
- annotations:
- description: '{{ $labels.node }} is unreachable and some workloads may be rescheduled.'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodeunreachable
- summary: Node is unreachable.
- expr: (kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1
- for: 15m
- labels:
- severity: warning
- - alert: KubeletTooManyPods
- annotations:
- description: Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubelettoomanypods
- summary: Kubelet is running at capacity.
- expr: |-
- count by (cluster, node) (
- (kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on (instance,pod,namespace,cluster) group_left(node) topk by (instance,pod,namespace,cluster) (1, kube_pod_info{job="kube-state-metrics"})
- )
- /
- max by (cluster, node) (
- kube_node_status_capacity{job="kube-state-metrics",resource="pods"} != 1
- ) > 0.95
- for: 15m
- labels:
- severity: info
- - alert: KubeNodeReadinessFlapping
- annotations:
- description: The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodereadinessflapping
- summary: Node readiness status is flapping.
- expr: sum(changes(kube_node_status_condition{job="kube-state-metrics",status="true",condition="Ready"}[15m])) by (cluster, node) > 2
- for: 15m
- labels:
- severity: warning
- - alert: KubeletPlegDurationHigh
- annotations:
- description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletplegdurationhigh
- summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist.
- expr: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10
- for: 5m
- labels:
- severity: warning
- - alert: KubeletPodStartUpLatencyHigh
- annotations:
- description: Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletpodstartuplatencyhigh
- summary: Kubelet Pod startup latency is too high.
- expr: histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (cluster, instance, le)) * on (cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"} > 60
- for: 15m
- labels:
- severity: warning
- - alert: KubeletClientCertificateExpiration
- annotations:
- description: Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificateexpiration
- summary: Kubelet client certificate is about to expire.
- expr: kubelet_certificate_manager_client_ttl_seconds < 604800
- labels:
- severity: warning
- - alert: KubeletClientCertificateExpiration
- annotations:
- description: Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificateexpiration
- summary: Kubelet client certificate is about to expire.
- expr: kubelet_certificate_manager_client_ttl_seconds < 86400
- labels:
- severity: critical
- - alert: KubeletServerCertificateExpiration
- annotations:
- description: Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificateexpiration
- summary: Kubelet server certificate is about to expire.
- expr: kubelet_certificate_manager_server_ttl_seconds < 604800
- labels:
- severity: warning
- - alert: KubeletServerCertificateExpiration
- annotations:
- description: Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificateexpiration
- summary: Kubelet server certificate is about to expire.
- expr: kubelet_certificate_manager_server_ttl_seconds < 86400
- labels:
- severity: critical
- - alert: KubeletClientCertificateRenewalErrors
- annotations:
- description: Kubelet on node {{ $labels.node }} has failed to renew its client certificate ({{ $value | humanize }} errors in the last 5 minutes).
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificaterenewalerrors
- summary: Kubelet has failed to renew its client certificate.
- expr: increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0
- for: 15m
- labels:
- severity: warning
- - alert: KubeletServerCertificateRenewalErrors
- annotations:
- description: Kubelet on node {{ $labels.node }} has failed to renew its server certificate ({{ $value | humanize }} errors in the last 5 minutes).
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificaterenewalerrors
- summary: Kubelet has failed to renew its server certificate.
- expr: increase(kubelet_server_expiration_renew_errors[5m]) > 0
- for: 15m
- labels:
- severity: warning
- - alert: KubeletDown
- annotations:
- description: Kubelet has disappeared from Prometheus target discovery.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletdown
- summary: Target disappeared from Prometheus target discovery.
- expr: absent(up{job="kubelet", metrics_path="/metrics"} == 1)
- for: 15m
- labels:
- severity: critical
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-kubernetes-system-scheduler
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: kubernetes-system-scheduler
- rules:
- - alert: KubeSchedulerDown
- annotations:
- description: KubeScheduler has disappeared from Prometheus target discovery.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeschedulerdown
- summary: Target disappeared from Prometheus target discovery.
- expr: absent(up{job="kube-scheduler"} == 1)
- for: 15m
- labels:
- severity: critical
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kubernetes-system.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-kubernetes-system
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: kubernetes-system
- rules:
- - alert: KubeVersionMismatch
- annotations:
- description: There are {{ $value }} different semantic versions of Kubernetes components running.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeversionmismatch
- summary: Different semantic versions of Kubernetes components running.
- expr: count by (cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1
- for: 15m
- labels:
- severity: warning
- - alert: KubeClientErrors
- annotations:
- description: Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value | humanizePercentage }} errors.'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclienterrors
- summary: Kubernetes API server client is experiencing errors.
- expr: |-
- (sum(rate(rest_client_requests_total{job="apiserver",code=~"5.."}[5m])) by (cluster, instance, job, namespace)
- /
- sum(rate(rest_client_requests_total{job="apiserver"}[5m])) by (cluster, instance, job, namespace))
- > 0.01
- for: 15m
- labels:
- severity: warning
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/node-exporter.rules.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-node-exporter.rules
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: node-exporter.rules
- rules:
- - expr: |-
- count without (cpu, mode) (
- node_cpu_seconds_total{job="node-exporter",mode="idle"}
- )
- record: instance:node_num_cpu:sum
- - expr: |-
- 1 - avg without (cpu) (
- sum without (mode) (rate(node_cpu_seconds_total{job="node-exporter", mode=~"idle|iowait|steal"}[5m]))
- )
- record: instance:node_cpu_utilisation:rate5m
- - expr: |-
- (
- node_load1{job="node-exporter"}
- /
- instance:node_num_cpu:sum{job="node-exporter"}
- )
- record: instance:node_load1_per_cpu:ratio
- - expr: |-
- 1 - (
- (
- node_memory_MemAvailable_bytes{job="node-exporter"}
- or
- (
- node_memory_Buffers_bytes{job="node-exporter"}
- +
- node_memory_Cached_bytes{job="node-exporter"}
- +
- node_memory_MemFree_bytes{job="node-exporter"}
- +
- node_memory_Slab_bytes{job="node-exporter"}
- )
- )
- /
- node_memory_MemTotal_bytes{job="node-exporter"}
- )
- record: instance:node_memory_utilisation:ratio
- - expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m])
- record: instance:node_vmstat_pgmajfault:rate5m
- - expr: rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m])
- record: instance_device:node_disk_io_time_seconds:rate5m
- - expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m])
- record: instance_device:node_disk_io_time_weighted_seconds:rate5m
- - expr: |-
- sum without (device) (
- rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[5m])
- )
- record: instance:node_network_receive_bytes_excluding_lo:rate5m
- - expr: |-
- sum without (device) (
- rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[5m])
- )
- record: instance:node_network_transmit_bytes_excluding_lo:rate5m
- - expr: |-
- sum without (device) (
- rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[5m])
- )
- record: instance:node_network_receive_drop_excluding_lo:rate5m
- - expr: |-
- sum without (device) (
- rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[5m])
- )
- record: instance:node_network_transmit_drop_excluding_lo:rate5m
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/node-exporter.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-node-exporter
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: node-exporter
- rules:
- - alert: NodeFilesystemSpaceFillingUp
- annotations:
- description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup
- summary: Filesystem is predicted to run out of space within the next 24 hours.
- expr: |-
- (
- node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 15
- and
- predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0
- and
- node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
- )
- for: 1h
- labels:
- severity: warning
- - alert: NodeFilesystemSpaceFillingUp
- annotations:
- description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up fast.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup
- summary: Filesystem is predicted to run out of space within the next 4 hours.
- expr: |-
- (
- node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 10
- and
- predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0
- and
- node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
- )
- for: 1h
- labels:
- severity: critical
- - alert: NodeFilesystemAlmostOutOfSpace
- annotations:
- description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace
- summary: Filesystem has less than 5% space left.
- expr: |-
- (
- node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5
- and
- node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
- )
- for: 30m
- labels:
- severity: warning
- - alert: NodeFilesystemAlmostOutOfSpace
- annotations:
- description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace
- summary: Filesystem has less than 3% space left.
- expr: |-
- (
- node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3
- and
- node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
- )
- for: 30m
- labels:
- severity: critical
- - alert: NodeFilesystemFilesFillingUp
- annotations:
- description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup
- summary: Filesystem is predicted to run out of inodes within the next 24 hours.
- expr: |-
- (
- node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 40
- and
- predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0
- and
- node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
- )
- for: 1h
- labels:
- severity: warning
- - alert: NodeFilesystemFilesFillingUp
- annotations:
- description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up fast.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup
- summary: Filesystem is predicted to run out of inodes within the next 4 hours.
- expr: |-
- (
- node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 20
- and
- predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0
- and
- node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
- )
- for: 1h
- labels:
- severity: critical
- - alert: NodeFilesystemAlmostOutOfFiles
- annotations:
- description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles
- summary: Filesystem has less than 5% inodes left.
- expr: |-
- (
- node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5
- and
- node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
- )
- for: 1h
- labels:
- severity: warning
- - alert: NodeFilesystemAlmostOutOfFiles
- annotations:
- description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles
- summary: Filesystem has less than 3% inodes left.
- expr: |-
- (
- node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3
- and
- node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
- )
- for: 1h
- labels:
- severity: critical
- - alert: NodeNetworkReceiveErrs
- annotations:
- description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} receive errors in the last two minutes.'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworkreceiveerrs
- summary: Network interface is reporting many receive errors.
- expr: rate(node_network_receive_errs_total{job="node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="node-exporter"}[2m]) > 0.01
- for: 1h
- labels:
- severity: warning
- - alert: NodeNetworkTransmitErrs
- annotations:
- description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} transmit errors in the last two minutes.'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworktransmiterrs
- summary: Network interface is reporting many transmit errors.
- expr: rate(node_network_transmit_errs_total{job="node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="node-exporter"}[2m]) > 0.01
- for: 1h
- labels:
- severity: warning
- - alert: NodeHighNumberConntrackEntriesUsed
- annotations:
- description: '{{ $value | humanizePercentage }} of conntrack entries are used.'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodehighnumberconntrackentriesused
- summary: Number of conntrack are getting close to the limit.
- expr: (node_nf_conntrack_entries{job="node-exporter"} / node_nf_conntrack_entries_limit) > 0.75
- labels:
- severity: warning
- - alert: NodeTextFileCollectorScrapeError
- annotations:
- description: Node Exporter text file collector on {{ $labels.instance }} failed to scrape.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodetextfilecollectorscrapeerror
- summary: Node Exporter text file collector failed to scrape.
- expr: node_textfile_scrape_error{job="node-exporter"} == 1
- labels:
- severity: warning
- - alert: NodeClockSkewDetected
- annotations:
- description: Clock at {{ $labels.instance }} is out of sync by more than 0.05s. Ensure NTP is configured correctly on this host.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodeclockskewdetected
- summary: Clock skew detected.
- expr: |-
- (
- node_timex_offset_seconds{job="node-exporter"} > 0.05
- and
- deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) >= 0
- )
- or
- (
- node_timex_offset_seconds{job="node-exporter"} < -0.05
- and
- deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) <= 0
- )
- for: 10m
- labels:
- severity: warning
- - alert: NodeClockNotSynchronising
- annotations:
- description: Clock at {{ $labels.instance }} is not synchronising. Ensure NTP is configured on this host.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodeclocknotsynchronising
- summary: Clock not synchronising.
- expr: |-
- min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0
- and
- node_timex_maxerror_seconds{job="node-exporter"} >= 16
- for: 10m
- labels:
- severity: warning
- - alert: NodeRAIDDegraded
- annotations:
- description: RAID array '{{ $labels.device }}' at {{ $labels.instance }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddegraded
- summary: RAID Array is degraded.
- expr: node_md_disks_required{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} - ignoring (state) (node_md_disks{state="active",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}) > 0
- for: 15m
- labels:
- severity: critical
- - alert: NodeRAIDDiskFailure
- annotations:
- description: At least one device in RAID array at {{ $labels.instance }} failed. Array '{{ $labels.device }}' needs attention and possibly a disk swap.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddiskfailure
- summary: Failed device in RAID array.
- expr: node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} > 0
- labels:
- severity: warning
- - alert: NodeFileDescriptorLimit
- annotations:
- description: File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit
- summary: Kernel is predicted to exhaust file descriptors limit soon.
- expr: |-
- (
- node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70
- )
- for: 15m
- labels:
- severity: warning
- - alert: NodeFileDescriptorLimit
- annotations:
- description: File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit
- summary: Kernel is predicted to exhaust file descriptors limit soon.
- expr: |-
- (
- node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90
- )
- for: 15m
- labels:
- severity: critical
- - alert: NodeCPUHighUsage
- annotations:
- description: 'CPU usage at {{ $labels.instance }} has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.
-
- '
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodecpuhighusage
- summary: High CPU usage.
- expr: sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node-exporter", mode!="idle"}[2m]))) * 100 > 90
- for: 15m
- labels:
- severity: info
- - alert: NodeSystemSaturation
- annotations:
- description: 'System load per core at {{ $labels.instance }} has been above 2 for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.
-
- This might indicate this instance resources saturation and can cause it becoming unresponsive.
-
- '
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemsaturation
- summary: System saturated, load per core is very high.
- expr: |-
- node_load1{job="node-exporter"}
- / count without (cpu, mode) (node_cpu_seconds_total{job="node-exporter", mode="idle"}) > 2
- for: 15m
- labels:
- severity: warning
- - alert: NodeMemoryMajorPagesFaults
- annotations:
- description: 'Memory major pages are occurring at very high rate at {{ $labels.instance }}, 500 major page faults per second for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.
-
- Please check that there is enough memory available at this instance.
-
- '
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodememorymajorpagesfaults
- summary: Memory major page faults are occurring at very high rate.
- expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) > 500
- for: 15m
- labels:
- severity: warning
- - alert: NodeMemoryHighUtilization
- annotations:
- description: 'Memory is filling up at {{ $labels.instance }}, has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.
-
- '
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodememoryhighutilization
- summary: Host is running out of memory.
- expr: 100 - (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"} * 100) > 90
- for: 15m
- labels:
- severity: warning
- - alert: NodeDiskIOSaturation
- annotations:
- description: 'Disk IO queue (aqu-sq) is high on {{ $labels.device }} at {{ $labels.instance }}, has been above 10 for the last 30 minutes, is currently at {{ printf "%.2f" $value }}.
-
- This symptom might indicate disk saturation.
-
- '
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodediskiosaturation
- summary: Disk IO queue is high.
- expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) > 10
- for: 30m
- labels:
- severity: warning
- - alert: NodeSystemdServiceFailed
- annotations:
- description: Systemd service {{ $labels.name }} has entered failed state at {{ $labels.instance }}
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemdservicefailed
- summary: Systemd service has entered failed state.
- expr: node_systemd_unit_state{job="node-exporter", state="failed"} == 1
- for: 5m
- labels:
- severity: warning
- - alert: NodeBondingDegraded
- annotations:
- description: Bonding interface {{ $labels.master }} on {{ $labels.instance }} is in degraded state due to one or more slave failures.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodebondingdegraded
- summary: Bonding interface is degraded
- expr: (node_bonding_slaves - node_bonding_active) != 0
- for: 5m
- labels:
- severity: warning
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/node-network.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-node-network
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: node-network
- rules:
- - alert: NodeNetworkInterfaceFlapping
- annotations:
- description: Network interface "{{ $labels.device }}" changing its up status often on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/nodenetworkinterfaceflapping
- summary: Network interface is often changing its status
- expr: changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2
- for: 2m
- labels:
- severity: warning
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/node.rules.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-node.rules
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: node.rules
- rules:
- - expr: |-
- topk by (cluster, namespace, pod) (1,
- max by (cluster, node, namespace, pod) (
- label_replace(kube_pod_info{job="kube-state-metrics",node!=""}, "pod", "$1", "pod", "(.*)")
- ))
- record: 'node_namespace_pod:kube_pod_info:'
- - expr: |-
- count by (cluster, node) (
- node_cpu_seconds_total{mode="idle",job="node-exporter"}
- * on (cluster, namespace, pod) group_left(node)
- topk by (cluster, namespace, pod) (1, node_namespace_pod:kube_pod_info:)
- )
- record: node:node_num_cpu:sum
- - expr: |-
- sum(
- node_memory_MemAvailable_bytes{job="node-exporter"} or
- (
- node_memory_Buffers_bytes{job="node-exporter"} +
- node_memory_Cached_bytes{job="node-exporter"} +
- node_memory_MemFree_bytes{job="node-exporter"} +
- node_memory_Slab_bytes{job="node-exporter"}
- )
- ) by (cluster)
- record: :node_memory_MemAvailable_bytes:sum
- - expr: |-
- avg by (cluster, node) (
- sum without (mode) (
- rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",job="node-exporter"}[5m])
- )
- )
- record: node:node_cpu_utilization:ratio_rate5m
- - expr: |-
- avg by (cluster) (
- node:node_cpu_utilization:ratio_rate5m
- )
- record: cluster:node_cpu:ratio_rate5m
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/prometheus-operator.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-prometheus-operator
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: prometheus-operator
- rules:
- - alert: PrometheusOperatorListErrors
- annotations:
- description: Errors while performing List operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorlisterrors
- summary: Errors while performing list operations in controller.
- expr: (sum by (cluster,controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="release-name-kps-operator",namespace="default"}[10m])) / sum by (cluster,controller,namespace) (rate(prometheus_operator_list_operations_total{job="release-name-kps-operator",namespace="default"}[10m]))) > 0.4
- for: 15m
- labels:
- severity: warning
- - alert: PrometheusOperatorWatchErrors
- annotations:
- description: Errors while performing watch operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorwatcherrors
- summary: Errors while performing watch operations in controller.
- expr: (sum by (cluster,controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="release-name-kps-operator",namespace="default"}[5m])) / sum by (cluster,controller,namespace) (rate(prometheus_operator_watch_operations_total{job="release-name-kps-operator",namespace="default"}[5m]))) > 0.4
- for: 15m
- labels:
- severity: warning
- - alert: PrometheusOperatorSyncFailed
- annotations:
- description: Controller {{ $labels.controller }} in {{ $labels.namespace }} namespace fails to reconcile {{ $value }} objects.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorsyncfailed
- summary: Last controller reconciliation failed
- expr: min_over_time(prometheus_operator_syncs{status="failed",job="release-name-kps-operator",namespace="default"}[5m]) > 0
- for: 10m
- labels:
- severity: warning
- - alert: PrometheusOperatorReconcileErrors
- annotations:
- description: '{{ $value | humanizePercentage }} of reconciling operations failed for {{ $labels.controller }} controller in {{ $labels.namespace }} namespace.'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorreconcileerrors
- summary: Errors while reconciling objects.
- expr: (sum by (cluster,controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="release-name-kps-operator",namespace="default"}[5m]))) / (sum by (cluster,controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="release-name-kps-operator",namespace="default"}[5m]))) > 0.1
- for: 10m
- labels:
- severity: warning
- - alert: PrometheusOperatorStatusUpdateErrors
- annotations:
- description: '{{ $value | humanizePercentage }} of status update operations failed for {{ $labels.controller }} controller in {{ $labels.namespace }} namespace.'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorstatusupdateerrors
- summary: Errors while updating objects status.
- expr: (sum by (cluster,controller,namespace) (rate(prometheus_operator_status_update_errors_total{job="release-name-kps-operator",namespace="default"}[5m]))) / (sum by (cluster,controller,namespace) (rate(prometheus_operator_status_update_operations_total{job="release-name-kps-operator",namespace="default"}[5m]))) > 0.1
- for: 10m
- labels:
- severity: warning
- - alert: PrometheusOperatorNodeLookupErrors
- annotations:
- description: Errors while reconciling Prometheus in {{ $labels.namespace }} Namespace.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatornodelookuperrors
- summary: Errors while reconciling Prometheus.
- expr: rate(prometheus_operator_node_address_lookup_errors_total{job="release-name-kps-operator",namespace="default"}[5m]) > 0.1
- for: 10m
- labels:
- severity: warning
- - alert: PrometheusOperatorNotReady
- annotations:
- description: Prometheus operator in {{ $labels.namespace }} namespace isn't ready to reconcile {{ $labels.controller }} resources.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatornotready
- summary: Prometheus operator not ready
- expr: min by (cluster,controller,namespace) (max_over_time(prometheus_operator_ready{job="release-name-kps-operator",namespace="default"}[5m]) == 0)
- for: 5m
- labels:
- severity: warning
- - alert: PrometheusOperatorRejectedResources
- annotations:
- description: Prometheus operator in {{ $labels.namespace }} namespace rejected {{ printf "%0.0f" $value }} {{ $labels.controller }}/{{ $labels.resource }} resources.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorrejectedresources
- summary: Resources rejected by Prometheus operator
- expr: min_over_time(prometheus_operator_managed_resources{state="rejected",job="release-name-kps-operator",namespace="default"}[5m]) > 0
- for: 5m
- labels:
- severity: warning
----
-# Source: mimir/charts/kps/templates/prometheus/rules-1.14/prometheus.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: release-name-kps-prometheus
- namespace: default
- labels:
- app: kps
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- groups:
- - name: prometheus
- rules:
- - alert: PrometheusBadConfig
- annotations:
- description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to reload its configuration.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusbadconfig
- summary: Failed Prometheus configuration reload.
- expr: |-
- # Without max_over_time, failed scrapes could create false negatives, see
- # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
- max_over_time(prometheus_config_last_reload_successful{job="release-name-kps-prometheus",namespace="default"}[5m]) == 0
- for: 10m
- labels:
- severity: critical
- - alert: PrometheusSDRefreshFailure
- annotations:
- description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to refresh SD with mechanism {{$labels.mechanism}}.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheussdrefreshfailure
- summary: Failed Prometheus SD refresh.
- expr: increase(prometheus_sd_refresh_failures_total{job="release-name-kps-prometheus",namespace="default"}[10m]) > 0
- for: 20m
- labels:
- severity: warning
- - alert: PrometheusNotificationQueueRunningFull
- annotations:
- description: Alert notification queue of Prometheus {{$labels.namespace}}/{{$labels.pod}} is running full.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusnotificationqueuerunningfull
- summary: Prometheus alert notification queue predicted to run full in less than 30m.
- expr: |-
- # Without min_over_time, failed scrapes could create false negatives, see
- # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
- (
- predict_linear(prometheus_notifications_queue_length{job="release-name-kps-prometheus",namespace="default"}[5m], 60 * 30)
- >
- min_over_time(prometheus_notifications_queue_capacity{job="release-name-kps-prometheus",namespace="default"}[5m])
- )
- for: 15m
- labels:
- severity: warning
- - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers
- annotations:
- description: '{{ printf "%.1f" $value }}% errors while sending alerts from Prometheus {{$labels.namespace}}/{{$labels.pod}} to Alertmanager {{$labels.alertmanager}}.'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheuserrorsendingalertstosomealertmanagers
- summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager.
- expr: |-
- (
- rate(prometheus_notifications_errors_total{job="release-name-kps-prometheus",namespace="default"}[5m])
- /
- rate(prometheus_notifications_sent_total{job="release-name-kps-prometheus",namespace="default"}[5m])
- )
- * 100
- > 1
- for: 15m
- labels:
- severity: warning
- - alert: PrometheusNotConnectedToAlertmanagers
- annotations:
- description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not connected to any Alertmanagers.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusnotconnectedtoalertmanagers
- summary: Prometheus is not connected to any Alertmanagers.
- expr: |-
- # Without max_over_time, failed scrapes could create false negatives, see
- # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
- max_over_time(prometheus_notifications_alertmanagers_discovered{job="release-name-kps-prometheus",namespace="default"}[5m]) < 1
- for: 10m
- labels:
- severity: warning
- - alert: PrometheusTSDBReloadsFailing
- annotations:
- description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value | humanize}} reload failures over the last 3h.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustsdbreloadsfailing
- summary: Prometheus has issues reloading blocks from disk.
- expr: increase(prometheus_tsdb_reloads_failures_total{job="release-name-kps-prometheus",namespace="default"}[3h]) > 0
- for: 4h
- labels:
- severity: warning
- - alert: PrometheusTSDBCompactionsFailing
- annotations:
- description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value | humanize}} compaction failures over the last 3h.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustsdbcompactionsfailing
- summary: Prometheus has issues compacting blocks.
- expr: increase(prometheus_tsdb_compactions_failed_total{job="release-name-kps-prometheus",namespace="default"}[3h]) > 0
- for: 4h
- labels:
- severity: warning
- - alert: PrometheusNotIngestingSamples
- annotations:
- description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not ingesting samples.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusnotingestingsamples
- summary: Prometheus is not ingesting samples.
- expr: |-
- (
- sum without(type) (rate(prometheus_tsdb_head_samples_appended_total{job="release-name-kps-prometheus",namespace="default"}[5m])) <= 0
- and
- (
- sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="release-name-kps-prometheus",namespace="default"}) > 0
- or
- sum without(rule_group) (prometheus_rule_group_rules{job="release-name-kps-prometheus",namespace="default"}) > 0
- )
- )
- for: 10m
- labels:
- severity: warning
- - alert: PrometheusDuplicateTimestamps
- annotations:
- description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf "%.4g" $value }} samples/s with different values but duplicated timestamp.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusduplicatetimestamps
- summary: Prometheus is dropping samples with duplicate timestamps.
- expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="release-name-kps-prometheus",namespace="default"}[5m]) > 0
- for: 10m
- labels:
- severity: warning
- - alert: PrometheusOutOfOrderTimestamps
- annotations:
- description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf "%.4g" $value }} samples/s with timestamps arriving out of order.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusoutofordertimestamps
- summary: Prometheus drops samples with out-of-order timestamps.
- expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="release-name-kps-prometheus",namespace="default"}[5m]) > 0
- for: 10m
- labels:
- severity: warning
- - alert: PrometheusRemoteStorageFailures
- annotations:
- description: Prometheus {{$labels.namespace}}/{{$labels.pod}} failed to send {{ printf "%.1f" $value }}% of the samples to {{ $labels.remote_name}}:{{ $labels.url }}
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusremotestoragefailures
- summary: Prometheus fails to send samples to remote storage.
- expr: |-
- (
- (rate(prometheus_remote_storage_failed_samples_total{job="release-name-kps-prometheus",namespace="default"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="release-name-kps-prometheus",namespace="default"}[5m]))
- /
- (
- (rate(prometheus_remote_storage_failed_samples_total{job="release-name-kps-prometheus",namespace="default"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="release-name-kps-prometheus",namespace="default"}[5m]))
- +
- (rate(prometheus_remote_storage_succeeded_samples_total{job="release-name-kps-prometheus",namespace="default"}[5m]) or rate(prometheus_remote_storage_samples_total{job="release-name-kps-prometheus",namespace="default"}[5m]))
- )
- )
- * 100
- > 1
- for: 15m
- labels:
- severity: critical
- - alert: PrometheusRemoteWriteBehind
- annotations:
- description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write is {{ printf "%.1f" $value }}s behind for {{ $labels.remote_name}}:{{ $labels.url }}.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusremotewritebehind
- summary: Prometheus remote write is behind.
- expr: |-
- # Without max_over_time, failed scrapes could create false negatives, see
- # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
- (
- max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="release-name-kps-prometheus",namespace="default"}[5m])
- - ignoring(remote_name, url) group_right
- max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="release-name-kps-prometheus",namespace="default"}[5m])
- )
- > 120
- for: 15m
- labels:
- severity: critical
- - alert: PrometheusRemoteWriteDesiredShards
- annotations:
- description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write desired shards calculation wants to run {{ $value }} shards for queue {{ $labels.remote_name}}:{{ $labels.url }}, which is more than the max of {{ printf `prometheus_remote_storage_shards_max{instance="%s",job="release-name-kps-prometheus",namespace="default"}` $labels.instance | query | first | value }}.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusremotewritedesiredshards
- summary: Prometheus remote write desired shards calculation wants to run more than configured max shards.
- expr: |-
- # Without max_over_time, failed scrapes could create false negatives, see
- # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
- (
- max_over_time(prometheus_remote_storage_shards_desired{job="release-name-kps-prometheus",namespace="default"}[5m])
- >
- max_over_time(prometheus_remote_storage_shards_max{job="release-name-kps-prometheus",namespace="default"}[5m])
- )
- for: 15m
- labels:
- severity: warning
- - alert: PrometheusRuleFailures
- annotations:
- description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to evaluate {{ printf "%.0f" $value }} rules in the last 5m.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusrulefailures
- summary: Prometheus is failing rule evaluations.
- expr: increase(prometheus_rule_evaluation_failures_total{job="release-name-kps-prometheus",namespace="default"}[5m]) > 0
- for: 15m
- labels:
- severity: critical
- - alert: PrometheusMissingRuleEvaluations
- annotations:
- description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has missed {{ printf "%.0f" $value }} rule group evaluations in the last 5m.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusmissingruleevaluations
- summary: Prometheus is missing rule evaluations due to slow rule group evaluation.
- expr: increase(prometheus_rule_group_iterations_missed_total{job="release-name-kps-prometheus",namespace="default"}[5m]) > 0
- for: 15m
- labels:
- severity: warning
- - alert: PrometheusTargetLimitHit
- annotations:
- description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ printf "%.0f" $value }} targets because the number of targets exceeded the configured target_limit.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustargetlimithit
- summary: Prometheus has dropped targets because some scrape configs have exceeded the targets limit.
- expr: increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="release-name-kps-prometheus",namespace="default"}[5m]) > 0
- for: 15m
- labels:
- severity: warning
- - alert: PrometheusLabelLimitHit
- annotations:
- description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ printf "%.0f" $value }} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheuslabellimithit
- summary: Prometheus has dropped targets because some scrape configs have exceeded the labels limit.
- expr: increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="release-name-kps-prometheus",namespace="default"}[5m]) > 0
- for: 15m
- labels:
- severity: warning
- - alert: PrometheusScrapeBodySizeLimitHit
- annotations:
- description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed {{ printf "%.0f" $value }} scrapes in the last 5m because some targets exceeded the configured body_size_limit.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusscrapebodysizelimithit
- summary: Prometheus has dropped some targets that exceeded body size limit.
- expr: increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="release-name-kps-prometheus",namespace="default"}[5m]) > 0
- for: 15m
- labels:
- severity: warning
- - alert: PrometheusScrapeSampleLimitHit
- annotations:
- description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed {{ printf "%.0f" $value }} scrapes in the last 5m because some targets exceeded the configured sample_limit.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusscrapesamplelimithit
- summary: Prometheus has failed scrapes that have exceeded the configured sample limit.
- expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="release-name-kps-prometheus",namespace="default"}[5m]) > 0
- for: 15m
- labels:
- severity: warning
- - alert: PrometheusTargetSyncFailure
- annotations:
- description: '{{ printf "%.0f" $value }} targets in Prometheus {{$labels.namespace}}/{{$labels.pod}} have failed to sync because invalid configuration was supplied.'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustargetsyncfailure
- summary: Prometheus has failed to sync targets.
- expr: increase(prometheus_target_sync_failed_total{job="release-name-kps-prometheus",namespace="default"}[30m]) > 0
- for: 5m
- labels:
- severity: critical
- - alert: PrometheusHighQueryLoad
- annotations:
- description: Prometheus {{$labels.namespace}}/{{$labels.pod}} query API has less than 20% available capacity in its query engine for the last 15 minutes.
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheushighqueryload
- summary: Prometheus is reaching its maximum capacity serving concurrent requests.
- expr: avg_over_time(prometheus_engine_queries{job="release-name-kps-prometheus",namespace="default"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="release-name-kps-prometheus",namespace="default"}[5m]) > 0.8
- for: 15m
- labels:
- severity: warning
- - alert: PrometheusErrorSendingAlertsToAnyAlertmanager
- annotations:
- description: '{{ printf "%.1f" $value }}% minimum errors while sending alerts from Prometheus {{$labels.namespace}}/{{$labels.pod}} to any Alertmanager.'
- runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheuserrorsendingalertstoanyalertmanager
- summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager.
- expr: |-
- min without (alertmanager) (
- rate(prometheus_notifications_errors_total{job="release-name-kps-prometheus",namespace="default",alertmanager!~``}[5m])
- /
- rate(prometheus_notifications_sent_total{job="release-name-kps-prometheus",namespace="default",alertmanager!~``}[5m])
- )
- * 100
- > 3
- for: 15m
- labels:
- severity: critical
----
-# Source: mimir/charts/kps/charts/kube-state-metrics/templates/servicemonitor.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-kube-state-metrics
- namespace: default
- labels:
- helm.sh/chart: kube-state-metrics-5.16.0
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: metrics
- app.kubernetes.io/part-of: kube-state-metrics
- app.kubernetes.io/name: kube-state-metrics
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.10.1"
- release: release-name
-spec:
- jobLabel: app.kubernetes.io/name
- selector:
- matchLabels:
- app.kubernetes.io/name: kube-state-metrics
- app.kubernetes.io/instance: release-name
- endpoints:
- - port: http
- honorLabels: true
----
-# Source: mimir/charts/kps/charts/prometheus-node-exporter/templates/servicemonitor.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-prometheus-node-exporter
- namespace: default
- labels:
- helm.sh/chart: prometheus-node-exporter-4.30.2
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: metrics
- app.kubernetes.io/part-of: prometheus-node-exporter
- app.kubernetes.io/name: prometheus-node-exporter
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "1.7.0"
- jobLabel: node-exporter
- release: release-name
-spec:
- jobLabel: jobLabel
-
- selector:
- matchLabels:
- app.kubernetes.io/name: prometheus-node-exporter
- app.kubernetes.io/instance: release-name
- attachMetadata:
- node: false
- endpoints:
- - port: http-metrics
- scheme: http
----
-# Source: mimir/charts/kps/templates/exporters/core-dns/servicemonitor.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-kps-coredns
- namespace: default
- labels:
- app: kps-coredns
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- jobLabel: jobLabel
-
- selector:
- matchLabels:
- app: kps-coredns
- release: "release-name"
- namespaceSelector:
- matchNames:
- - "kube-system"
- endpoints:
- - port: http-metrics
- bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
----
-# Source: mimir/charts/kps/templates/exporters/kube-api-server/servicemonitor.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-kps-apiserver
- namespace: default
- labels:
- app: kps-apiserver
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
-
- endpoints:
- - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
- port: https
- scheme: https
- metricRelabelings:
- - action: drop
- regex: apiserver_request_duration_seconds_bucket;(0.15|0.2|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2|3|3.5|4|4.5|6|7|8|9|15|25|40|50)
- sourceLabels:
- - __name__
- - le
- tlsConfig:
- caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
- serverName: kubernetes
- insecureSkipVerify: false
- jobLabel: component
- namespaceSelector:
- matchNames:
- - default
- selector:
- matchLabels:
- component: apiserver
- provider: kubernetes
----
-# Source: mimir/charts/kps/templates/exporters/kube-controller-manager/servicemonitor.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-kps-kube-controller-manager
- namespace: default
- labels:
- app: kps-kube-controller-manager
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- jobLabel: jobLabel
-
- selector:
- matchLabels:
- app: kps-kube-controller-manager
- release: "release-name"
- namespaceSelector:
- matchNames:
- - "kube-system"
- endpoints:
- - port: http-metrics
- bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
- scheme: https
- tlsConfig:
- caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
- insecureSkipVerify: true
----
-# Source: mimir/charts/kps/templates/exporters/kube-etcd/servicemonitor.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-kps-kube-etcd
- namespace: default
- labels:
- app: kps-kube-etcd
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- jobLabel: jobLabel
-
- selector:
- matchLabels:
- app: kps-kube-etcd
- release: "release-name"
- namespaceSelector:
- matchNames:
- - "kube-system"
- endpoints:
- - port: http-metrics
- bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
----
-# Source: mimir/charts/kps/templates/exporters/kube-scheduler/servicemonitor.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-kps-kube-scheduler
- namespace: default
- labels:
- app: kps-kube-scheduler
-
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
- jobLabel: jobLabel
-
- selector:
- matchLabels:
- app: kps-kube-scheduler
- release: "release-name"
- namespaceSelector:
- matchNames:
- - "kube-system"
- endpoints:
- - port: http-metrics
- bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
- scheme: https
- tlsConfig:
- caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
- insecureSkipVerify: true
----
-# Source: mimir/charts/kps/templates/exporters/kubelet/servicemonitor.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-kps-kubelet
- namespace: default
- labels:
- app: kps-kubelet
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "56.9.0"
- app.kubernetes.io/part-of: kps
- chart: kps-56.9.0
- release: "release-name"
- heritage: "Helm"
-spec:
-
- attachMetadata:
- node: false
- endpoints:
- - port: https-metrics
- scheme: https
- tlsConfig:
- caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
- insecureSkipVerify: true
- bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
- honorLabels: true
- honorTimestamps: true
- relabelings:
- - action: replace
- sourceLabels:
- - __metrics_path__
- targetLabel: metrics_path
- - port: https-metrics
- scheme: https
- path: /metrics/cadvisor
- honorLabels: true
- honorTimestamps: true
- tlsConfig:
- caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
- insecureSkipVerify: true
- bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
- metricRelabelings:
- - action: drop
- regex: container_cpu_(cfs_throttled_seconds_total|load_average_10s|system_seconds_total|user_seconds_total)
- sourceLabels:
- - __name__
- - action: drop
- regex: container_fs_(io_current|io_time_seconds_total|io_time_weighted_seconds_total|reads_merged_total|sector_reads_total|sector_writes_total|writes_merged_total)
- sourceLabels:
- - __name__
- - action: drop
- regex: container_memory_(mapped_file|swap)
- sourceLabels:
- - __name__
- - action: drop
- regex: container_(file_descriptors|tasks_state|threads_max)
- sourceLabels:
- - __name__
- - action: drop
- regex: container_spec.*
- sourceLabels:
- - __name__
- - action: drop
- regex: .+;
- sourceLabels:
- - id
- - pod
- relabelings:
- - action: replace
- sourceLabels:
- - __metrics_path__
- targetLabel: metrics_path
- - port: https-metrics
- scheme: https
- path: /metrics/probes
- honorLabels: true
- honorTimestamps: true
- tlsConfig:
- caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
- insecureSkipVerify: true
- bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
- relabelings:
- - action: replace
- sourceLabels:
- - __metrics_path__
- targetLabel: metrics_path
- jobLabel: k8s-app
- namespaceSelector:
- matchNames:
- - kube-system
- selector:
- matchLabels:
- app.kubernetes.io/name: kubelet
- k8s-app: kubelet
----
-# Source: mimir/charts/mimir/templates/alertmanager/alertmanager-servmon.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-mimir-alertmanager
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: alertmanager
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-spec:
- namespaceSelector:
- matchNames:
- - default
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: alertmanager
- matchExpressions:
- - key: prometheus.io/service-monitor
- operator: NotIn
- values:
- - "false"
- endpoints:
- - port: http-metrics
- relabelings:
- - sourceLabels: [job]
- replacement: "default/alertmanager"
- targetLabel: job
- - replacement: "release-name"
- targetLabel: cluster
- scheme: http
----
-# Source: mimir/charts/mimir/templates/compactor/compactor-servmon.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-mimir-compactor
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: compactor
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-spec:
- namespaceSelector:
- matchNames:
- - default
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: compactor
- matchExpressions:
- - key: prometheus.io/service-monitor
- operator: NotIn
- values:
- - "false"
- endpoints:
- - port: http-metrics
- relabelings:
- - sourceLabels: [job]
- replacement: "default/compactor"
- targetLabel: job
- - replacement: "release-name"
- targetLabel: cluster
- scheme: http
----
-# Source: mimir/charts/mimir/templates/distributor/distributor-servmon.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-mimir-distributor
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: distributor
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-spec:
- namespaceSelector:
- matchNames:
- - default
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: distributor
- matchExpressions:
- - key: prometheus.io/service-monitor
- operator: NotIn
- values:
- - "false"
- endpoints:
- - port: http-metrics
- relabelings:
- - sourceLabels: [job]
- replacement: "default/distributor"
- targetLabel: job
- - replacement: "release-name"
- targetLabel: cluster
- scheme: http
----
-# Source: mimir/charts/mimir/templates/index-cache/index-cache-servmon.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-mimir-index-cache
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: index-cache
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-spec:
- namespaceSelector:
- matchNames:
- - default
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: index-cache
- matchExpressions:
- - key: prometheus.io/service-monitor
- operator: NotIn
- values:
- - "false"
- endpoints:
- - port: http-metrics
- relabelings:
- - sourceLabels: [job]
- replacement: "default/index-cache"
- targetLabel: job
- - replacement: "release-name"
- targetLabel: cluster
- scheme: http
----
-# Source: mimir/charts/mimir/templates/ingester/ingester-servmon.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-mimir-ingester
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-spec:
- namespaceSelector:
- matchNames:
- - default
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ingester
- matchExpressions:
- - key: prometheus.io/service-monitor
- operator: NotIn
- values:
- - "false"
- endpoints:
- - port: http-metrics
- relabelings:
- - sourceLabels: [job]
- replacement: "default/ingester"
- targetLabel: job
- - replacement: "release-name"
- targetLabel: cluster
- scheme: http
----
-# Source: mimir/charts/mimir/templates/metadata-cache/metadata-cache-servmon.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-mimir-metadata-cache
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: metadata-cache
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-spec:
- namespaceSelector:
- matchNames:
- - default
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: metadata-cache
- matchExpressions:
- - key: prometheus.io/service-monitor
- operator: NotIn
- values:
- - "false"
- endpoints:
- - port: http-metrics
- relabelings:
- - sourceLabels: [job]
- replacement: "default/metadata-cache"
- targetLabel: job
- - replacement: "release-name"
- targetLabel: cluster
- scheme: http
----
-# Source: mimir/charts/mimir/templates/overrides-exporter/overrides-exporter-servmon.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-mimir-overrides-exporter
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: overrides-exporter
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-spec:
- namespaceSelector:
- matchNames:
- - default
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: overrides-exporter
- matchExpressions:
- - key: prometheus.io/service-monitor
- operator: NotIn
- values:
- - "false"
- endpoints:
- - port: http-metrics
- relabelings:
- - sourceLabels: [job]
- replacement: "default/overrides-exporter"
- targetLabel: job
- - replacement: "release-name"
- targetLabel: cluster
- scheme: http
----
-# Source: mimir/charts/mimir/templates/querier/querier-servmon.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-mimir-querier
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: querier
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-spec:
- namespaceSelector:
- matchNames:
- - default
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: querier
- matchExpressions:
- - key: prometheus.io/service-monitor
- operator: NotIn
- values:
- - "false"
- endpoints:
- - port: http-metrics
- relabelings:
- - sourceLabels: [job]
- replacement: "default/querier"
- targetLabel: job
- - replacement: "release-name"
- targetLabel: cluster
- scheme: http
----
-# Source: mimir/charts/mimir/templates/query-frontend/query-frontend-servmon.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-mimir-query-frontend
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-frontend
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-spec:
- namespaceSelector:
- matchNames:
- - default
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-frontend
- matchExpressions:
- - key: prometheus.io/service-monitor
- operator: NotIn
- values:
- - "false"
- endpoints:
- - port: http-metrics
- relabelings:
- - sourceLabels: [job]
- replacement: "default/query-frontend"
- targetLabel: job
- - replacement: "release-name"
- targetLabel: cluster
- scheme: http
----
-# Source: mimir/charts/mimir/templates/query-scheduler/query-scheduler-servmon.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-mimir-query-scheduler
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-scheduler
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-spec:
- namespaceSelector:
- matchNames:
- - default
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: query-scheduler
- matchExpressions:
- - key: prometheus.io/service-monitor
- operator: NotIn
- values:
- - "false"
- endpoints:
- - port: http-metrics
- relabelings:
- - sourceLabels: [job]
- replacement: "default/query-scheduler"
- targetLabel: job
- - replacement: "release-name"
- targetLabel: cluster
- scheme: http
----
-# Source: mimir/charts/mimir/templates/results-cache/results-cache-servmon.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-mimir-results-cache
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: results-cache
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-spec:
- namespaceSelector:
- matchNames:
- - default
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: results-cache
- matchExpressions:
- - key: prometheus.io/service-monitor
- operator: NotIn
- values:
- - "false"
- endpoints:
- - port: http-metrics
- relabelings:
- - sourceLabels: [job]
- replacement: "default/results-cache"
- targetLabel: job
- - replacement: "release-name"
- targetLabel: cluster
- scheme: http
----
-# Source: mimir/charts/mimir/templates/ruler/ruler-servmon.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-mimir-ruler
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ruler
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-spec:
- namespaceSelector:
- matchNames:
- - default
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: ruler
- matchExpressions:
- - key: prometheus.io/service-monitor
- operator: NotIn
- values:
- - "false"
- endpoints:
- - port: http-metrics
- relabelings:
- - sourceLabels: [job]
- replacement: "default/ruler"
- targetLabel: job
- - replacement: "release-name"
- targetLabel: cluster
- scheme: http
----
-# Source: mimir/charts/mimir/templates/store-gateway/store-gateway-servmon.yaml
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
- name: release-name-mimir-store-gateway
- namespace: "default"
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- app.kubernetes.io/part-of: memberlist
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
-spec:
- namespaceSelector:
- matchNames:
- - default
- selector:
- matchLabels:
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: store-gateway
- matchExpressions:
- - key: prometheus.io/service-monitor
- operator: NotIn
- values:
- - "false"
- endpoints:
- - port: http-metrics
- relabelings:
- - sourceLabels: [job]
- replacement: "default/store-gateway"
- targetLabel: job
- - replacement: "release-name"
- targetLabel: cluster
- scheme: http
----
-# Source: mimir/charts/mimir/charts/minio/templates/post-job.yaml
-apiVersion: batch/v1
-kind: Job
-metadata:
- name: release-name-minio-post-job
- labels:
- app: minio-post-job
- chart: minio-5.0.14
- release: release-name
- heritage: Helm
- annotations:
- "helm.sh/hook": post-install,post-upgrade
- "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
-spec:
- template:
- metadata:
- labels:
- app: minio-job
- release: release-name
- spec:
- restartPolicy: OnFailure
- volumes:
- - name: etc-path
- emptyDir: {}
- - name: tmp
- emptyDir: {}
- - name: minio-configuration
- projected:
- sources:
- - configMap:
- name: release-name-minio
- - secret:
- name: release-name-minio
- serviceAccountName: minio-sa
- containers:
- - name: minio-make-bucket
- image: "quay.io/minio/mc:RELEASE.2023-09-29T16-41-22Z"
- imagePullPolicy: IfNotPresent
- command: [ "/bin/sh", "/config/initialize" ]
- env:
- - name: MINIO_ENDPOINT
- value: release-name-minio
- - name: MINIO_PORT
- value: "9000"
- volumeMounts:
- - name: etc-path
- mountPath: /etc/minio/mc
- - name: tmp
- mountPath: /tmp
- - name: minio-configuration
- mountPath: /config
- resources:
- requests:
- memory: 128Mi
- - name: minio-make-user
- image: "quay.io/minio/mc:RELEASE.2023-09-29T16-41-22Z"
- imagePullPolicy: IfNotPresent
- command: [ "/bin/sh", "/config/add-user" ]
- env:
- - name: MINIO_ENDPOINT
- value: release-name-minio
- - name: MINIO_PORT
- value: "9000"
- volumeMounts:
- - name: etc-path
- mountPath: /etc/minio/mc
- - name: tmp
- mountPath: /tmp
- - name: minio-configuration
- mountPath: /config
- resources:
- requests:
- memory: 128Mi
----
-# Source: mimir/charts/mimir/templates/smoke-test/smoke-test-job.yaml
-apiVersion: batch/v1
-kind: Job
-metadata:
- name: release-name-mimir-smoke-test
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/component: smoke-test
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- annotations:
- "helm.sh/hook": test
- namespace: "default"
-spec:
- backoffLimit: 5
- completions: 1
- parallelism: 1
- selector:
- template:
- metadata:
- labels:
- helm.sh/chart: mimir-5.2.1
- app.kubernetes.io/name: mimir
- app.kubernetes.io/instance: release-name
- app.kubernetes.io/version: "2.11.0"
- app.kubernetes.io/managed-by: Helm
- app.kubernetes.io/component: smoke-test
- spec:
- serviceAccountName: release-name-mimir
- securityContext:
- fsGroup: 10001
- runAsGroup: 10001
- runAsNonRoot: true
- runAsUser: 10001
- seccompProfile:
- type: RuntimeDefault
- initContainers:
- []
- containers:
- - name: smoke-test
- image: "grafana/mimir-continuous-test:2.11.0"
- imagePullPolicy: IfNotPresent
- args:
- - "-tests.smoke-test"
- - "-tests.write-endpoint=http://release-name-mimir-nginx.default.svc:80"
- - "-tests.read-endpoint=http://release-name-mimir-nginx.default.svc:80/prometheus"
- - "-tests.tenant-id="
- - "-tests.write-read-series-test.num-series=1000"
- - "-tests.write-read-series-test.max-query-age=48h"
- - "-server.metrics-port=8080"
- volumeMounts:
- env:
- envFrom:
- restartPolicy: OnFailure
- volumes:
diff --git a/charts/incubator/mimir/questions.yaml b/charts/incubator/mimir/questions.yaml
deleted file mode 100644
index 488aa7360d1..00000000000
--- a/charts/incubator/mimir/questions.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-# Include{groups}
-portals:
- open:
-# Include{portalLink}
-questions:
-
-# Include{documentation}
diff --git a/charts/incubator/mimir/templates/NOTES.txt b/charts/incubator/mimir/templates/NOTES.txt
deleted file mode 100644
index efcb74cb772..00000000000
--- a/charts/incubator/mimir/templates/NOTES.txt
+++ /dev/null
@@ -1 +0,0 @@
-{{- include "tc.v1.common.lib.chart.notes" $ -}}
diff --git a/charts/incubator/mimir/templates/common.yaml b/charts/incubator/mimir/templates/common.yaml
deleted file mode 100644
index b51394e00a4..00000000000
--- a/charts/incubator/mimir/templates/common.yaml
+++ /dev/null
@@ -1 +0,0 @@
-{{ include "tc.v1.common.loader.all" . }}
diff --git a/charts/incubator/mimir/values.yaml b/charts/incubator/mimir/values.yaml
deleted file mode 100644
index 958cde9253e..00000000000
--- a/charts/incubator/mimir/values.yaml
+++ /dev/null
@@ -1,375 +0,0 @@
-image:
- repository: tccr.io/truecharts/scratch
- pullPolicy: IfNotPresent
- tag: 18.0.1@sha256:49df5708d7fc085acf76c7868f10f25fa7ba947c8a7d3354a97207ba69c85cc3
-
-service:
- main:
- enabled: false
- ports:
- main:
- enabled: false
-
-workload:
- main:
- enabled: false
- podSpec:
- containers:
- main:
- enabled: false
-
-portal:
- open:
- enabled: false
-
-configmap:
- grafana-datasource:
- enabled: "{{ if .Values.grafana.datasource.enabled }}true{{ else }}false{{ end }}"
- labels:
- grafana_datasources: "1"
- data:
- datasource.yaml: |-
- apiVersion: 1
- datasources:
- - name: Mimir
- type: prometheus
- uid: {{ .Values.grafana.datasource.uid | default "prometheus" }}
- url: http://{{ if ne .Release.Name "mimir" }}{{ .Release.Name }}-{{ end }}mimir-nginx.{{ .Release.Namespace }}/prometheus
- access: proxy
- isDefault: {{ .Values.grafana.datasource.default | default true }}
- jsonData:
- httpMethod: {{ .Values.grafana.datasource.httpMethod | default "POST" }}
- timeInterval: {{ .Values.grafana.datasource.scrapeInterval | default "30s" }}
- {{- if .Values.grafana.datasource.timeout }}
- timeout: {{ .Values.grafana.datasource.timeout }}
- {{- end }}
-
-grafana:
- datasource:
- enabled: true
- default: true
- httpMethod: "POST"
- scrapeInterval: "30s"
- uid: "prometheus"
-
-# -- Mimir chart values. Resources are set to a minimum by default.
-mimir:
- metaMonitoring:
- # Dashboard configuration for deploying Grafana dashboards for Mimir
- dashboards:
- enabled: true
- annotations: {}
- serviceMonitor:
- enabled: true
-
- grafanaAgent:
- enabled: true
- installOperator: false
- metrics:
- remote:
- url: http://{{ if ne .Release.Name "mimir" }}{{ .Release.Name }}-{{ end }}mimir-nginx.{{ .Release.Namespace }}/prometheus
- headers: {}
- scrapeK8s:
- enabled: false
-
- alertmanager:
- persistentVolume:
- enabled: true
- replicas: 2
- resources:
- limits:
- cpu: 1.4
- memory: 1.4Gi
- requests:
- cpu: 20m
- memory: 10Mi
- statefulSet:
- enabled: true
-
- compactor:
- persistentVolume:
- size: 20Gi
- resources:
- limits:
- cpu: 1.4
- memory: 2.1Gi
- requests:
- cpu: 20m
- memory: 10Mi
-
- distributor:
- replicas: 2
- resources:
- limits:
- cpu: 3.5
- memory: 5.7Gi
- requests:
- cpu: 20m
- memory: 10Mi
-
- ingester:
- persistentVolume:
- size: 50Gi
- replicas: 3
- resources:
- limits:
- cpu: 5
- memory: 12Gi
- requests:
- cpu: 20m
- memory: 10Mi
- topologySpreadConstraints: {}
- affinity:
- podAntiAffinity:
- requiredDuringSchedulingIgnoredDuringExecution:
- - labelSelector:
- matchExpressions:
- - key: target
- operator: In
- values:
- - ingester
- topologyKey: 'kubernetes.io/hostname'
-
- - labelSelector:
- matchExpressions:
- - key: app.kubernetes.io/component
- operator: In
- values:
- - ingester
- topologyKey: 'kubernetes.io/hostname'
- zoneAwareReplication:
- topologyKey: 'kubernetes.io/hostname'
-
- admin-cache:
- enabled: true
- replicas: 2
-
- chunks-cache:
- enabled: true
- replicas: 2
- resources:
- requests:
- cpu: 20m
- memory: 10Mi
-
- index-cache:
- enabled: true
- replicas: 3
-
- metadata-cache:
- enabled: true
-
- results-cache:
- enabled: true
-
- minio:
- enabled: true
-
- overrides_exporter:
- replicas: 1
- resources:
- limits:
- cpu: 100m
- memory: 128Mi
- requests:
- cpu: 20m
- memory: 10Mi
-
- querier:
- replicas: 1
- resources:
- limits:
- cpu: 2.8
- memory: 5.6Gi
- requests:
- cpu: 20m
- memory: 10Mi
-
- query_frontend:
- replicas: 1
- resources:
- limits:
- cpu: 2.8
- memory: 2.8Gi
- requests:
- cpu: 20m
- memory: 10Mi
-
- ruler:
- replicas: 1
- resources:
- limits:
- cpu: 1.4
- memory: 2.8Gi
- requests:
- cpu: 20m
- memory: 10Mi
-
- store_gateway:
- persistentVolume:
- size: 10Gi
- replicas: 3
- resources:
- limits:
- cpu: 1.4
- memory: 2.1Gi
- requests:
- cpu: 20m
- memory: 10Mi
- topologySpreadConstraints: {}
- affinity:
- podAntiAffinity:
- requiredDuringSchedulingIgnoredDuringExecution:
- - labelSelector:
- matchExpressions:
- - key: target
- operator: In
- values:
- - store-gateway
- topologyKey: 'kubernetes.io/hostname'
-
- - labelSelector:
- matchExpressions:
- - key: app.kubernetes.io/component
- operator: In
- values:
- - store-gateway
- topologyKey: 'kubernetes.io/hostname'
-
- zoneAwareReplication:
- topologyKey: 'kubernetes.io/hostname'
-
- nginx:
- replicas: 1
- resources:
- limits:
- cpu: 1.4
- memory: 731Mi
- requests:
- cpu: 20m
- memory: 10Mi
-
-
-kps:
- ## Install Prometheus Operator CRDs
- ##
- crds:
- enabled: false
-
- ## Manages Prometheus and Alertmanager components
- ##
- prometheusOperator:
- enabled: false
-
- ####
- ##
- ## Everything down here, explicitly disables everything BUT the operator itself
- ##
- ####
-
- ##
- global:
- rbac:
- create: true
-
- ## Create default rules for monitoring the cluster
- ##
- defaultRules:
- create: true
-
- windowsMonitoring:
- ## Deploys the windows-exporter and Windows-specific dashboards and rules (job name must be 'windows-exporter')
- enabled: false
-
- ## Configuration for prometheus-windows-exporter
- ## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-windows-exporter
- ##
- prometheus-windows-exporter:
- ## Enable ServiceMonitor and set Kubernetes label to use as a job label
- ##
- prometheus:
- monitor:
- enabled: false
-
- ## Configuration for alertmanager
- ## ref: https://prometheus.io/docs/alerting/alertmanager/
- ##
- alertmanager:
- ## Deploy alertmanager
- ##
- enabled: false
-
- ## Using default values from https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml
- ##
- grafana:
- enabled: false
- forceDeployDashboards: true
- defaultDashboardsEnabled: true
-
- ## Flag to disable all the kubernetes component scrapers
- ##
- kubernetesServiceMonitors:
- enabled: true
-
- ## Component scraping the kube api server
- ##
- kubeApiServer:
- enabled: true
-
- ## Component scraping the kubelet and kubelet-hosted cAdvisor
- ##
- kubelet:
- enabled: true
-
- ## Component scraping the kube controller manager
- ##
- kubeControllerManager:
- enabled: true
-
- ## Component scraping coreDns. Use either this or kubeDns
- ##
- coreDns:
- enabled: true
-
- ## Component scraping kubeDns. Use either this or coreDns
- ##
- kubeDns:
- enabled: false
-
- ## Component scraping etcd
- ##
- kubeEtcd:
- enabled: true
-
- ## Component scraping kube scheduler
- ##
- kubeScheduler:
- enabled: true
-
- ## Component scraping kube proxy
- ##
- kubeProxy:
- enabled: false
-
- ## Component scraping kube state metrics
- ##
- kubeStateMetrics:
- enabled: true
-
- ## dontDeploy node exporter as a daemonset to all nodes
- ##
- nodeExporter:
- enabled: true
-
- ## dont Deploy a Prometheus instance
- ##
- prometheus:
- enabled: false
-
- ## Configuration for thanosRuler
- ## ref: https://thanos.io/tip/components/rule.md/
- ##
- thanosRuler:
- ## Dont Deploy thanosRuler
- ##
- enabled: false