diff --git a/stacks/_templates/prometheus-service-monitor.yaml b/stacks/_templates/prometheus-service-monitor.yaml deleted file mode 100644 index 3d597a7b..00000000 --- a/stacks/_templates/prometheus-service-monitor.yaml +++ /dev/null @@ -1,39 +0,0 @@ ---- -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: scrape-label - labels: - stackable.tech/vendor: Stackable - release: prometheus -spec: - endpoints: - - port: metrics - selector: - matchLabels: - prometheus.io/scrape: "true" - namespaceSelector: - any: true - jobLabel: app.kubernetes.io/instance ---- -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: scrape-minio - labels: - stackable.tech/vendor: Stackable - release: prometheus -spec: - endpoints: - - path: /minio/v2/metrics/node - port: http - scheme: http - - path: /minio/v2/metrics/cluster - port: http - scheme: http - selector: - matchLabels: - app: minio - monitoring: "true" - namespaceSelector: - any: true diff --git a/stacks/monitoring/create-prometheus-tls-certificate-serviceaccount.yaml b/stacks/monitoring/create-prometheus-tls-certificate-serviceaccount.yaml new file mode 100644 index 00000000..3b79bba5 --- /dev/null +++ b/stacks/monitoring/create-prometheus-tls-certificate-serviceaccount.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: create-prometheus-tls-certificate-serviceaccount +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: create-prometheus-tls-certificate-rolebinding +subjects: +- kind: ServiceAccount + name: create-prometheus-tls-certificate-serviceaccount + namespace: {{ NAMESPACE }} +roleRef: + kind: Role + name: create-prometheus-tls-certificate-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: create-prometheus-tls-certificate-role +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "patch"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["delete"] diff --git a/stacks/monitoring/create-prometheus-tls-certificate.yaml b/stacks/monitoring/create-prometheus-tls-certificate.yaml new file mode 100644 index 00000000..faecd89b --- /dev/null +++ b/stacks/monitoring/create-prometheus-tls-certificate.yaml @@ -0,0 +1,76 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: create-prometheus-tls-certificate + labels: + app: create-prometheus-tls-certificate +spec: + replicas: 1 + selector: + matchLabels: + app: create-prometheus-tls-certificate + template: + metadata: + labels: + app: create-prometheus-tls-certificate + spec: + serviceAccountName: create-prometheus-tls-certificate-serviceaccount + containers: + - name: create-prometheus-tls-certificate + image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + command: + - bash + - -euo + - pipefail + - -c + - | + # "kubectl create secret" fails on existing Secrets, so we "kubectl apply" instead + kubectl create secret generic prometheus-tls-certificate \ + --from-file=/prometheus-tls-certificate/ca.crt \ + --from-file=/prometheus-tls-certificate/tls.crt \ + --from-file=/prometheus-tls-certificate/tls.key \ + --dry-run=client -o yaml \ + | kubectl apply -f - + + echo Sleeping 6 hours before deleting my own Pod + sleep 21600 # 6 * 60 * 60 + + echo "Deleting our own Pod, so that it gets re-created and secret-operator issues a new certificate (only crash-looping the container is not enough!)" + kubectl --namespace "$POD_NAMESPACE" delete pod "$POD_NAME" + exit 0 + volumeMounts: + - name: prometheus-tls-certificate + mountPath: /prometheus-tls-certificate + volumes: + - name: prometheus-tls-certificate + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + # Highly professional tests have shown that Prometheus is able to handle the + # certificate rotation :) + # You can change the certificate lifetime here for easier testing: + # secrets.stackable.tech/backend.autotls.cert.lifetime: "1d" + secrets.stackable.tech/class: "tls" + secrets.stackable.tech/format: "tls-pem" + secrets.stackable.tech/scope: "service=prometheus" + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech + volumeMode: Filesystem + securityContext: + fsGroup: 1000 diff --git a/stacks/monitoring/grafana-dashboards.yaml b/stacks/monitoring/grafana-dashboards.yaml index 523c2ed7..21967e5c 100644 --- a/stacks/monitoring/grafana-dashboards.yaml +++ b/stacks/monitoring/grafana-dashboards.yaml @@ -5,6 +5,289 @@ kind: ConfigMap metadata: name: stackable-grafana-dashboards data: + metric-collection-overview.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 35, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 6, + "panels": [], + "title": "Stacklets", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 4, + "maxPerRow": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "repeat": "product", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(count by (app_kubernetes_io_instance) ({app_kubernetes_io_name=\"${product}\"}))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "${product}", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 5, + "panels": [], + "title": "Pods", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 7, + "maxPerRow": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "repeat": "product", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count(count by (pod) ({app_kubernetes_io_name=\"${product}\",pod!=\"\"}))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "${product}", + "type": "stat" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "allowCustomValue": true, + "current": { + "text": "$__all", + "value": "$__all" + }, + "description": "", + "includeAll": true, + "multi": true, + "name": "product", + "options": [ + { + "selected": false, + "text": "airflow", + "value": "airflow" + }, + { + "selected": false, + "text": "druid", + "value": "druid" + }, + { + "selected": false, + "text": "hbase", + "value": "hbase" + }, + { + "selected": false, + "text": "hdfs", + "value": "hdfs" + }, + { + "selected": false, + "text": "hive", + "value": "hive" + }, + { + "selected": false, + "text": "nifi", + "value": "nifi" + }, + { + "selected": false, + "text": "opensearch", + "value": "opensearch" + }, + { + "selected": false, + "text": "spark", + "value": "spark" + }, + { + "selected": false, + "text": "superset", + "value": "superset" + }, + { + "selected": false, + "text": "trino", + "value": "trino" + }, + { + "selected": false, + "text": "zookeeper", + "value": "zookeeper" + }, + { + "selected": false, + "text": "opa", + "value": "opa" + } + ], + "query": "airflow,druid,hbase,hdfs,hive,nifi,opensearch,spark-k8s,superset,trino,zookeeper,opa", + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Metric collection overview", + "uid": "289ad709-e51e-4b91-86a1-6e426300d0eb", + "version": 1 + } trino.json: | { "annotations": { @@ -32,9 +315,8 @@ data: "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 2, + "id": 8, "links": [], - "liveNow": false, "panels": [ { "datasource": { @@ -52,8 +334,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -76,6 +357,7 @@ data: "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -83,9 +365,11 @@ data: "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -118,8 +402,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -142,6 +425,7 @@ data: "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -149,9 +433,11 @@ data: "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -184,8 +470,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -209,6 +494,7 @@ data: "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -216,9 +502,11 @@ data: "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -236,9 +524,9 @@ data: "type": "stat" }, { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "fieldConfig": { + "defaults": {}, + "overrides": [] }, "gridPos": { "h": 3, @@ -256,20 +544,7 @@ data: "content": "

$cluster

", "mode": "html" }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], + "pluginVersion": "12.0.2", "title": "Cluster name", "type": "text" }, @@ -288,8 +563,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -312,6 +586,7 @@ data: "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -319,9 +594,11 @@ data: "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -330,7 +607,7 @@ data: }, "editorMode": "code", "exemplar": false, - "expr": "trino_memory_clustermemorypool_nodes{namespace=~\"$namespace\",job=~\"$cluster\"} - 1 # Subtract 1 for coordinator", + "expr": "trino_memory_clustermemorypool_nodes{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"} - 1 # Subtract 1 for coordinator", "instant": true, "legendFormat": "__auto", "range": false, @@ -351,12 +628,14 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", @@ -365,6 +644,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" @@ -389,8 +669,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -466,10 +745,12 @@ data: "showLegend": false }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -522,12 +803,14 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", @@ -536,6 +819,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -557,8 +841,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -635,10 +918,12 @@ data: "showLegend": false }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -679,12 +964,14 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "opacity", @@ -693,6 +980,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -714,8 +1002,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -790,10 +1077,12 @@ data: "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -834,11 +1123,13 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "opacity", @@ -847,6 +1138,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -868,8 +1160,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -944,10 +1235,12 @@ data: "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -955,7 +1248,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (namespae, job) (trino_execution_querymanager_runningqueries{namespace=~\"$namespace\",job=~\"$cluster\"})", + "expr": "sum by (namespae, job) (trino_execution_querymanager_runningqueries{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", "legendFormat": "running", "range": true, "refId": "A" @@ -966,7 +1259,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (namespae, job) (trino_execution_querymanager_queuedqueries{namespace=~\"$namespace\",job=~\"$cluster\"})", + "expr": "sum by (namespae, job) (trino_execution_querymanager_queuedqueries{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", "hide": false, "legendFormat": "queueud", "range": true, @@ -987,11 +1280,13 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1000,6 +1295,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1021,8 +1317,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1096,10 +1391,12 @@ data: "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -1107,7 +1404,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (namespae, job) (trino_execution_querymanager_completedqueries_totalcount{namespace=~\"$namespace\",job=~\"$cluster\"})", + "expr": "sum by (namespae, job) (trino_execution_querymanager_completedqueries_totalcount{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", "legendFormat": "completed", "range": true, "refId": "A" @@ -1118,7 +1415,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (namespae, job) (trino_execution_querymanager_failedqueries_totalcount{namespace=~\"$namespace\",job=~\"$cluster\"})", + "expr": "sum by (namespae, job) (trino_execution_querymanager_failedqueries_totalcount{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", "hide": false, "legendFormat": "failed", "range": true, @@ -1130,7 +1427,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (namespae, job) (trino_execution_querymanager_abandonedqueries_totalcount{namespace=~\"$namespace\",job=~\"$cluster\"})", + "expr": "sum by (namespae, job) (trino_execution_querymanager_abandonedqueries_totalcount{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", "hide": false, "legendFormat": "abandoned", "range": true, @@ -1142,7 +1439,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (namespae, job) (trino_execution_querymanager_canceledqueries_totalcount{namespace=~\"$namespace\",job=~\"$cluster\"})", + "expr": "sum by (namespae, job) (trino_execution_querymanager_canceledqueries_totalcount{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", "hide": false, "legendFormat": "canceled", "range": true, @@ -1163,12 +1460,14 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", @@ -1177,6 +1476,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1198,8 +1498,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1276,10 +1575,12 @@ data: "showLegend": false }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -1287,7 +1588,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "trino_memory_clustermemorymanager_clustertotalmemoryreservation{namespace=~\"$namespace\",job=~\"$cluster\"}", + "expr": "trino_memory_clustermemorymanager_clustertotalmemoryreservation{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}", "hide": false, "legendFormat": "used", "range": true, @@ -1299,7 +1600,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "trino_memory_clustermemorymanager_clustermemorybytes{namespace=~\"$namespace\",job=~\"$cluster\"}", + "expr": "trino_memory_clustermemorymanager_clustermemorybytes{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}", "hide": false, "legendFormat": "limit", "range": true, @@ -1320,12 +1621,14 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "opacity", @@ -1334,6 +1637,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1355,8 +1659,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1429,10 +1732,12 @@ data: "showLegend": false }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -1440,7 +1745,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "rate(trino_execution_querymanager_consumedinputrows_totalcount{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval])", + "expr": "rate(trino_execution_querymanager_consumedinputrows_totalcount{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}[$__rate_interval])", "hide": false, "legendFormat": "read", "range": true, @@ -1461,12 +1766,14 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "opacity", @@ -1475,6 +1782,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1496,8 +1804,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1570,10 +1877,12 @@ data: "showLegend": false }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -1581,7 +1890,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "rate(trino_execution_querymanager_consumedinputbytes_totalcount{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval])*8", + "expr": "rate(trino_execution_querymanager_consumedinputbytes_totalcount{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}[$__rate_interval])*8", "hide": false, "legendFormat": "read", "range": true, @@ -1592,14 +1901,13 @@ data: "type": "timeseries" } ], - "schemaVersion": 37, - "style": "dark", + "preload": false, + "schemaVersion": 41, "tags": [], "templating": { "list": [ { "current": { - "selected": false, "text": "default", "value": "default" }, @@ -1608,10 +1916,8 @@ data: "uid": "prometheus" }, "definition": "label_values(trino_memory_clustermemorypool_nodes, namespace)", - "hide": 0, "includeAll": false, "label": "Namespace", - "multi": false, "name": "namespace", "options": [], "query": { @@ -1620,13 +1926,11 @@ data: }, "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": { - "selected": false, "text": "trino", "value": "trino" }, @@ -1634,28 +1938,30 @@ data: "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(trino_memory_clustermemorypool_nodes{namespace=~\"$namespace\"}, job)", - "hide": 0, + "definition": "label_values(trino_memory_clustermemorypool_nodes{namespace=~\"$namespace\"},app_kubernetes_io_instance)", "includeAll": false, "label": "Cluster", - "multi": false, "name": "cluster", "options": [], "query": { - "query": "label_values(trino_memory_clustermemorypool_nodes{namespace=~\"$namespace\"}, job)", - "refId": "StandardVariableQuery" + "qryType": 1, + "query": "label_values(trino_memory_clustermemorypool_nodes{namespace=~\"$namespace\"},app_kubernetes_io_instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "regex": "", - "skipUrlSync": false, "sort": 1, "type": "query" }, { + "current": { + "text": "trino", + "value": "trino" + }, "hide": 2, "name": "product", "query": "trino", - "skipUrlSync": false, + "skipUrlSync": true, "type": "constant" } ] @@ -1668,8 +1974,7 @@ data: "timezone": "", "title": "Trino", "uid": "-mVc4yO4k", - "version": 5, - "weekStart": "" + "version": 3 } kafka.json: | { @@ -1698,9 +2003,8 @@ data: "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 3, + "id": 5, "links": [], - "liveNow": false, "panels": [ { "datasource": { @@ -1718,8 +2022,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1742,6 +2045,7 @@ data: "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -1749,9 +2053,11 @@ data: "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -1784,8 +2090,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1808,6 +2113,7 @@ data: "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -1815,9 +2121,11 @@ data: "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -1850,8 +2158,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1875,6 +2182,7 @@ data: "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -1882,9 +2190,11 @@ data: "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -1902,9 +2212,9 @@ data: "type": "stat" }, { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "fieldConfig": { + "defaults": {}, + "overrides": [] }, "gridPos": { "h": 3, @@ -1922,20 +2232,7 @@ data: "content": "

$cluster

", "mode": "html" }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], + "pluginVersion": "12.0.2", "title": "Cluster name", "type": "text" }, @@ -1954,8 +2251,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1978,6 +2274,7 @@ data: "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -1985,9 +2282,11 @@ data: "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -1996,7 +2295,7 @@ data: }, "editorMode": "code", "exemplar": false, - "expr": "max(kafka_controller_kafkacontroller_activebrokercount{namespace=~\"$namespace\",job=~\"$cluster\"})", + "expr": "max(kafka_controller_kafkacontroller_activebrokercount{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", "instant": true, "legendFormat": "__auto", "range": false, @@ -2021,8 +2320,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2045,6 +2343,7 @@ data: "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -2052,9 +2351,11 @@ data: "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -2063,7 +2364,7 @@ data: }, "editorMode": "code", "exemplar": false, - "expr": "count(count by (topic) (kafka_server_brokertopicmetrics_messagesin_total{namespace=~\"$namespace\",job=~\"$cluster\",topic!=\"\"}))", + "expr": "count(count by (topic) (kafka_server_brokertopicmetrics_messagesin_total{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\",topic!=\"\"}))", "instant": true, "legendFormat": "__auto", "range": false, @@ -2084,12 +2385,14 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", @@ -2098,6 +2401,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" @@ -2122,8 +2426,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2199,10 +2502,12 @@ data: "showLegend": false }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -2255,12 +2560,14 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", @@ -2269,6 +2576,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2290,8 +2598,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2368,10 +2675,12 @@ data: "showLegend": false }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -2412,12 +2721,14 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "opacity", @@ -2426,6 +2737,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2447,8 +2759,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2523,10 +2834,12 @@ data: "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -2567,11 +2880,13 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2580,6 +2895,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2601,8 +2917,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2633,10 +2948,12 @@ data: "sortDesc": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -2644,7 +2961,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by(topic)(rate(kafka_server_brokertopicmetrics_messagesin_total{namespace=~\"$namespace\",job=~\"$cluster\",topic!=\"\"}[$__rate_interval]))", + "expr": "sum by(topic)(rate(kafka_server_brokertopicmetrics_messagesin_total{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\",topic!=\"\"}[$__rate_interval]))", "legendFormat": "{{topic}}", "range": true, "refId": "A" @@ -2664,11 +2981,13 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2677,6 +2996,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2698,8 +3018,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2730,10 +3049,12 @@ data: "sortDesc": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -2741,7 +3062,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by(topic)(rate(kafka_server_brokertopicmetrics_bytesin_total{namespace=~\"$namespace\",job=~\"$cluster\",topic!=\"\"}[$__rate_interval]))*8", + "expr": "sum by(topic)(rate(kafka_server_brokertopicmetrics_bytesin_total{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\",topic!=\"\"}[$__rate_interval]))*8", "legendFormat": "{{topic}}", "range": true, "refId": "A" @@ -2761,11 +3082,13 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2774,6 +3097,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2795,8 +3119,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2827,10 +3150,12 @@ data: "sortDesc": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -2838,7 +3163,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by(topic)(kafka_log_log_size{namespace=~\"$namespace\",job=~\"$cluster\",topic!=\"\"})", + "expr": "sum by(topic)(kafka_log_log_size{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\",topic!=\"\"})", "legendFormat": "{{topic}}", "range": true, "refId": "A" @@ -2858,12 +3183,14 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", @@ -2872,6 +3199,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2893,8 +3221,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2971,10 +3298,12 @@ data: "showLegend": false }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -3015,12 +3344,14 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "opacity", @@ -3029,6 +3360,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -3050,8 +3382,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3127,10 +3458,12 @@ data: "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -3138,7 +3471,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(kafka_server_kafkaserver_linux_disk_read_bytes{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval]))*8", + "expr": "sum(rate(kafka_server_kafkaserver_linux_disk_read_bytes{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}[$__rate_interval]))*8", "hide": false, "legendFormat": "rx", "range": true, @@ -3150,7 +3483,7 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "-sum(rate(kafka_server_kafkaserver_linux_disk_write_bytes{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval]))*8", + "expr": "-sum(rate(kafka_server_kafkaserver_linux_disk_write_bytes{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}[$__rate_interval]))*8", "hide": false, "legendFormat": "tx", "range": true, @@ -3161,22 +3494,25 @@ data: "type": "timeseries" } ], + "preload": false, "refresh": false, - "schemaVersion": 37, - "style": "dark", + "schemaVersion": 41, "tags": [], "templating": { "list": [ { + "current": { + "text": "kafka", + "value": "kafka" + }, "hide": 2, "name": "product", "query": "kafka", - "skipUrlSync": false, + "skipUrlSync": true, "type": "constant" }, { "current": { - "selected": false, "text": "default", "value": "default" }, @@ -3185,10 +3521,8 @@ data: "uid": "prometheus" }, "definition": "label_values(kafka_server_kafkaserver_brokerstate, namespace)", - "hide": 0, "includeAll": false, "label": "Namespace", - "multi": false, "name": "namespace", "options": [], "query": { @@ -3197,34 +3531,30 @@ data: }, "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": { - "selected": false, - "text": "kafka", - "value": "kafka" + "text": "simple-kafka", + "value": "simple-kafka" }, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(kafka_server_kafkaserver_brokerstate{namespace=~\"$namespace\"}, job)", - "hide": 0, + "definition": "label_values(kafka_server_kafkaserver_brokerstate{namespace=~\"$namespace\"},app_kubernetes_io_instance)", "includeAll": false, "label": "Cluster", - "multi": false, "name": "cluster", "options": [], "query": { - "query": "label_values(kafka_server_kafkaserver_brokerstate{namespace=~\"$namespace\"}, job)", - "refId": "StandardVariableQuery" + "qryType": 1, + "query": "label_values(kafka_server_kafkaserver_brokerstate{namespace=~\"$namespace\"},app_kubernetes_io_instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "regex": "", - "skipUrlSync": false, "sort": 1, "type": "query" } @@ -3238,8 +3568,7 @@ data: "timezone": "", "title": "Kafka", "uid": "C_8qIX04k", - "version": 4, - "weekStart": "" + "version": 3 } minio.json: | { @@ -3268,21 +3597,17 @@ data: "description": "MinIO Grafana Dashboard - https://min.io/", "editable": true, "fiscalYearStartMonth": 0, - "gnetId": 13502, "graphTooltip": 0, - "id": 29, + "id": 35, "links": [ { "icon": "external link", "includeVars": true, "keepTime": true, - "tags": [ - "minio" - ], + "tags": [], "type": "dashboards" } ], - "liveNow": false, "panels": [ { "datasource": { @@ -3307,8 +3632,7 @@ data: "mode": "percentage", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -3323,13 +3647,13 @@ data: "y": 0 }, "id": 1, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "mean" @@ -3337,10 +3661,12 @@ data: "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -3348,7 +3674,7 @@ data: "uid": "prometheus" }, "exemplar": true, - "expr": "time() - max(minio_node_process_starttime_seconds{job=\"$scrape_jobs\"})", + "expr": "time() - max(minio_node_process_starttime_seconds{job=~\"$scrape_jobs\"})", "format": "time_series", "instant": true, "interval": "", @@ -3385,8 +3711,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -3401,13 +3726,13 @@ data: "y": 0 }, "id": 65, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "last" @@ -3415,19 +3740,20 @@ data: "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", "exemplar": true, - "expr": "sum (minio_s3_traffic_received_bytes{job=\"$scrape_jobs\"})", + "expr": "sum by (instance) (minio_s3_traffic_received_bytes{job=~\"$scrape_jobs\"})", "format": "table", "hide": false, "instant": false, @@ -3439,7 +3765,7 @@ data: "step": 60 } ], - "title": "Total S3 Traffic Inbound", + "title": "Total S3 Ingress", "type": "stat" }, { @@ -3514,7 +3840,6 @@ data: }, "id": 50, "interval": "1m", - "links": [], "maxDataPoints": 100, "options": { "displayLabels": [], @@ -3535,22 +3860,21 @@ data: "values": false }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "8.2.1", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", "exemplar": true, - "expr": "topk(1, max(sum(minio_cluster_capacity_usable_total_bytes{job=\"$scrape_jobs\"}) by (instance))) - topk(1, max(sum(minio_cluster_capacity_usable_free_bytes{job=\"$scrape_jobs\"}) by (instance)))", + "expr": "topk(1, sum(minio_cluster_capacity_usable_total_bytes{job=~\"$scrape_jobs\"}) by (instance)) - topk(1, sum(minio_cluster_capacity_usable_free_bytes{job=~\"$scrape_jobs\"}) by (instance))", "format": "time_series", - "hide": false, "instant": false, "interval": "1m", "intervalFactor": 1, @@ -3563,13 +3887,11 @@ data: "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", "exemplar": true, - "expr": "topk(1, max(sum(minio_cluster_capacity_usable_free_bytes{job=\"$scrape_jobs\"}) by (instance)))", + "expr": "topk(1, sum(minio_cluster_capacity_usable_free_bytes{job=~\"$scrape_jobs\"}) by (instance)) ", "hide": false, "interval": "1m", "legendFormat": "Free", - "range": true, "refId": "B" } ], @@ -3587,11 +3909,13 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -3600,6 +3924,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -3621,8 +3946,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3689,11 +4013,12 @@ data: "showLegend": false }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "8.2.1", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -3702,7 +4027,7 @@ data: }, "editorMode": "code", "exemplar": true, - "expr": "max(sum(minio_bucket_usage_total_bytes{job=\"$scrape_jobs\"}) by (instance))", + "expr": "max(minio_cluster_usage_total_bytes{job=~\"$scrape_jobs\"})", "interval": "", "legendFormat": "Usage", "range": true, @@ -3719,17 +4044,15 @@ data: }, "fieldConfig": { "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "semi-dark-red", - "value": 80 + "color": "green" } ] } @@ -3743,43 +4066,54 @@ data: "y": 0 }, "id": 52, - "links": [], "options": { - "displayMode": "basic", - "minVizHeight": 10, - "minVizWidth": 0, + "displayMode": "lcd", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ - "mean" + "lastNotNull" ], "fields": "", "values": false }, - "showUnfilled": false, - "text": {} + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, + "disableTextWrap": false, "editorMode": "code", - "exemplar": false, - "expr": "max by (range) (minio_bucket_objects_size_distribution{job=\"$scrape_jobs\"})", + "exemplar": true, + "expr": "minio_cluster_objects_size_distribution{job=~\"$scrape_jobs\"}", "format": "time_series", - "instant": true, + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{range}}", - "range": false, "refId": "A", - "step": 300 + "step": 300, + "useBackend": false } ], - "title": "Object size distribution", + "title": "Object Size Distribution", "type": "bargauge" }, { @@ -3790,31 +4124,55 @@ data: "description": "", "fieldConfig": { "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" } - ], + }, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", - "value": 2000 + "value": 80 } ] - }, - "unit": "short" + } }, "overrides": [] }, @@ -3825,45 +4183,43 @@ data: "y": 0 }, "id": 61, - "links": [], "maxDataPoints": 100, "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false }, - "text": {}, - "textMode": "auto" + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, + "editorMode": "code", "exemplar": true, - "expr": "sum (minio_node_file_descriptor_open_total{job=\"$scrape_jobs\"})", - "format": "table", + "expr": "minio_node_file_descriptor_open_total{job=~\"$scrape_jobs\"}", + "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "", + "legendFormat": "{{server}}", "metric": "process_start_time_seconds", "refId": "A", "step": 60 } ], - "title": "Total Open FDs", - "type": "stat" + "title": "Open FDs ", + "type": "timeseries" }, { "datasource": { @@ -3888,8 +4244,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -3904,13 +4259,13 @@ data: "y": 3 }, "id": 64, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "last" @@ -3918,19 +4273,20 @@ data: "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", "exemplar": true, - "expr": "sum (minio_s3_traffic_sent_bytes{job=\"$scrape_jobs\"})", + "expr": "sum by (instance) (minio_s3_traffic_sent_bytes{job=~\"$scrape_jobs\"})", "format": "table", "hide": false, "instant": false, @@ -3942,7 +4298,7 @@ data: "step": 60 } ], - "title": "Total S3 Traffic Outbound", + "title": "Total S3 Egress", "type": "stat" }, { @@ -3953,31 +4309,55 @@ data: "description": "", "fieldConfig": { "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" } - ], + }, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", - "value": 2000 + "value": 80 } ] - }, - "unit": "short" + } }, "overrides": [] }, @@ -3988,24 +4368,21 @@ data: "y": 3 }, "id": 62, - "links": [], "maxDataPoints": 100, "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false }, - "text": {}, - "textMode": "auto" + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -4014,54 +4391,38 @@ data: }, "editorMode": "code", "exemplar": true, - "expr": "sum without (server,instance,pod) (minio_node_go_routine_total{job=\"$scrape_jobs\"})", - "format": "table", + "expr": "minio_node_go_routine_total{job=~\"$scrape_jobs\"}", + "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "", + "legendFormat": "{{server}}", "metric": "process_start_time_seconds", "refId": "A", "step": 60 } ], - "title": "Total Goroutines", - "type": "stat" + "title": "Goroutines", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "", "fieldConfig": { "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "color": "green" } ] }, - "unit": "short" + "unit": "bool_on_off" }, "overrides": [] }, @@ -4071,45 +4432,45 @@ data: "x": 0, "y": 6 }, - "id": 53, - "links": [], - "maxDataPoints": 100, + "id": 94, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ - "mean" + "lastNotNull" ], "fields": "", "values": false }, - "text": {}, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, + "disableTextWrap": false, + "editorMode": "code", "exemplar": true, - "expr": "minio_cluster_nodes_online_total{job=\"$scrape_jobs\"}", - "format": "table", - "hide": false, - "instant": true, + "expr": "minio_cluster_health_status{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "process_start_time_seconds", + "legendFormat": "Pool: {{pool}} Set: {{set}}", + "range": true, "refId": "A", - "step": 60 + "useBackend": false } ], - "title": "Total Online Servers", + "title": "Cluster Health Status", "type": "stat" }, { @@ -4120,59 +4481,42 @@ data: "description": "", "fieldConfig": { "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], + "mappings": [], "thresholds": { - "mode": "absolute", + "mode": "percentage", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "color": "green" } ] - }, - "unit": "short" + } }, "overrides": [] }, "gridPos": { - "h": 2, + "h": 4, "w": 3, "x": 3, "y": 6 }, - "id": 9, - "links": [], + "id": 78, "maxDataPoints": 100, "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", + "minVizHeight": 75, + "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ - "mean" + "lastNotNull" ], "fields": "", "values": false }, - "text": {}, - "textMode": "auto" + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -4181,21 +4525,36 @@ data: }, "editorMode": "code", "exemplar": false, - "expr": "min(minio_cluster_disk_online_total{job=\"$scrape_jobs\"})", - "format": "table", + "expr": "max(minio_cluster_drive_online_total{job=~\"$scrape_jobs\"})", + "format": "time_series", "hide": false, "instant": true, "interval": "", "intervalFactor": 1, - "legendFormat": "Total online disks in MinIO Cluster", + "legendFormat": ".", "metric": "process_start_time_seconds", "range": false, "refId": "A", "step": 60 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(minio_cluster_drive_offline_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "hide": false, + "instant": true, + "legendFormat": ".", + "range": false, + "refId": "B" } ], - "title": "Total Online Disks", - "type": "stat" + "title": "Total Online/Offline Drives", + "type": "gauge" }, { "datasource": { @@ -4219,8 +4578,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "dark-yellow", @@ -4243,13 +4601,13 @@ data: "y": 6 }, "id": 66, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -4257,18 +4615,21 @@ data: "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, + "editorMode": "code", "exemplar": true, - "expr": "count(count by (bucket) (minio_bucket_usage_total_bytes{job=\"$scrape_jobs\"}))", + "expr": "max(minio_cluster_bucket_total{job=~\"$scrape_jobs\"})", "format": "time_series", "instant": false, "interval": "1m", @@ -4281,230 +4642,152 @@ data: "type": "stat" }, { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { - "unit": "bits" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 7, - "x": 9, - "y": 6 - }, - "hiddenSeries": false, - "id": 63, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "color": { + "mode": "palette-classic" }, - "editorMode": "code", - "exemplar": true, - "expr": "sum by (server) (rate(minio_s3_traffic_received_bytes{job=\"$scrape_jobs\"}[$__rate_interval]))*8", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "Data Received [{{server}}]", - "range": true, - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "S3 API Data Received Rate ", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "bits", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "unit": "bits" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { "h": 6, - "w": 8, - "x": 16, + "w": 7, + "x": 9, "y": 6 }, - "hiddenSeries": false, - "id": 70, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "id": 63, "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", "exemplar": true, - "expr": "sum by (server) (rate(minio_s3_traffic_sent_bytes{job=\"$scrape_jobs\"}[$__rate_interval]))*8", + "expr": "sum by (server) (rate(minio_s3_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", "interval": "1m", "intervalFactor": 2, - "legendFormat": "Data Sent [{{server}}]", - "range": true, + "legendFormat": "Data Received [{{server}}]", "refId": "A" } ], - "thresholds": [], - "timeRegions": [], - "title": "S3 API Data Sent Rate ", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "bits", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } + "title": "S3 API Ingress Rate ", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "", "fieldConfig": { "defaults": { - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" } - ], + }, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4512,35 +4795,31 @@ data: } ] }, - "unit": "short" + "unit": "binBps" }, "overrides": [] }, "gridPos": { - "h": 2, - "w": 3, - "x": 0, - "y": 8 + "h": 6, + "w": 8, + "x": 16, + "y": 6 }, - "id": 69, - "links": [], - "maxDataPoints": 100, + "id": 70, "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false }, - "text": {}, - "textMode": "auto" + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -4548,20 +4827,15 @@ data: "uid": "prometheus" }, "exemplar": true, - "expr": "minio_cluster_nodes_offline_total{job=\"$scrape_jobs\"}", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 + "expr": "sum by (server) (rate(minio_s3_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Data Sent [{{server}}]", + "refId": "A" } ], - "title": "Total Offline Servers", - "type": "stat" + "title": "S3 API Egress Rate ", + "type": "timeseries" }, { "datasource": { @@ -4586,8 +4860,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4602,17 +4875,17 @@ data: "gridPos": { "h": 2, "w": 3, - "x": 3, + "x": 0, "y": 8 }, - "id": 78, - "links": [], + "id": 53, "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "mean" @@ -4620,19 +4893,20 @@ data: "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", "exemplar": true, - "expr": "max(minio_cluster_disk_offline_total{job=\"$scrape_jobs\"})", + "expr": "max(minio_cluster_nodes_online_total{job=~\"$scrape_jobs\"})", "format": "table", "hide": false, "instant": true, @@ -4644,7 +4918,7 @@ data: "step": 60 } ], - "title": "Total Offline Disks", + "title": "Total Online Servers", "type": "stat" }, { @@ -4669,8 +4943,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "dark-yellow", @@ -4693,13 +4966,13 @@ data: "y": 9 }, "id": 44, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -4707,10 +4980,12 @@ data: "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -4719,7 +4994,7 @@ data: }, "editorMode": "code", "exemplar": true, - "expr": "topk(1, max(sum(minio_bucket_usage_object_total{job=\"$scrape_jobs\"}) by (instance)))", + "expr": "max(minio_cluster_usage_object_total{job=~\"$scrape_jobs\"})", "format": "time_series", "instant": false, "interval": "1m", @@ -4747,8 +5022,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -4763,13 +5037,13 @@ data: "y": 10 }, "id": 80, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "last" @@ -4777,30 +5051,31 @@ data: "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", "exemplar": true, - "expr": "min(minio_heal_time_last_activity_nano_seconds{job=\"$scrape_jobs\"})", + "expr": "max(minio_heal_time_last_activity_nano_seconds{job=~\"$scrape_jobs\"})", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, - "legendFormat": "{{server}}", + "legendFormat": "", "metric": "process_start_time_seconds", "refId": "A", "step": 60 } ], - "title": "Time Since Last Heal Activity", + "title": "Time Since Last Heal", "type": "stat" }, { @@ -4819,8 +5094,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -4835,13 +5109,13 @@ data: "y": 10 }, "id": 81, - "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "last" @@ -4849,30 +5123,31 @@ data: "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", "exemplar": true, - "expr": "min(minio_usage_last_activity_nano_seconds{job=\"$scrape_jobs\"})", + "expr": "max(minio_usage_last_activity_nano_seconds{job=~\"$scrape_jobs\"})", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, - "legendFormat": "{{server}}", + "legendFormat": "", "metric": "process_start_time_seconds", "refId": "A", "step": 60 } ], - "title": "Time Since Last Scan Activity", + "title": "Time Since Last Scan", "type": "stat" }, { @@ -4880,34 +5155,36 @@ data: "type": "prometheus", "uid": "prometheus" }, - "description": "Total s3 bytes sent per bucket", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 5, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "smooth", + "insertNulls": false, + "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 2, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": true, + "showPoints": "never", + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -4921,8 +5198,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4930,48 +5206,78 @@ data: } ] }, - "unit": "binbps" + "unit": "none" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { - "h": 10, - "w": 12, + "h": 6, + "w": 9, "x": 0, "y": 12 }, - "id": 90, + "id": 60, "options": { + "alertThreshold": true, "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", - "expr": "sum by(bucket) (rate(minio_bucket_traffic_sent_bytes{job=\"$scrape_jobs\"}[$__rate_interval]))*8", - "legendFormat": "__auto", - "range": true, + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", "refId": "A" } ], - "title": "Bucket Traffic Sent", + "title": "S3 API Request Rate", "type": "timeseries" }, { @@ -4979,34 +5285,36 @@ data: "type": "prometheus", "uid": "prometheus" }, - "description": "Total s3 bytes received per bucket", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 5, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "smooth", + "insertNulls": false, + "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 2, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "always", - "spanNulls": true, + "showPoints": "never", + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -5020,8 +5328,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5029,46 +5336,78 @@ data: } ] }, - "unit": "binbps" + "unit": "none" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { - "h": 10, - "w": 12, - "x": 12, + "h": 6, + "w": 7, + "x": 9, "y": 12 }, - "id": 92, + "id": 88, "options": { + "alertThreshold": true, "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", + "calcs": [], + "displayMode": "list", + "placement": "bottom", "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", - "expr": "sum by(bucket) (rate(minio_bucket_traffic_received_bytes{job=\"$scrape_jobs\"}[$__rate_interval]))*8", - "legendFormat": "__auto", - "range": true, + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_4xx_errors_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", "refId": "A" } ], - "title": "Bucket Traffic Received", + "title": "S3 API Request Error Rate (4xx)", "type": "timeseries" }, { @@ -5076,34 +5415,36 @@ data: "type": "prometheus", "uid": "prometheus" }, - "description": "Total s3 bytes sent per bucket", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 5, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "smooth", + "insertNulls": false, + "lineInterpolation": "linear", "lineWidth": 1, - "pointSize": 2, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": true, + "showPoints": "never", + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -5117,8 +5458,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5128,47 +5468,76 @@ data: }, "unit": "none" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 22 + "h": 6, + "w": 8, + "x": 16, + "y": 12 }, - "id": 93, + "id": 86, "options": { + "alertThreshold": true, "legend": { - "calcs": [ - "max", - "last" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", - "expr": "max by (bucket) (minio_bucket_usage_object_total{job=\"$scrape_jobs\"})", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "B" + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_5xx_errors_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", + "refId": "A" } ], - "title": "Bucket objects", + "title": "S3 API Request Error Rate (5xx)", "type": "timeseries" }, { @@ -5176,331 +5545,306 @@ data: "type": "prometheus", "uid": "prometheus" }, - "description": "Total s3 bytes sent per bucket", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 5, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 2, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "fixed" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "color": "green" } ] - }, - "unit": "bytes" + } }, "overrides": [] }, "gridPos": { - "h": 10, + "h": 8, "w": 12, - "x": 12, - "y": 22 + "x": 0, + "y": 18 }, - "id": 94, + "id": 99, "options": { + "displayMode": "lcd", "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { "calcs": [ - "max", - "last" + "lastNotNull" ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true + "fields": "", + "values": false }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", - "expr": "max by (bucket) (minio_bucket_usage_total_bytes{job=\"$scrape_jobs\"})", + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_online_drives{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, "hide": false, - "legendFormat": "__auto", + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Online Drives", "range": true, - "refId": "B" - } - ], - "title": "Bucket size", - "type": "timeseries" - }, - { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 32 - }, - "hiddenSeries": false, - "id": 60, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "refId": "A", + "useBackend": false + }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "exemplar": true, - "expr": "sum by (server,api) (increase(minio_s3_requests_total{job=\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{server,api}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "S3 API Request Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_read_quorum{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Read Quorum", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_write_quorum{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Write Quorum", + "range": true, + "refId": "C", + "useBackend": false + }, { - "$$hashKey": "object:331", - "format": "none", - "logBase": 1, - "show": true + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_healing_drives{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Healing Drives", + "range": true, + "refId": "D", + "useBackend": false }, { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_status{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Status", + "range": true, + "refId": "E", + "useBackend": false } ], - "yaxis": { - "align": false - } + "title": "Health Breakdown", + "type": "bargauge" }, { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, "w": 12, "x": 12, - "y": 32 - }, - "hiddenSeries": false, - "id": 71, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "y": 18 + }, + "id": 76, "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "displayMode": "lcd", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "exemplar": true, - "expr": "sum by (server,api) (increase(minio_s3_requests_errors_total{job=\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{server,api}}", + "editorMode": "code", + "exemplar": false, + "expr": "minio_node_process_resident_memory_bytes{job=~\"$scrape_jobs\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{server}}", + "range": true, "refId": "A" } ], - "thresholds": [], - "timeRegions": [], - "title": "S3 API Request Error Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } + "title": "Memory Usage ", + "type": "bargauge" }, { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fill": 1, - "fillGradient": 0, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, "gridPos": { - "h": 10, + "h": 7, "w": 12, "x": 0, - "y": 42 + "y": 26 }, - "hiddenSeries": false, - "id": 86, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "id": 73, "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -5508,194 +5852,179 @@ data: "uid": "prometheus" }, "exemplar": true, - "expr": "sum by (server,api) (increase(minio_s3_requests_5xx_errors_total{job=\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{server,api}}", + "expr": "rate(minio_node_io_rchar_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Node RChar [{{server}}]", "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "S3 API Request Error Rate (5xx)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "none", - "logBase": 1, - "show": true }, { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "rate(minio_node_io_wchar_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Node WChar [{{server}}]", + "refId": "B" } ], - "yaxis": { - "align": false - } + "title": "Read, Write I/O", + "type": "timeseries" }, { - "aliasColors": { - "S3 Errors": "light-red", - "S3 Requests": "light-green" - }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fill": 1, - "fillGradient": 0, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, "gridPos": { - "h": 10, + "h": 7, "w": 12, "x": 12, - "y": 42 + "y": 26 }, - "hiddenSeries": false, - "id": 88, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "id": 77, "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, + "editorMode": "code", "exemplar": true, - "expr": "sum by (server,api) (increase(minio_s3_requests_4xx_errors_total{job=\"$scrape_jobs\"}[$__rate_interval]))", - "interval": "1m", - "intervalFactor": 2, - "legendFormat": "{{server,api}}", + "expr": "rate(minio_node_process_cpu_total_seconds{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{server}}", + "range": true, "refId": "A" } ], - "thresholds": [], - "timeRegions": [], - "title": "S3 API Request Error Rate (4xx)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:331", - "format": "none", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:332", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } + "title": "CPU Usage", + "type": "gauge" }, { - "aliasColors": { - "10.13.1.25:9000 DELETE": "red", - "10.13.1.25:9000 GET": "green", - "10.13.1.25:9000 POST": "blue" - }, - "bars": true, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "Total number of bytes received and sent among all MinIO server instances", + "description": "Total number of bytes received and sent on MinIO cluster", "fieldConfig": { "defaults": { - "links": [], - "unit": "binbps" + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" }, "overrides": [] }, - "fill": 10, - "fillGradient": 1, "gridPos": { - "h": 9, + "h": 7, "w": 12, "x": 0, - "y": 52 + "y": 33 }, - "hiddenSeries": false, "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.4", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -5704,7 +6033,7 @@ data: }, "editorMode": "code", "exemplar": true, - "expr": "rate(minio_inter_node_traffic_sent_bytes{job=\"$scrape_jobs\"}[$__rate_interval])*8", + "expr": "rate(minio_inter_node_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -5719,90 +6048,132 @@ data: "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", "exemplar": true, - "expr": "rate(minio_inter_node_traffic_received_bytes{job=\"$scrape_jobs\"}[$__rate_interval])*8", + "expr": "rate(minio_inter_node_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", "interval": "", "legendFormat": "Internode Bytes Sent [{{server}}]", - "range": true, "refId": "B" } ], - "thresholds": [], - "timeRegions": [], - "title": "Internode Data Transfer", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:211", - "format": "binbps", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:212", - "format": "s", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "title": "Internode Traffic", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fill": 1, - "fillGradient": 0, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "available 10.13.1.25:9000" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used 10.13.1.25:9000" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, "gridPos": { - "h": 9, + "h": 7, "w": 12, "x": 12, - "y": 52 - }, - "hiddenSeries": false, - "id": 84, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "y": 33 + }, + "id": 8, "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -5810,22 +6181,147 @@ data: "uid": "prometheus" }, "exemplar": true, - "expr": "sum by (instance) (minio_heal_objects_heal_total{job=\"$scrape_jobs\"})", + "expr": "minio_node_file_descriptor_open_total{job=~\"$scrape_jobs\"}", "interval": "", - "legendFormat": "Objects healed in current self heal run", - "refId": "A" + "legendFormat": "Open FDs [{{server}}]", + "refId": "B" + } + ], + "title": "File Descriptors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of online drives per MinIO Server", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Offline 10.13.1.25:9000" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total 10.13.1.25:9000" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 11, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "exemplar": true, - "expr": "sum by (instance) (minio_heal_objects_error_total{job=\"$scrape_jobs\"})", - "hide": false, + "expr": "rate(minio_node_syscall_read_total{job=~\"$scrape_jobs\"}[$__rate_interval])", + "format": "time_series", "interval": "", - "legendFormat": "Heal errors in current self heal run", - "refId": "B" + "intervalFactor": 2, + "legendFormat": "Read Syscalls [{{server}}]", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 }, { "datasource": { @@ -5833,669 +6329,3127 @@ data: "uid": "prometheus" }, "exemplar": true, - "expr": "sum by (instance) (minio_heal_objects_total{job=\"$scrape_jobs\"}) ", - "hide": false, + "expr": "rate(minio_node_syscall_write_total{job=~\"$scrape_jobs\"}[$__rate_interval])", "interval": "", - "legendFormat": "Objects scanned in current self heal run", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Healing", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:846", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:847", - "format": "short", - "logBase": 1, - "show": true + "legendFormat": "Write Syscalls [{{server}}]", + "refId": "B" } ], - "yaxis": { - "align": false - } + "title": "Syscalls", + "type": "timeseries" }, { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 61 - }, - "hiddenSeries": false, - "id": 77, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 95, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "rate(minio_node_scanner_objects_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "title": "Scanned Objects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 75, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "rate(minio_node_scanner_versions_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "title": "Scanned Versions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 96, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "rate(minio_node_scanner_directories_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "title": "Scanned Directories", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 54 + }, + "id": 89, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "minio_cluster_kms_uptime{job=~\"$scrape_jobs\"}", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "minio_cluster_kms_uptime", + "refId": "A", + "step": 60 + } + ], + "title": "KMS Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 54 + }, + "id": 91, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum by (server) (increase(minio_cluster_kms_request_error{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "KMS Request 4xx Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bool_on_off" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 54 + }, + "id": 90, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum by (server) (minio_cluster_kms_online{job=~\"$scrape_jobs\"})", + "interval": "1m", + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "KMS Online(1)/Offline(0)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 54 + }, + "id": 98, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "rate(minio_node_scanner_bucket_scans_finished{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "title": "Bucket Scans Finished", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 58 + }, + "id": 92, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum by (server) (increase(minio_cluster_kms_request_failure{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "KMS Request 5xx Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 58 + }, + "id": 93, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "sum by (server) (rate(minio_cluster_kms_request_success{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "KMS Request Success [{{server}}]", + "refId": "A" + } + ], + "title": "KMS Request Success Rate ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 61 + }, + "id": 97, + "options": { + "alertThreshold": true, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "exemplar": true, + "expr": "rate(minio_node_scanner_bucket_scans_started{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "title": "Bucket Scans Started", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(job)", + "includeAll": true, + "multi": true, + "name": "scrape_jobs", + "options": [], + "query": { + "query": "label_values(job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "MinIO", + "uid": "TgmJnqnnk", + "version": 1 + } + hdfs.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_requests{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU request", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 0 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_limits{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "CPU limit", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kube_pod_container_resource_requests{resource=\"memory\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory limit", + "type": "stat" + }, + { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 9, + "y": 0 + }, + "id": 2, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "

$cluster

", + "mode": "html" + }, + "pluginVersion": "12.0.2", + "title": "Cluster name", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 14, + "y": 0 + }, + "id": 25, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(journal_node_bytes_written{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Journalnodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 17, + "y": 0 + }, + "id": 26, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(name_node_name_node_state{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Namenodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 20, + "y": 0 + }, + "id": 27, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count(datanode_data_node_packet_responder_count{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Datanodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 25 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 3 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(rate(container_cpu_usage_seconds_total{container!=\"\"}[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(kube_pod_container_resource_limits{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(kube_pod_container_resource_requests{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "request", + "range": true, + "refId": "C" + } + ], + "title": "CPU usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 25 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 3 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(container_memory_working_set_bytes{container!=\"\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(kube_pod_container_resource_requests{resource=\"memory\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))", + "hide": false, + "legendFormat": "limit", + "range": true, + "refId": "C" + } + ], + "title": "Memory usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 3 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, pod)(rate(container_network_receive_bytes_total[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))*8", + "hide": false, + "legendFormat": "rx", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "-sum(max by (namespace, pod)(rate(container_network_transmit_bytes_total[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))*8", + "hide": false, + "legendFormat": "tx", + "range": true, + "refId": "B" + } + ], + "title": "Network usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "dark-green", + "mode": "fixed" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "remaining" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 0, + "y": 10 + }, + "id": 29, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "values": [ + "percent", + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min(fs_namesystem_capacity_remaining{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "legendFormat": "remaining", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(fs_namesystem_capacity_used{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "hide": false, + "legendFormat": "used", + "range": true, + "refId": "B" + } + ], + "title": "DFS usage", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 4, + "y": 10 + }, + "id": 28, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min(fs_namesystem_capacity_total{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "DFS capacity", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 7, + "x": 7, + "y": 10 + }, + "id": 37, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by(pod)(rate(datanode_bytes_read{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}[$__rate_interval])))*8", + "hide": false, + "legendFormat": "rx", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "-sum(max by(pod)(rate(datanode_bytes_written{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}[$__rate_interval])))*8", + "hide": false, + "legendFormat": "tx", + "range": true, + "refId": "B" + } + ], + "title": "HDFS cluster I/O", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 10, + "x": 14, + "y": 10 + }, + "id": 36, + "options": { + "legend": { + "calcs": [ + "max", + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(fs_namesystem_files_total{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "legendFormat": "total", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(fs_namesystem_num_files_under_construction{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "hide": false, + "legendFormat": "under construction", + "range": true, + "refId": "B" + } + ], + "title": "Files", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 4, + "y": 13 + }, + "id": 30, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min(fs_namesystem_capacity_used{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "DFS used", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 4, + "y": 16 + }, + "id": 31, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min(fs_namesystem_capacity_remaining{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "DFS remaining", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "corrupt" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "mssing" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "underreplicated" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "excess" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 10, + "x": 14, + "y": 17 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + "max", + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "exemplar": true, - "expr": "rate(minio_node_process_cpu_total_seconds{job=\"$scrape_jobs\"}[$__rate_interval])", - "interval": "", - "legendFormat": "CPU Usage Rate [{{server}}]", + "editorMode": "code", + "expr": "max(fs_namesystem_blocks_total{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "legendFormat": "total", + "range": true, "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Node CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1043", - "format": "none", - "logBase": 1, - "show": true }, - { - "$$hashKey": "object:1044", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 61 - }, - "hiddenSeries": false, - "id": 76, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "exemplar": true, - "expr": "minio_node_process_resident_memory_bytes{job=\"$scrape_jobs\"}", - "interval": "", - "legendFormat": "Memory Used [{{server}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Node Memory Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:1043", - "format": "bytes", - "logBase": 1, - "show": true + "editorMode": "code", + "expr": "max(fs_namesystem_missing_blocks{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "hide": false, + "legendFormat": "mssing", + "range": true, + "refId": "B" }, - { - "$$hashKey": "object:1044", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 70 - }, - "hiddenSeries": false, - "id": 74, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "exemplar": true, - "expr": "minio_node_disk_used_bytes{job=\"$scrape_jobs\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Used Capacity [{{server}}:{{disk}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Drive Used Capacity", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:381", - "format": "bytes", - "logBase": 1, - "show": true + "editorMode": "code", + "expr": "max(fs_namesystem_corrupt_blocks{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "hide": false, + "legendFormat": "corrupt", + "range": true, + "refId": "C" }, - { - "$$hashKey": "object:382", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 70 - }, - "hiddenSeries": false, - "id": 82, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "exemplar": true, - "expr": "minio_cluster_disk_free_inodes{job=\"$scrape_jobs\"}", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Free Inodes [{{server}}:{{disk}}]", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Drives Free Inodes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:381", - "format": "none", - "logBase": 1, - "show": true + "editorMode": "code", + "expr": "max(fs_namesystem_under_replicated_blocks{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "hide": false, + "legendFormat": "underreplicated", + "range": true, + "refId": "D" }, { - "$$hashKey": "object:382", - "format": "short", - "logBase": 1, - "show": true + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(fs_namesystem_excess_blocks{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "hide": false, + "legendFormat": "excess", + "range": true, + "refId": "E" } ], - "yaxis": { - "align": false - } + "title": "Blocks", + "type": "timeseries" }, { - "aliasColors": { - "Offline 10.13.1.25:9000": "dark-red", - "Total 10.13.1.25:9000": "blue" - }, - "bars": true, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "Number of online disks per MinIO Server", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 9, - "w": 12, + "h": 5, + "w": 7, "x": 0, - "y": 78 + "y": 19 }, - "hiddenSeries": false, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null", + "id": 33, "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "exemplar": true, - "expr": "rate(minio_node_syscall_read_total{job=\"$scrape_jobs\"}[$__rate_interval])", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "Read Syscalls [{{server}}]", - "metric": "process_start_time_seconds", - "refId": "A", - "step": 60 + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "exemplar": true, - "expr": "rate(minio_node_syscall_write_total{job=\"$scrape_jobs\"}[$__rate_interval])", - "interval": "", - "legendFormat": "Write Syscalls [{{server}}]", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Node Syscalls", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:185", - "decimals": 0, - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:186", - "format": "short", - "logBase": 1, - "show": true + "editorMode": "code", + "expr": "max(fs_namesystem_total_load{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" } ], - "yaxis": { - "align": false - } + "title": "Client connections", + "type": "timeseries" }, { - "aliasColors": { - "available 10.13.1.25:9000": "green", - "used 10.13.1.25:9000": "blue" - }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 25 + } + ] + } + ] }, - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 78 + "h": 5, + "w": 7, + "x": 7, + "y": 19 }, - "hiddenSeries": false, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", + "id": 23, "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "exemplar": true, - "expr": "minio_node_file_descriptor_open_total{job=\"$scrape_jobs\"}", - "interval": "", - "legendFormat": "Open FDs [{{server}}]", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Node File Descriptors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:212", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:213", - "format": "none", - "logBase": 1, - "min": "0", - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "unit": "binbps" + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 87 + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } }, - "hiddenSeries": false, - "id": 73, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.2.4", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -6503,12 +9457,10 @@ data: "uid": "prometheus" }, "editorMode": "code", - "exemplar": true, - "expr": "rate(minio_node_io_rchar_bytes{job=\"$scrape_jobs\"}[$__rate_interval])*8", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "Node RChar [{{server}}]", + "expr": "sum(max by (namespace, persistentvolumeclaim)(kubelet_volume_stats_used_bytes) * on (namespace, persistentvolumeclaim) group_left() (max by (namespace, persistentvolumeclaim) (kube_persistentvolumeclaim_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "hide": false, + "legendFormat": "used", + "range": true, "refId": "A" }, { @@ -6517,80 +9469,80 @@ data: "uid": "prometheus" }, "editorMode": "code", - "exemplar": true, - "expr": "rate(minio_node_io_wchar_bytes{job=\"$scrape_jobs\"}[$__rate_interval])*8", - "interval": "", - "legendFormat": "Node WChar [{{server}}]", + "expr": "sum(max by (namespace, persistentvolumeclaim)(kubelet_volume_stats_capacity_bytes) * on (namespace, persistentvolumeclaim) group_left() (max by (namespace, persistentvolumeclaim) (kube_persistentvolumeclaim_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "hide": false, + "legendFormat": "limit", "range": true, - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Node IO", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:381", - "format": "binbps", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:382", - "format": "short", - "logBase": 1, - "show": true + "refId": "C" } ], - "yaxis": { - "align": false - } + "title": "PVC usage", + "type": "timeseries" } ], + "preload": false, "refresh": false, - "schemaVersion": 37, - "style": "dark", - "tags": [ - "minio" - ], + "schemaVersion": 41, + "tags": [], "templating": { "list": [ { "current": { - "selected": false, - "text": "minio", - "value": "minio" + "text": "hdfs", + "value": "hdfs" + }, + "hide": 2, + "name": "product", + "query": "hdfs", + "skipUrlSync": true, + "type": "constant" + }, + { + "current": { + "text": "default", + "value": "default" }, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(minio_cluster_disk_total,job)", - "hide": 0, + "definition": "label_values(fs_namesystem_blocks_total,namespace)", "includeAll": false, - "label": "Instance", - "multi": false, - "name": "scrape_jobs", + "label": "Namespace", + "name": "namespace", "options": [], "query": { - "query": "label_values(minio_cluster_disk_total,job)", - "refId": "StandardVariableQuery" + "qryType": 1, + "query": "label_values(fs_namesystem_blocks_total,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 0, + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "simple-hdfs", + "value": "simple-hdfs" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(fs_namesystem_blocks_total{namespace=~\"$namespace\"},app_kubernetes_io_instance)", + "includeAll": false, + "label": "Cluster", + "name": "cluster", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(fs_namesystem_blocks_total{namespace=~\"$namespace\"},app_kubernetes_io_instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "sort": 1, "type": "query" } ] @@ -6599,37 +9551,13 @@ data: "from": "now-3h", "to": "now" }, - "timepicker": { - "refresh_intervals": [ - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, + "timepicker": {}, "timezone": "", - "title": "MinIO", - "uid": "TgmJnqnnk", - "version": 2, - "weekStart": "" + "title": "HDFS", + "uid": "qjZGZ3AVz", + "version": 4 } - hdfs.json: | + hbase.json: | { "annotations": { "list": [ @@ -6658,7 +9586,6 @@ data: "graphTooltip": 0, "id": 3, "links": [], - "liveNow": false, "panels": [ { "datasource": { @@ -6668,82 +9595,14 @@ data: "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 0, - "y": 0 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(kube_pod_container_resource_requests{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "CPU request", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "fixed" + "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6757,15 +9616,16 @@ data: "gridPos": { "h": 3, "w": 3, - "x": 3, + "x": 0, "y": 0 }, - "id": 7, + "id": 17, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -6773,9 +9633,11 @@ data: "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -6783,13 +9645,15 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(kube_pod_container_resource_limits{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "exemplar": false, + "expr": "count(procedure_num_master_wa_ls{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "instant": true, "legendFormat": "__auto", - "range": true, + "range": false, "refId": "A" } ], - "title": "CPU limit", + "title": "Masters", "type": "stat" }, { @@ -6800,39 +9664,37 @@ data: "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", - "mode": "fixed" + "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - }, - "unit": "bytes" + } }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, - "x": 6, + "x": 3, "y": 0 }, - "id": 8, + "id": 18, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -6840,9 +9702,11 @@ data: "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -6850,53 +9714,17 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(kube_pod_container_resource_requests{resource=\"memory\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "exemplar": false, + "expr": "max(server_num_region_servers{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "instant": true, "legendFormat": "__auto", - "range": true, + "range": false, "refId": "A" } ], - "title": "Memory limit", + "title": "Region servers", "type": "stat" }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "gridPos": { - "h": 3, - "w": 5, - "x": 9, - "y": 0 - }, - "id": 2, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "

$cluster

", - "mode": "html" - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Cluster name", - "type": "text" - }, { "datasource": { "type": "prometheus", @@ -6912,8 +9740,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6927,15 +9754,16 @@ data: "gridPos": { "h": 3, "w": 3, - "x": 14, + "x": 6, "y": 0 }, - "id": 25, + "id": 26, "options": { "colorMode": "value", - "graphMode": "none", + "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -6943,9 +9771,11 @@ data: "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -6954,16 +9784,41 @@ data: }, "editorMode": "code", "exemplar": false, - "expr": "count(hadoop_journalnode_version{namespace=~\"$namespace\",job=~\"$cluster\"})", - "instant": true, + "expr": "sum(max by (namespace, pod)(server_region_count{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}))", + "instant": false, "legendFormat": "__auto", - "range": false, + "range": true, "refId": "A" } ], - "title": "Journalnodes", + "title": "Regions", "type": "stat" }, + { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 9, + "y": 0 + }, + "id": 2, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "

$cluster

", + "mode": "html" + }, + "pluginVersion": "12.0.2", + "title": "Cluster name", + "type": "text" + }, { "datasource": { "type": "prometheus", @@ -6979,8 +9834,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6994,15 +9848,16 @@ data: "gridPos": { "h": 3, "w": 3, - "x": 17, + "x": 14, "y": 0 }, - "id": 26, + "id": 25, "options": { "colorMode": "value", - "graphMode": "none", + "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -7010,9 +9865,12 @@ data: "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -7020,15 +9878,13 @@ data: "uid": "prometheus" }, "editorMode": "code", - "exemplar": false, - "expr": "count(hadoop_namenode_version{namespace=~\"$namespace\",job=~\"$cluster\"})", - "instant": true, + "expr": "sum(max by (namespace, pod)(assignment_manager_rit_count{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}))", "legendFormat": "__auto", - "range": false, + "range": true, "refId": "A" } ], - "title": "Namenodes", + "title": "Regions in transition", "type": "stat" }, { @@ -7039,6 +9895,7 @@ data: "fieldConfig": { "defaults": { "color": { + "fixedColor": "green", "mode": "thresholds" }, "mappings": [], @@ -7046,30 +9903,35 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" + }, + { + "color": "dark-yellow", + "value": 600000 }, { "color": "red", - "value": 80 + "value": 3600000 } ] - } + }, + "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, - "x": 20, + "x": 17, "y": 0 }, - "id": 27, + "id": 31, "options": { "colorMode": "value", - "graphMode": "none", + "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -7077,9 +9939,12 @@ data: "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -7087,15 +9952,13 @@ data: "uid": "prometheus" }, "editorMode": "code", - "exemplar": false, - "expr": "count(hadoop_datanode_version{namespace=~\"$namespace\",job=~\"$cluster\"})", - "instant": true, + "expr": "max(max by (pod)(assignment_manager_rit_oldest_age{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}))", "legendFormat": "__auto", - "range": false, + "range": true, "refId": "A" } ], - "title": "Datanodes", + "title": "Oldest regions in transition", "type": "stat" }, { @@ -7109,12 +9972,14 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", @@ -7123,6 +9988,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" @@ -7147,8 +10013,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7224,10 +10089,12 @@ data: "showLegend": false }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -7280,12 +10147,14 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", @@ -7294,6 +10163,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -7315,8 +10185,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7393,10 +10262,12 @@ data: "showLegend": false }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -7437,12 +10308,14 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "opacity", @@ -7451,6 +10324,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -7472,8 +10346,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7542,133 +10415,18 @@ data: "legend": { "calcs": [ "last" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(max by (namespace, pod)(rate(container_network_receive_bytes_total[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))*8", - "hide": false, - "legendFormat": "rx", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "-sum(max by (namespace, pod)(rate(container_network_transmit_bytes_total[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))*8", - "hide": false, - "legendFormat": "tx", - "range": true, - "refId": "B" - } - ], - "title": "Network usage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "dark-green", - "mode": "fixed" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [], - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "remaining" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "used" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 4, - "x": 0, - "y": 10 - }, - "id": 29, - "options": { - "displayLabels": [ - "percent" - ], - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "values": [ - "percent", - "value" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + ], + "displayMode": "table", + "placement": "right", + "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -7676,8 +10434,9 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "min(hadoop_namenode_capacity_remaining{namespace=~\"$namespace\",job=~\"$cluster\"})", - "legendFormat": "remaining", + "expr": "sum(max by (namespace, pod)(rate(container_network_receive_bytes_total[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))*8", + "hide": false, + "legendFormat": "rx", "range": true, "refId": "A" }, @@ -7687,15 +10446,15 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "max(hadoop_namenode_capacity_used{namespace=~\"$namespace\",job=~\"$cluster\"})", + "expr": "-sum(max by (namespace, pod)(rate(container_network_transmit_bytes_total[$__rate_interval]) * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"}))))*8", "hide": false, - "legendFormat": "used", + "legendFormat": "tx", "range": true, "refId": "B" } ], - "title": "DFS usage", - "type": "piechart" + "title": "Network usage", + "type": "timeseries" }, { "datasource": { @@ -7705,49 +10464,82 @@ data: "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", - "mode": "fixed" + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - }, - "unit": "bytes" + } }, "overrides": [] }, "gridPos": { - "h": 3, - "w": 3, - "x": 4, + "h": 8, + "w": 8, + "x": 0, "y": 10 }, - "id": 28, + "id": 33, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { + "legend": { "calcs": [ - "lastNotNull" + "last" ], - "fields": "", - "values": false + "displayMode": "table", + "placement": "right", + "showLegend": false, + "sortBy": "Last", + "sortDesc": true }, - "textMode": "auto" + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -7755,14 +10547,14 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "min(hadoop_namenode_capacity_total{namespace=~\"$namespace\",job=~\"$cluster\"})", - "legendFormat": "__auto", + "expr": "sum(max by (pod)(master_num_active_handler{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}))", + "legendFormat": "active handlers", "range": true, "refId": "A" } ], - "title": "DFS capacity", - "type": "stat" + "title": "Active handlers", + "type": "timeseries" }, { "datasource": { @@ -7775,12 +10567,14 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "opacity", @@ -7789,6 +10583,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -7810,8 +10605,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7819,7 +10613,7 @@ data: } ] }, - "unit": "binbps" + "unit": "reqps" }, "overrides": [ { @@ -7870,12 +10664,12 @@ data: ] }, "gridPos": { - "h": 9, - "w": 7, - "x": 7, + "h": 8, + "w": 8, + "x": 8, "y": 10 }, - "id": 37, + "id": 28, "options": { "legend": { "calcs": [ @@ -7886,10 +10680,12 @@ data: "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -7897,9 +10693,9 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(max by(pod)(rate(hadoop_datanode_bytes_read{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval])))*8", + "expr": "sum(max by(pod)(rate(server_read_request_count{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}[$__rate_interval])))*8", "hide": false, - "legendFormat": "rx", + "legendFormat": "read", "range": true, "refId": "A" }, @@ -7909,14 +10705,14 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "-sum(max by(pod)(rate(hadoop_datanode_bytes_written{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval])))*8", + "expr": "-sum(max by(pod)(rate(server_write_request_count{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}[$__rate_interval])))*8", "hide": false, - "legendFormat": "tx", + "legendFormat": "write", "range": true, "refId": "B" } ], - "title": "HDFS cluster I/O", + "title": "Requests", "type": "timeseries" }, { @@ -7930,11 +10726,13 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -7943,6 +10741,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -7964,8 +10763,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7977,27 +10775,30 @@ data: "overrides": [] }, "gridPos": { - "h": 5, - "w": 10, - "x": 14, + "h": 8, + "w": 8, + "x": 16, "y": 10 }, - "id": 36, + "id": 32, "options": { "legend": { "calcs": [ - "max", "last" ], "displayMode": "table", "placement": "right", - "showLegend": true + "showLegend": true, + "sortBy": "Last", + "sortDesc": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -8005,8 +10806,8 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "max(hadoop_namenode_files_total{namespace=~\"$namespace\",job=~\"$cluster\"})", - "legendFormat": "total", + "expr": "sum(max by (pod)(rate(server_slow_get_count{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}[$__rate_interval])))", + "legendFormat": "get", "range": true, "refId": "A" }, @@ -8016,82 +10817,51 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "max(hadoop_namenode_num_files_under_construction{namespace=~\"$namespace\",job=~\"$cluster\"})", + "expr": "sum(max by (pod)(rate(server_slow_put_count{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}[$__rate_interval])))", "hide": false, - "legendFormat": "under construction", + "legendFormat": "put", "range": true, "refId": "B" - } - ], - "title": "Files", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" }, - "unit": "bytes" + "editorMode": "code", + "expr": "sum(max by (pod)(rate(server_slow_append_count{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}[$__rate_interval])))", + "hide": false, + "legendFormat": "append", + "range": true, + "refId": "C" }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 4, - "y": 13 - }, - "id": 30, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (pod)(rate(server_slow_delete_count{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}[$__rate_interval])))", + "hide": false, + "legendFormat": "delete", + "range": true, + "refId": "D" }, - "textMode": "auto" - }, - "pluginVersion": "9.2.4", - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "min(hadoop_namenode_capacity_used{namespace=~\"$namespace\",job=~\"$cluster\"})", - "legendFormat": "__auto", + "expr": "sum(max by (pod)(rate(server_slow_increment_count{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}[$__rate_interval])))", + "hide": false, + "legendFormat": "increment", "range": true, - "refId": "A" + "refId": "E" } ], - "title": "DFS used", - "type": "stat" + "title": "Slow operations", + "type": "timeseries" }, { "datasource": { @@ -8104,11 +10874,13 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -8117,6 +10889,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -8138,101 +10911,42 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "corrupt" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "mssing" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "underreplicated" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "excess" - }, - "properties": [ + "color": "green" + }, { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } + "color": "red", + "value": 80 } ] } - ] + }, + "overrides": [] }, "gridPos": { - "h": 5, - "w": 10, - "x": 14, - "y": 15 + "h": 8, + "w": 8, + "x": 0, + "y": 18 }, - "id": 35, + "id": 23, "options": { "legend": { "calcs": [ - "max", "last" ], "displayMode": "table", "placement": "right", - "showLegend": true + "showLegend": true, + "sortBy": "Last", + "sortDesc": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -8240,63 +10954,28 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "max(hadoop_namenode_blocks_total{namespace=~\"$namespace\",job=~\"$cluster\"})", - "legendFormat": "total", + "expr": "sum(max by (pod)(assignment_manager_rit_count{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}))", + "legendFormat": "In transition", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max(hadoop_namenode_missing_blocks{namespace=~\"$namespace\",job=~\"$cluster\"})", - "hide": false, - "legendFormat": "mssing", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max(hadoop_namenode_corrupt_blocks{namespace=~\"$namespace\",job=~\"$cluster\"})", - "hide": false, - "legendFormat": "corrupt", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max(hadoop_namenode_under_replicated_blocks{namespace=~\"$namespace\",job=~\"$cluster\"})", - "hide": false, - "legendFormat": "underreplicated", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "max(hadoop_namenode_excess_blocks{namespace=~\"$namespace\",job=~\"$cluster\"})", - "hide": false, - "legendFormat": "excess", - "range": true, - "refId": "E" } ], - "title": "Blocks", + "title": "Regions", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 30, + "panels": [], + "title": "Queues", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -8305,49 +10984,82 @@ data: "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", - "mode": "fixed" + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - }, - "unit": "bytes" + } }, "overrides": [] }, "gridPos": { - "h": 3, - "w": 3, - "x": 4, - "y": 16 + "h": 8, + "w": 8, + "x": 0, + "y": 27 }, - "id": 31, + "id": 20, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { + "legend": { "calcs": [ - "lastNotNull" + "last" ], - "fields": "", - "values": false + "displayMode": "table", + "placement": "right", + "showLegend": false, + "sortBy": "Last", + "sortDesc": true }, - "textMode": "auto" + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -8355,14 +11067,14 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "min(hadoop_namenode_capacity_remaining{namespace=~\"$namespace\",job=~\"$cluster\"})", - "legendFormat": "__auto", + "expr": "max by (pod)(server_compaction_queue_length{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], - "title": "DFS remaining", - "type": "stat" + "title": "Compaction queue length", + "type": "timeseries" }, { "datasource": { @@ -8375,11 +11087,13 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -8388,6 +11102,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -8409,8 +11124,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -8422,24 +11136,30 @@ data: "overrides": [] }, "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 19 + "h": 8, + "w": 8, + "x": 8, + "y": 27 }, - "id": 33, + "id": 21, "options": { "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "sortBy": "Last", + "sortDesc": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -8447,13 +11167,13 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "max(hadoop_namenode_total_load{namespace=~\"$namespace\",job=~\"$cluster\"})", - "legendFormat": "__auto", + "expr": "max by (pod)(server_flush_queue_length{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], - "title": "Client connections", + "title": "Flush queue length", "type": "timeseries" }, { @@ -8467,20 +11187,22 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, - "gradientMode": "opacity", + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -8502,88 +11224,42 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - }, - "unit": "bytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "request" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "used" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 25 - } - ] } - ] + }, + "overrides": [] }, "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 19 + "h": 8, + "w": 8, + "x": 16, + "y": 27 }, - "id": 23, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "id": 22, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "sortBy": "Last", + "sortDesc": true }, "tooltip": { - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -8591,45 +11267,35 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(max by (namespace, persistentvolumeclaim)(kubelet_volume_stats_used_bytes) * on (namespace, persistentvolumeclaim) group_left() (max by (namespace, persistentvolumeclaim) (kube_persistentvolumeclaim_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", - "hide": false, - "legendFormat": "used", + "expr": "max by (pod)(server_split_queue_length{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "legendFormat": "{{pod}}", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(max by (namespace, persistentvolumeclaim)(kubelet_volume_stats_capacity_bytes) * on (namespace, persistentvolumeclaim) group_left() (max by (namespace, persistentvolumeclaim) (kube_persistentvolumeclaim_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", - "hide": false, - "legendFormat": "limit", - "range": true, - "refId": "C" } ], - "title": "PVC usage", + "title": "Split queue length", "type": "timeseries" } ], + "preload": false, "refresh": false, - "schemaVersion": 37, - "style": "dark", + "schemaVersion": 41, "tags": [], "templating": { "list": [ { + "current": { + "text": "hbase", + "value": "hbase" + }, "hide": 2, "name": "product", - "query": "hdfs", - "skipUrlSync": false, + "query": "hbase", + "skipUrlSync": true, "type": "constant" }, { "current": { - "selected": false, "text": "default", "value": "default" }, @@ -8637,47 +11303,42 @@ data: "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(hadoop_namenode_blocks_total, namespace)", - "hide": 0, + "definition": "label_values(tables_num_tables,namespace)", "includeAll": false, "label": "Namespace", - "multi": false, "name": "namespace", "options": [], "query": { - "query": "label_values(hadoop_namenode_blocks_total, namespace)", - "refId": "StandardVariableQuery" + "qryType": 1, + "query": "label_values(tables_num_tables,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": { - "selected": false, - "text": "hdfs", - "value": "hdfs" + "text": "simple-hbase", + "value": "simple-hbase" }, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(hadoop_namenode_blocks_total{namespace=~\"$namespace\"}, job)", - "hide": 0, + "definition": "label_values(tables_num_tables{namespace=~\"$namespace\"},app_kubernetes_io_instance)", "includeAll": false, "label": "Cluster", - "multi": false, "name": "cluster", "options": [], "query": { - "query": "label_values(hadoop_namenode_blocks_total{namespace=~\"$namespace\"}, job)", - "refId": "StandardVariableQuery" + "qryType": 1, + "query": "label_values(tables_num_tables{namespace=~\"$namespace\"},app_kubernetes_io_instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "regex": "", - "skipUrlSync": false, "sort": 1, "type": "query" } @@ -8689,12 +11350,11 @@ data: }, "timepicker": {}, "timezone": "", - "title": "HDFS", - "uid": "qjZGZ3AVz", - "version": 4, - "weekStart": "" + "title": "HBase", + "uid": "vVpFeeAVz", + "version": 6 } - hbase.json: | + nifi.json: | { "annotations": { "list": [ @@ -8721,9 +11381,8 @@ data: "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 29, + "id": 5, "links": [], - "liveNow": false, "panels": [ { "datasource": { @@ -8733,15 +11392,15 @@ data: "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "fixedColor": "green", + "mode": "fixed" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -8758,12 +11417,13 @@ data: "x": 0, "y": 0 }, - "id": 17, + "id": 6, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -8771,9 +11431,11 @@ data: "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -8781,15 +11443,13 @@ data: "uid": "prometheus" }, "editorMode": "code", - "exemplar": false, - "expr": "count(Hadoop_HBase_numMasterWALs{namespace=~\"$namespace\",job=~\"$cluster\"})", - "instant": true, + "expr": "sum(kube_pod_container_resource_requests{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", "legendFormat": "__auto", - "range": false, + "range": true, "refId": "A" } ], - "title": "Masters", + "title": "CPU request", "type": "stat" }, { @@ -8800,15 +11460,15 @@ data: "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "fixedColor": "green", + "mode": "fixed" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -8825,12 +11485,13 @@ data: "x": 3, "y": 0 }, - "id": 18, + "id": 7, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -8838,9 +11499,11 @@ data: "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -8848,15 +11511,13 @@ data: "uid": "prometheus" }, "editorMode": "code", - "exemplar": false, - "expr": "max(Hadoop_HBase_numRegionServers{namespace=~\"$namespace\",job=~\"$cluster\"})", - "instant": true, + "expr": "sum(kube_pod_container_resource_limits{resource=\"cpu\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", "legendFormat": "__auto", - "range": false, + "range": true, "refId": "A" } ], - "title": "Region servers", + "title": "CPU limit", "type": "stat" }, { @@ -8867,22 +11528,23 @@ data: "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "fixedColor": "green", + "mode": "fixed" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [] }, @@ -8892,12 +11554,13 @@ data: "x": 6, "y": 0 }, - "id": 26, + "id": 8, "options": { "colorMode": "value", - "graphMode": "area", + "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -8905,9 +11568,11 @@ data: "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -8915,21 +11580,19 @@ data: "uid": "prometheus" }, "editorMode": "code", - "exemplar": false, - "expr": "sum(max by (namespace, pod)(Hadoop_HBase_regionCount{namespace=~\"$namespace\",job=~\"$cluster\"}))", - "instant": false, + "expr": "sum(kube_pod_container_resource_requests{resource=\"memory\"} * on (namespace, pod) group_left() (max by (namespace, pod) (kube_pod_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Regions", + "title": "Memory limit", "type": "stat" }, { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "fieldConfig": { + "defaults": {}, + "overrides": [] }, "gridPos": { "h": 3, @@ -8947,20 +11610,7 @@ data: "content": "

$cluster

", "mode": "html" }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], + "pluginVersion": "12.0.2", "title": "Cluster name", "type": "text" }, @@ -8979,8 +11629,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -8997,84 +11646,13 @@ data: "x": 14, "y": 0 }, - "id": 25, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "sum(max by (namespace, pod)(Hadoop_HBase_ritCount{namespace=~\"$namespace\",job=~\"$cluster\"}))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Regions in transition", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "dark-yellow", - "value": 600000 - }, - { - "color": "red", - "value": 3600000 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 17, - "y": 0 - }, - "id": 31, + "id": 3, "options": { "colorMode": "value", - "graphMode": "area", + "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -9082,10 +11660,11 @@ data: "fields": "", "values": false }, - "text": {}, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.2.4", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -9093,13 +11672,15 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "max(max by (pod)(Hadoop_HBase_ritOldestAge{namespace=~\"$namespace\",job=~\"$cluster\"}))", + "exemplar": false, + "expr": "count(count by (pod)(nifi_amount_bytes_read{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}))", + "instant": true, "legendFormat": "__auto", - "range": true, + "range": false, "refId": "A" } ], - "title": "Oldest regions in transition", + "title": "Nodes", "type": "stat" }, { @@ -9113,12 +11694,14 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", @@ -9127,6 +11710,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" @@ -9151,8 +11735,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -9228,10 +11811,12 @@ data: "showLegend": false }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -9284,12 +11869,14 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "opacity", @@ -9298,6 +11885,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -9319,8 +11907,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -9397,10 +11984,12 @@ data: "showLegend": false }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -9441,12 +12030,14 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "opacity", @@ -9455,6 +12046,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -9476,8 +12068,7 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -9552,10 +12143,12 @@ data: "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -9596,19 +12189,22 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 50, - "gradientMode": "opacity", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -9630,25 +12226,25 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - } + }, + "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 8, + "w": 12, "x": 0, "y": 10 }, - "id": 33, + "id": 24, "options": { "legend": { "calcs": [ @@ -9656,15 +12252,17 @@ data: ], "displayMode": "table", "placement": "right", - "showLegend": false, + "showLegend": true, "sortBy": "Last", "sortDesc": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -9672,13 +12270,13 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(max by (pod)(Hadoop_HBase_numActiveHandler{namespace=~\"$namespace\",job=~\"$cluster\"}))", - "legendFormat": "active handlers", + "expr": "max by(pod)(nifi_amount_threads_active{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "legendFormat": "{{pod}}", "range": true, "refId": "A" } ], - "title": "Active handlers", + "title": "Active threads", "type": "timeseries" }, { @@ -9692,20 +12290,23 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMin": 0, + "axisSoftMax": 1, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 25, - "gradientMode": "opacity", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -9719,7 +12320,7 @@ data: "mode": "none" }, "thresholdsStyle": { - "mode": "off" + "mode": "dashed" } }, "mappings": [], @@ -9727,72 +12328,25 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", - "value": 80 + "value": 0.9 } ] }, - "unit": "reqps" + "unit": "percentunit" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "limit" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "request" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "used" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, - "w": 8, - "x": 8, + "w": 12, + "x": 12, "y": 10 }, - "id": 28, + "id": 18, "options": { "legend": { "calcs": [ @@ -9800,13 +12354,17 @@ data: ], "displayMode": "table", "placement": "right", - "showLegend": true + "showLegend": true, + "sortBy": "Last", + "sortDesc": true }, "tooltip": { - "mode": "multi", + "hideZeros": false, + "mode": "single", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -9814,26 +12372,13 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(max by(pod)(rate(Hadoop_HBase_readRequestCount{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval])))*8", - "hide": false, - "legendFormat": "read", + "expr": "max by(pod)(nifi_jvm_heap_usage{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "legendFormat": "{{pod}}", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "-sum(max by(pod)(rate(Hadoop_HBase_writeRequestCount{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval])))*8", - "hide": false, - "legendFormat": "write", - "range": true, - "refId": "B" } ], - "title": "Requests", + "title": "JVM heap usage", "type": "timeseries" }, { @@ -9847,11 +12392,13 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -9860,6 +12407,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -9881,25 +12429,25 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 8, - "x": 16, - "y": 10 + "w": 12, + "x": 0, + "y": 18 }, - "id": 32, + "id": 25, "options": { "legend": { "calcs": [ @@ -9912,10 +12460,12 @@ data: "sortDesc": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -9923,8 +12473,8 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(max by (pod)(rate(Hadoop_HBase_slowGetCount{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval])))", - "legendFormat": "get", + "expr": "sum(nifi_amount_bytes_read{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "legendFormat": "read", "range": true, "refId": "A" }, @@ -9934,9 +12484,9 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(max by (pod)(rate(Hadoop_HBase_slowPutCount{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval])))", + "expr": "sum(nifi_amount_bytes_written{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", "hide": false, - "legendFormat": "put", + "legendFormat": "written", "range": true, "refId": "B" }, @@ -9946,9 +12496,9 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(max by (pod)(rate(Hadoop_HBase_slowAppendCount{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval])))", + "expr": "sum(nifi_amount_bytes_received{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", "hide": false, - "legendFormat": "append", + "legendFormat": "received", "range": true, "refId": "C" }, @@ -9958,9 +12508,9 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(max by (pod)(rate(Hadoop_HBase_slowDeleteCount{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval])))", + "expr": "sum(nifi_amount_bytes_sent{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", "hide": false, - "legendFormat": "delete", + "legendFormat": "sent", "range": true, "refId": "D" }, @@ -9970,14 +12520,14 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(max by (pod)(rate(Hadoop_HBase_slowIncrementCount{namespace=~\"$namespace\",job=~\"$cluster\"}[$__rate_interval])))", + "expr": "sum(nifi_amount_bytes_transferred{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", "hide": false, - "legendFormat": "increment", + "legendFormat": "transferred", "range": true, "refId": "E" } ], - "title": "Slow operations", + "title": "Amount of bytes", "type": "timeseries" }, { @@ -9991,11 +12541,13 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -10004,6 +12556,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10025,25 +12578,25 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - } + }, + "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 8, - "x": 0, + "w": 12, + "x": 12, "y": 18 }, - "id": 23, + "id": 26, "options": { "legend": { "calcs": [ @@ -10056,10 +12609,12 @@ data: "sortDesc": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -10067,28 +12622,39 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(max by (pod)(Hadoop_HBase_ritCount{namespace=~\"$namespace\",job=~\"$cluster\"}))", - "legendFormat": "In transition", + "expr": "sum(nifi_amount_items_input{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "legendFormat": "input", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(nifi_amount_items_output{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "hide": false, + "legendFormat": "output", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(nifi_amount_items_queued{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "hide": false, + "legendFormat": "queued", + "range": true, + "refId": "C" } ], - "title": "Regions", + "title": "Items", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 26 - }, - "id": 30, - "panels": [], - "title": "Queues", - "type": "row" - }, { "datasource": { "type": "prometheus", @@ -10100,11 +12666,13 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -10113,6 +12681,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10134,25 +12703,25 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 8, + "w": 9, "x": 0, - "y": 27 + "y": 26 }, - "id": 20, + "id": 27, "options": { "legend": { "calcs": [ @@ -10160,15 +12729,17 @@ data: ], "displayMode": "table", "placement": "right", - "showLegend": false, + "showLegend": true, "sortBy": "Last", "sortDesc": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -10176,13 +12747,37 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (pod)(Hadoop_HBase_compactionQueueLength{namespace=~\"$namespace\",job=~\"$cluster\"})", - "legendFormat": "{{pod}}", + "expr": "max(nifi_content_repo_used_space_bytes{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "legendFormat": "content", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(nifi_flow_file_repo_used_space_bytes{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "hide": false, + "legendFormat": "flow file", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(nifi_provenance_repo_used_space_bytes{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "hide": false, + "legendFormat": "provenance", + "range": true, + "refId": "C" } ], - "title": "Compaction queue length", + "title": "Used repository space", "type": "timeseries" }, { @@ -10196,11 +12791,13 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -10209,6 +12806,7 @@ data: "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10230,25 +12828,25 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 8, - "x": 8, - "y": 27 + "w": 9, + "x": 9, + "y": 26 }, - "id": 21, + "id": 28, "options": { "legend": { "calcs": [ @@ -10256,15 +12854,17 @@ data: ], "displayMode": "table", "placement": "right", - "showLegend": false, + "showLegend": true, "sortBy": "Last", - "sortDesc": true + "sortDesc": false }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -10272,13 +12872,37 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (pod)(Hadoop_HBase_flushQueueLength{namespace=~\"$namespace\",job=~\"$cluster\"})", - "legendFormat": "{{pod}}", + "expr": "max(nifi_content_repo_free_space_bytes{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "legendFormat": "content", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(nifi_flow_file_repo_free_space_bytes{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "hide": false, + "legendFormat": "flow file", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(nifi_provenance_repo_free_space_bytes{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"})", + "hide": false, + "legendFormat": "provenance", + "range": true, + "refId": "C" } ], - "title": "Flush queue length", + "title": "Free repository space", "type": "timeseries" }, { @@ -10292,19 +12916,23 @@ data: "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", + "axisSoftMin": 0, "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, - "gradientMode": "none", + "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10326,41 +12954,89 @@ data: "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - } + }, + "unit": "bytes" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 25 + } + ] + } + ] }, "gridPos": { "h": 8, - "w": 8, - "x": 16, - "y": 27 + "w": 6, + "x": 18, + "y": 26 }, - "id": 22, + "id": 23, "options": { "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "right", - "showLegend": false, - "sortBy": "Last", - "sortDesc": true + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, "tooltip": { - "mode": "single", + "hideZeros": false, + "mode": "multi", "sort": "none" } }, + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -10368,32 +13044,48 @@ data: "uid": "prometheus" }, "editorMode": "code", - "expr": "max by (pod)(Hadoop_HBase_splitQueueLength{namespace=~\"$namespace\",job=~\"$cluster\"})", - "legendFormat": "{{pod}}", + "expr": "sum(max by (namespace, persistentvolumeclaim)(kubelet_volume_stats_used_bytes) * on (namespace, persistentvolumeclaim) group_left() (max by (namespace, persistentvolumeclaim) (kube_persistentvolumeclaim_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "hide": false, + "legendFormat": "used", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max by (namespace, persistentvolumeclaim)(kubelet_volume_stats_capacity_bytes) * on (namespace, persistentvolumeclaim) group_left() (max by (namespace, persistentvolumeclaim) (kube_persistentvolumeclaim_labels{namespace=~\"$namespace\",label_app_kubernetes_io_name=~\"$product\",label_app_kubernetes_io_component=~\".*\",label_app_kubernetes_io_instance=~\"$cluster\"})))", + "hide": false, + "legendFormat": "limit", + "range": true, + "refId": "C" } ], - "title": "Split queue length", + "title": "PVC usage", "type": "timeseries" } ], + "preload": false, "refresh": false, - "schemaVersion": 37, - "style": "dark", + "schemaVersion": 41, "tags": [], "templating": { "list": [ { + "current": { + "text": "nifi", + "value": "nifi" + }, "hide": 2, "name": "product", - "query": "hbase", - "skipUrlSync": false, + "query": "nifi", + "skipUrlSync": true, "type": "constant" }, { "current": { - "selected": false, "text": "default", "value": "default" }, @@ -10401,47 +13093,42 @@ data: "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(Hadoop_HBase_numTables, namespace)", - "hide": 0, + "definition": "label_values(nifi_amount_bytes_read,namespace)", "includeAll": false, "label": "Namespace", - "multi": false, "name": "namespace", "options": [], "query": { - "query": "label_values(Hadoop_HBase_numTables, namespace)", - "refId": "StandardVariableQuery" + "qryType": 1, + "query": "label_values(nifi_amount_bytes_read,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": { - "selected": false, - "text": "hbase", - "value": "hbase" + "text": "nifi", + "value": "nifi" }, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(Hadoop_HBase_numTables{namespace=~\"$namespace\"}, job)", - "hide": 0, + "definition": "label_values(nifi_amount_bytes_read{namespace=~\"$namespace\"},app_kubernetes_io_instance)", "includeAll": false, "label": "Cluster", - "multi": false, "name": "cluster", "options": [], "query": { - "query": "label_values(Hadoop_HBase_numTables{namespace=~\"$namespace\"}, job)", - "refId": "StandardVariableQuery" + "qryType": 1, + "query": "label_values(nifi_amount_bytes_read{namespace=~\"$namespace\"},app_kubernetes_io_instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "regex": "", - "skipUrlSync": false, "sort": 1, "type": "query" } @@ -10453,9 +13140,8 @@ data: }, "timepicker": {}, "timezone": "", - "title": "HBase", - "uid": "vVpFeeAVz", - "version": 17, - "weekStart": "" + "title": "NiFi", + "uid": "C_8qIX04l", + "version": 13 } # {% endraw %} diff --git a/stacks/monitoring/prometheus-service-monitors.yaml b/stacks/monitoring/prometheus-service-monitors.yaml new file mode 100644 index 00000000..3a6bb88d --- /dev/null +++ b/stacks/monitoring/prometheus-service-monitors.yaml @@ -0,0 +1,268 @@ +# Use something like this to check for metrics: +# count by (app_kubernetes_io_name, app_kubernetes_io_instance, pod) ({app_kubernetes_io_name!="",pod!=""}) +# +# [x] Airflow - exporter +# [x] Druid - native +# [x] HBase - native +# [x] Hadoop HDFS - native +# [x] Hive - exporter +# [x] Kafka - exporter +# [x] NiFi 1 - native +# [x] NiFi 2 - native +# [ ] OpenSearch - not officially part of the platform yet +# [ ] Spark - native - was too lazy, no idea if we even expose metrics +# [x] Superset - exporter +# [x] Trino - native +# [x] ZooKeeper - native +# [x] OPA - native +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: stackable + labels: + stackable.tech/vendor: Stackable + release: prometheus +spec: + namespaceSelector: + any: true + selector: + matchLabels: + stackable.tech/vendor: Stackable + prometheus.io/scrape: "true" + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - airflow + - druid + - hive + - kafka + - nifi # This only works for NiFi 1, NiFi 2 has a special ServiceMonitor below + - opa + - superset + - trino + endpoints: + - scheme: http + port: metrics + path: /metrics + podTargetLabels: + - app.kubernetes.io/name + - app.kubernetes.io/instance + - app.kubernetes.io/component + - app.kubernetes.io/role-group + - app.kubernetes.io/version +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: stackable-native-metrics + labels: + stackable.tech/vendor: Stackable + release: prometheus +spec: + namespaceSelector: + any: true + selector: + matchLabels: + stackable.tech/vendor: Stackable + prometheus.io/scrape: "true" + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - zookeeper + endpoints: + - scheme: http + port: native-metrics + path: /metrics + podTargetLabels: + - app.kubernetes.io/name + - app.kubernetes.io/instance + - app.kubernetes.io/component + - app.kubernetes.io/role-group + - app.kubernetes.io/version +--- +# Kafka is special in that the operator totally messes up services: +# 1. The metrics Service is missing +# 2. The role level simple-kafka-broker-default has the prometheus.io/scrape label, but exposes no ports... +# 3. The role level simple-kafka-broker-default is labeled with app.kubernetes.io/name: listener??? +# So we have a dedicated config for it +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: stackable-kafka + labels: + stackable.tech/vendor: Stackable + release: prometheus +spec: + namespaceSelector: + any: true + selector: + matchLabels: + stackable.tech/vendor: Stackable + app.kubernetes.io/name: listener # Dafuq? + app.kubernetes.io/component: broker # We need to filter on brokers instead, as the app.kubernetes.io/name is messed up + endpoints: + - scheme: http + port: metrics + path: /metrics + podTargetLabels: + - app.kubernetes.io/name + - app.kubernetes.io/instance + - app.kubernetes.io/component + - app.kubernetes.io/role-group + - app.kubernetes.io/version +--- +# We prefer the native metrics over the statsd-exporter +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: stackable-hdfs + labels: + stackable.tech/vendor: Stackable + release: prometheus +spec: + namespaceSelector: + any: true + selector: + matchLabels: + stackable.tech/vendor: Stackable + prometheus.io/scrape: "true" + app.kubernetes.io/name: hdfs + endpoints: + - scheme: http + port: http # Don't use metrics! + path: /prom + podTargetLabels: + - app.kubernetes.io/name + - app.kubernetes.io/instance + - app.kubernetes.io/component + - app.kubernetes.io/role-group + - app.kubernetes.io/version +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: stackable-hbase + labels: + stackable.tech/vendor: Stackable + release: prometheus +spec: + namespaceSelector: + any: true + selector: + matchLabels: + stackable.tech/vendor: Stackable + prometheus.io/scrape: "true" + app.kubernetes.io/name: hbase + endpoints: + - scheme: http + port: ui-http + path: /prometheus + podTargetLabels: + - app.kubernetes.io/name + - app.kubernetes.io/instance + - app.kubernetes.io/component + - app.kubernetes.io/role-group + - app.kubernetes.io/version +--- +# NiFI 2 is a beast on it's own... +# We need to use mTLS (otherwise we get a 401) and can not use the PodIP +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: stackable-nifi-2 + labels: + stackable.tech/vendor: Stackable + release: prometheus +spec: + namespaceSelector: + any: true + selector: + matchLabels: + stackable.tech/vendor: Stackable + prometheus.io/scrape: "true" + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - nifi + endpoints: + - scheme: https + port: https + path: /nifi-api/flow/metrics/prometheus + # See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api-reference/api.md#monitoring.coreos.com/v1.TLSConfig + tlsConfig: + ca: + secret: + name: prometheus-tls-certificate + key: ca.crt + cert: + secret: + name: prometheus-tls-certificate + key: tls.crt + keySecret: + name: prometheus-tls-certificate + key: tls.key + # We need to talk to the Pod via the FQDN of the Pod because of the stupid SNI check of NiFi. + # We can not use the typical PodIP, as it is not contained in the NiFi certificate, + # see https://github.com/stackabletech/secret-operator/issues/620 + relabelings: + - sourceLabels: + - __meta_kubernetes_pod_name + - __meta_kubernetes_service_name + - __meta_kubernetes_namespace + - __meta_kubernetes_pod_container_port_number + targetLabel: __address__ + replacement: ${1}.${2}-headless.${3}.svc.cluster.local:${4} + regex: (.+);(.+?)(?:-metrics)?;(.+);(.+) + podTargetLabels: + - app.kubernetes.io/name + - app.kubernetes.io/instance + - app.kubernetes.io/component + - app.kubernetes.io/role-group + - app.kubernetes.io/version +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: stackable-minio-http + labels: + stackable.tech/vendor: Stackable + release: prometheus +spec: + namespaceSelector: + any: true + selector: + matchLabels: + # stackable.tech/vendor: Stackable # This is not always set, e.g. missing in the nifi-kafka-druid-water-level-data demo + app: minio + monitoring: "true" + endpoints: + - scheme: http + port: http + path: /minio/v2/metrics/cluster +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: stackable-minio-https + labels: + stackable.tech/vendor: Stackable + release: prometheus +spec: + namespaceSelector: + any: true + selector: + matchLabels: + # stackable.tech/vendor: Stackable # This is not always set, e.g. missing in the nifi-kafka-druid-water-level-data demo + app: minio + monitoring: "true" + endpoints: + - scheme: https + port: https + path: /minio/v2/metrics/cluster + # Prevent "tls: failed to verify certificate: x509: cannot validate certificate for 100.96.234.154 because it doesn't contain any IP SANs" + tlsConfig: + insecureSkipVerify: true diff --git a/stacks/_templates/prometheus.yaml b/stacks/monitoring/prometheus.yaml similarity index 100% rename from stacks/_templates/prometheus.yaml rename to stacks/monitoring/prometheus.yaml diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index a4329400..9dd32287 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -6,14 +6,17 @@ stacks: stackableOperators: - commons - listener + - secret labels: - monitoring - prometheus - grafana manifests: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/monitoring/grafana-dashboards.yaml - - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/prometheus.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/prometheus-service-monitor.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/monitoring/create-prometheus-tls-certificate-serviceaccount.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/monitoring/create-prometheus-tls-certificate.yaml + - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/monitoring/prometheus.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/monitoring/prometheus-service-monitors.yaml supportedNamespaces: [] resourceRequests: cpu: 1750m