diff --git a/docs/architecture.md b/docs/architecture.md index 57b1d95c0..48e181ac7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -75,7 +75,7 @@ Soperator consists of the following components: - **soperator-activechecks**. This chart is optional. It is used to deploy Kubernetes resources of `kind: ActiveCheck`, which perform active jobs to monitor and validate the state of a Slurm cluster. - **soperator-crds**. It deploys the schema of the `SlurmCluster` custom resource, it should always be applied when you upgrade the version of Soperator in your cluster. -- **soperator-dcgm-exporter**. This chart is optional. It is a custom Helm chart with `hpc-jobs-dir` where Slurm stores information about jobs. +- **soperator-dcgm-exporter**. This chart is optional. It deploys the NVIDIA DCGM exporter with soperator's curated metric set and Slurm-aware relabelings (NVLink instance group and NodeSet labels). - **soperator-fluxcd**. This chart is optional. It is an umbrella chart with various components designed for convenient use in a cluster with soperator. It includes `kind: HelmRelease` and `kind: HelmRepository`. This approach was chosen because customizing components through Kustomize is difficult, and Helm dependencies lack flexibility in configuration. Therefore, this umbrella chart was created to deploy HelmReleases with properly configured dependencies. - **soperatorchecks**. This chart is optional. It is used to deploy additional controller that runs Slurm and Kubernetes jobs to actively check the state of the Slurm cluster. diff --git a/docs/metrics-pipeline.md b/docs/metrics-pipeline.md index 9552f24db..a7e444e0b 100644 --- a/docs/metrics-pipeline.md +++ b/docs/metrics-pipeline.md @@ -67,6 +67,7 @@ The exporter applies metric relabeling to drop volatile Kubernetes labels (`pod` - Metrics: GPU temperature, power, utilization, memory, errors - Scrape Interval: 15s - DaemonSet: Runs on nodes with `nvidia.com/gpu.deploy.dcgm-exporter=true` +- Job attribution: DCGM metrics carry no job labels; per-job GPU views join them with the Slurm exporter's `slurm_node_job` metric on `node_name`, which is added to DCGM series at scrape time from the worker Pod name (see [slurm-exporter.md](slurm-exporter.md)) Connection Example: ```bash diff --git a/docs/slurm-exporter.md b/docs/slurm-exporter.md index 9fccd0668..da09615b9 100644 --- a/docs/slurm-exporter.md +++ b/docs/slurm-exporter.md @@ -140,6 +140,20 @@ max by (user_name, nvlink_instance_group, nodeset_name) ( ) ``` +### Joining DCGM GPU metrics with jobs + +DCGM metrics carry no job labels, but they carry `node_name` — the Slurm node name, copied at scrape time from the worker Pod name. Attribute GPU metrics to jobs by joining it with `slurm_node_job`. The join is node-granular — when several jobs share one node, each job inherits the whole node's GPU signal. + +```promql +# Average GPU utilization per running job +avg by (job_id) ( + slurm_node_job + * on (node_name) group_left() + avg by (node_name) (DCGM_FI_DEV_GPU_UTIL) +) +and on (job_id) slurm_job_info{job_state="RUNNING"} +``` + ### Controller RPC Metrics These metrics provide insights into SLURM controller performance, similar to the output of the `sdiag` command, and were implemented to address [issue #1027](https://github.com/nebius/soperator/issues/1027). diff --git a/helm/slurm-cluster/slurm_scripts/map_job_dcgm.sh b/helm/slurm-cluster/slurm_scripts/map_job_dcgm.sh deleted file mode 100644 index 80699e219..000000000 --- a/helm/slurm-cluster/slurm_scripts/map_job_dcgm.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -echo "[$(date)] Map the Slurm job with DCGM metrics" - -# set in hpcJobMapDir in soperator/helm/soperator-fluxcd/values.yaml -# and dcgm_job_map_dir in nebius-solution-library/soperator/modules/slurm/variables.tf -# check those before changing it here -metrics_dir="/var/run/nebius/slurm" - -if [[ -z "${SLURM_JOB_GPUS:-}" ]]; then - echo "No GPU devices are requested by user" >&2 - exit 0 -fi - -if [[ ! -d "$metrics_dir" ]]; then - if ! mkdir -p "$metrics_dir"; then - echo "Unable to create the data dir metrics_dir" >&2 - exit 0 - fi -fi - -IFS=',' read -ra cuda_devs <<< "$SLURM_JOB_GPUS" - -for gpu_id in "${cuda_devs[@]}"; do - [[ -z "$gpu_id" ]] && continue - echo "Writing $metrics_dir/$gpu_id" - if ! printf "%s" "${SLURM_JOB_ID:-0}" > "$metrics_dir/$gpu_id"; then - echo "Unable to write job file $metrics_dir/$gpu_id" - fi -done diff --git a/helm/slurm-cluster/slurm_scripts/map_job_dcgm.sh.json b/helm/slurm-cluster/slurm_scripts/map_job_dcgm.sh.json deleted file mode 100644 index 48d58416c..000000000 --- a/helm/slurm-cluster/slurm_scripts/map_job_dcgm.sh.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "map_job_dcgm", - "command": "./map_job_dcgm.sh", - "platforms": ["8xGPU", "1xGPU", "4xGPU"], - "skip_for_cpu_jobs": true, - "skip_for_partial_gpu_jobs": false, - "contexts": ["prolog"], - "node_states": ["any"], - "on_fail": "none", - "on_ok": "none", - "reason_base": "", - "reason_append_details": false, - "run_in_jail": false, - "log": "slurm_scripts/$worker.$name.$context.out", - "need_env": [] -} diff --git a/helm/slurm-cluster/slurm_scripts/unmap_job_dcgm.sh b/helm/slurm-cluster/slurm_scripts/unmap_job_dcgm.sh deleted file mode 100644 index a4c675bbd..000000000 --- a/helm/slurm-cluster/slurm_scripts/unmap_job_dcgm.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -echo "[$(date)] Unmap the Slurm job with DCGM metrics" - -# set in hpcJobMapDir in soperator/helm/soperator-fluxcd/values.yaml -# and dcgm_job_map_dir in nebius-solution-library/soperator/modules/slurm/variables.tf -# check those before changing it here -metrics_dir="/var/run/nebius/slurm" - -if [[ -z "${SLURM_JOB_GPUS:-}" ]]; then - echo "No GPU devices are requested by user" >&2 - exit 0 -fi - -IFS=',' read -ra cuda_devs <<< "$SLURM_JOB_GPUS" - -for gpu_id in "${cuda_devs[@]}"; do - [[ -z "$gpu_id" ]] && continue - echo "Removing $metrics_dir/${gpu_id:-99}" - rm -f "${metrics_dir:-}/${gpu_id:-99}" || echo "Unable to remove file ${metrics_dir:-}/${gpu_id:-99}" -done diff --git a/helm/slurm-cluster/slurm_scripts/unmap_job_dcgm.sh.json b/helm/slurm-cluster/slurm_scripts/unmap_job_dcgm.sh.json deleted file mode 100644 index a5df263b1..000000000 --- a/helm/slurm-cluster/slurm_scripts/unmap_job_dcgm.sh.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "unmap_job_dcgm", - "command": "./unmap_job_dcgm.sh", - "platforms": ["8xGPU", "1xGPU", "4xGPU"], - "skip_for_cpu_jobs": true, - "skip_for_partial_gpu_jobs": false, - "contexts": ["epilog"], - "node_states": ["any"], - "on_fail": "none", - "on_ok": "none", - "reason_base": "", - "reason_append_details": false, - "run_in_jail": false, - "log": "slurm_scripts/$worker.$name.$context.out", - "need_env": [] -} diff --git a/helm/slurm-cluster/values.yaml b/helm/slurm-cluster/values.yaml index d7ea5a7ba..d6478fec6 100644 --- a/helm/slurm-cluster/values.yaml +++ b/helm/slurm-cluster/values.yaml @@ -874,14 +874,6 @@ slurmScripts: enabled: true customContent: null customConfig: null - map_job_dcgm.sh: - enabled: true - customContent: null - customConfig: null - unmap_job_dcgm.sh: - enabled: true - customContent: null - customConfig: null extra: null # extra: # example.sh: diff --git a/helm/soperator-dcgm-exporter/templates/daemonset.yaml b/helm/soperator-dcgm-exporter/templates/daemonset.yaml index c09d8b714..b800b2a5f 100644 --- a/helm/soperator-dcgm-exporter/templates/daemonset.yaml +++ b/helm/soperator-dcgm-exporter/templates/daemonset.yaml @@ -24,8 +24,6 @@ spec: value: "true" - name: DCGM_EXPORTER_COLLECTORS value: /etc/dcgm-exporter/dcgm-metrics.csv - - name: DCGM_HPC_JOB_MAPPING_DIR - value: {{ .Values.dcgmHpcJobMappingDir }} - name: NODE_NAME valueFrom: fieldRef: @@ -55,30 +53,9 @@ spec: name: metrics-config readOnly: true subPath: dcgm-metrics.csv - - mountPath: {{ .Values.dcgmHpcJobMappingDir }} - mountPropagation: HostToContainer - name: hpc-jobs-dir dnsPolicy: ClusterFirst - initContainers: - - name: create-hpc-jobs-dir - command: - - sh - - -c - - | - mkdir -p {{ .Values.dcgmHpcJobMappingDir }} - env: - - name: KUBERNETES_CLUSTER_DOMAIN - value: {{ quote .Values.kubernetesClusterDomain }} - image: {{ .Values.daemonSet.createHpcJobsDir.image.repository }}:{{ .Values.daemonSet.createHpcJobsDir.image.tag }} - imagePullPolicy: {{ .Values.daemonSet.createHpcJobsDir.image.pullPolicy | default "IfNotPresent" }} - resources: {} - securityContext: - privileged: true - volumeMounts: - - mountPath: /var - mountPropagation: HostToContainer - name: host-var {{- if .Values.validateToolkit }} + initContainers: - name: toolkit-validation args: - until [ -f /run/nvidia/validations/toolkit-ready ]; do echo waiting for nvidia container stack to be setup; sleep 5; done @@ -126,14 +103,6 @@ spec: type: "" name: run-nvidia {{- end }} - - hostPath: - path: {{ .Values.dcgmHpcJobMappingDir }} - type: "" - name: hpc-jobs-dir - - hostPath: - path: /var - type: "" - name: host-var - configMap: defaultMode: 420 items: diff --git a/helm/soperator-dcgm-exporter/values.yaml b/helm/soperator-dcgm-exporter/values.yaml index 400780b18..53fc5b004 100644 --- a/helm/soperator-dcgm-exporter/values.yaml +++ b/helm/soperator-dcgm-exporter/values.yaml @@ -1,7 +1,5 @@ kubernetesClusterDomain: cluster.local -dcgmHpcJobMappingDir: /var/run/nebius/slurm - validateToolkit: true metricsPort: 9400 @@ -16,12 +14,6 @@ daemonSet: tag: v24.6.2 pullPolicy: IfNotPresent - createHpcJobsDir: - image: - repository: cr.eu-north1.nebius.cloud/soperator/busybox - tag: latest - pullPolicy: IfNotPresent - nvidiaDcgmExporter: image: repository: cr.eu-north1.nebius.cloud/soperator/dcgm-exporter @@ -43,6 +35,14 @@ serviceMonitor: regex: feature_node_kubernetes_io_.*|kubernetes_io_.*|nvidia_com_gpu.* - action: labeldrop regex: UUID|DCGM_FI_PROCESS_NAME|DCGM_FI_DEV_SERIAL|DCGM_FI_DEV_NAME|job|pci_bus_id|prometheus + # The worker Pod name (exported_pod after the scrape resolves the pod-label + # conflict) equals the Slurm node name; expose it as node_name so DCGM series + # join with the Slurm exporter metrics without label rewriting. + - action: replace + regex: (.+) + sourceLabels: + - exported_pod + targetLabel: node_name relabelings: - action: replace regex: nvidia-dcgm-exporter diff --git a/helm/soperator-fluxcd/templates/dcgm-exporter.yaml b/helm/soperator-fluxcd/templates/dcgm-exporter.yaml index 29d7544aa..5e2990864 100644 --- a/helm/soperator-fluxcd/templates/dcgm-exporter.yaml +++ b/helm/soperator-fluxcd/templates/dcgm-exporter.yaml @@ -27,7 +27,6 @@ spec: upgrade: {{- toYaml .Values.observability.dcgmExporter.upgrade | nindent 4 }} values: - dcgmHpcJobMappingDir: {{ .Values.observability.dcgmExporter.values.hpcJobMapDir }} validateToolkit: {{ .Values.observability.dcgmExporter.values.validateToolkit }} {{- if .Values.observability.dcgmExporter.values.resources }} daemonSet: diff --git a/helm/soperator-fluxcd/values.yaml b/helm/soperator-fluxcd/values.yaml index 69f2b32f8..bc7321dae 100644 --- a/helm/soperator-fluxcd/values.yaml +++ b/helm/soperator-fluxcd/values.yaml @@ -648,7 +648,6 @@ observability: retries: 6 remediateLastFailure: true values: - hpcJobMapDir: /var/run/nebius/slurm validateToolkit: true resources: limits: diff --git a/helm/soperator-monitoring-dashboards/dashboards/cluster_health.json b/helm/soperator-monitoring-dashboards/dashboards/cluster_health.json index 75b48aed4..3662862e0 100644 --- a/helm/soperator-monitoring-dashboards/dashboards/cluster_health.json +++ b/helm/soperator-monitoring-dashboards/dashboards/cluster_health.json @@ -3909,7 +3909,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "with (\n filters = {}\n)\n# GPU Util\navg(\n label_move(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", filters}\n , 'hpc_job','job_id')\n) by (job_id)\nand on (job_id)\nslurm_job_info{cluster=~\"$cluster\", \n filters,\n job_state=\"RUNNING\",\n user_name=~\"$user_name\", \n slurm_partition${partition_match_exp}, \n}", + "expr": "with (\n filters = {}\n)\n# GPU Util\navg(\n slurm_node_job{cluster=~\"$cluster\"}\n * on(cluster, node_name) group_left()\n avg(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", filters}\n ) by (cluster, node_name)\n) by (job_id)\nand on (job_id)\nslurm_job_info{cluster=~\"$cluster\",\n filters,\n job_state=\"RUNNING\",\n user_name=~\"$user_name\",\n slurm_partition${partition_match_exp},\n}", "format": "table", "hide": false, "instant": true, @@ -3924,7 +3924,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "with (\n filters = {}\n)\n# SM Util\navg(\n label_move(\n DCGM_FI_PROF_SM_ACTIVE{cluster=~\"$cluster\", filters}\n , 'hpc_job','job_id')\n) by (job_id)\nand on (job_id)\nslurm_job_info{cluster=~\"$cluster\", \n filters,\n job_state=\"RUNNING\",\n user_name=~\"$user_name\", \n slurm_partition${partition_match_exp}, \n}", + "expr": "with (\n filters = {}\n)\n# SM Util\navg(\n slurm_node_job{cluster=~\"$cluster\"}\n * on(cluster, node_name) group_left()\n avg(\n DCGM_FI_PROF_SM_ACTIVE{cluster=~\"$cluster\", filters}\n ) by (cluster, node_name)\n) by (job_id)\nand on (job_id)\nslurm_job_info{cluster=~\"$cluster\",\n filters,\n job_state=\"RUNNING\",\n user_name=~\"$user_name\",\n slurm_partition${partition_match_exp},\n}", "format": "table", "hide": false, "instant": true, @@ -4734,13 +4734,13 @@ "disableTextWrap": false, "editorMode": "code", "exemplar": false, - "expr": "floor(\n avg_over_time(\n (\n (\n (\n round((count(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", hpc_job!=\"\"} > 2) by(hpc_job) / 8) > 2)\n -\n round(\n (count(DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\", hpc_job!=\"\"} > 120) by(hpc_job) / 8)\n >\n 2\n )\n )\n /\n round((count(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", hpc_job!=\"\", } > 2) by(hpc_job) / 8) > 2)\n )\n *\n 100\n )[1h:1m]\n) >= 70\n)\n and on(hpc_job)\nlabel_move(slurm_job_info{cluster=~\"$cluster\", job_state=\"RUNNING\"}, 'job_id', 'hpc_job')\nand on (hpc_job)\ncount_over_time( #filter out short durations\navg_over_time((\n (\n (\n (\n round((count(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", hpc_job!=\"\"} > 2) by(hpc_job) / 8) > 2)\n -\n round(\n (count(DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\", hpc_job!=\"\"} > 120) by(hpc_job) / 8)\n >\n 2\n )\n )\n /\n round((count(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", hpc_job!=\"\"} > 2) by(hpc_job) / 8) > 2)\n )\n *\n 100\n )[1h:1m]\n) > 70\n and on(hpc_job)\nlabel_move(slurm_job_info{cluster=~\"$cluster\", job_state=\"RUNNING\"}, 'job_id', 'hpc_job')\n)[1h:1m]\n) > 20", + "expr": "with (\n gpus_total = count(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\"}) by (cluster, node_name),\n mem_frac = count(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\"} > 2) by (cluster, node_name) / gpus_total,\n power_frac = count(DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\"} > 120) by (cluster, node_name) / gpus_total,\n workers_with_mem = round(sum(slurm_node_job{cluster=~\"$cluster\"} * on(cluster, node_name) group_left() mem_frac) by (job_id) > 2),\n workers_with_power = round(sum(slurm_node_job{cluster=~\"$cluster\"} * on(cluster, node_name) group_left() power_frac) by (job_id) > 2),\n underutil_pct = (workers_with_mem - workers_with_power) / workers_with_mem * 100,\n running_jobs = slurm_job_info{cluster=~\"$cluster\", job_state=\"RUNNING\"}\n)\nfloor(avg_over_time(underutil_pct[1h:1m]) >= 70)\nand on(job_id) running_jobs\nand on (job_id)\ncount_over_time( #filter out short durations\n (avg_over_time(underutil_pct[1h:1m]) > 70 and on(job_id) running_jobs)[1h:1m]\n) > 20", "format": "time_series", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "intervalFactor": 2, - "legendFormat": "{{hpc_job}}", + "legendFormat": "{{job_id}}", "range": true, "refId": "A", "useBackend": false, @@ -5948,7 +5948,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "with (\n filters = {}\n)\navg(\n label_move(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", filters, hpc_job!=\"\"}, 'hpc_job','job_id')\n * on (job_id) group_left(user_name)\n slurm_job_info{cluster=~\"$cluster\", \n filters,\n job_state=\"RUNNING\",\n user_name=~\"$user_name\", \n slurm_partition${partition_match_exp}\n }\n) by (user_name)", + "expr": "with (\n filters = {}\n)\navg(\n (\n slurm_node_job{cluster=~\"$cluster\"}\n * on(cluster, node_name) group_left()\n avg(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", filters}\n ) by (cluster, node_name)\n )\n * on (job_id) group_left(user_name)\n slurm_job_info{cluster=~\"$cluster\",\n filters,\n job_state=\"RUNNING\",\n user_name=~\"$user_name\",\n slurm_partition${partition_match_exp}\n }\n) by (user_name)", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", diff --git a/helm/soperator-monitoring-dashboards/dashboards/jobs_overview.json b/helm/soperator-monitoring-dashboards/dashboards/jobs_overview.json index 28bee7537..dc33d852c 100644 --- a/helm/soperator-monitoring-dashboards/dashboards/jobs_overview.json +++ b/helm/soperator-monitoring-dashboards/dashboards/jobs_overview.json @@ -160,7 +160,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg (\n (1 - rate(node_cpu_seconds_total{cluster=~\"$cluster\", mode=\"idle\"}[1m]))\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n) * 100", + "expr": "avg (\n (1 - rate(node_cpu_seconds_total{cluster=~\"$cluster\", mode=\"idle\"}[1m]))\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move((DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"}), \"Hostname\", \"nodename\")), 1, 1)\n) * 100", "instant": false, "legendFormat": "__auto", "range": true, @@ -237,7 +237,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_SM_CLOCK{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}*1000000)", + "expr": "avg(DCGM_FI_DEV_SM_CLOCK{cluster=~\"$cluster\", exported_pod=~\"$worker\"}*1000000 and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -325,7 +325,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "sum(DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -411,7 +411,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_MEM_COPY_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "avg(DCGM_FI_DEV_MEM_COPY_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -497,7 +497,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(\n (\n node_memory_MemTotal_bytes{cluster=~\"$cluster\"} - \n node_memory_MemFree_bytes{cluster=~\"$cluster\"} - \n node_memory_Buffers_bytes{cluster=~\"$cluster\"} - \n node_memory_Cached_bytes{cluster=~\"$cluster\"}\n ) / \n node_memory_MemTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n) * 100", + "expr": "avg(\n (\n node_memory_MemTotal_bytes{cluster=~\"$cluster\"} - \n node_memory_MemFree_bytes{cluster=~\"$cluster\"} - \n node_memory_Buffers_bytes{cluster=~\"$cluster\"} - \n node_memory_Cached_bytes{cluster=~\"$cluster\"}\n ) / \n node_memory_MemTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move((DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"}), \"Hostname\", \"nodename\")), 1, 1)\n) * 100", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -576,7 +576,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_MEM_CLOCK{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}*1000000)", + "expr": "avg(DCGM_FI_DEV_MEM_CLOCK{cluster=~\"$cluster\", exported_pod=~\"$worker\"}*1000000 and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -666,7 +666,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "avg(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -752,7 +752,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "avg(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -1424,7 +1424,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "max(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -1439,7 +1439,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "avg(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -1454,7 +1454,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "min(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -1554,7 +1554,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"} * 1979)", + "expr": "sum(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{cluster=~\"$cluster\", exported_pod=~\"$worker\"} * 1979 and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -1569,7 +1569,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"} * 989.7)", + "expr": "sum(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{cluster=~\"$cluster\", exported_pod=~\"$worker\"} * 989.7 and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -1741,7 +1741,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max (DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "max (DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -1756,7 +1756,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg (DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "avg (DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -1771,7 +1771,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min (DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "min (DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -1923,7 +1923,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(\n (1 - rate(node_cpu_seconds_total{cluster=~\"$cluster\", mode=\"idle\", instance=~\"$ip\"}[$__rate_interval]))\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", + "expr": "max(\n (1 - rate(node_cpu_seconds_total{cluster=~\"$cluster\", mode=\"idle\", instance=~\"$ip\"}[$__rate_interval]))\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move((DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"}), \"Hostname\", \"nodename\")), 1, 1)\n)", "instant": false, "legendFormat": "max", "range": true, @@ -1935,7 +1935,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(\n (1 - rate(node_cpu_seconds_total{cluster=~\"$cluster\", mode=\"idle\", instance=~\"$ip\"}[$__rate_interval]))\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", + "expr": "avg(\n (1 - rate(node_cpu_seconds_total{cluster=~\"$cluster\", mode=\"idle\", instance=~\"$ip\"}[$__rate_interval]))\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move((DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"}), \"Hostname\", \"nodename\")), 1, 1)\n)", "hide": false, "instant": false, "legendFormat": "avg", @@ -1948,7 +1948,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(\n (1 - rate(node_cpu_seconds_total{cluster=~\"$cluster\", mode=\"idle\", instance=~\"$ip\"}[$__rate_interval]))\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", + "expr": "min(\n (1 - rate(node_cpu_seconds_total{cluster=~\"$cluster\", mode=\"idle\", instance=~\"$ip\"}[$__rate_interval]))\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move((DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"}), \"Hostname\", \"nodename\")), 1, 1)\n)", "hide": false, "instant": false, "legendFormat": "min", @@ -2109,7 +2109,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}/(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}+DCGM_FI_DEV_FB_FREE{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}))", + "expr": "max(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\"}/(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\"}+DCGM_FI_DEV_FB_FREE{cluster=~\"$cluster\", exported_pod=~\"$worker\"}) and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -2124,7 +2124,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}/(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}+DCGM_FI_DEV_FB_FREE{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}))", + "expr": "avg(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\"}/(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\"}+DCGM_FI_DEV_FB_FREE{cluster=~\"$cluster\", exported_pod=~\"$worker\"}) and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -2139,7 +2139,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}/(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}+DCGM_FI_DEV_FB_FREE{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}))", + "expr": "min(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\"}/(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\"}+DCGM_FI_DEV_FB_FREE{cluster=~\"$cluster\", exported_pod=~\"$worker\"}) and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -2279,7 +2279,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n rate(node_infiniband_port_data_transmitted_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", + "expr": "sum(\n rate(node_infiniband_port_data_transmitted_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move((DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"}), \"Hostname\", \"nodename\")), 1, 1)\n)", "format": "time_series", "hide": false, "interval": "", @@ -2294,7 +2294,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n rate(node_infiniband_port_data_received_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", + "expr": "sum(\n rate(node_infiniband_port_data_received_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move((DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"}), \"Hostname\", \"nodename\")), 1, 1)\n)", "format": "time_series", "hide": false, "instant": false, @@ -2454,7 +2454,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(\n (\n (\n node_memory_MemTotal_bytes{cluster=~\"$cluster\"} - \n node_memory_MemFree_bytes{cluster=~\"$cluster\"} - \n node_memory_Buffers_bytes{cluster=~\"$cluster\"} - \n node_memory_Cached_bytes{cluster=~\"$cluster\"}\n )\n /\n node_memory_MemTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n )\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", + "expr": "max(\n (\n (\n node_memory_MemTotal_bytes{cluster=~\"$cluster\"} - \n node_memory_MemFree_bytes{cluster=~\"$cluster\"} - \n node_memory_Buffers_bytes{cluster=~\"$cluster\"} - \n node_memory_Cached_bytes{cluster=~\"$cluster\"}\n )\n /\n node_memory_MemTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n )\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move((DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"}), \"Hostname\", \"nodename\")), 1, 1)\n)", "format": "time_series", "hide": false, "instant": false, @@ -2469,7 +2469,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(\n (\n (\n node_memory_MemTotal_bytes{cluster=~\"$cluster\"} - \n node_memory_MemFree_bytes{cluster=~\"$cluster\"} - \n node_memory_Buffers_bytes{cluster=~\"$cluster\"} - \n node_memory_Cached_bytes{cluster=~\"$cluster\"}\n )\n /\n node_memory_MemTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n )\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", + "expr": "avg(\n (\n (\n node_memory_MemTotal_bytes{cluster=~\"$cluster\"} - \n node_memory_MemFree_bytes{cluster=~\"$cluster\"} - \n node_memory_Buffers_bytes{cluster=~\"$cluster\"} - \n node_memory_Cached_bytes{cluster=~\"$cluster\"}\n )\n /\n node_memory_MemTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n )\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move((DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"}), \"Hostname\", \"nodename\")), 1, 1)\n)", "format": "time_series", "hide": false, "instant": false, @@ -2484,7 +2484,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(\n (\n (\n node_memory_MemTotal_bytes{cluster=~\"$cluster\"} - \n node_memory_MemFree_bytes{cluster=~\"$cluster\"} - \n node_memory_Buffers_bytes{cluster=~\"$cluster\"} - \n node_memory_Cached_bytes{cluster=~\"$cluster\"}\n )\n /\n node_memory_MemTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n )\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", + "expr": "min(\n (\n (\n node_memory_MemTotal_bytes{cluster=~\"$cluster\"} - \n node_memory_MemFree_bytes{cluster=~\"$cluster\"} - \n node_memory_Buffers_bytes{cluster=~\"$cluster\"} - \n node_memory_Cached_bytes{cluster=~\"$cluster\"}\n )\n /\n node_memory_MemTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n )\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move((DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"}), \"Hostname\", \"nodename\")), 1, 1)\n)", "format": "time_series", "hide": false, "instant": false, @@ -2604,7 +2604,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg by (exported_pod) (DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "avg by (exported_pod) (DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -2708,7 +2708,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg by (exported_pod) (DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}/(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}+DCGM_FI_DEV_FB_FREE{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}))", + "expr": "avg by (exported_pod) (DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\"}/(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\"}+DCGM_FI_DEV_FB_FREE{cluster=~\"$cluster\", exported_pod=~\"$worker\"}) and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -2814,7 +2814,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg by (exported_pod) (DCGM_FI_DEV_MEM_COPY_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "avg by (exported_pod) (DCGM_FI_DEV_MEM_COPY_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -2915,7 +2915,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (exported_pod) (DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "sum by (exported_pod) (DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -3019,7 +3019,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg by (exported_pod) (DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "avg by (exported_pod) (DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"} and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -3134,7 +3134,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (err_msg, exported_pod) (rate(DCGM_FI_DEV_XID_ERRORS{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}))", + "expr": "sum by (err_msg, exported_pod) (rate(DCGM_FI_DEV_XID_ERRORS{cluster=~\"$cluster\", exported_pod=~\"$worker\"}) and on(cluster, node_name) slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"})", "instant": false, "legendFormat": "{{exported_pod}}: {{err_msg}}", "range": true, @@ -3265,7 +3265,6 @@ "type": "query" }, { - "allValue": ".*", "current": { "text": "All", "value": "$__all" @@ -3274,7 +3273,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"},exported_pod)", + "definition": "label_values(slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"},node_name)", "description": "Filtered by selected Jobs", "includeAll": true, "label": "Workers", @@ -3283,7 +3282,7 @@ "options": [], "query": { "qryType": 1, - "query": "label_values(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"},exported_pod)", + "query": "label_values(slurm_node_job{cluster=~\"$cluster\", job_id=~\"$slurm_job\"},node_name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, diff --git a/helm/soperator-monitoring-dashboards/dashboards/slurm_controller.json b/helm/soperator-monitoring-dashboards/dashboards/slurm_controller.json index 14e8c2ce3..f5134ee43 100644 --- a/helm/soperator-monitoring-dashboards/dashboards/slurm_controller.json +++ b/helm/soperator-monitoring-dashboards/dashboards/slurm_controller.json @@ -710,7 +710,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Share of GPUs Slurm is currently using, as reported by the DCGM exporter via the `hpc_job` label (set by soperator's `map_job_dcgm.sh` prolog/epilog).\n\nBoth numerator and denominator are joined with `slurm_node_info{is_unavailable=\"\"}` on `exported_pod` \u2194 `node_name`, so only GPUs on *available* nodes (not DOWN or DRAINED) are counted. This means drained/down hosts don't dilute the score on either side \u2014 no manual ratio correction needed.\n\nRequires the DCGM exporter and soperator's GPU-mapping scripts (default in the chart). CPU-only clusters or non-soperator deployments will show \"No data\".\n\nValue shown is the **1h moving average**, not the instantaneous reading \u2014 smooths out scrape jitter and short bursts.\n\n**Note on join keys**: DCGM's `pod` label is the DaemonSet-pod name and conflicts with the Prometheus scrape target's `pod`, so the scrape rules relabel the original (worker pod) to `exported_pod`. That's the label we join on.", + "description": "Share of GPUs Slurm is currently using: a GPU counts as allocated when its node's Slurm base state is ALLOCATED or MIXED (from `slurm_node_info`).\n\nBoth numerator and denominator are joined with `slurm_node_info{is_unavailable=\"\"}` on `exported_pod` \u2194 `node_name`, so only GPUs on *available* nodes (not DOWN or DRAINED) are counted. This means drained/down hosts don't dilute the score on either side \u2014 no manual ratio correction needed.\n\nRequires the DCGM exporter and the soperator Slurm exporter (both default in the chart). CPU-only clusters or non-soperator deployments will show \"No data\".\n\nValue shown is the **1h moving average**, not the instantaneous reading \u2014 smooths out scrape jitter and short bursts.\n\n**Note on join keys**: DCGM's `pod` label is the DaemonSet-pod name and conflicts with the Prometheus scrape target's `pod`, so the scrape rules relabel the original (worker pod) to `exported_pod` and copy it to `node_name` — the worker pod name equals the Slurm node name. That's the label we join on.", "fieldConfig": { "defaults": { "color": { @@ -770,7 +770,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg_over_time((100 * (count(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job!=\"\"} and on(exported_pod) label_replace(slurm_node_info{cluster=~\"$cluster\", is_unavailable=\"\"}, \"exported_pod\", \"$1\", \"node_name\", \"(.*)\")) or vector(0)) / count(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\"} and on(exported_pod) label_replace(slurm_node_info{cluster=~\"$cluster\", is_unavailable=\"\"}, \"exported_pod\", \"$1\", \"node_name\", \"(.*)\")))[1h:])", + "expr": "avg_over_time((100 * (count(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\"} and on(node_name) slurm_node_info{cluster=~\"$cluster\", is_unavailable=\"\", state_base=~\"ALLOCATED|MIXED\"}) or vector(0)) / count(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\"} and on(node_name) slurm_node_info{cluster=~\"$cluster\", is_unavailable=\"\"}))[1h:])", "legendFormat": "allocated %", "range": true, "refId": "A" @@ -1995,7 +1995,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Share of GPUs Slurm is currently using, per-scrape. Both numerator and denominator are joined with `slurm_node_info{is_unavailable=\"\"}` on `exported_pod` \u2194 `node_name`, so only GPUs on *available* nodes (not DOWN or DRAINED) are counted \u2014 drained/down hosts don't dilute the score on either side.\n\nComplement to the GPU Utilization % panel \u2014 together they show scheduler packing vs actual GPU work.\n\n**Note on join keys**: DCGM's `pod` label is the DaemonSet-pod name and conflicts with the Prometheus scrape target's `pod`, so the scrape rules relabel the original (worker pod) to `exported_pod`. That's the label we join on.", + "description": "Share of GPUs Slurm is currently using, per-scrape. Both numerator and denominator are joined with `slurm_node_info{is_unavailable=\"\"}` on `exported_pod` \u2194 `node_name`, so only GPUs on *available* nodes (not DOWN or DRAINED) are counted \u2014 drained/down hosts don't dilute the score on either side.\n\nComplement to the GPU Utilization % panel \u2014 together they show scheduler packing vs actual GPU work.\n\n**Note on join keys**: DCGM's `pod` label is the DaemonSet-pod name and conflicts with the Prometheus scrape target's `pod`, so the scrape rules relabel the original (worker pod) to `exported_pod` and copy it to `node_name` — the worker pod name equals the Slurm node name. That's the label we join on.", "fieldConfig": { "defaults": { "color": { @@ -2081,7 +2081,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "100 * (count(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job!=\"\"} and on(exported_pod) label_replace(slurm_node_info{cluster=~\"$cluster\", is_unavailable=\"\"}, \"exported_pod\", \"$1\", \"node_name\", \"(.*)\")) or vector(0)) / count(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\"} and on(exported_pod) label_replace(slurm_node_info{cluster=~\"$cluster\", is_unavailable=\"\"}, \"exported_pod\", \"$1\", \"node_name\", \"(.*)\"))", + "expr": "100 * (count(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\"} and on(node_name) slurm_node_info{cluster=~\"$cluster\", is_unavailable=\"\", state_base=~\"ALLOCATED|MIXED\"}) or vector(0)) / count(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\"} and on(node_name) slurm_node_info{cluster=~\"$cluster\", is_unavailable=\"\"})", "legendFormat": "allocated", "range": true, "refId": "A" diff --git a/helm/soperator-monitoring-dashboards/dashboards/workers_detailed_stats.json b/helm/soperator-monitoring-dashboards/dashboards/workers_detailed_stats.json index f0f65674c..524e07995 100644 --- a/helm/soperator-monitoring-dashboards/dashboards/workers_detailed_stats.json +++ b/helm/soperator-monitoring-dashboards/dashboards/workers_detailed_stats.json @@ -1131,7 +1131,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n rate(node_infiniband_port_data_transmitted_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n)\n", + "expr": "sum(\n rate(node_infiniband_port_data_transmitted_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n)\n", "format": "time_series", "hide": false, "interval": "", @@ -1146,7 +1146,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n rate(node_infiniband_port_data_received_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", + "expr": "sum(\n rate(node_infiniband_port_data_received_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", "format": "time_series", "hide": false, "instant": false, @@ -1685,7 +1685,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"system\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", + "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"system\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -1700,7 +1700,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"user\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", + "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"user\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", "format": "time_series", "intervalFactor": 1, "legendFormat": "User - Normal processes executing in user mode", @@ -1714,7 +1714,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"nice\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", + "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"nice\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Nice - Niced processes executing in user mode", @@ -1728,7 +1728,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"iowait\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", + "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"iowait\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Iowait - Waiting for I/O to complete", @@ -1742,7 +1742,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"irq\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", + "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"irq\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Irq - Servicing interrupts", @@ -1756,7 +1756,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"softirq\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", + "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"softirq\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Softirq - Servicing softirqs", @@ -1770,7 +1770,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"steal\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", + "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"steal\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment", @@ -1784,7 +1784,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"idle\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", + "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"idle\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left() clamp(count by (nodename) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", "format": "time_series", "hide": false, "intervalFactor": 1,