Add WebhookCertificateRotationDeferredByUnhealthyMRs alert

charts/controlplane-operations/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: controlplane-operations
-version: 1.1.13
+version: 1.1.14
 description: A set of Plutono dashboards and Prometheus alerting rules combined with playbooks to ensure effective operations of Controlplane clusters.
 maintainers:
   - name: Vladimir Videlov (d051408)

charts/controlplane-operations/alerts/controlplane-remote.yaml

@@ -179,3 +179,19 @@ groups:
       summary: "gardener.cloud/operation=reconcile patch failing on shoot {{`{{ $labels.namespace }}`}}."
       description: "webhook-injector stamped a ManagedResource Secret in namespace {{`{{ $labels.namespace }}`}} but failed to patch the gardener.cloud/operation=reconcile annotation on the parent MR. gardener-resource-manager may take up to its own watch/sync interval to notice the change rather than reconciling immediately — caBundle propagation to the shoot will still happen, but with extra latency. Check the webhook-injector logs in namespace {{`{{ $labels.namespace }}`}} for the affected MR name."
 {{- end }}
+
+{{- if not (.Values.prometheusRules.disabled.WebhookCertificateRotationDeferredByUnhealthyMRs | default false) }}
+  - alert: WebhookCertificateRotationDeferredByUnhealthyMRs
+    expr: |
+      sum by (namespace) (rate(webhook_injector_certificate_rotation_deferred_unhealthy_mrs_total[5m])) > 0
+    for: {{ dig "WebhookCertificateRotationDeferredByUnhealthyMRs" "for" "15m" .Values.prometheusRules }}
+    labels:
+      {{ include "controlplane-operations.additionalRuleLabels" . }}
+      severity: {{ dig "WebhookCertificateRotationDeferredByUnhealthyMRs" "severity" "warning" .Values.prometheusRules }}
+      playbook: https://operations.global.cloud.sap/docs/support/playbook/webhook-certificate-rotation-deferred-by-unhealthy-mrs/ #TODO: add playbook
+      service: {{ dig "WebhookCertificateRotationDeferredByUnhealthyMRs" "service" .Values.prometheusRules.defaultService .Values.prometheusRules }}
+      support_group: {{ dig "WebhookCertificateRotationDeferredByUnhealthyMRs" "support_group" .Values.prometheusRules.defaultSupportGroup .Values.prometheusRules }}
+    annotations:
+      summary: "Certificate rotation on shoot {{`{{ $labels.namespace }}`}} is being held back by unhealthy ManagedResources."
+      description: "webhook-injector observed a non-force certificate rotation reason (expired/san_mismatch/etc.) on shoot {{`{{ $labels.namespace }}`}} but is deferring the rotation because at least one labeled ManagedResource has ResourcesApplied or ResourcesHealthy != True. The existing cert keeps being served; rotation will resume automatically when the MR(s) recover. If the cert reaches NotAfter before that happens, the rotation will proceed anyway with a louder warning log — but webhook calls on shoot-side CRDs may TLS-fail during the gap. Check the webhook-injector logs in namespace {{`{{ $labels.namespace }}`}} for the offending MR names, and the GRM/MR status for the underlying cause."
+{{- end }}

charts/controlplane-operations/plugindefinition.yaml

@@ -3,15 +3,15 @@ kind: PluginDefinition
 metadata:
   name: controlplane-operations
 spec:
-  version: 1.1.13
+  version: 1.1.14
   displayName: Controlplane operations bundle
   description: Operations bundle for Controlplane clusters
   docMarkDownUrl: https://raw.githubusercontent.com/cloudoperators/controlplane-operations/main/README.md
   icon: https://raw.githubusercontent.com/cloudoperators/controlplane-operations/main/charts/controlplane-operations/kubernetes-logo.png
   helmChart:
     name: controlplane-operations
     repository: oci://ghcr.io/cloudoperators/controlplane-operations/charts
-    version: 1.1.13
+    version: 1.1.14
   options:
     - name: prometheusRules.create
       description: Create Prometheus rules