From 803de063c82a17c3729cefcc9bc83100e6a2dcea Mon Sep 17 00:00:00 2001 From: Vladimir Videlov Date: Fri, 26 Jun 2026 14:37:04 +0200 Subject: [PATCH 1/2] Add WebhookCertificateRotationDeferredByUnhealthyMRs alert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit webhook-injector now defers a non-force cert rotation (expired, san_mismatch, etc.) when at least one labeled ManagedResource has a ResourcesApplied or ResourcesHealthy condition with Status != True. The cert keeps being served from the existing valid leaf; rotation resumes automatically when the MR(s) recover. This counter fires when that deferral is active. A sustained non-zero rate over 15m means a labeled MR has been unhealthy long enough that a cert rotation is now being held back by it — combined with the existing WebhookManagedResourceUnhealthy alert it tells on-call exactly what to fix (and the CertificateAboutToExpire alert is the safety net if the underlying issue isn't fixed before the cert reaches NotAfter; at that point the gate's escape hatch lets the rotation proceed regardless and the counter keeps ticking, so this alert stays firing throughout the under-duress rotation as well). Style follows the existing webhook-injector propagation alerts: sum by (namespace), 5m rate window, 15m for, warning severity, label-enriched summary/description naming the affected shoot. Patch-bump chart + plugin to 1.1.14. Verified: - helm lint: clean - promtool check rules: SUCCESS, 12 rules found (was 11; added one). --- charts/controlplane-operations/Chart.yaml | 2 +- .../alerts/controlplane-remote.yaml | 16 ++++++++++++++++ .../plugindefinition.yaml | 4 ++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/charts/controlplane-operations/Chart.yaml b/charts/controlplane-operations/Chart.yaml index 687d4cd..fb2b31e 100644 --- a/charts/controlplane-operations/Chart.yaml +++ b/charts/controlplane-operations/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: controlplane-operations -version: 1.1.17 +version: 1.1.18 description: A set of Plutono dashboards and Prometheus alerting rules combined with playbooks to ensure effective operations of Controlplane clusters. maintainers: - name: Vladimir Videlov (d051408) diff --git a/charts/controlplane-operations/alerts/controlplane-remote.yaml b/charts/controlplane-operations/alerts/controlplane-remote.yaml index d19e4ba..2f073de 100644 --- a/charts/controlplane-operations/alerts/controlplane-remote.yaml +++ b/charts/controlplane-operations/alerts/controlplane-remote.yaml @@ -195,3 +195,19 @@ groups: summary: "gardener.cloud/operation=reconcile patch failing on shoot {{`{{ $labels.namespace }}`}}." description: "webhook-injector stamped a ManagedResource Secret in namespace {{`{{ $labels.namespace }}`}} but failed to patch the gardener.cloud/operation=reconcile annotation on the parent MR. gardener-resource-manager may take up to its own watch/sync interval to notice the change rather than reconciling immediately — caBundle propagation to the shoot will still happen, but with extra latency. Check the webhook-injector logs in namespace {{`{{ $labels.namespace }}`}} for the affected MR name." {{- end }} + +{{- if not (.Values.prometheusRules.disabled.WebhookCertificateRotationDeferredByUnhealthyMRs | default false) }} + - alert: WebhookCertificateRotationDeferredByUnhealthyMRs + expr: | + sum by (namespace) (rate(webhook_injector_certificate_rotation_deferred_unhealthy_mrs_total[5m])) > 0 + for: {{ dig "WebhookCertificateRotationDeferredByUnhealthyMRs" "for" "15m" .Values.prometheusRules }} + labels: + {{ include "controlplane-operations.additionalRuleLabels" . }} + severity: {{ dig "WebhookCertificateRotationDeferredByUnhealthyMRs" "severity" "warning" .Values.prometheusRules }} + playbook: https://operations.global.cloud.sap/docs/support/playbook/webhook-certificate-rotation-deferred-by-unhealthy-mrs/ #TODO: add playbook + service: {{ dig "WebhookCertificateRotationDeferredByUnhealthyMRs" "service" .Values.prometheusRules.defaultService .Values.prometheusRules }} + support_group: {{ dig "WebhookCertificateRotationDeferredByUnhealthyMRs" "support_group" .Values.prometheusRules.defaultSupportGroup .Values.prometheusRules }} + annotations: + summary: "Certificate rotation on shoot {{`{{ $labels.namespace }}`}} is being held back by unhealthy ManagedResources." + description: "webhook-injector observed a non-force certificate rotation reason (expired/san_mismatch/etc.) on shoot {{`{{ $labels.namespace }}`}} but is deferring the rotation because at least one labeled ManagedResource has ResourcesApplied or ResourcesHealthy != True. The existing cert keeps being served; rotation will resume automatically when the MR(s) recover. If the cert reaches NotAfter before that happens, the rotation will proceed anyway with a louder warning log — but webhook calls on shoot-side CRDs may TLS-fail during the gap. Check the webhook-injector logs in namespace {{`{{ $labels.namespace }}`}} for the offending MR names, and the GRM/MR status for the underlying cause." +{{- end }} diff --git a/charts/controlplane-operations/plugindefinition.yaml b/charts/controlplane-operations/plugindefinition.yaml index e42fe5a..0f58fd6 100644 --- a/charts/controlplane-operations/plugindefinition.yaml +++ b/charts/controlplane-operations/plugindefinition.yaml @@ -3,7 +3,7 @@ kind: PluginDefinition metadata: name: controlplane-operations spec: - version: 1.1.17 + version: 1.1.18 displayName: Controlplane operations bundle description: Operations bundle for Controlplane clusters docMarkDownUrl: https://raw.githubusercontent.com/cloudoperators/controlplane-operations/main/README.md @@ -11,7 +11,7 @@ spec: helmChart: name: controlplane-operations repository: oci://ghcr.io/cloudoperators/controlplane-operations/charts - version: 1.1.17 + version: 1.1.18 options: - name: prometheusRules.create description: Create Prometheus rules From 1762bbc9df59af581038d94757da260867b97c6d Mon Sep 17 00:00:00 2001 From: Vladimir Videlov Date: Thu, 24 Sep 2026 12:35:26 +0200 Subject: [PATCH 2/2] Resolve playbook URL for WebhookCertificateRotationDeferredByUnhealthyMRs The playbook now exists in documentation-operation, so drop the '#TODO: add playbook' marker from the alert's playbook annotation. Refs cc/unified-kubernetes#1305 --- charts/controlplane-operations/alerts/controlplane-remote.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/controlplane-operations/alerts/controlplane-remote.yaml b/charts/controlplane-operations/alerts/controlplane-remote.yaml index 2f073de..d6d39c2 100644 --- a/charts/controlplane-operations/alerts/controlplane-remote.yaml +++ b/charts/controlplane-operations/alerts/controlplane-remote.yaml @@ -204,7 +204,7 @@ groups: labels: {{ include "controlplane-operations.additionalRuleLabels" . }} severity: {{ dig "WebhookCertificateRotationDeferredByUnhealthyMRs" "severity" "warning" .Values.prometheusRules }} - playbook: https://operations.global.cloud.sap/docs/support/playbook/webhook-certificate-rotation-deferred-by-unhealthy-mrs/ #TODO: add playbook + playbook: https://operations.global.cloud.sap/docs/support/playbook/webhook-certificate-rotation-deferred-by-unhealthy-mrs/ service: {{ dig "WebhookCertificateRotationDeferredByUnhealthyMRs" "service" .Values.prometheusRules.defaultService .Values.prometheusRules }} support_group: {{ dig "WebhookCertificateRotationDeferredByUnhealthyMRs" "support_group" .Values.prometheusRules.defaultSupportGroup .Values.prometheusRules }} annotations: