Merge pull request #399 from justinblalock87/cleanup-metrics

Add metrics for cleanup controller

Author: k8s-ci-robot

cmd/node-cleanup/main.go

@@ -18,15 +18,23 @@ package main
 
 import (
 	"context"
+	"log"
+	"net/http"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	flag "github.com/spf13/pflag"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 
+	"k8s.io/component-base/metrics/legacyregistry"
+	_ "k8s.io/component-base/metrics/prometheus/clientgo" // for client metric registration
+
 	"k8s.io/klog/v2"
+	metrics "sigs.k8s.io/sig-storage-local-static-provisioner/pkg/metrics/node-cleanup"
 	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/node-cleanup/controller"
 	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/node-cleanup/deleter"
 )
@@ -40,6 +48,8 @@ var (
 	workerThreads            = flag.Uint("worker-threads", 10, "Number of controller worker threads.")
 	pvcDeletionDelay         = flag.Duration("pvc-deletion-delay", 60*time.Second, "Duration, in seconds, to wait after Node deletion for PVC cleanup.")
 	stalePVDiscoveryInterval = flag.Duration("stale-pv-discovery-interval", 10*time.Second, "Duration, in seconds, the PV Deleter should wait between tries to clean up stale PVs.")
+	listenAddress            = flag.String("listen-address", ":8080", "The TCP network address where the prometheus metrics endpoint will listen (example: `:8080`).")
+	metricsPath              = flag.String("metrics-path", "/metrics", "The HTTP path where prometheus metrics will be exposed.")
 )
 
 func main() {
@@ -77,6 +87,28 @@ func main() {
 
 	factory.Start(ctx.Done())
 
+	// Prepare http endpoint for metrics
+	if *listenAddress != "" {
+		reg := prometheus.NewRegistry()
+		reg.MustRegister([]prometheus.Collector{
+			metrics.PersistentVolumeDeleteTotal,
+			metrics.PersistentVolumeDeleteFailedTotal,
+			metrics.PersistentVolumeClaimDeleteTotal,
+			metrics.PersistentVolumeClaimDeleteFailedTotal,
+		}...)
+		gatherers := prometheus.Gatherers{
+			reg,
+			legacyregistry.DefaultGatherer,
+		}
+
+		http.Handle(*metricsPath, promhttp.HandlerFor(gatherers, promhttp.HandlerOpts{}))
+
+		go func() {
+			klog.Infof("Starting metrics server at %s\n", *listenAddress)
+			log.Fatal(http.ListenAndServe(*listenAddress, nil))
+		}()
+	}
+
 	// Start Deleter
 	go deleter.Run(ctx, *stalePVDiscoveryInterval)
 

deployment/kubernetes/example/node-cleanup-controller/deployment.yaml

@@ -19,4 +19,7 @@ spec:
         args:
           - "--storageclass-names=nvme-ssd-block"
           - "--pvc-deletion-delay=60s"
-          - "--stale-pv-discovery-interval=10s"
+          - "--stale-pv-discovery-interval=10s"
+        ports:
+          - name: metrics
+            containerPort: 8080

docs/node-cleanup-controller.md

@@ -26,6 +26,8 @@ Please see the example [deployment](../deployment/kubernetes/example/node-cleanu
 * `--kube-api-endpoint`: Master URL to build a client config from. Either this or kubeconfig needs to be set if the provisioner is being run out of cluster.
 * `--resync`: Duration, in minutes, of the resync interval of the controller. Defaults to 10 minutes.
 * `--worker-threads`: Number of controller worker threads. Defaults to 10.
+* `--listen-address`: The TCP network address where the prometheus metrics endpoint will listen. Defaults to `:8080`.
+* `--metrics-path`: The HTTP path where prometheus metrics will be exposed. Defaults to "/metrics".
 
 ## Design
 

pkg/metrics/node-cleanup/metrics.go

@@ -0,0 +1,65 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	// LocalVolumeNodeCleanupSubsystem is prometheus subsystem name.
+	LocalVolumeNodeCleanupSubsystem = "local_volume_node_cleanup"
+)
+
+var (
+	// PersistentVolumeDeleteTotal is used to collect accumulated count of persistent volume delete attempts.
+	// This metric will report as higher than the true amount of persistent volumes deleted if
+	// the node-cleanup deleter has a short sync period.
+	PersistentVolumeDeleteTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: LocalVolumeNodeCleanupSubsystem,
+			Name:      "persistentvolume_delete_total",
+			Help:      "Total number of successful persistent volume delete *attempts*. Broken down by persistent volume phase.",
+		},
+		[]string{"phase"},
+	)
+	// PersistentVolumeDeleteFailedTotal is used to collect accumulated count of persistent volume delete failed attempts.
+	PersistentVolumeDeleteFailedTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: LocalVolumeNodeCleanupSubsystem,
+			Name:      "persistentvolume_delete_failed_total",
+			Help:      "Total number of persistent volume delete failed attempts. Broken down by persistent volume phase.",
+		},
+		[]string{"phase"},
+	)
+	// PersistentVolumeClaimDeleteTotal is used to collect accumulated count of persistent volume claims deleted.
+	PersistentVolumeClaimDeleteTotal = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Subsystem: LocalVolumeNodeCleanupSubsystem,
+			Name:      "persistentvolumeclaim_delete_total",
+			Help:      "Total number of persistent volume claims deleted.",
+		},
+	)
+	// PersistentVolumeClaimDeleteFailedTotal is used to collect accumulated count of persistent volume claim delete failed attempts.
+	PersistentVolumeClaimDeleteFailedTotal = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Subsystem: LocalVolumeNodeCleanupSubsystem,
+			Name:      "persistentvolumeclaim_delete_failed_total",
+			Help:      "Total number of persistent volume claim delete failed attempts.",
+		},
+	)
+)

pkg/node-cleanup/controller/controller.go

@@ -38,6 +38,7 @@ import (
 	"k8s.io/klog/v2"
 
 	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/common"
+	cleanupmetrics "sigs.k8s.io/sig-storage-local-static-provisioner/pkg/metrics/node-cleanup"
 )
 
 // CleanupController handles the deletion of PVCs that reference deleted Nodes.
@@ -235,10 +236,12 @@ func (c *CleanupController) syncHandler(ctx context.Context, pvName string) erro
 
 	err = c.deletePVC(ctx, pvc)
 	if err != nil {
+		cleanupmetrics.PersistentVolumeClaimDeleteFailedTotal.Inc()
 		klog.Errorf("failed to delete pvc %q in namespace &q: %w", pvClaimRef.Name, pvClaimRef.Namespace, err)
 		return err
 	}
 
+	cleanupmetrics.PersistentVolumeClaimDeleteTotal.Inc()
 	klog.Infof("Deleted PVC %q that pointed to Node %q", pvClaimRef.Name, nodeName)
 	return nil
 }

pkg/node-cleanup/deleter/deleter.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/klog/v2"
 
 	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/common"
+	cleanupmetrics "sigs.k8s.io/sig-storage-local-static-provisioner/pkg/metrics/node-cleanup"
 )
 
 // Deleter handles cleanup of local PVs with an affinity to a deleted Node.
@@ -91,15 +92,22 @@ func (d *Deleter) DeletePVs(ctx context.Context) {
 			continue
 		}
 
+		phase := pv.Status.Phase
+		reclaimPolicy := pv.Spec.PersistentVolumeReclaimPolicy
 		// PV is a stale object since it references a deleted Node.
 		// Therefore it can safely be deleted in the two following cases.
-		isReleasedWithDeleteReclaim := pv.Status.Phase == v1.VolumeReleased && pv.Spec.PersistentVolumeReclaimPolicy == v1.PersistentVolumeReclaimDelete
-		isAvailable := pv.Status.Phase == v1.VolumeAvailable
+		isReleasedWithDeleteReclaim := phase == v1.VolumeReleased && reclaimPolicy == v1.PersistentVolumeReclaimDelete
+		isAvailable := phase == v1.VolumeAvailable
 		if isReleasedWithDeleteReclaim || isAvailable {
-			klog.Infof("Deleting PV that has NodeAffinity to deleted Node, pv: %s", pv.Name)
+			klog.Infof("Attempting to delete PV that has NodeAffinity to deleted Node, pv: %s", pv.Name)
 			if err = d.deletePV(ctx, pv.Name); err != nil {
+				cleanupmetrics.PersistentVolumeDeleteFailedTotal.WithLabelValues(string(phase)).Inc()
 				klog.Errorf("Error deleting PV: %s", pv.Name)
+				continue
 			}
+			// TODO: Cache successful deletion to avoid multiple delete calls
+			// when there is a short sync period
+			cleanupmetrics.PersistentVolumeDeleteTotal.WithLabelValues(string(phase)).Inc()
 		}
 	}
 }