Add metrics for cleanup controller

Author: justinblalock87

cmd/node-cleanup/main.go

@@ -18,15 +18,20 @@ package main
 
 import (
 	"context"
+	"log"
+	"net/http"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	flag "github.com/spf13/pflag"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 
 	"k8s.io/klog/v2"
+	metrics "sigs.k8s.io/sig-storage-local-static-provisioner/pkg/metrics/node-cleanup"
 	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/node-cleanup/controller"
 	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/node-cleanup/deleter"
 )
@@ -40,6 +45,8 @@ var (
 	workerThreads            = flag.Uint("worker-threads", 10, "Number of controller worker threads.")
 	pvcDeletionDelay         = flag.Duration("pvc-deletion-delay", 60*time.Second, "Duration, in seconds, to wait after Node deletion for PVC cleanup.")
 	stalePVDiscoveryInterval = flag.Duration("stale-pv-discovery-interval", 10*time.Second, "Duration, in seconds, the PV Deleter should wait between tries to clean up stale PVs.")
+	listenAddress            = flag.String("listen-address", ":8080", "The TCP network address where the prometheus metrics endpoint will listen (example: `:8080`).")
+	metricsPath              = flag.String("metrics-path", "/metrics", "The HTTP path where prometheus metrics will be exposed.")
 )
 
 func main() {
@@ -77,6 +84,22 @@ func main() {
 
 	factory.Start(ctx.Done())
 
+	// Prepare http endpoint for metrics
+	if *listenAddress != "" {
+		prometheus.MustRegister([]prometheus.Collector{
+			metrics.APIServerRequestsTotal,
+			metrics.PersistentVolumeDeleteFailedTotal,
+			metrics.PersistentVolumeClaimDeleteTotal,
+			metrics.PersistentVolumeClaimDeleteFailedTotal,
+		}...)
+		http.Handle(*metricsPath, promhttp.Handler())
+
+		go func() {
+			klog.Infof("Starting metrics server at %s\n", *listenAddress)
+			log.Fatal(http.ListenAndServe(*listenAddress, nil))
+		}()
+	}
+
 	// Start Deleter
 	go deleter.Run(ctx, *stalePVDiscoveryInterval)
 

deployment/kubernetes/example/node-cleanup-controller/deployment.yaml

@@ -19,4 +19,7 @@ spec:
         args:
           - "--storageclass-names=nvme-ssd-block"
           - "--pvc-deletion-delay=60s"
-          - "--stale-pv-discovery-interval=10s"
+          - "--stale-pv-discovery-interval=10s"
+        ports:
+          - name: metrics
+            containerPort: 8080

docs/node-cleanup-controller.md

@@ -26,6 +26,8 @@ Please see the example [deployment](../deployment/kubernetes/example/node-cleanu
 * `--kube-api-endpoint`: Master URL to build a client config from. Either this or kubeconfig needs to be set if the provisioner is being run out of cluster.
 * `--resync`: Duration, in minutes, of the resync interval of the controller. Defaults to 10 minutes.
 * `--worker-threads`: Number of controller worker threads. Defaults to 10.
+* `--listen-address`: The TCP network address where the prometheus metrics endpoint will listen. Defaults to `:8080`.
+* `--metrics-path`: The HTTP path where prometheus metrics will be exposed. Defaults to "/metrics".
 
 ## Design
 

pkg/metrics/node-cleanup/metrics.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	// LocalVolumeNodeCleanupSubsystem is prometheus subsystem name.
+	LocalVolumeNodeCleanupSubsystem = "local_volume_node_cleanup"
+)
+
+var (
+	// APIServerRequestsTotal is used to collect accumulated count of apiserver requests.
+	APIServerRequestsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: LocalVolumeNodeCleanupSubsystem,
+			Name:      "apiserver_requests_total",
+			Help:      "Total number of apiserver requests. Broken down by method.",
+		},
+		[]string{"method"},
+	)
+	// PersistentVolumeDeleteFailedTotal is used to collect accumulated count of persistent volume delete failed attempts.
+	PersistentVolumeDeleteFailedTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: LocalVolumeNodeCleanupSubsystem,
+			Name:      "persistentvolume_delete_failed_total",
+			Help:      "Total number of persistent volume delete failed attempts. Broken down by persistent volume status and reclaim policy.",
+		},
+		[]string{"status", "reclaim"},
+	)
+	// PersistentVolumeClaimDeleteTotal is used to collect accumulated count of persistent volume claims deleted.
+	PersistentVolumeClaimDeleteTotal = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Subsystem: LocalVolumeNodeCleanupSubsystem,
+			Name:      "persistentvolumeclaim_delete_total",
+			Help:      "Total number of persistent volume claims deleted.",
+		},
+	)
+	// PersistentVolumeClaimDeleteFailedTotal is used to collect accumulated count of persistent volume claim delete failed attempts.
+	PersistentVolumeClaimDeleteFailedTotal = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Subsystem: LocalVolumeNodeCleanupSubsystem,
+			Name:      "persistentvolumeclaim_delete_failed_total",
+			Help:      "Total number of persistent volume claim delete failed attempts.",
+		},
+	)
+)

pkg/node-cleanup/controller/controller.go

@@ -38,6 +38,8 @@ import (
 	"k8s.io/klog/v2"
 
 	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/common"
+	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/metrics"
+	cleanupmetrics "sigs.k8s.io/sig-storage-local-static-provisioner/pkg/metrics/node-cleanup"
 )
 
 // CleanupController handles the deletion of PVCs that reference deleted Nodes.
@@ -235,10 +237,12 @@ func (c *CleanupController) syncHandler(ctx context.Context, pvName string) erro
 
 	err = c.deletePVC(ctx, pvc)
 	if err != nil {
+		cleanupmetrics.PersistentVolumeClaimDeleteFailedTotal.Inc()
 		klog.Errorf("failed to delete pvc %q in namespace &q: %w", pvClaimRef.Name, pvClaimRef.Namespace, err)
 		return err
 	}
 
+	cleanupmetrics.PersistentVolumeClaimDeleteTotal.Inc()
 	klog.Infof("Deleted PVC %q that pointed to Node %q", pvClaimRef.Name, nodeName)
 	return nil
 }
@@ -296,6 +300,7 @@ func (c *CleanupController) deletePVC(ctx context.Context, pvc *v1.PersistentVol
 	options := metav1.DeleteOptions{
 		Preconditions: &metav1.Preconditions{UID: &pvc.UID},
 	}
+	cleanupmetrics.APIServerRequestsTotal.WithLabelValues(metrics.APIServerRequestDelete).Inc()
 	err := c.client.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(ctx, pvc.Name, options)
 	if err != nil && errors.IsNotFound(err) {
 		// The PVC could already be deleted by some other process

pkg/node-cleanup/deleter/deleter.go

@@ -30,6 +30,8 @@ import (
 	"k8s.io/klog/v2"
 
 	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/common"
+	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/metrics"
+	cleanupmetrics "sigs.k8s.io/sig-storage-local-static-provisioner/pkg/metrics/node-cleanup"
 )
 
 // Deleter handles cleanup of local PVs with an affinity to a deleted Node.
@@ -91,14 +93,18 @@ func (d *Deleter) DeletePVs(ctx context.Context) {
 			continue
 		}
 
+		phase := pv.Status.Phase
+		reclaimPolicy := pv.Spec.PersistentVolumeReclaimPolicy
 		// PV is a stale object since it references a deleted Node.
 		// Therefore it can safely be deleted in the two following cases.
-		isReleasedWithDeleteReclaim := pv.Status.Phase == v1.VolumeReleased && pv.Spec.PersistentVolumeReclaimPolicy == v1.PersistentVolumeReclaimDelete
-		isAvailable := pv.Status.Phase == v1.VolumeAvailable
+		isReleasedWithDeleteReclaim := phase == v1.VolumeReleased && reclaimPolicy == v1.PersistentVolumeReclaimDelete
+		isAvailable := phase == v1.VolumeAvailable
 		if isReleasedWithDeleteReclaim || isAvailable {
-			klog.Infof("Deleting PV that has NodeAffinity to deleted Node, pv: %s", pv.Name)
+			klog.Infof("Attempting to delete PV that has NodeAffinity to deleted Node, pv: %s", pv.Name)
 			if err = d.deletePV(ctx, pv.Name); err != nil {
+				cleanupmetrics.PersistentVolumeDeleteFailedTotal.WithLabelValues(string(phase), string(reclaimPolicy)).Inc()
 				klog.Errorf("Error deleting PV: %s", pv.Name)
+				continue
 			}
 		}
 	}
@@ -127,6 +133,7 @@ func (d *Deleter) referencesNonExistentNode(localPV *v1.PersistentVolume) (bool,
 }
 
 func (d *Deleter) deletePV(ctx context.Context, pvName string) error {
+	cleanupmetrics.APIServerRequestsTotal.WithLabelValues(metrics.APIServerRequestDelete).Inc()
 	err := d.client.CoreV1().PersistentVolumes().Delete(ctx, pvName, metav1.DeleteOptions{})
 	if err != nil && errors.IsNotFound(err) {
 		klog.Warningf("PV %q no longer exists", pvName)