feat: add metric for sync durations

add syncDuration metric
add syncDuration metric unit test

Signed-off-by: Jack-R-lantern <[email protected]>

Author: Jack-R-lantern

controller/appcontroller.go

@@ -1587,6 +1587,7 @@ func (ctrl *ApplicationController) setOperationState(app *appv1.Application, sta
 			destServer = destCluster.Server
 		}
 		ctrl.metricsServer.IncSync(app, destServer, state)
+		ctrl.metricsServer.IncAppSyncDuration(app, destServer, state)
 	}
 }
 

controller/metrics/metrics.go

@@ -33,6 +33,7 @@ import (
 type MetricsServer struct {
 	*http.Server
 	syncCounter                       *prometheus.CounterVec
+	syncDuration                      *prometheus.CounterVec
 	kubectlExecCounter                *prometheus.CounterVec
 	kubectlExecPendingGauge           *prometheus.GaugeVec
 	orphanedResourcesGauge            *prometheus.GaugeVec
@@ -76,6 +77,14 @@ var (
 		append(descAppDefaultLabels, "dest_server", "phase", "dry_run"),
 	)
 
+	syncDuration = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "argocd_app_sync_duration_seconds_total",
+			Help: "Application sync performance in seconds total.",
+		},
+		append(descAppDefaultLabels, "dest_server"),
+	)
+
 	k8sRequestCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "argocd_app_k8s_request_total",
@@ -188,6 +197,7 @@ func NewMetricsServer(addr string, appLister applister.ApplicationLister, appFil
 	healthz.ServeHealthCheck(mux, healthCheck)
 
 	registry.MustRegister(syncCounter)
+	registry.MustRegister(syncDuration)
 	registry.MustRegister(k8sRequestCounter)
 	registry.MustRegister(kubectlExecCounter)
 	registry.MustRegister(kubectlExecPendingGauge)
@@ -209,6 +219,7 @@ func NewMetricsServer(addr string, appLister applister.ApplicationLister, appFil
 			Handler: mux,
 		},
 		syncCounter:                       syncCounter,
+		syncDuration:                      syncDuration,
 		k8sRequestCounter:                 k8sRequestCounter,
 		kubectlExecCounter:                kubectlExecCounter,
 		kubectlExecPendingGauge:           kubectlExecPendingGauge,
@@ -243,6 +254,14 @@ func (m *MetricsServer) IncSync(app *argoappv1.Application, destServer string, s
 	m.syncCounter.WithLabelValues(app.Namespace, app.Name, app.Spec.GetProject(), destServer, string(state.Phase), strconv.FormatBool(isDryRun)).Inc()
 }
 
+// IncAppSyncDuration observes app sync duration
+func (m *MetricsServer) IncAppSyncDuration(app *argoappv1.Application, destServer string, state *argoappv1.OperationState) {
+	if state.FinishedAt != nil {
+		m.syncDuration.WithLabelValues(app.Namespace, app.Name, app.Spec.GetProject(), destServer).
+			Add(float64(time.Duration(state.FinishedAt.Unix() - state.StartedAt.Unix())))
+	}
+}
+
 func (m *MetricsServer) IncKubectlExec(command string) {
 	m.kubectlExecCounter.WithLabelValues(m.hostname, command).Inc()
 }
@@ -314,6 +333,7 @@ func (m *MetricsServer) SetExpiration(cacheExpiration time.Duration) error {
 	_, err := m.cron.AddFunc(fmt.Sprintf("@every %s", cacheExpiration), func() {
 		log.Infof("Reset Prometheus metrics based on existing expiration '%v'", cacheExpiration)
 		m.syncCounter.Reset()
+		m.syncDuration.Reset()
 		m.kubectlExecCounter.Reset()
 		m.kubectlExecPendingGauge.Reset()
 		m.orphanedResourcesGauge.Reset()

controller/metrics/metrics_test.go

@@ -178,6 +178,63 @@ status:
     status: Healthy
 `
 
+const fakeAppOperationRunning = `
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: my-app
+  namespace: argocd
+  labels:
+    team-name: my-team
+    team-bu: bu-id
+    argoproj.io/cluster: test-cluster
+spec:
+  destination:
+    namespace: dummy-namespace
+    name: cluster1
+  project: important-project
+  source:
+    path: some/path
+    repoURL: https://github.com/argoproj/argocd-example-apps.git
+status:
+  sync:
+    status: OutOfSync
+  health:
+    status: Progressing
+  operationState:
+    phase: Running
+    startedAt: "2025-01-29T08:42:34Z"
+`
+
+const fakeAppOperationFinished = `
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: my-app
+  namespace: argocd
+  labels:
+    team-name: my-team
+    team-bu: bu-id
+    argoproj.io/cluster: test-cluster
+spec:
+  destination:
+    namespace: dummy-namespace
+    name: cluster1
+  project: important-project
+  source:
+    path: some/path
+    repoURL: https://github.com/argoproj/argocd-example-apps.git
+status:
+  sync:
+    status: Synced
+  health:
+    status: Healthy
+  operationState:
+    phase: Succeeded
+    startedAt: "2025-01-29T08:42:34Z"
+    finishedAt: "2025-01-29T08:42:35Z"
+`
+
 var noOpHealthCheck = func(_ *http.Request) error {
 	return nil
 }
@@ -468,6 +525,43 @@ func assertMetricsNotPrinted(t *testing.T, expectedLines, body string) {
 	}
 }
 
+func TestMetricsSyncDuration(t *testing.T) {
+	cancel, appLister := newFakeLister()
+	defer cancel()
+	mockDB := mocks.NewArgoDB(t)
+	metricsServ, err := NewMetricsServer("localhost:8082", appLister, appFilter, noOpHealthCheck, []string{}, []string{}, mockDB)
+	require.NoError(t, err)
+
+	fakeAppOperationRunning := newFakeApp(fakeAppOperationRunning)
+	metricsServ.IncAppSyncDuration(fakeAppOperationRunning, "https://localhost:6443", fakeAppOperationRunning.Status.OperationState)
+
+	req, err := http.NewRequest(http.MethodGet, "/metrics", nil)
+	require.NoError(t, err)
+	rr := httptest.NewRecorder()
+	metricsServ.Handler.ServeHTTP(rr, req)
+	assert.Equal(t, http.StatusOK, rr.Code)
+	body := rr.Body.String()
+	log.Println(body)
+	assert.NotContains(t, body, "argocd_app_sync_duration_seconds_total")
+
+	fakeAppOperationFinished := newFakeApp(fakeAppOperationFinished)
+	metricsServ.IncAppSyncDuration(fakeAppOperationFinished, "https://localhost:6443", fakeAppOperationFinished.Status.OperationState)
+
+	req, err = http.NewRequest(http.MethodGet, "/metrics", nil)
+	require.NoError(t, err)
+	rr = httptest.NewRecorder()
+	metricsServ.Handler.ServeHTTP(rr, req)
+	assert.Equal(t, http.StatusOK, rr.Code)
+	body = rr.Body.String()
+	appSyncDurationTotal := `
+# HELP argocd_app_sync_duration_seconds_total Application sync performance in seconds total.
+# TYPE argocd_app_sync_duration_seconds_total counter
+argocd_app_sync_duration_seconds_total{dest_server="https://localhost:6443",name="my-app",namespace="argocd",project="important-project"} 1
+`
+	log.Println(body)
+	assertMetricsPrinted(t, appSyncDurationTotal, body)
+}
+
 func TestReconcileMetrics(t *testing.T) {
 	cancel, appLister := newFakeLister()
 	defer cancel()