Refactor openstack migrations syncer (#311)

Refactor nova syncer for migrations to fetch complete history + minor
adjustment to comments and general clean up

Author: SoWieMarkus

internal/extractor/plugins/shared/vm_host_residency.sql

@@ -21,7 +21,7 @@ WITH durations AS (
             )) AS BIGINT)
         ) AS duration
     FROM openstack_migrations AS migrations
-    LEFT JOIN openstack_servers_v2 AS servers ON servers.id = migrations.instance_uuid
+    LEFT JOIN openstack_servers AS servers ON servers.id = migrations.instance_uuid
     LEFT JOIN openstack_flavors_v2 AS flavors ON flavors.name = servers.flavor_name
 )
 SELECT

internal/extractor/plugins/vmware/vrops_hostsystem_resolver.sql

@@ -3,5 +3,5 @@ SELECT DISTINCT
     m.hostsystem AS vrops_hostsystem,
     s.os_ext_srv_attr_host AS nova_compute_host
 FROM vrops_vm_metrics m
-LEFT JOIN openstack_servers_v2 s ON m.instance_uuid = s.id
+LEFT JOIN openstack_servers s ON m.instance_uuid = s.id
 WHERE s.os_ext_srv_attr_host IS NOT NULL;

internal/extractor/plugins/vmware/vrops_project_noisiness.sql

@@ -19,7 +19,7 @@ host_cpu_usage AS (
         s.tenant_id,
         h.service_host,
         AVG(p.avg_cpu) AS avg_cpu_of_project
-    FROM openstack_servers_v2 s
+    FROM openstack_servers s
     JOIN vrops_vm_metrics m ON s.id = m.instance_uuid
     JOIN projects_avg_cpu p ON s.tenant_id = p.tenant_id
     JOIN openstack_hypervisors h ON s.os_ext_srv_attr_hypervisor_hostname = h.hostname

internal/sync/openstack/nova/nova_api.go

@@ -32,8 +32,8 @@ type NovaAPI interface {
 	GetAllHypervisors(ctx context.Context) ([]Hypervisor, error)
 	// Get all nova flavors.
 	GetAllFlavors(ctx context.Context) ([]Flavor, error)
-	// Get all changed nova migrations since the timestamp.
-	GetChangedMigrations(ctx context.Context, changedSince *time.Time) ([]Migration, error)
+	// Get all nova migrations.
+	GetAllMigrations(ctx context.Context) ([]Migration, error)
 	// Get all aggregates.
 	GetAllAggregates(ctx context.Context) ([]Aggregate, error)
 }
@@ -110,6 +110,11 @@ func (api *novaAPI) GetAllServers(ctx context.Context) ([]Server, error) {
 }
 
 // Get all deleted Nova servers.
+// Note on Nova terminology: Nova uses "instance" internally in its database and code,
+// but exposes these as "server" objects through the public API.
+// Server lifecycle and cleanup:
+//   - In SAP Cloud Infrastructure's Nova fork, orphaned servers are purged after 3 weeks
+//   - This means historical server data is limited to 3 weeks
 func (api *novaAPI) GetDeletedServers(ctx context.Context, since time.Time) ([]DeletedServer, error) {
 	label := DeletedServer{}.TableName()
 
@@ -226,10 +231,19 @@ func (api *novaAPI) GetAllFlavors(ctx context.Context) ([]Flavor, error) {
 	return data.Flavors, nil
 }
 
-// Get all changed Nova migrations.
-func (api *novaAPI) GetChangedMigrations(ctx context.Context, changedSince *time.Time) ([]Migration, error) {
+// Get all Nova migrations from the OpenStack API.
+//
+// Note on Nova terminology: Nova uses "instance" internally in its database and code,
+// but exposes these as "server" objects through the public API.
+//
+// Migration lifecycle and cleanup:
+//   - Migrations are automatically deleted when their associated server is deleted
+//     (see Nova source: https://github.com/openstack/nova/blob/1508cb39a2b12ef2d4f706b9c303a744ce40e707/nova/db/main/api.py#L1337-L1358)
+//   - In SAP Cloud Infrastructure's Nova fork, orphaned migrations are purged after 3 weeks
+//   - This means historical migration data has limited retention
+func (api *novaAPI) GetAllMigrations(ctx context.Context) ([]Migration, error) {
 	label := Migration{}.TableName()
-	slog.Info("fetching nova data", "label", label, "changedSince", changedSince)
+	slog.Info("fetching nova data", "label", label)
 	// Note: currently we need to fetch this without gophercloud.
 	// See: https://github.com/gophercloud/gophercloud/pull/3244
 	if api.mon.PipelineRequestTimer != nil {
@@ -238,11 +252,6 @@ func (api *novaAPI) GetChangedMigrations(ctx context.Context, changedSince *time
 		defer timer.ObserveDuration()
 	}
 	initialURL := api.sc.Endpoint + "os-migrations"
-	// It is important to omit the changes-since parameter if it is nil.
-	// Otherwise Nova may return huge amounts of data since the beginning of time.
-	if changedSince != nil {
-		initialURL += "?changes-since=" + changedSince.Format(time.RFC3339)
-	}
 	var nextURL = &initialURL
 	var migrations []Migration
 	for nextURL != nil {

internal/sync/openstack/nova/nova_api_test.go

@@ -191,60 +191,45 @@ func TestNovaAPI_GetAllFlavors(t *testing.T) {
 	}
 }
 
-func TestNovaAPI_GetChangedMigrations(t *testing.T) {
-	tests := []struct {
-		name string
-		time *time.Time
-	}{
-		{"nil", nil},
-		{"time", &time.Time{}},
-	}
-	for _, tt := range tests {
-		handler := func(w http.ResponseWriter, r *http.Request) {
-			if tt.time == nil {
-				// Check that the changes-since query parameter is not set.
-				if r.URL.Query().Get("changes-since") != "" {
-					t.Fatalf("expected no changes-since query parameter, got %s", r.URL.Query().Get("changes-since"))
-				}
-			} else {
-				if r.URL.Query().Get("changes-since") != tt.time.Format(time.RFC3339) {
-					t.Fatalf("expected changes-since query parameter to be %s, got %s", tt.time.Format(time.RFC3339), r.URL.Query().Get("changes-since"))
-				}
-			}
-			w.Header().Add("Content-Type", "application/json")
-			w.WriteHeader(http.StatusOK)
-			resp := struct {
-				Migrations []Migration `json:"migrations"`
-				Links      []struct {
-					Rel  string `json:"rel"`
-					Href string `json:"href"`
-				} `json:"migrations_links"`
-			}{
-				Migrations: []Migration{{ID: 1, SourceCompute: "host1", DestCompute: "host2", Status: "completed"}},
-			}
-			if err := json.NewEncoder(w).Encode(resp); err != nil {
-				t.Fatalf("failed to write response: %v", err)
-			}
+func TestNovaAPI_GetAllMigrations(t *testing.T) {
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("changes-since") != "" {
+			t.Fatalf("expected no changes-since query parameter, got %s", r.URL.Query().Get("changes-since"))
 		}
-		server, k := setupNovaMockServer(handler)
-		defer server.Close()
+		w.Header().Add("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		resp := struct {
+			Migrations []Migration `json:"migrations"`
+			Links      []struct {
+				Rel  string `json:"rel"`
+				Href string `json:"href"`
+			} `json:"migrations_links"`
+		}{
+			Migrations: []Migration{{ID: 1, SourceCompute: "host1", DestCompute: "host2", Status: "completed"}},
+		}
+		if err := json.NewEncoder(w).Encode(resp); err != nil {
+			t.Fatalf("failed to write response: %v", err)
+		}
+	}
 
-		mon := sync.Monitor{}
-		conf := NovaConf{Availability: "public"}
+	server, k := setupNovaMockServer(handler)
+	defer server.Close()
 
-		api := NewNovaAPI(mon, k, conf).(*novaAPI)
-		api.Init(t.Context())
+	mon := sync.Monitor{}
+	conf := NovaConf{Availability: "public"}
 
-		ctx := t.Context()
-		migrations, err := api.GetChangedMigrations(ctx, tt.time)
-		if err != nil {
-			t.Fatalf("expected no error, got %v", err)
-		}
-		if len(migrations) != 1 {
-			t.Fatalf("expected 1 migration, got %d", len(migrations))
-		}
-		if migrations[0].ID != 1 || migrations[0].SourceCompute != "host1" || migrations[0].DestCompute != "host2" || migrations[0].Status != "completed" {
-			t.Errorf("unexpected migration data: %+v", migrations[0])
-		}
+	api := NewNovaAPI(mon, k, conf).(*novaAPI)
+	api.Init(t.Context())
+
+	ctx := t.Context()
+	migrations, err := api.GetAllMigrations(ctx)
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+	if len(migrations) != 1 {
+		t.Fatalf("expected 1 migration, got %d", len(migrations))
+	}
+	if migrations[0].ID != 1 || migrations[0].SourceCompute != "host1" || migrations[0].DestCompute != "host2" || migrations[0].Status != "completed" {
+		t.Errorf("unexpected migration data: %+v", migrations[0])
 	}
 }