Add testing and fixed scorecard ingestion (#140)

* Add testing and fixed scorecard ingestion

- Added testing for scorecard ingestion in the service and in pkg
- Fixed scorecard ingestion
- Moved LoadDataFromPath to cmd, cutting the dependency of cmd on pkg

Signed-off-by: neilnaveen <[email protected]>

* Deal with errors better

Signed-off-by: neilnaveen <[email protected]>

---------

Signed-off-by: neilnaveen <[email protected]>

Author: neilnaveen

api/v1/service_test.go

@@ -9,7 +9,6 @@ import (
 	"github.com/RoaringBitmap/roaring"
 	service "github.com/bitbomdev/minefield/gen/api/v1"
 	"github.com/bitbomdev/minefield/pkg/graph"
-	"github.com/bitbomdev/minefield/pkg/tools/ingest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/protobuf/types/known/emptypb"
@@ -73,23 +72,26 @@ func TestGetNodesByGlob(t *testing.T) {
 func TestQueriesIngestAndCache(t *testing.T) {
 	s := setupService()
 
-	result, err := ingest.LoadDataFromPath("../../testdata/osv-sboms/google_agi.sbom.json")
+	// Read and ingest SBOM file
+	sbomData, err := os.ReadFile("../../testdata/osv-sboms/google_agi.sbom.json")
 	require.NoError(t, err)
-	for _, data := range result {
-		sbomReq := connect.NewRequest(&service.IngestSBOMRequest{
-			Sbom: data.Data,
-		})
-		_, err = s.IngestSBOM(context.Background(), sbomReq)
-		require.NoError(t, err)
-	}
 
-	content, err := os.ReadFile("../../testdata/osv-vulns/GHSA-cx63-2mw6-8hw5.json")
+	sbomReq := connect.NewRequest(&service.IngestSBOMRequest{
+		Sbom: sbomData,
+	})
+	_, err = s.IngestSBOM(context.Background(), sbomReq)
 	require.NoError(t, err)
+
+	// Read and ingest vulnerability file
+	vulnData, err := os.ReadFile("../../testdata/osv-vulns/GHSA-cx63-2mw6-8hw5.json")
+	require.NoError(t, err)
+
 	vulnReq := connect.NewRequest(&service.IngestVulnerabilityRequest{
-		Vulnerability: content,
+		Vulnerability: vulnData,
 	})
 	_, err = s.IngestVulnerability(context.Background(), vulnReq)
 	require.NoError(t, err)
+
 	// Check if the node with name "pkg:pypi/[email protected]" exists
 	graphReq := connect.NewRequest(&service.GetNodeByNameRequest{Name: "pkg:pypi/[email protected]"})
 	resp, err := s.GetNodeByName(context.Background(), graphReq)
@@ -179,6 +181,17 @@ func TestIngestVulnerability(t *testing.T) {
 	require.NoError(t, err)
 }
 
+func TestIngestScorecard(t *testing.T) {
+	s := setupService()
+	content, err := os.ReadFile("../../testdata/scorecards/scorecards.json")
+	require.NoError(t, err)
+	req := connect.NewRequest(&service.IngestScorecardRequest{
+		Scorecard: content,
+	})
+	_, err = s.IngestScorecard(context.Background(), req)
+	require.NoError(t, err)
+}
+
 func TestHealthCheck(t *testing.T) {
 	s := setupService()
 	req := connect.NewRequest(&emptypb.Empty{})

pkg/tools/ingest/loader.go renamed to cmd/helpers/fileLoader.go

@@ -1,4 +1,4 @@
-package ingest
+package helpers
 
 import (
 	"archive/zip"

cmd/ingest/osv/osv.go

@@ -6,16 +6,15 @@ import (
 	"net/http"
 
 	"connectrpc.com/connect"
+	"github.com/bitbomdev/minefield/cmd/helpers"
 	apiv1 "github.com/bitbomdev/minefield/gen/api/v1"
 	"github.com/bitbomdev/minefield/gen/api/v1/apiv1connect"
 	"github.com/bitbomdev/minefield/pkg/tools"
-	"github.com/bitbomdev/minefield/pkg/tools/ingest"
 	"github.com/spf13/cobra"
 )
 
 type options struct {
-	addr string // Address of the minefield server
-
+	addr                string // Address of the minefield server
 	ingestServiceClient apiv1connect.IngestServiceClient
 }
 
@@ -35,7 +34,7 @@ func (o *options) Run(_ *cobra.Command, args []string) error {
 	}
 	vulnsPath := args[0]
 	// Ingest vulnerabilities
-	result, err := ingest.LoadDataFromPath(vulnsPath)
+	result, err := helpers.LoadDataFromPath(vulnsPath)
 	if err != nil {
 		return fmt.Errorf("failed to load vulnerabilities: %w", err)
 	}

cmd/ingest/sbom/sbom.go

@@ -6,10 +6,10 @@ import (
 	"net/http"
 
 	"connectrpc.com/connect"
+	"github.com/bitbomdev/minefield/cmd/helpers"
 	apiv1 "github.com/bitbomdev/minefield/gen/api/v1"
 	"github.com/bitbomdev/minefield/gen/api/v1/apiv1connect"
 	"github.com/bitbomdev/minefield/pkg/tools"
-	"github.com/bitbomdev/minefield/pkg/tools/ingest"
 	"github.com/spf13/cobra"
 )
 
@@ -36,7 +36,7 @@ func (o *options) Run(_ *cobra.Command, args []string) error {
 	}
 	sbomPath := args[0]
 	// Ingest SBOM
-	result, err := ingest.LoadDataFromPath(sbomPath)
+	result, err := helpers.LoadDataFromPath(sbomPath)
 	if err != nil {
 		return fmt.Errorf("failed to ingest SBOM: %w", err)
 	}

cmd/ingest/scorecard/scorecard.go

@@ -6,16 +6,15 @@ import (
 	"net/http"
 
 	"connectrpc.com/connect"
+	"github.com/bitbomdev/minefield/cmd/helpers"
 	apiv1 "github.com/bitbomdev/minefield/gen/api/v1"
 	"github.com/bitbomdev/minefield/gen/api/v1/apiv1connect"
 	"github.com/bitbomdev/minefield/pkg/tools"
-	"github.com/bitbomdev/minefield/pkg/tools/ingest"
 	"github.com/spf13/cobra"
 )
 
 type options struct {
-	addr string // Address of the minefield server
-
+	addr                string // Address of the minefield server
 	ingestServiceClient apiv1connect.IngestServiceClient
 }
 
@@ -36,7 +35,7 @@ func (o *options) Run(_ *cobra.Command, args []string) error {
 	}
 	scorecardPath := args[0]
 
-	result, err := ingest.LoadDataFromPath(scorecardPath)
+	result, err := helpers.LoadDataFromPath(scorecardPath)
 	if err != nil {
 		return fmt.Errorf("failed to ingest SBOM: %w", err)
 	}

pkg/storages/e2e_test.go

@@ -3,6 +3,7 @@ package storages
 import (
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/bitbomdev/minefield/pkg/graph"
@@ -19,19 +20,38 @@ func TestParseAndExecute_E2E(t *testing.T) {
 	sbomPath := filepath.Join("..", "..", "testdata", "sboms")
 	vulnsPath := filepath.Join("..", "..", "testdata", "osv-vulns")
 
-	// Ingest data from the folder
-	result, err := ingest.LoadDataFromPath(sbomPath)
+	// Process SBOM files
+	sbomFiles, err := os.ReadDir(sbomPath)
 	assert.NoError(t, err)
-	for _, data := range result {
-		if err := ingest.SBOM(redisStorage, data.Data); err != nil {
-			t.Fatalf("Failed to load SBOM from data: %v", err)
+
+	for _, file := range sbomFiles {
+		if !strings.HasSuffix(file.Name(), ".json") {
+			continue
+		}
+
+		data, err := os.ReadFile(filepath.Join(sbomPath, file.Name()))
+		assert.NoError(t, err)
+
+		err = ingest.SBOM(redisStorage, data)
+		if err != nil {
+			t.Fatalf("Failed to load SBOM from file %s: %v", file.Name(), err)
 		}
 	}
-	result, err = ingest.LoadDataFromPath(vulnsPath)
+	// Process vulnerability files
+	vulnFiles, err := os.ReadDir(vulnsPath)
 	assert.NoError(t, err)
-	for _, data := range result {
-		if err := ingest.Vulnerabilities(redisStorage, data.Data); err != nil {
-			t.Fatalf("Failed to load vulnerabilities from data: %v", err)
+
+	for _, file := range vulnFiles {
+		if !strings.HasSuffix(file.Name(), ".json") {
+			continue
+		}
+
+		data, err := os.ReadFile(filepath.Join(vulnsPath, file.Name()))
+		assert.NoError(t, err)
+
+		err = ingest.Vulnerabilities(redisStorage, data)
+		if err != nil {
+			t.Fatalf("Failed to load vulnerabilities from file %s: %v", file.Name(), err)
 		}
 	}
 

pkg/tools/ingest/sbom_test.go

@@ -2,118 +2,47 @@ package ingest
 
 import (
 	"os"
-	"sort"
+	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/bitbomdev/minefield/pkg/graph"
 )
 
 func TestIngestSBOM(t *testing.T) {
-	createTestFiles(t)
-	tests := []struct {
-		name     string
-		sbomPath string
-		want     map[uint32]*graph.Node
-		wantErr  bool
-	}{
-		{
-			name:     "non-existent file",
-			sbomPath: "non_existent_file.json",
-			wantErr:  true,
-		},
-		{
-			name:     "empty directory",
-			sbomPath: "../../../empty_dir",
-			want:     map[uint32]*graph.Node{},
-			wantErr:  false,
-		},
-		{
-			name:     "SBOM with no components",
-			sbomPath: "../../../no_components_sbom.json",
-			want:     map[uint32]*graph.Node{},
-		},
-	}
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			storage := graph.NewMockStorage()
-			result, err := LoadDataFromPath(test.sbomPath)
-			if test.wantErr != (err != nil) {
-				t.Errorf("Sbom() error = %v, wantErr = %v", err, test.wantErr)
-			}
-
-			for _, data := range result {
-				if err := SBOM(storage, data.Data); err != nil {
-					if test.wantErr != (err != nil) {
-						t.Errorf("Sbom() error = %v, wantErr = %v", err, test.wantErr)
-					}
-				}
-			}
+	storage := graph.NewMockStorage()
 
-			keys, err := storage.GetAllKeys()
-			if err != nil {
-				t.Fatalf("Failed to get all keys, %v", err)
-			}
+	sbomDir := "../../../testdata/sboms"
 
-			sort.Slice(keys, func(i, j int) bool {
-				return keys[i] < keys[j]
-			})
-
-			for _, key := range keys {
-				node, err := storage.GetNode(key)
-				if err != nil {
-					t.Fatalf("Failed to get node, %v", err)
-				}
-				if !nodeEquals(node, test.want[key]) {
-					t.Fatalf("expected node %v, got %v", test.want[key], node)
-				}
-			}
-		})
-	}
-	if err := os.RemoveAll("../../../empty_dir"); err != nil {
-		t.Fatal(err)
-	}
-	if err := os.Remove("../../../invalid_sbom.json"); err != nil {
-		t.Fatal(err)
-	}
-	if err := os.Remove("../../../no_components_sbom.json"); err != nil {
-		t.Fatal(err)
+	// Read SBOM files
+	sbomFiles, err := os.ReadDir(sbomDir)
+	if err != nil {
+		t.Fatalf("Failed to read SBOM directory: %v", err)
 	}
-}
 
-func nodeEquals(n, n2 *graph.Node) bool {
-	if ((n == nil || n2 == nil) && n != n2) ||
-		(n != nil && (n.ID != n2.ID || n.Type != n2.Type || n.Name != n2.Name)) {
-		return false
-	}
-	return true
-}
+	for _, file := range sbomFiles {
+		if !strings.HasSuffix(file.Name(), ".json") {
+			continue
+		}
 
-func createTestFiles(t *testing.T) {
-	t.Helper()
+		data, err := os.ReadFile(filepath.Join(sbomDir, file.Name()))
+		if err != nil {
+			t.Fatalf("Failed to read SBOM file %s: %v", file.Name(), err)
+		}
 
-	// Create an empty directory
-	err := os.MkdirAll("../../../empty_dir", 0o755)
-	if err != nil {
-		t.Fatalf("Failed to create empty directory: %v", err)
+		if err := SBOM(storage, data); err != nil {
+			t.Fatalf("Failed to process SBOM from file %s: %v", file.Name(), err)
+		}
 	}
 
-	// Create an invalid SBOM file
-	invalidSBOM := []byte(`{"invalid": "json"}`)
-	err = os.WriteFile("../../../invalid_sbom.json", invalidSBOM, 0o644)
+	keys, err := storage.GetAllKeys()
 	if err != nil {
-		t.Fatalf("Failed to create invalid SBOM file: %v", err)
+		t.Fatalf("Failed to get all keys: %v", err)
 	}
 
-	// Create a SBOM file with no components
-	noComponentsSBOM := []byte(`{
-		"bomFormat": "CycloneDX",
-		"specVersion": "1.5",
-		"version": 1,
-		"metadata": {},
-		"components": []
-	}`)
-	err = os.WriteFile("../../../no_components_sbom.json", noComponentsSBOM, 0o644)
-	if err != nil {
-		t.Fatalf("Failed to create no components SBOM file: %v", err)
+	// Verify we have the expected number of nodes
+	if len(keys) != 1600 {
+		t.Fatalf("Expected 1600 nodes to be created from SBOM ingestion, got %d", len(keys))
 	}
+
 }

pkg/tools/ingest/scorecard.go

@@ -3,11 +3,13 @@ package ingest
 import (
 	"encoding/json"
 	"fmt"
+	"log"
 
 	"strings"
 
 	"github.com/bitbomdev/minefield/pkg/graph"
 	"github.com/bitbomdev/minefield/pkg/tools"
+	"github.com/package-url/packageurl-go"
 )
 
 type Repo struct {
@@ -45,7 +47,6 @@ type ScorecardResult struct {
 
 // Scorecard processes the Scorecard JSON data and stores it in the graph.
 func Scorecards(storage graph.Storage, data []byte) error {
-
 	if len(data) == 0 {
 		return fmt.Errorf("data is empty")
 	}
@@ -61,7 +62,19 @@ func Scorecards(storage graph.Storage, data []byte) error {
 		if !result.Success {
 			continue
 		}
-		scorecardResults[result.PURL] = append(scorecardResults[result.PURL], result)
+		purl, err := packageurl.FromString(result.PURL)
+		if err != nil {
+			// Log and skip invalid PURLs instead of failing
+			log.Printf("Warning: Invalid PURL %q: %v", result.PURL, err)
+			continue
+		}
+
+		if purl.Name == "" {
+			log.Printf("Warning: Empty package name in PURL %q", result.PURL)
+			continue
+		}
+
+		scorecardResults[purl.Name] = append(scorecardResults[purl.Name], result)
 	}
 
 	keys, err := storage.GetAllKeys()