Add Falco rules tracker (#514)

Closes #239

Signed-off-by: Sergio Castaño Arteaga <[email protected]>

.github/workflows/ci.yml

@@ -89,6 +89,9 @@ jobs:
       - name: Build hub
         working-directory: ./cmd/hub
         run: go build -v
+      - name: Build falco-tracker
+        working-directory: ./cmd/falco-tracker
+        run: go build -v
       - name: Build helm-tracker
         working-directory: ./cmd/helm-tracker
         run: go build -v
@@ -143,6 +146,13 @@ jobs:
             -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/db-migrator:$GITHUB_SHA .
       - name: Push db-migrator image
         run: docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/db-migrator:$GITHUB_SHA
+      - name: Build falco-tracker image
+        run: |
+          docker build \
+            -f cmd/falco-tracker/Dockerfile \
+            -t $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/falco-tracker:$GITHUB_SHA .
+      - name: Push falco-tracker image
+        run: docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/falco-tracker:$GITHUB_SHA
       - name: Build helm-tracker image
         run: |
           docker build \

.github/workflows/release.yml

@@ -30,6 +30,14 @@ jobs:
             -t artifacthub/db-migrator:latest .
       - name: Push db-migrator image
         run: docker push artifacthub/db-migrator
+      - name: Build falco-tracker image
+        run: |
+          docker build \
+            -f cmd/falco-tracker/Dockerfile \
+            -t artifacthub/falco-tracker:${{steps.extract_tag_name.outputs.tag}} \
+            -t artifacthub/falco-tracker:latest .
+      - name: Push falco-tracker image
+        run: docker push artifacthub/falco-tracker
       - name: Build helm-tracker image
         run: |
           docker build \

README.md

@@ -78,9 +78,10 @@ When the parameter `dbMigrator.loadSampleData` is set to true (default) a **demo
 
 ### Populating packages
 
-The chart installs some `cronjobs` in charge of launching periodically (every 30m) the trackers, which index packages from the registered repositories. Some sample repositories are added by default when `dbMigrator.loadSampleData` is set to true. If you don't want to wait until the jobs are triggered by the cronjob, you can create them manually using the following commands:
+The chart installs some `cronjobs` in charge of launching periodically (every 30m) the trackers, which index packages from the registered repositories. Some sample repositories are added by default when `dbMigrator.loadSampleData` is set to true. If you don't want to wait until the jobs are triggered by the cronjob, you can create some or all of them manually using the following commands:
 
 ```bash
+$ kubectl create job initial-falco-tracker-job --from=cronjob/falco-tracker
 $ kubectl create job initial-helm-tracker-job --from=cronjob/helm-tracker
 $ kubectl create job initial-olm-tracker-job --from=cronjob/olm-tracker
 ```

chart/templates/falco_tracker_cronjob.yaml

@@ -0,0 +1,41 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: falco-tracker
+spec:
+  schedule: "*/30 * * * *"
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      template:
+        spec:
+        {{- with .Values.imagePullSecrets }}
+          imagePullSecrets:
+            {{- toYaml . | nindent 8 }}
+        {{- end }}
+          restartPolicy: Never
+          initContainers:
+          - name: check-db-ready
+            image: {{ .Values.postgresql.image.repository }}:{{ .Values.postgresql.image.tag }}
+            imagePullPolicy: {{ .Values.pullPolicy }}
+            resources:
+              {{- toYaml .Values.falcoTracker.cronjob.resources | nindent 14 }}
+            env:
+              - name: PGHOST
+                value: {{ .Values.db.host }}
+              - name: PGPORT
+                value: "{{ .Values.db.port }}"
+            command: ['sh', '-c', 'until pg_isready; do echo waiting for database; sleep 2; done;']
+          containers:
+          - name: falco-tracker
+            image: {{ .Values.falcoTracker.cronjob.image.repository }}:{{ .Values.imageTag }}
+            imagePullPolicy: {{ .Values.pullPolicy }}
+            volumeMounts:
+            - name: falco-tracker-config
+              mountPath: "/home/falco-tracker/.cfg"
+              readOnly: true
+          volumes:
+          - name: falco-tracker-config
+            secret:
+              secretName: falco-tracker-config

chart/templates/falco_tracker_secret.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: falco-tracker-config
+type: Opaque
+stringData:
+  falco-tracker.yaml: |-
+    log:
+      level: {{ .Values.log.level }}
+      pretty: {{ .Values.log.pretty }}
+    db:
+      host: {{ .Values.db.host }}
+      port: {{ .Values.db.port }}
+      database: {{ .Values.db.database }}
+      user: {{ .Values.db.user }}
+      password: {{ .Values.db.password }}
+    tracker:
+      concurrency: {{ .Values.falcoTracker.concurrency }}
+      repositoriesNames: {{ .Values.falcoTracker.repositories }}
+      imageStore: {{ .Values.falcoTracker.imageStore }}
+      bypassDigestCheck: {{ .Values.falcoTracker.bypassDigestCheck }}

chart/values-production.yaml

@@ -45,6 +45,16 @@ hub:
       google:
         redirectURL: https://artifacthub.io/oauth/google/callback
 
+falcoTracker:
+  cronjob:
+    resources:
+      requests:
+        cpu: 2
+        memory: 8000Mi
+  concurrency: 10
+  repositories: []
+  imageStore: pg
+
 helmTracker:
   cronjob:
     resources:

chart/values-staging.yaml

@@ -46,6 +46,16 @@ hub:
       google:
         redirectURL: https://staging.artifacthub.io/oauth/google/callback
 
+falcoTracker:
+  cronjob:
+    resources:
+      requests:
+        cpu: 2
+        memory: 2000Mi
+  concurrency: 10
+  repositories: []
+  imageStore: pg
+
 helmTracker:
   cronjob:
     resources:

chart/values.yaml

@@ -70,6 +70,19 @@ hub:
   analytics:
     gaTrackingID: ""
 
+falcoTracker:
+  cronjob:
+    image:
+      repository: artifacthub/falco-tracker
+    resources:
+      requests:
+        cpu: 100m
+        memory: 500Mi
+  concurrency: 10
+  repositories: []
+  imageStore: pg
+  bypassDigestCheck: false
+
 helmTracker:
   cronjob:
     image:

cmd/falco-tracker/Dockerfile

@@ -0,0 +1,15 @@
+# Build falco-tracker
+FROM golang:1.14-alpine AS builder
+WORKDIR /go/src/github.com/artifacthub/hub
+COPY go.* ./
+COPY cmd/falco-tracker cmd/falco-tracker
+COPY internal internal
+RUN cd cmd/falco-tracker && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /falco-tracker .
+
+# Final stage
+FROM alpine:latest
+RUN apk --no-cache add ca-certificates && addgroup -S falco-tracker && adduser -S falco-tracker -G falco-tracker
+USER falco-tracker
+WORKDIR /home/falco-tracker
+COPY --from=builder /falco-tracker ./
+CMD ["./falco-tracker"]

cmd/falco-tracker/main.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+
+	"github.com/artifacthub/hub/internal/hub"
+	"github.com/artifacthub/hub/internal/pkg"
+	"github.com/artifacthub/hub/internal/repo"
+	"github.com/artifacthub/hub/internal/tracker"
+	"github.com/artifacthub/hub/internal/util"
+	"github.com/rs/zerolog/log"
+	"golang.org/x/time/rate"
+)
+
+func main() {
+	// Setup configuration and logger
+	cfg, err := util.SetupConfig("falco-tracker")
+	if err != nil {
+		log.Fatal().Err(err).Msg("configuration setup failed")
+	}
+	fields := map[string]interface{}{
+		"cmd": "falco-tracker",
+	}
+	if err := util.SetupLogger(cfg, fields); err != nil {
+		log.Fatal().Err(err).Msg("logger setup failed")
+	}
+
+	// Shutdown gracefully when SIGINT or SIGTERM signal is received
+	log.Info().Int("pid", os.Getpid()).Msg("falco tracker started")
+	ctx, cancel := context.WithCancel(context.Background())
+	shutdown := make(chan os.Signal, 1)
+	signal.Notify(shutdown, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-shutdown
+		cancel()
+		log.Info().Msg("falco tracker shutting down..")
+	}()
+
+	// Setup internal services required
+	db, err := util.SetupDB(cfg)
+	if err != nil {
+		log.Fatal().Err(err).Msg("database setup failed")
+	}
+	rm := repo.NewManager(db)
+	pm := pkg.NewManager(db)
+	is, err := util.SetupImageStore(cfg, db)
+	if err != nil {
+		log.Fatal().Err(err).Msg("image store setup failed")
+	}
+
+	// Track registered Falco repositories
+	repos, err := tracker.GetRepositories(cfg, rm, hub.Falco)
+	if err != nil {
+		log.Fatal().Err(err).Send()
+	}
+	ec := tracker.NewDBErrorsCollector(ctx, rm, repos)
+	concurrency := cfg.GetInt("tracker.concurrency")
+	limiter := rate.NewLimiter(rate.Limit(concurrency), concurrency)
+	var wg sync.WaitGroup
+	for _, r := range repos {
+		_ = limiter.Wait(ctx)
+		wg.Add(1)
+		w := NewTracker(ctx, cfg, r, rm, pm, is, ec)
+		go func(r *hub.Repository) {
+			if err := w.Track(&wg); err != nil {
+				ec.Append(r.RepositoryID, err)
+				w.Logger.Err(err).Send()
+			}
+		}(r)
+	}
+	wg.Wait()
+	ec.Flush()
+	log.Info().Msg("falco tracker finished")
+}

cmd/falco-tracker/testdata/path3/test.yaml

@@ -0,0 +1 @@
+version: invalid

cmd/falco-tracker/testdata/path4/test.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: FalcoRules
+vendor: Sample provider
+name: test
+shortDescription: Short description
+version: 0.1.0
+description: Description
+keywords:
+  - kw1
+  - kw2
+rules:
+  - raw: Falco rules in YAML