feat(external_ref): Make external_ref fields configurable

Signed-off-by: deo002 <[email protected]>

Author: deo002

.github/workflows/api-swagger.yml

@@ -23,6 +23,10 @@ jobs:
         check-latest: true
         cache: true
 
+    - name: Code Generation
+      run: |
+        cp external_ref_fields.example.yaml external_ref_fields.yaml && go generate ./...
+
     - name: Install swag
       run: |
         go install github.com/swaggo/swag/cmd/swag@latest
@@ -33,7 +37,7 @@ jobs:
 
     - name: Check doc diff
       run: |
-        diff swag/docs/docs.go cmd/laas/docs/docs.go > swagger-diff.txt
+        diff swag/docs/docs.go cmd/laas/docs/docs.go > swagger-diff.txt || true
         # Check if file swagger-diff.txt is empty
         if [ -s swagger-diff.txt ]
         then

.github/workflows/codeql.yml

@@ -85,7 +85,7 @@ jobs:
     - if: matrix.build-mode == 'manual'
       shell: bash
       run: |
-        go build ./cmd/laas
+        cp external_ref_fields.example.yaml external_ref_fields.yaml && go generate ./... && go build ./cmd/laas
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v3

.github/workflows/go.yml

@@ -34,7 +34,7 @@ jobs:
         fi
 
     - name: Build
-      run: go build -v ./...
+      run: cp external_ref_fields.example.yaml external_ref_fields.yaml && go generate ./... && go build -v ./...
 
     # - name: Test
     #   run: go test -v ./...

.github/workflows/golangci.yml

@@ -27,6 +27,9 @@ jobs:
         check-latest: true
         cache: true
 
+    - name: Code Generation
+      run: cp external_ref_fields.example.yaml external_ref_fields.yaml && go generate ./...
+
     - name: lint
       uses: golangci/golangci-lint-action@v3
       with:

.gitignore

@@ -29,3 +29,7 @@ licenseRef.json
 # IDE related files
 /.idea
 /LicenseDb.iml
+
+external_ref_fields.yaml
+
+pkg/models/external_ref_structs.go

Dockerfile

@@ -10,10 +10,11 @@ RUN go mod download
 COPY cmd/ cmd/
 COPY pkg/ pkg/
 COPY docs/ docs/
+COPY external_ref_fields.example.yaml external_ref_fields.yaml
 
 RUN wget https://raw.githubusercontent.com/fossology/fossology/master/install/db/licenseRef.json -O licenseRef.json
 
-RUN CGO_ENABLED=0 GOOS=linux go build -a -o laas ./cmd/laas
+RUN CGO_ENABLED=0 GOOS=linux go generate ./cmd/laas && go build -a -o laas ./cmd/laas
 
 # Release Stage
 FROM alpine:3.20 AS build-release
@@ -22,7 +23,7 @@ WORKDIR /app
 
 COPY entrypoint.sh /app/entrypoint.sh
 
-RUN apk add --no-cache openssl bash \
+RUN apk add --no-cache openssl bash libc6-compat \
     && addgroup -S noroot \
     && adduser -S noroot -G noroot
 

README.md

@@ -71,6 +71,20 @@ git clone https://github.com/fossology/LicenseDb.git
 cd LicenseDb
 ```
 
+- Create the `external_ref_fields.yaml` file in the root directory of the project and change the
+  values of the extra license json keys as per your requirement.
+
+```bash
+cp external_ref_fields.yaml.example external_ref_fields.yaml
+vim external_ref_fields.yaml
+```
+
+- Generate Go struct for the extra fields listed in the external_ref_fields.yaml.
+
+```bash
+go generate ./...
+```
+
 - Build the project using following command.
 
 ```bash

cmd/laas/gen.go

@@ -0,0 +1,8 @@
+// SPDX-FileCopyrightText: 2024 Dearsh Oberoi <[email protected]>
+// SPDX-FileCopyrightText: 2024 Siemens AG
+//
+// SPDX-License-Identifier: GPL-2.0-only
+
+package main
+
+//go:generate go run gen_external_ref_schema.go

cmd/laas/gen_external_ref_schema.go

@@ -0,0 +1,87 @@
+// SPDX-FileCopyrightText: 2024 Dearsh Oberoi <[email protected]>
+// SPDX-FileCopyrightText: 2024 Siemens AG
+//
+// SPDX-License-Identifier: GPL-2.0-only
+
+//go:build ignore
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"os"
+
+	"path/filepath"
+
+	"github.com/dave/jennifer/jen"
+	"gopkg.in/yaml.v3"
+)
+
+var (
+	PATH_EXTERNAL_REF_CONFIG_FILE = filepath.FromSlash("../../external_ref_fields.yaml")
+	PATH_EXTERNAL_REF_STRUCT_FILE = filepath.FromSlash("../../pkg/models/external_ref_structs.go")
+)
+
+// ExternalRefFieldMetadata is the metadata about the extra fields saved as json in the license external_ref column
+type ExternalRefFieldMetaData struct {
+	StructFieldName string `yaml:"struct_field_name"`
+	Type            string `yaml:"type"`
+	Name            string `yaml:"name"`
+}
+
+// ExternalRefFields is the list of metadata of all extra fields
+type ExternalRefFields struct {
+	Fields []ExternalRefFieldMetaData `yaml:"fields"`
+}
+
+func main() {
+	externalRefFields := ExternalRefFields{}
+
+	fieldsMetadata, err := os.ReadFile(PATH_EXTERNAL_REF_CONFIG_FILE)
+	if err != nil {
+		log.Fatalf("Failed to instantiate json schema for external ref in license: %v", err)
+	}
+
+	err = yaml.Unmarshal(fieldsMetadata, &externalRefFields)
+	if err != nil {
+		log.Fatalf("Failed to instantiate json schema for external ref in license: %v", err)
+	}
+
+	// REUSE-IgnoreStart
+
+	f := jen.NewFile("models")
+	f.Comment("SPDX-FileCopyrightText: FOSSology Community\nSPDX-License-Identifier: GPL-2.0-only\n\nThis file is autogenerated. Please do not edit it.\n")
+
+	// REUSE-IgnoreStart
+
+	var fields []jen.Code
+
+	for _, f := range externalRefFields.Fields {
+		field := jen.Id(f.StructFieldName).Op("*")
+		if f.StructFieldName == "" {
+			err = errors.New("field struct_field_name is missing in external_ref_fields.yaml")
+		}
+		switch f.Type {
+		case "boolean":
+			field = field.Bool()
+		case "string":
+			field = field.String()
+		case "int":
+			field = field.Int64()
+		default:
+			err = fmt.Errorf("type %s in external_ref_fields.yaml is not supported", f.Type)
+		}
+		if err != nil {
+			log.Fatalf("Failed to instantiate json schema for external ref in license: %v", err)
+			return
+		}
+		field = field.Tag(map[string]string{"json": fmt.Sprintf("%s,omitempty", f.Name)})
+		fields = append(fields, field)
+	}
+
+	f.Type().Id("LicenseDBSchemaExtension").Struct(fields...)
+	f.Save(PATH_EXTERNAL_REF_STRUCT_FILE)
+
+}

cmd/laas/main.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/joho/godotenv"
 
+	_ "github.com/dave/jennifer/jen"
 	_ "github.com/fossology/LicenseDb/cmd/laas/docs"
 	"github.com/fossology/LicenseDb/pkg/api"
 	"github.com/fossology/LicenseDb/pkg/db"