main.go

// Ortelius v11 Vulnerability Microservice that handles creating Vulnerability from OSV.dev
// Runs as a cronjob
package main

import (
	"archive/zip"
	"bytes"
	"context"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/arangodb/go-driver/v2/arangodb"
	mapset "github.com/deckarep/golang-set/v2"
	"github.com/ortelius/scec-commons/database"
	_ "github.com/ortelius/scec-vulnerability/docs"
)

type vulnID struct {
	ID       string `json:"id"`
	Modified string `json:"modified"`
}

type cveText struct {
	Description string `json:"cvetext"`
}

var logger = database.InitLogger()
var dbconn = database.InitializeDatabase()

func getMitreURL() string {
	// Get the environment variable value for MITER_MAPPING_URL
	mitreURL := os.Getenv("MITER_MAPPING_URL")

	// Check if deppkgURL is empty
	if len(mitreURL) == 0 {
		// Get the environment variable value for SCEC_DEPPKG_SERVICE_HOST
		mitreHost := os.Getenv("SCEC_MITER_MAPPING_SERVICE_HOST")
		if mitreHost == "" {
			mitreHost = "127.0.0.1"
		}

		// Resolve the host name
		addrs, err := net.LookupAddr(mitreHost)
		if err != nil || len(addrs) == 0 {
			logger.Sugar().Infoln("Error resolving host:", err)
			return ""
		}
		host := addrs[0]

		// Get the environment variable value for SCEC_DEPPKG_SERVICE_PORT
		mitrePort := os.Getenv("SCEC_MITER_MAPPING_SERVICE_PORT")
		if mitrePort == "" {
			mitrePort = "80"
		}

		// Construct the URL
		mitreURL = fmt.Sprintf("http://%s:%s/msapi/mitre", host, mitrePort)
	}

	return mitreURL
}

// unpackAndLoad
func unpackAndLoad(src string, currentVulns mapset.Set[string]) error {

	r, err := zip.OpenReader(src)
	if err != nil {
		return err
	}
	defer func() {
		if err := r.Close(); err != nil {
			panic(err)
		}
	}()

	// Closure to address file descriptors issue with all the deferred .Close() methods
	extractAndWriteFile := func(f *zip.File) error {

		// Check for ZipSlip: https://snyk.io/research/zip-slip-vulnerability
		if strings.Contains(f.Name, "/") {
			return fmt.Errorf("%s: illegal file path", f.Name)
		}

		rc, err := f.Open()
		if err != nil {
			return err
		}
		defer func() {
			if err := rc.Close(); err != nil {
				panic(err)
			}
		}()

		if !f.FileInfo().IsDir() {
			var vulnJSON strings.Builder

			_, err = io.Copy(&vulnJSON, rc) // #nosec G110
			if err != nil {
				return err
			}

			var vulnIDMod vulnID
			var vulnStr = vulnJSON.String()
			if err := json.Unmarshal([]byte(vulnStr), &vulnIDMod); err != nil {
				logger.Sugar().Infoln(err)
			}

			key := vulnIDMod.ID + vulnIDMod.Modified
			if currentVulns.Contains(key) {
				return nil
			}

			logger.Sugar().Infof("%s %s %s", src, vulnIDMod.ID, vulnIDMod.Modified)
			// Add json to db
			if err := newVuln(vulnStr); err != nil {
				logger.Sugar().Infoln(err)
				logger.Sugar().Infoln(vulnStr)
			}
			currentVulns.Add(key)
		}
		return nil
	}

	for _, f := range r.File {
		err := extractAndWriteFile(f)
		if err != nil {
			return err
		}
	}

	return nil
}

// LoadFromOSVDev retrieves the vulns from osv.dev and adds them to the Arangodb
func LoadFromOSVDev() {

	if currentVulns, err := getVulns(); err == nil {

		baseURL := "https://www.googleapis.com/download/storage/v1/b/osv-vulnerabilities/o/ecosystems.txt?alt=media"

		client := http.Client{}

		resp, err := client.Get(baseURL)
		if err != nil {
			logger.Sugar().Fatal(err)
		}

		// We Read the response body on the line below.
		body, err := io.ReadAll(resp.Body)
		if err != nil {
			logger.Sugar().Fatalln(err)
		}
		resp.Body.Close()

		// Convert the body to type string
		ecosystems := strings.Split(string(body), "\n")

		for _, platform := range ecosystems {
			if len(strings.TrimSpace(platform)) == 0 {
				continue
			}

			tr := &http.Transport{
				TLSClientConfig: &tls.Config{
					InsecureSkipVerify: false,
					MinVersion:         tls.VersionTLS12,
				},
			}
			client := &http.Client{Transport: tr}

			url := fmt.Sprintf("https://www.googleapis.com/download/storage/v1/b/osv-vulnerabilities/o/%s%%2Fall.zip?alt=media", url.PathEscape(platform))
			logger.Sugar().Infof("%s", url)
			if resp, err := client.Get(url); err == nil {
				filename := fmt.Sprintf("%s.zip", platform)
				if out, err := os.Create(filename); err == nil {
					if _, err := io.Copy(out, resp.Body); err != nil {
						logger.Sugar().Infoln(err)
					}
					out.Close()
				} else {
					logger.Sugar().Infoln(err)
				}

				if err := unpackAndLoad(filename, currentVulns); err != nil {
					logger.Sugar().Infoln(err)
					logger.Sugar().Fatalln(filename)
				}
				os.Remove(filename)
				resp.Body.Close()
			} else {
				logger.Sugar().Infoln(err)
			}
		}
	} else {
		logger.Sugar().Fatalln(err)
	}
}

// getVulns godoc
// @Summary Get a list of Vulns aleady in the database
// @Description Returns a Set of vuln ids
// @Tags vulns
func getVulns() (mapset.Set[string], error) {

	var cursor arangodb.Cursor     // db cursor for rows
	var err error                  // for error handling
	var ctx = context.Background() // use default database context

	currentVulns := mapset.NewSet[string]()

	// query all the domains in the collection
	aql := `FOR vuln in vulns
			RETURN {id: vuln._key, modified: vuln.modified}`

	// execute the query with no parameters
	if cursor, err = dbconn.Database.Query(ctx, aql, nil); err != nil {
		logger.Sugar().Errorf("Failed to run query: %v", err) // log error
		return currentVulns, err
	}

	defer cursor.Close() // close the cursor when returning from this function

	for cursor.HasMore() { // loop thru all of the documents

		var vulnIDMod vulnID

		// fetch a document from the cursor
		if _, err = cursor.ReadDocument(ctx, &vulnIDMod); err != nil {
			logger.Sugar().Errorf("Failed to read document: %v", err)
			return currentVulns, err
		}
		key := vulnIDMod.ID + vulnIDMod.Modified
		currentVulns.Add(key)
	}

	return currentVulns, nil
}

// newVuln godoc
// @Summary Create a Vulnerability
// @Description Create a new Vulnerability and persist it
// @Tags vulnerability
func newVuln(vulnJSON string) error {

	var err error                  // for error handling
	var ctx = context.Background() // use default database context

	// Unmarshal the raw message into a map so we can access the summary and details keys
	var content map[string]interface{}

	if err = json.Unmarshal([]byte(strings.Replace(vulnJSON, "\"id\":", "\"_key\":", 1)), &content); err != nil {
		logger.Sugar().Infoln(err)
	} else {

		combinedJSON := vulnJSON

		summary := ""
		details := ""

		if val, ok := content["summary"]; ok {
			if s, ok := val.(string); ok {
				summary = s
			}
		}

		if val, ok := content["details"]; ok {
			if d, ok := val.(string); ok {
				details = d
			}
		}

		cve := cveText{
			Description: summary + " " + details,
		}

		jsonData, err := json.Marshal(cve)
		if err != nil {
			logger.Sugar().Errorln("Error marshaling JSON:", err)
			return err
		}

		mitreURL := getMitreURL()

		if len(mitreURL) > 0 {
			// #nosec G107
			resp, err := http.Post(mitreURL, "application/json", bytes.NewBuffer(jsonData))
			if err == nil {
				defer resp.Body.Close()

				if resp.StatusCode == 200 {
					if body, err := io.ReadAll(resp.Body); err == nil {
						techniqueJSON := ", \"techniques\":" + string(body)
						lastBraceIndex := strings.LastIndex(vulnJSON, "}")
						if lastBraceIndex != -1 {
							combinedJSON = vulnJSON[:lastBraceIndex] + techniqueJSON + "}"
						}

						// convert the combined json string into the map so we can persist it in Arango
						combinedJSON = strings.Replace(combinedJSON, "\"id\":", "\"_key\":", 1)
					}
				}
			} else {
				logger.Sugar().Errorln("Error sending POST request:", err)
			}
		}

		if err := json.Unmarshal([]byte(combinedJSON), &content); err != nil {
			logger.Sugar().Infoln(err)
		}

		//	overwriteMode := arangodb.CollectionDocumentCreateOverwriteModeConflict
		overwrite := true
		options := &arangodb.CollectionDocumentCreateOptions{
			Overwrite: &overwrite,
		}

		// update existing docs and add if missing
		if _, err = dbconn.Collections["vulns"].CreateDocumentWithOptions(ctx, content, options); err != nil {
			logger.Sugar().Errorf("Failed to create document: %v", err)
		}

		aql := `FOR vuln IN vulns
					FILTER vuln._key == @vulnKey
					LET purls = UNIQUE(
						FOR affected IN vuln.affected
						FILTER LENGTH(affected.package.purl) > 0
						LET purl = FIRST(SPLIT(affected.package.purl, '?'))
						RETURN purl
					)
					FILTER LENGTH(purls) > 0
					FOR purl IN purls
						// Upsert the purl and capture the _key of the upserted purl document
						LET upsertedPurl = FIRST(
							UPSERT { purl: purl }
							INSERT { purl: purl }
							UPDATE {} IN purls
							RETURN NEW
						)

						// Ensure the upsertedPurl is properly captured
						LET purlKey = upsertedPurl._key

						// Check for existing edge using the purlKey
						LET existingEdge = FIRST(
							FOR edge IN purl2vulns
							FILTER edge._from == CONCAT("purls/", purlKey) AND edge._to == CONCAT("vulns/", vuln._key)
							RETURN edge
						)

						// Insert edge if it does not exist
						FILTER existingEdge == NULL
						INSERT { _from: CONCAT("purls/", purlKey), _to: CONCAT("vulns/", vuln._key) } INTO purl2vulns
		`

		// Define bind variables
		parameters := map[string]interface{}{
			"vulnKey": content["_key"],
		}

		// Execute the query
		ctx := context.Background()
		cursor, err := dbconn.Database.Query(ctx, aql, &arangodb.QueryOptions{BindVars: parameters})
		if err != nil {
			logger.Sugar().Errorf("Failed to execute query %s: %v", aql, err)
		}
		cursor.Close()
	}
	return nil
}

// @title Ortelius v11 Vulnerability Microservice
// @version 11.0.0
// @description Vulnerabilities from osv.dev
// @description ![Release](https://img.shields.io/github/v/release/ortelius/scec-vulnerability?sort=semver)
// @description ![license](https://img.shields.io/github/license/ortelius/.github)
// @description
// @description ![Build](https://img.shields.io/github/actions/workflow/status/ortelius/scec-vulnerability/build-push-chart.yml)
// @description [![MegaLinter](https://github.com/ortelius/scec-vulnerability/workflows/MegaLinter/badge.svg?branch=main)](https://github.com/ortelius/scec-vulnerability/actions?query=workflow%3AMegaLinter+branch%3Amain)
// @description ![CodeQL](https://github.com/ortelius/scec-vulnerability/workflows/CodeQL/badge.svg)
// @description [![OpenSSF-Scorecard](https://api.securityscorecards.dev/projects/github.com/ortelius/scec-vulnerability/badge)](https://api.securityscorecards.dev/projects/github.com/ortelius/scec-vulnerability)
// @description
// @description ![Discord](https://img.shields.io/discord/722468819091849316)

// @termsOfService http://swagger.io/terms/
// @contact.name Ortelius Google Group
// @contact.email ortelius-dev@googlegroups.com
// @license.name Apache 2.0
// @license.url http://www.apache.org/licenses/LICENSE-2.0.html
// @host localhost:8080
func main() {
	LoadFromOSVDev()
}