save mps features into db

.github/workflows/create-image.yml

@@ -0,0 +1,43 @@
+name: build docker image and publish to ghcr
+
+on:
+  push:
+    branches:
+      - main
+      - mps-fix
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get short SHA
+        id: vars
+        run: echo "SHORT_SHA=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: ./cloud_webserver_v2
+          file: ./cloud_webserver_v2/Dockerfile
+          push: true
+          tags: |
+            ghcr.io/hytech-racing/data_acq_cloud:latest
+            ghcr.io/hytech-racing/data_acq_cloud:${{ steps.vars.outputs.SHORT_SHA }}
+

cloud_webserver_v2/Dockerfile

@@ -32,24 +32,15 @@ WORKDIR /app
 # Copy the server binary from the build stage
 COPY --from=go-builder /app/cmd/cloud_webserver_v2/server .
 
-# Read in build arguments and save them to .env
-ARG MONGODB_URI
-ARG AWS_REGION
-ARG AWS_S3_RUN_BUCKET
-ARG AWS_S3_ENDPOINT
-ARG AWS_ACCESS_KEY
-ARG AWS_SECRET_KEY
-ARG MATLAB_URI
-ARG ENV
-
-RUN echo "MONGODB_URI=${MONGODB_URI}" >> .env && \
-    echo "AWS_REGION=${AWS_REGION}" >> .env && \
-    echo "AWS_S3_RUN_BUCKET=${AWS_S3_RUN_BUCKET}" >> .env && \
-    echo "AWS_S3_ENDPOINT=${AWS_S3_ENDPOINT}" >> .env && \
-    echo "AWS_ACCESS_KEY=${AWS_ACCESS_KEY}" >> .env && \
-    echo "AWS_SECRET_KEY=${AWS_SECRET_KEY}" >> .env && \
-    echo "ENV=${ENV}" >> .env && \
-    echo "MATLAB_URI=${MATLAB_URI}" >> .env
+# Declare runtime environment variables
+ENV MONGODB_URI=""
+ENV AWS_REGION=""
+ENV AWS_S3_RUN_BUCKET=""
+ENV AWS_S3_ENDPOINT=""
+ENV AWS_ACCESS_KEY=""
+ENV AWS_SECRET_KEY=""
+ENV MATLAB_URI=""
+ENV ENV=""
 
 # Expose the port
 EXPOSE 8080

cloud_webserver_v2/cmd/cloud_webserver_v2/main.go

@@ -55,14 +55,10 @@ func main() {
 	// load .env file
 	err := godotenv.Load(".env")
 	if err != nil {
-		log.Fatalf("Error loading .env file %s", err)
+		log.Println("No .env file found; continuing with environment variables.")
 	}
 	log.Println("Loaded .env file...")
 
-	mpsURI := os.Getenv("MATLAB_URI")
-	mpsClient := mps.NewMatlabClient(mpsURI)
-	mpsClient.PollForResults()
-
 	// Setup database our database connection
 	uri := os.Getenv("MONGODB_URI")
 	if uri == "" {
@@ -74,6 +70,10 @@ func main() {
 	}
 	log.Println("Connected to database...")
 
+	// Setup MPS
+	mpsURI := os.Getenv("MATLAB_URI")
+	mpsClient := mps.NewMatlabClient(dbClient, mpsURI, 1*time.Second)
+
 	// Setup aws s3 connection
 	awsRegion := os.Getenv("AWS_REGION")
 	if awsRegion == "" {

cloud_webserver_v2/docker-compose.yml

@@ -2,23 +2,19 @@ services:
   cloud_webserver_v2:
     container_name: cloud_webserver_v2
     volumes:
+      - matlab_mps_data:/mps_data
       - run_metadata:/data/run_metadata
       - ./htmls:/app/files
       - ./logs:/app/logs
     build:
       context: .
-      args:
-        MONGODB_URI: ${MONGODB_URI}
-        AWS_REGION: ${AWS_REGION}
-        AWS_S3_RUN_BUCKET: ${AWS_S3_RUN_BUCKET}
-        AWS_S3_ENDPOINT: ${AWS_S3_ENDPOINT}
-        AWS_ACCESS_KEY: ${AWS_ACCESS_KEY}
-        AWS_SECRET_KEY: ${AWS_SECRET_KEY}
-        ENV: ${ENV}
-        MATLAB_URI: ${MATLAB_URI}
+    env_file: 
+      - .env
     ports:
       - "8080:8080"
     restart: unless-stopped
 
 volumes:
   run_metadata:
+  matlab_mps_data:
+    external: true

cloud_webserver_v2/go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/go-chi/chi/v5 v5.1.0
 	github.com/go-chi/cors v1.2.1
 	github.com/go-chi/render v1.0.3
+	github.com/google/go-github v17.0.0+incompatible
 	github.com/jhump/protoreflect v1.16.0
 	github.com/joho/godotenv v1.5.1
 	go-hep.org/x/hep v0.35.0
@@ -48,7 +49,6 @@ require (
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/gonuts/binary v0.2.0 // indirect
-	github.com/google/go-github v17.0.0+incompatible // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/klauspost/compress v1.17.9 // indirect

cloud_webserver_v2/internal/background/mcap_jobs.go

@@ -187,7 +187,7 @@ func (p *PostProcessMCAPUploadJob) Process(fp *FileProcessor, job *FileJob) erro
 		McapFiles:    mcapFiles,
 		MatFiles:     matFiles,
 		ContentFiles: contentFiles,
-		MpsRecord:    make(map[string]interface{}),
+		MpsRecord:    models.MpsRecordModel{},
 		Id:           recordId,
 	}
 

cloud_webserver_v2/internal/delivery/http/mcap_handler.go

@@ -285,6 +285,11 @@ func (h *mcapHandler) ProcessMatlabJob(w http.ResponseWriter, r *http.Request) *
 	}
 	scripts := strings.Split(scriptsParam, ",")
 
+	versionParam := r.URL.Query().Get("version")
+	if versionParam == "" {
+		return NewHandlerError("invalid request, must pass in archive version in query params", http.StatusBadRequest)
+	}
+
 	mcapId := chi.URLParam(r, "id")
 	if mcapId == "" {
 		return NewHandlerError("invalid request, must pass in mcap id", http.StatusBadRequest)
@@ -310,12 +315,8 @@ func (h *mcapHandler) ProcessMatlabJob(w http.ResponseWriter, r *http.Request) *
 		return NewHandlerError("no h5 files found", http.StatusFailedDependency)
 	}
 
-	// TODO: check if matfile exists on filesystem, if not download it from S3
-	// currently assume it exists on filesystem
-	for _, matFile := range matFiles {
-		for _, script := range scripts {
-			h.mpsClient.SubmitMatlabJob(matFile.FileName, script, script)
-		}
+	for _, script := range scripts {
+		h.mpsClient.SubmitMatlabJob(ctx, h.s3Repository, mcapId, versionParam, script)
 	}
 
 	render.JSON(w, r, "jobs submitted")
@@ -349,16 +350,18 @@ func (h *mcapHandler) UpdateMetadataRecordFromID(w http.ResponseWriter, r *http.
 
 	for key, values := range r.Form {
 		if strings.HasPrefix(key, "mps.") {
-			mpsmetadata := make(map[string]interface{})
-			mpsmetadata[strings.TrimPrefix(key, "mps.")] = values[0]
+			// TODO: Figure out if updating MPS with an HTTP request is needed
 
-			if runModel.MpsRecord == nil {
-				runModel.MpsRecord = make(map[string]interface{})
-			}
+			// mpsMetadata := make(map[string]interface{})
+			// mpsMetadata[strings.TrimPrefix(key, "mps.")] = values[0]
 
-			for function, record := range mpsmetadata {
-				runModel.MpsRecord[function] = record
-			}
+			// if runModel.MpsRecord == nil {
+			// 	runModel.MpsRecord = make(map[string]models.MpsScripts)
+			// }
+
+			// for function, record := range mpsMetadata {
+			// 	runModel.MpsRecord[function] = record
+			// }
 		} else {
 			switch key {
 			case "date":
@@ -419,8 +422,9 @@ func (h *mcapHandler) ResetMetadataRecordFromID(w http.ResponseWriter, r *http.R
 		runModel.Location = nil
 	case "event_type":
 		runModel.EventType = nil
-	case "mps_record":
-		runModel.MpsRecord = make(map[string]interface{})
+	// TODO: Figure out if updating MPS with an HTTP request is needed
+	// case "mps_record":
+	// 	runModel.MpsRecord = make(map[string]interface{})
 	case "car_model":
 		runModel.CarModel = ""
 	default:

cloud_webserver_v2/internal/models/vehicle_run_model.go

@@ -21,6 +21,31 @@ type FileModelResponse struct {
 	FileName  string `json:"file_name"`
 }
 
+// MpsScriptResultType defines the type of result that can be returned by a MATLAB script
+// Can be "mat", "image", or "text" for now
+type MpsScriptResultType string
+
+const (
+	Mat   MpsScriptResultType = "mat"
+	Image MpsScriptResultType = "image"
+	Text  MpsScriptResultType = "text"
+)
+
+// MpsScriptResultModel represents the schema of the returned result of a MATLAB script
+type MpsScriptResultModel struct {
+	Type MpsScriptResultType `json:"type"`
+
+	// If Type is "mat" or "image", Result will be a path to the file as a string
+	// If Type is "text"", Result will be the result as a string
+	Result string `json:"result"`
+}
+
+// MpsScriptModel represents a map of MATLAB script names to their results
+type MpsScriptModel map[string]MpsScriptResultModel
+
+// MpsRecord represents a map of MATLAB package names to their scripts
+type MpsRecordModel map[string]MpsScriptModel
+
 type VehicleRunModel struct {
 	Id             primitive.ObjectID     `bson:"_id,omitempty"`
 	ContentFiles   map[string][]FileModel `bson:"content_files,omitempty"`
@@ -33,7 +58,7 @@ type VehicleRunModel struct {
 	CarModel       string                 `bson:"car_model,omitempty"`
 	Date           time.Time              `bson:"date"`
 	MatFiles       []FileModel            `bson:"mat_files,omitempty"`
-	MpsRecord      map[string]interface{} `bson:"mps_record,omitempty"`
+	MpsRecord      MpsRecordModel         `bson:"mps_record,omitempty"`
 }
 
 type VehicleRunModelResponse struct {
@@ -48,7 +73,7 @@ type VehicleRunModelResponse struct {
 	Location       *string                        `json:"location"`
 	EventType      *string                        `json:"event_type"`
 	DynamicFields  map[string]interface{}         `json:"dynamic_fields"`
-	MpsRecord      map[string]interface{}         `json:"mps_record"`
+	MpsRecord      MpsRecordModel                 `json:"mps_record"`
 }
 
 func VehicleRunSerialize(ctx context.Context, s3Repo *s3.S3Repository, model VehicleRunModel) VehicleRunModelResponse {