Joseph huang local environments

cloud_webserver_v2/.air.toml

@@ -0,0 +1,8 @@
+root = "."
+tmp_dir = "/tmp/air"
+
+[build]
+  cmd = "go build -o /tmp/air/server ./cmd/cloud_webserver_v2"
+  bin = "/tmp/air/server"
+  include_ext = ["go", "toml", "env"]
+  exclude_dir = ["vendor", "uploads", "garage"]

cloud_webserver_v2/.env.example

@@ -0,0 +1,11 @@
+# rename this file to .env.dev
+# get variables from ./garage/bootstrap.sh
+AWS_S3_ENDPOINT=
+AWS_ACCESS_KEY=
+AWS_SECRET_KEY=
+AWS_S3_RUN_BUCKET=
+AWS_REGION=
+
+MONGODB_URI=mongodb://test:test@mongo:27017
+MATLAB_URI=https://mps.hytechracing.duckdns.org
+ENV=DEVELOPMENT

cloud_webserver_v2/.gitignore

@@ -1,4 +1,5 @@
 .env
+.env.dev
 *.png
 *.mat
 uploads/

cloud_webserver_v2/Dockerfile.dev

@@ -0,0 +1,17 @@
+FROM golang:1.23
+
+RUN apt-get update && apt-get install -y \
+    libhdf5-serial-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN go install github.com/air-verse/air@v1.61.7
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+EXPOSE 8080
+
+ENTRYPOINT ["air"]

cloud_webserver_v2/docker-compose.override.yml

@@ -0,0 +1,62 @@
+services:
+  garage:
+    container_name: garage
+    image: dxflrs/garage:v1.1.0
+    volumes:
+      - ./garage/garage.toml:/etc/garage.toml
+      - garage_data:/var/lib/garage/data
+      - garage_meta:/var/lib/garage/meta
+    ports:
+      - "3900:3900"
+      - "3902:3902"
+    healthcheck:
+      test: ["CMD", "/garage", "stats", "-a"]
+      interval: 5s
+      timeout: 5s
+      retries: 3
+
+  mongo:
+    image: mongo:7
+    container_name: mongo
+    ports:
+      - "27017:27017"
+    volumes:
+      - mongo-data:/data/db
+    command: ["--quiet"]
+    environment:
+      - MONGO_INITDB_ROOT_USERNAME=test
+      - MONGO_INITDB_ROOT_PASSWORD=test
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "mongosh --quiet --eval 'db.runCommand({ ping: 1 }).ok' localhost:27017 | grep -q 1",
+        ]
+      interval: 5s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+
+  cloud_webserver_v2:
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+    env_file:
+      - path: .env.dev
+        required: true
+    depends_on:
+      mongo:
+        condition: service_healthy
+      garage:
+        condition: service_healthy
+    volumes:
+      - .:/app
+      - matlab_mps_data:/mps_data
+      - run_metadata:/data/run_metadata
+      - dops_htmls:/app/files
+      - dops_logs:/app/logs
+
+volumes:
+  garage_data:
+  garage_meta:
+  mongo-data:

cloud_webserver_v2/docker-compose.yml

@@ -1,20 +1,22 @@
 services:
   cloud_webserver_v2:
     container_name: cloud_webserver_v2
-    volumes:
-      - matlab_mps_data:/mps_data
-      - run_metadata:/data/run_metadata
-      - ./htmls:/app/files
-      - ./logs:/app/logs
     build:
       context: .
-    env_file: 
-      - .env
+    env_file:
+      - path: .env
+        required: false
     ports:
       - "8080:8080"
     restart: unless-stopped
+    volumes:
+      - matlab_mps_data:/mps_data
+      - run_metadata:/data/run_metadata
+      - dops_htmls:/app/files
+      - dops_logs:/app/logs
 
 volumes:
-  run_metadata:
   matlab_mps_data:
-    external: true
+  run_metadata:
+  dops_htmls:
+  dops_logs:

cloud_webserver_v2/garage/bootstrap.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+set -euo pipefail
+
+ADMIN_API="http://localhost:3902"
+ADMIN_TOKEN="admin"
+BUCKET="dops-dev"
+ACCESS_KEY_NAME="dev"
+
+header=(-H "Authorization: Bearer ${ADMIN_TOKEN}")
+
+echo "==> Getting node ID..."
+node_status=$(curl -s "${header[@]}" "${ADMIN_API}/v1/status")
+node_id=$(echo "$node_status" | jq -r '.node')
+
+echo "==> Assigning layout..."
+curl -s "${header[@]}" -X POST "${ADMIN_API}/v1/layout" \
+  -H "Content-Type: application/json" \
+  -d "[{\"id\": \"${node_id}\", \"zone\": \"dc1\", \"capacity\": 1073741824, \"tags\": []}]" > /dev/null
+
+echo "==> Applying layout..."
+layout=$(curl -s "${header[@]}" "${ADMIN_API}/v1/layout")
+version=$(echo "$layout" | jq -r '.version')
+next_version=$((version + 1))
+curl -s "${header[@]}" -X POST "${ADMIN_API}/v1/layout/apply" \
+  -H "Content-Type: application/json" \
+  -d "{\"version\": ${next_version}}" > /dev/null
+
+echo "==> Creating bucket '${BUCKET}'..."
+curl -s "${header[@]}" -X POST "${ADMIN_API}/v1/bucket" \
+  -H "Content-Type: application/json" \
+  -d "{\"globalAlias\": \"${BUCKET}\"}" > /dev/null
+
+echo "==> Creating access key '${ACCESS_KEY_NAME}'..."
+key_result=$(curl -s "${header[@]}" -X POST "${ADMIN_API}/v1/key" \
+  -H "Content-Type: application/json" \
+  -d "{\"name\": \"${ACCESS_KEY_NAME}\"}")
+key_id=$(echo "$key_result" | jq -r '.accessKeyId')
+key_secret=$(echo "$key_result" | jq -r '.secretAccessKey')
+
+echo "==> Granting key read/write on bucket..."
+bucket_result=$(curl -s "${header[@]}" "${ADMIN_API}/v1/bucket?globalAlias=${BUCKET}")
+bucket_id=$(echo "$bucket_result" | jq -r '.id')
+curl -s "${header[@]}" -X POST "${ADMIN_API}/v1/bucket/allow" \
+  -H "Content-Type: application/json" \
+  -d "{\"bucketId\": \"${bucket_id}\", \"accessKeyId\": \"${key_id}\", \"permissions\": {\"read\": true, \"write\": true, \"owner\": true}}" > /dev/null
+
+echo ""
+echo "Garage is ready! Set these in your .env:"
+echo "  AWS_S3_ENDPOINT=http://garage:3900"
+echo "  AWS_ACCESS_KEY=${key_id}"
+echo "  AWS_SECRET_KEY=${key_secret}"
+echo "  AWS_S3_RUN_BUCKET=${BUCKET}"
+echo "  AWS_REGION=us-east-1"

cloud_webserver_v2/garage/garage.toml

@@ -0,0 +1,21 @@
+metadata_dir = "/var/lib/garage/meta"
+data_dir = "/var/lib/garage/data"
+db_engine = "sqlite"
+
+replication_factor = 1
+
+rpc_bind_addr = "[::]:3901"
+rpc_secret = "0000000000000000000000000000000000000000000000000000000000000000"
+
+[s3_api]
+s3_region = "us-east-1"
+api_bind_addr = "[::]:3900"
+root_domain = ".s3.garage.localhost"
+
+[s3_web]
+bind_addr = "[::]:3903"
+root_domain = ".web.garage.localhost"
+
+[admin]
+api_bind_addr = "[::]:3902"
+admin_token = "admin"

cloud_webserver_v2/internal/s3/s3.go

@@ -3,6 +3,7 @@ package s3
 import (
 	"context"
 	"log"
+	"os"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
@@ -27,11 +28,23 @@ func NewS3Session(accessKey string, secretKey string, region string, bucket stri
 		log.Fatalf("could not load config: %v", err)
 	}
 
+	isDev := os.Getenv("ENV") == "DEVELOPMENT"
+
 	// Create an aws s3 service client
 	client := s3.NewFromConfig(cfg, func(o *s3.Options) {
 		o.BaseEndpoint = aws.String(endpoint)
+		o.UsePathStyle = isDev
 	})
+
+	// In dev, presigned URLs need to use localhost so they're accessible from the browser
 	presignClient := s3.NewPresignClient(client)
+	if isDev {
+		publicClient := s3.NewFromConfig(cfg, func(o *s3.Options) {
+			o.BaseEndpoint = aws.String("http://localhost:3900")
+			o.UsePathStyle = true
+		})
+		presignClient = s3.NewPresignClient(publicClient)
+	}
 
 	session := &s3Session{
 		client:        client,