diff --git a/cloud_webserver_v2/.air.toml b/cloud_webserver_v2/.air.toml new file mode 100644 index 0000000..95b1157 --- /dev/null +++ b/cloud_webserver_v2/.air.toml @@ -0,0 +1,8 @@ +root = "." +tmp_dir = "/tmp/air" + +[build] + cmd = "go build -o /tmp/air/server ./cmd/cloud_webserver_v2" + bin = "/tmp/air/server" + include_ext = ["go", "toml", "env"] + exclude_dir = ["vendor", "uploads", "garage"] diff --git a/cloud_webserver_v2/.env.example b/cloud_webserver_v2/.env.example new file mode 100644 index 0000000..188f6d4 --- /dev/null +++ b/cloud_webserver_v2/.env.example @@ -0,0 +1,11 @@ +# rename this file to .env.dev +# get variables from ./garage/bootstrap.sh +AWS_S3_ENDPOINT= +AWS_ACCESS_KEY= +AWS_SECRET_KEY= +AWS_S3_RUN_BUCKET= +AWS_REGION= + +MONGODB_URI=mongodb://test:test@mongo:27017 +MATLAB_URI=https://mps.hytechracing.duckdns.org +ENV=DEVELOPMENT diff --git a/cloud_webserver_v2/.gitignore b/cloud_webserver_v2/.gitignore index c6399b8..deea482 100644 --- a/cloud_webserver_v2/.gitignore +++ b/cloud_webserver_v2/.gitignore @@ -1,4 +1,5 @@ .env +.env.dev *.png *.mat uploads/ diff --git a/cloud_webserver_v2/Dockerfile.dev b/cloud_webserver_v2/Dockerfile.dev new file mode 100644 index 0000000..312136f --- /dev/null +++ b/cloud_webserver_v2/Dockerfile.dev @@ -0,0 +1,17 @@ +FROM golang:1.23 + +RUN apt-get update && apt-get install -y \ + libhdf5-serial-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN go install github.com/air-verse/air@v1.61.7 + +WORKDIR /app + +COPY go.mod go.sum ./ +RUN go mod download + +EXPOSE 8080 + +ENTRYPOINT ["air"] diff --git a/cloud_webserver_v2/docker-compose.override.yml b/cloud_webserver_v2/docker-compose.override.yml new file mode 100644 index 0000000..a0b6e58 --- /dev/null +++ b/cloud_webserver_v2/docker-compose.override.yml @@ -0,0 +1,62 @@ +services: + garage: + container_name: garage + image: dxflrs/garage:v1.1.0 + volumes: + - ./garage/garage.toml:/etc/garage.toml + - garage_data:/var/lib/garage/data + - garage_meta:/var/lib/garage/meta + ports: + - "3900:3900" + - "3902:3902" + healthcheck: + test: ["CMD", "/garage", "stats", "-a"] + interval: 5s + timeout: 5s + retries: 3 + + mongo: + image: mongo:7 + container_name: mongo + ports: + - "27017:27017" + volumes: + - mongo-data:/data/db + command: ["--quiet"] + environment: + - MONGO_INITDB_ROOT_USERNAME=test + - MONGO_INITDB_ROOT_PASSWORD=test + healthcheck: + test: + [ + "CMD-SHELL", + "mongosh --quiet --eval 'db.runCommand({ ping: 1 }).ok' localhost:27017 | grep -q 1", + ] + interval: 5s + timeout: 5s + retries: 3 + start_period: 10s + + cloud_webserver_v2: + build: + context: . + dockerfile: Dockerfile.dev + env_file: + - path: .env.dev + required: true + depends_on: + mongo: + condition: service_healthy + garage: + condition: service_healthy + volumes: + - .:/app + - matlab_mps_data:/mps_data + - run_metadata:/data/run_metadata + - dops_htmls:/app/files + - dops_logs:/app/logs + +volumes: + garage_data: + garage_meta: + mongo-data: diff --git a/cloud_webserver_v2/docker-compose.yml b/cloud_webserver_v2/docker-compose.yml index 6b4b71e..c408d16 100644 --- a/cloud_webserver_v2/docker-compose.yml +++ b/cloud_webserver_v2/docker-compose.yml @@ -1,20 +1,22 @@ services: cloud_webserver_v2: container_name: cloud_webserver_v2 - volumes: - - matlab_mps_data:/mps_data - - run_metadata:/data/run_metadata - - ./htmls:/app/files - - ./logs:/app/logs build: context: . - env_file: - - .env + env_file: + - path: .env + required: false ports: - "8080:8080" restart: unless-stopped + volumes: + - matlab_mps_data:/mps_data + - run_metadata:/data/run_metadata + - dops_htmls:/app/files + - dops_logs:/app/logs volumes: - run_metadata: matlab_mps_data: - external: true + run_metadata: + dops_htmls: + dops_logs: diff --git a/cloud_webserver_v2/garage/bootstrap.sh b/cloud_webserver_v2/garage/bootstrap.sh new file mode 100755 index 0000000..9d54425 --- /dev/null +++ b/cloud_webserver_v2/garage/bootstrap.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -euo pipefail + +ADMIN_API="http://localhost:3902" +ADMIN_TOKEN="admin" +BUCKET="dops-dev" +ACCESS_KEY_NAME="dev" + +header=(-H "Authorization: Bearer ${ADMIN_TOKEN}") + +echo "==> Getting node ID..." +node_status=$(curl -s "${header[@]}" "${ADMIN_API}/v1/status") +node_id=$(echo "$node_status" | jq -r '.node') + +echo "==> Assigning layout..." +curl -s "${header[@]}" -X POST "${ADMIN_API}/v1/layout" \ + -H "Content-Type: application/json" \ + -d "[{\"id\": \"${node_id}\", \"zone\": \"dc1\", \"capacity\": 1073741824, \"tags\": []}]" > /dev/null + +echo "==> Applying layout..." +layout=$(curl -s "${header[@]}" "${ADMIN_API}/v1/layout") +version=$(echo "$layout" | jq -r '.version') +next_version=$((version + 1)) +curl -s "${header[@]}" -X POST "${ADMIN_API}/v1/layout/apply" \ + -H "Content-Type: application/json" \ + -d "{\"version\": ${next_version}}" > /dev/null + +echo "==> Creating bucket '${BUCKET}'..." +curl -s "${header[@]}" -X POST "${ADMIN_API}/v1/bucket" \ + -H "Content-Type: application/json" \ + -d "{\"globalAlias\": \"${BUCKET}\"}" > /dev/null + +echo "==> Creating access key '${ACCESS_KEY_NAME}'..." +key_result=$(curl -s "${header[@]}" -X POST "${ADMIN_API}/v1/key" \ + -H "Content-Type: application/json" \ + -d "{\"name\": \"${ACCESS_KEY_NAME}\"}") +key_id=$(echo "$key_result" | jq -r '.accessKeyId') +key_secret=$(echo "$key_result" | jq -r '.secretAccessKey') + +echo "==> Granting key read/write on bucket..." +bucket_result=$(curl -s "${header[@]}" "${ADMIN_API}/v1/bucket?globalAlias=${BUCKET}") +bucket_id=$(echo "$bucket_result" | jq -r '.id') +curl -s "${header[@]}" -X POST "${ADMIN_API}/v1/bucket/allow" \ + -H "Content-Type: application/json" \ + -d "{\"bucketId\": \"${bucket_id}\", \"accessKeyId\": \"${key_id}\", \"permissions\": {\"read\": true, \"write\": true, \"owner\": true}}" > /dev/null + +echo "" +echo "Garage is ready! Set these in your .env:" +echo " AWS_S3_ENDPOINT=http://garage:3900" +echo " AWS_ACCESS_KEY=${key_id}" +echo " AWS_SECRET_KEY=${key_secret}" +echo " AWS_S3_RUN_BUCKET=${BUCKET}" +echo " AWS_REGION=us-east-1" diff --git a/cloud_webserver_v2/garage/garage.toml b/cloud_webserver_v2/garage/garage.toml new file mode 100644 index 0000000..d95942b --- /dev/null +++ b/cloud_webserver_v2/garage/garage.toml @@ -0,0 +1,21 @@ +metadata_dir = "/var/lib/garage/meta" +data_dir = "/var/lib/garage/data" +db_engine = "sqlite" + +replication_factor = 1 + +rpc_bind_addr = "[::]:3901" +rpc_secret = "0000000000000000000000000000000000000000000000000000000000000000" + +[s3_api] +s3_region = "us-east-1" +api_bind_addr = "[::]:3900" +root_domain = ".s3.garage.localhost" + +[s3_web] +bind_addr = "[::]:3903" +root_domain = ".web.garage.localhost" + +[admin] +api_bind_addr = "[::]:3902" +admin_token = "admin" diff --git a/cloud_webserver_v2/internal/s3/s3.go b/cloud_webserver_v2/internal/s3/s3.go index bfa9423..cb8f96b 100644 --- a/cloud_webserver_v2/internal/s3/s3.go +++ b/cloud_webserver_v2/internal/s3/s3.go @@ -3,6 +3,7 @@ package s3 import ( "context" "log" + "os" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" @@ -27,11 +28,23 @@ func NewS3Session(accessKey string, secretKey string, region string, bucket stri log.Fatalf("could not load config: %v", err) } + isDev := os.Getenv("ENV") == "DEVELOPMENT" + // Create an aws s3 service client client := s3.NewFromConfig(cfg, func(o *s3.Options) { o.BaseEndpoint = aws.String(endpoint) + o.UsePathStyle = isDev }) + + // In dev, presigned URLs need to use localhost so they're accessible from the browser presignClient := s3.NewPresignClient(client) + if isDev { + publicClient := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String("http://localhost:3900") + o.UsePathStyle = true + }) + presignClient = s3.NewPresignClient(publicClient) + } session := &s3Session{ client: client,