pushing multiple layers

Author: hackathon

.github/workflows/multi-layer.yaml

@@ -0,0 +1,72 @@
+name: Build Lambda Layers
+
+on:
+  push:
+    paths:
+      - 'layers/**'
+  workflow_dispatch:
+
+jobs:
+  build-and-publish-layers:
+    runs-on: ubuntu-latest
+
+    env:
+      S3_BUCKET: hackathon-lambda-ap-ai-cyberark
+      PYTHON_VERSION: python3.12
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      - name: Build and publish each layer
+        run: |
+          for dir in layers/*; do
+            if [ -f "$dir/requirements.txt" ]; then
+              LAYER_NAME=$(basename "$dir")
+              echo "Building layer: $LAYER_NAME"
+
+              # Create target directory
+              mkdir -p "$dir/python"
+
+              # Install Python packages
+              pip install --upgrade --no-cache-dir -r "$dir/requirements.txt" -t "$dir/python"
+
+              # Clean up unnecessary files
+              find "$dir/python" -type d -name "__pycache__" -exec rm -rf {} +
+              find "$dir/python" -type d -name "tests" -exec rm -rf {} +
+              find "$dir/python" -type f -name "*.pyc" -delete
+              find "$dir/python" -type d -name "*.dist-info" -exec rm -rf {} +
+
+              # Zip the layer
+              cd "$dir"
+              zip -r "../$LAYER_NAME.zip" python > /dev/null
+              cd ../..
+
+              # Upload to S3
+              echo "Uploading $LAYER_NAME.zip to S3"
+              aws s3 cp "$dir.zip" s3://$S3_BUCKET/layers/$LAYER_NAME.zip
+
+              # Publish to Lambda
+              echo "Publishing Lambda layer: $LAYER_NAME"
+              LAYER_ARN=$(aws lambda publish-layer-version \
+                --layer-name "$LAYER_NAME" \
+                --description "Layer for $LAYER_NAME" \
+                --content S3Bucket=$S3_BUCKET,S3Key=layers/$LAYER_NAME.zip \
+                --compatible-runtimes $PYTHON_VERSION \
+                --query 'LayerVersionArn' --output text)
+
+              echo "Published $LAYER_NAME: $LAYER_ARN"
+            fi
+          done

backend/layers/layer1/requirements.txt

@@ -0,0 +1,12 @@
+numpy
+pandas
+boto3
+python-docx
+fastapi[all]
+uvicorn[standard]
+pydantic
+python-dotenv
+requests_toolbelt
+sqlmodel
+psycopg2-binary==2.9.9
+mangum

backend/layers/layer2/requirements.txt

@@ -0,0 +1 @@
+sentence-transformers

backend/layers/layer3/requirements.txt

@@ -0,0 +1 @@
+faiss-cpu

backend/layers/layer4/requirements.txt

@@ -0,0 +1 @@
+scikit-learn