Merge pull request #28 from AllenInstitute/feature/upgrade-to-uv

Upgrade package to uv

Author: rpmcginty

.github/workflows/build.yml

@@ -7,6 +7,8 @@ on:
       - '**/_version.py'
   push:
     branches: [ main ]
+    tags-ignore:
+      - 'v*'
     paths-ignore:
       - '**/_version.py'
   workflow_dispatch:
@@ -20,22 +22,42 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+        uv-resolution: ["highest", "lowest"]
+        exclude:
+          - python-version: "3.11"
+            uv-resolution: "lowest"
+          - python-version: "3.12"
+            uv-resolution: "lowest"
+    env:
+      PYTHON_VERSION: ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4        
+      - name: Install uv
+        id: setup-uv
+        uses: astral-sh/setup-uv@v7
         with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
+          python-version: ${{ github.event.inputs.python-version }}
       - name: Set up AllenInstitute Repo Authorization
         uses: ./.github/actions/configure-org-repo-authorization
         with:
           token: ${{ secrets.AI_PACKAGES_TOKEN }}
           ssh_private_key: ${{ secrets.AIBSGITHUB_PRIVATE_KEY }}
       - name: Run Release
         run: |
-          make release
+          if [ ${{ matrix.uv-resolution }} == "highest" ]; then
+            make release            
+          else
+            # Install dependencies
+            uv sync --frozen --group dev --resolution ${{ matrix.uv-resolution }}
+            
+            # Linting 
+            uv run ruff check
+            uv run mypy ./
+
+            # Testing
+            uv run pytest -vv --durations=10
+          fi
         shell: bash
       - name: Upload coverage reports
         if: |
@@ -46,11 +68,10 @@ jobs:
               || (github.event_name == 'push' && github.ref_name == 'main')
               || github.event_name == 'workflow_dispatch'
             )
-            && matrix.python-version == '3.11'
+            && matrix.python-version == '3.12'
           }}
         uses: codecov/codecov-action@v5
         with:
           # https://github.com/codecov/codecov-action#arguments
           token: ${{ secrets.CODECOV_TOKEN }}
           env_vars: PYTHON_VERSION
-

.github/workflows/release.yml

@@ -19,7 +19,7 @@ on:
         type: choice
         required: true
         default: "3.11"
-        options: ["3.9", "3.10", "3.11", "3.12"]
+        options: ["3.10", "3.11", "3.12", "3.13"]
 
 jobs:
   bump:
@@ -41,11 +41,11 @@ jobs:
         uses: actions/checkout@v4
         with:
           ssh-key: ${{ secrets.AIBSGITHUB_PRIVATE_KEY }}
-      - name: Set up Python ${{ github.event.inputs.python-version }}
-        uses: actions/setup-python@v4        
+      - name: Install uv
+        id: setup-uv
+        uses: astral-sh/setup-uv@v7
         with:
           python-version: ${{ github.event.inputs.python-version }}
-          cache: 'pip'
       - name: Set up AllenInstitute Repo Authorization
         uses: ./.github/actions/configure-org-repo-authorization
         with:
@@ -63,7 +63,7 @@ jobs:
       - name: Bump version with bumpversion
         run: |
           source .venv/bin/activate  
-          bump-my-version bump ${{ github.event.inputs.part }}
+          uvx --from bump-my-version bump-my-version bump ${{ github.event.inputs.part }}
       - name: Commit and push with tags
         if: ${{ github.event.inputs.dry-run == 'false' }}
         run: |
@@ -106,11 +106,11 @@ jobs:
         with:
           ref: ${{ needs.bump.outputs.VERSION_TAG }}
           ssh-key: ${{ secrets.AIBSGITHUB_PRIVATE_KEY }}
-      - name: Set up Python ${{ github.event.inputs.python-version }}
-        uses: actions/setup-python@v4        
+      - name: Install uv
+        id: setup-uv
+        uses: astral-sh/setup-uv@v7
         with:
           python-version: ${{ github.event.inputs.python-version }}
-          cache: 'pip'
       - name: Set up AllenInstitute Repo Authorization
         uses: ./.github/actions/configure-org-repo-authorization
         with:

Makefile

@@ -13,7 +13,6 @@ COVERAGE_DIR := $(PACKAGE_DIR)/build/documentation/coverage/
 
 DEP_FILES := $(wildcard setup.*) $(wildcard requirements*.txt) $(wildcard pyproject.toml)
 
-
 ####################
 ##@ Helper Commands
 ####################
@@ -27,7 +26,6 @@ help:  ## Display this help
 ##@ Cleaning Commands
 ######################
 
-
 clean-venv:  ## Clean virtualenv
 	rm -rf $(VENV)/
 
@@ -60,37 +58,32 @@ obliterate: clean-venv clean  ## alias to clean, clean-venv
 
 .PHONY: clean-venv clean-install-stamp clean-build clean-pyc clean-test clean obliterate
 
-
 #########################
 ##@ Installation Commands
 #########################
 
-create-venv: $(PYTHON)  ## Creates virtualenv
-$(PYTHON):
-	python3 -m venv $(VENV) --prompt $(shell basename $(PACKAGE_DIR))
-	$(PYTHON) -m pip install --upgrade pip
+.uv: ## Check that uv is installed
+	@uv -V || echo 'Please install uv: https://docs.astral.sh/uv/getting-started/installation/'
 
-install: $(INSTALL_STAMP) ## Installs package dependencies
-$(INSTALL_STAMP): $(PYTHON) $(DEP_FILES)
-	@. $(VENV_BIN)/activate;\
-	$(PIP) install -e .[dev]; 
-	@touch $(INSTALL_STAMP)
+install: .uv  ## Installs development (dev/lint) related dependencies
+	uv sync --frozen --group dev
 
-install-release: clean-install-stamp $(PYTHON) $(DEP_FILES) ## Installs package for release
-	@. $(VENV_BIN)/activate;\
-	$(PIP) install .[release]
+install-release: .uv  ## Installs package dependencies
+	uv sync --frozen --group release
 
-install-force: clean-install-stamp install ## Force install package dependencies
+
+rebuild-lockfile: .uv  ## Rebuilds the lockfile
+	uv lock --upgrade
 
 link-packages: ## Link local packages to virtualenv  
 	@parent_dir=$$(dirname $$(pwd)); \
 	local_packages=$$(ls $$parent_dir); \
-	dependencies=$$($(PIP) list --format freeze --exclude-editable | awk -F '==' '{print $$1}');\
+	dependencies=$$(uv pip list --format freeze --exclude-editable | awk -F '==' '{print $$1}');\
 	for local_package in $$local_packages; do \
 		for dependency in $$dependencies; do \
 			if [ $$local_package == $$dependency ]; then \
 				echo "Reinstalling $$local_package dependency to local override"; \
-				$(PIP) install -e $$parent_dir/$$local_package --no-deps; \
+				uv add -v --editable --frozen $$parent_dir/$$local_package; \
 			fi \
 		done; \
 	done
@@ -99,72 +92,64 @@ unlink-packages: ## Unlink local packages from virtualenv
 	@parent_dir=$$(dirname $$(pwd)); \
 	this_package=$$(basename $$(pwd)); \
 	local_packages=$$(ls $$parent_dir); \
-	dependencies=$$($(PIP) list --format freeze --editable | awk -F '==' '{print $$1}');\
+	dependencies=$$(uv pip list --format freeze --editable | awk -F '==' '{print $$1}');\
 	is_found=0; \
 	for local_package in $$local_packages; do \
 		for dependency in $$dependencies; do \
 			if [ $$local_package == $$dependency ] && [ $$local_package != $$this_package ]; then \
 				is_found=1; \
+				uv remove --frozen $$local_package; \
 			fi; \
 		done \
 	done; \
 	if [ $$is_found == 1 ]; then \
 		echo "Found dependencies installed locally, reinstalling..."; \
-		make clean-install-stamp install; \
+		make install; \
 	fi
 
-.PHONY: create-venv install install-force link-packages unlink-packages
+.PHONY: .uv install install-release install rebuild-lockfile link-packages unlink-packages
 
 #######################
 ##@ Formatting Commands
 #######################
 
-lint-black: $(INSTALL_STAMP) ## Run black (check only)
-	$(VENV_BIN)/black ./ --check
-
-lint-isort: $(INSTALL_STAMP) ## Run isort (check only) 
-	$(VENV_BIN)/isort ./ --check
-
-lint-mypy: $(INSTALL_STAMP) ## Run mypy
-	$(VENV_BIN)/mypy ./
+lint-ruff: install ## Run ruff checker
+	uv run ruff check
 
+lint-mypy: install ## Run mypy
+	uv run mypy ./
 
-lint: lint-isort lint-black lint-mypy  ## Run all lint targets (black, isort, mypy)
+lint: lint-ruff lint-mypy  ## Run all lint targets (ruff, mypy)
 
 
-format-black: $(INSTALL_STAMP) ## Format code using black
-	$(VENV_BIN)/black ./
+format-ruff: install ## Run ruff formatter 
+	uv run ruff check --fix
+	uv run ruff format
 
-format-isort: $(INSTALL_STAMP) ## Format code using isort
-	$(VENV_BIN)/isort ./
+format: format-ruff  ## Run all formatters (ruff) 
 
-
-format: format-isort format-black  ## Run all formatters (black, isort) 
-
-.PHONY: lint-isort lint-black lint-mypy lint format-lint format-black format-mypy format
+.PHONY: lint-ruff lint-mypy lint format-ruff format-mypy format
 
 #####################
 ##@ Testing Commands
 #####################
 
-pytest: $(INSTALL_STAMP)  ## Run test (pytest)
-	$(VENV_BIN)/pytest -vv --durations=10
-
-tox: $(INSTALL_STAMP)  ## Run Test in tox environment
-	$(VENV_BIN)/tox
+pytest: install  ## Run test (pytest)
+	uv run pytest -vv --durations=10
 
 test: pytest  ## Run Standard Tests
 
-.PHONY: pytest tox test
-
+.PHONY: pytest test
 
 #####################
 ##@ Inspect Commands
 #####################
 
-coverage-server: $(INSTALL_STAMP) ## Run coverage server
+coverage-server: install ## Run coverage server
 	$(PYTHON) -m http.server $(COVERAGE_SERVER_PORT) -d $(COVERAGE_DIR)
 
+.PHONY: coverage-server
+
 
 #####################
 ##@ Docker Commands
@@ -183,8 +168,7 @@ docker-build: ## Build docker image
 #####################
 
 dist: install-release ## Build source and wheel package
-	@. $(VENV_BIN)/activate;\
-	$(PYTHON) -m build;
+	uv build
 
 reinstall: obliterate install ## Recreate environment and install
 
@@ -194,4 +178,4 @@ post-build: lint test  ## Run linters and tests
 release: pre-build build post-build  ## Runs pre-build, build, post-build
 run: release
 
-.PHONY: reinstall pre-build build post-build release run
+.PHONY: dist reinstall pre-build build post-build release run

docker/Dockerfile

@@ -1,12 +1,66 @@
-FROM public.ecr.aws/lambda/python:3.11
+# Pin the Python toolchain used in the builder; must match Lambda runtime major.minor
+ARG PYTHON_VERSION=3.11
+
 
 # Resources:
 # - https://gallery.ecr.aws/lambda/python
 # - https://docs.aws.amazon.com/lambda/latest/dg/python-image.html
 # - https://docs.aws.amazon.com/lambda/latest/dg/images-test.html
 # - https://github.com/aws/aws-lambda-python-runtime-interface-client
 
-# update system
+
+#####################################
+############## STAGE 0 ##############
+########### UV BINARIES #############
+#####################################
+FROM ghcr.io/astral-sh/uv:latest AS uvbin
+
+#####################################
+############## STAGE 1 ##############
+################ Builder ############
+#####################################
+FROM public.ecr.aws/lambda/python:${PYTHON_VERSION} AS builder
+
+# Minimal tools for resolving/installing deps from the monorepo
+RUN yum update -y \
+    && yum install -y \
+        git \
+        openssh-clients \
+    && yum clean all \
+    && rm -rf /var/cache/yum
+
+# Bring in uv from the official Astral image (builder only)
+COPY --from=uvbin /uv /uvx /usr/local/bin/
+
+# Prepare for private repos if needed
+RUN mkdir -p -m 0700 /root/.ssh && ssh-keyscan github.com >> /root/.ssh/known_hosts
+
+# Copy the monorepo so relative path installs work
+ENV REPO_ROOT=/asset-input
+COPY . ${REPO_ROOT}
+WORKDIR ${REPO_ROOT}
+
+# Vendor all Python deps (including awslambdaric and local pkgs) into a folder
+# that will become ${LAMBDA_TASK_ROOT} in the runtime image.
+ENV LAYER_DIR=/opt/lambda-task
+RUN --mount=type=ssh \
+    uv export --frozen --no-dev --group docker --no-editable -o requirements-autogen.txt && \
+    uv pip install -r requirements-autogen.txt --target ${LAYER_DIR}
+
+# Clean up build-time artifacts that might have been created (defensive)
+RUN find ${LAYER_DIR} -type d -name "__pycache__" -prune -exec rm -rf {} +
+
+# Copy entrypoint script alongside the vendored packages
+RUN cp ${REPO_ROOT}/docker/docker-entrypoint.sh ${LAYER_DIR}/docker-entrypoint.sh \
+    && chmod +x ${LAYER_DIR}/docker-entrypoint.sh
+
+#####################################
+############## STAGE 2 ##############
+############## Runtime ##############
+#####################################
+FROM public.ecr.aws/lambda/python:${PYTHON_VERSION}
+
+# Base image is Amazon Linux; install helpful tools (retain previous behavior)
 RUN yum update -y \
     && yum install -y \
         curl \
@@ -15,30 +69,20 @@ RUN yum update -y \
         wget \
         aws-cli \
         git \
-    && rm -rf /var/lib/apt/lists/*
+    && yum clean all \
+    && rm -rf /var/cache/yum
 
-RUN mkdir -p -m 0700 ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts
-
-# Base image has an old aws cli version (v1.x), this will replace that
+# Upgrade AWS CLI to v2
 ADD https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip /tmp
 RUN unzip /tmp/awscli-exe-linux-x86_64.zip -d /tmp \
     && rm /tmp/awscli-exe-linux-x86_64.zip \
     && /tmp/aws/install
 
-COPY . /asset-input
-
-RUN --mount=type=ssh cd /asset-input \
-    && python3 -m pip install --upgrade pip \
-    && pip3 install .[lambda] \
-        --no-cache-dir \
-        --target "${LAMBDA_TASK_ROOT}" \
-    && cp docker/docker-entrypoint.sh ${LAMBDA_TASK_ROOT}/docker-entrypoint.sh \
-    && chmod +x ${LAMBDA_TASK_ROOT}/docker-entrypoint.sh
+# Copy only the vendored application deps into the Lambda task root
+# (No uv, no build tools end up in the final image)
+COPY --from=builder /opt/lambda-task ${LAMBDA_TASK_ROOT}
 
 ENV PYTHONPATH="${LAMBDA_TASK_ROOT}:${PYTHONPATH}"
 ENV PATH="${LAMBDA_TASK_ROOT}/bin:${PATH}"
 
-RUN rm -rf /asset-input
-
 ENTRYPOINT [ "bash", "-c", "${LAMBDA_TASK_ROOT}/docker-entrypoint.sh" ]
-