Move /deps/node_modules to /data/olympia/node_modules (#22955)

* Make build-docker locally debuggable * Migrate build scripts to python with single script to orchestrate during make up - Replaced the locale compilation script from a shell script to a Python script (compile_locales.py) for better error handling and parallel processing. - Updated the update_assets target in Makefile-docker to use the new update_assets.py script. - Removed the obsolete compile-mo.sh script. - Introduced sync_host_files.py to streamline dependency updates and asset synchronization. * Better checks that static file routing works as expected - Removed obsolete file checks and user validation from Makefile-docker. - Updated Nginx configuration to improve static file handling and added headers for better traceability. - Refactored storage management commands in the Python codebase: - Renamed `clean_storage` to `make_storage` for clarity and added a `clean` parameter. - Updated command implementations to use the new `make_storage` method. - Introduced a new system check for Nginx configurations to ensure proper file accessibility and response validation. * Migrate dependencies to /data/olympia - Updated docker-compose.yml to mount dependencies in /data/olympia/deps for better organization. - Modified Dockerfile to set environment variables for dependency directories and ensure proper ownership. - Adjusted Makefile-docker to remove NODE_MODULES variable and streamline npm commands. - Updated documentation to reflect changes in dependency paths. - Refactored install_deps.py to clean up dependency directories and removed obsolete package.json copying logic. - Updated settings_base.py to reference new dependency paths. * Remove HOST_MOUNT concept and always mount host files - Updated docker-compose.yml to simplify volume mounts by using the current directory for all services. - Removed HOST_MOUNT and HOST_MOUNT_SOURCE environment variables from settings and entrypoint scripts, streamlining the configuration for local and production environments. - Refactored setup.py to eliminate the get_olympia_mount function, replacing it with a more straightforward approach to determine the Docker target. - Adjusted GitHub Actions workflows to remove references to the obsolete mount input, ensuring cleaner CI/CD processes. - Enhanced test cases to reflect changes in Docker configuration and environment variable handling.
mozilla · Jan 22, 2025 · 61eb931 · 61eb931
1 parent 05fdf33
commit 61eb931
Show file tree

Hide file tree

Showing 37 changed files with 601 additions and 431 deletions.
diff --git a/.github/actions/build-docker/action.yml b/.github/actions/build-docker/action.yml
@@ -37,6 +37,7 @@ runs:
 
         echo "git_build_url=$git_repo_url/actions/runs/${{ github.run_id }}" >> $GITHUB_OUTPUT
         echo "git_sha=${{ github.sha }}" >> $GITHUB_OUTPUT
+        echo "metadata_file=buildx-bake-metadata.json" >> $GITHUB_OUTPUT
 
         cat $GITHUB_OUTPUT
 
@@ -51,33 +52,28 @@ runs:
           # and to prevent multiple tags from being associated with a build
           type=raw,value=${{ inputs.version }}
 
-    - name: Create .env and version.json files
-      shell: bash
-      run: |
-        # We only build the production image in CI
-        echo "DOCKER_TARGET=production" >> $GITHUB_ENV
-        echo "DOCKER_VERSION=${{ steps.meta.outputs.version }}" >> $GITHUB_ENV
-        echo "DOCKER_COMMIT=${{ steps.context.outputs.git_sha }}" >> $GITHUB_ENV
-        echo "DOCKER_BUILD=${{ steps.context.outputs.git_build_url }}" >> $GITHUB_ENV
-        echo "TAGS_FILE=${{ steps.meta.outputs.bake-file-tags }}" >> $GITHUB_ENV
-        echo "ANNOTATIONS_FILE=${{ steps.meta.outputs.bake-file-annotations }}" >> $GITHUB_ENV
-        echo "DOCKER_METADATA_FILE=buildx-bake-metadata.json" >> $GITHUB_ENV
-
-        make setup
-
     - name: Build Image
       id: build
       shell: bash
+      env:
+        DOCKER_TAGS_FILE: ${{ steps.meta.outputs.bake-file-tags }}
+        DOCKER_ANNOTATIONS_FILE: ${{ steps.meta.outputs.bake-file-annotations }}
+        DOCKER_METADATA_FILE: ${{ steps.context.outputs.metadata_file }}
+        DOCKER_PUSH: ${{ inputs.push }}
       run: |
+        make setup \
+          DOCKER_TARGET="production" \
+          DOCKER_VERSION="${{ steps.meta.outputs.version }}"
+
         make docker_build_web \
-          ARGS="--file ${{ env.TAGS_FILE }} --file ${{ env.ANNOTATIONS_FILE }}" \
-          DOCKER_PUSH=${{ inputs.push }}
+          DOCKER_COMMIT="${{ steps.context.outputs.git_sha }}" \
+          DOCKER_BUILD="${{ steps.context.outputs.git_build_url }}"
 
     - name: Get image digest
       id: build_meta
       shell: bash
       run: |
-        metadata=$(cat $DOCKER_METADATA_FILE)
+        metadata=$(cat ${{ steps.context.outputs.metadata_file }})
         echo "digest=$(echo $metadata | jq -r '.web."containerimage.digest"')" >> $GITHUB_OUTPUT
         echo "tag=$(echo $metadata | jq -r '.web."image.name"')" >> $GITHUB_OUTPUT
 
diff --git a/.github/actions/run-docker/action.yml b/.github/actions/run-docker/action.yml
@@ -23,10 +23,6 @@ inputs:
     description: 'Docker target to run (development|production)'
     required: false
     default: 'production'
-  mount:
-    description: 'Mount host files at runtime? (development|production)'
-    required: false
-    default: 'production'
   deps:
     description: 'Which dependencies to install at runtime? (development|production)'
     required: false
@@ -44,7 +40,6 @@ runs:
           DOCKER_DIGEST="${{ inputs.digest }}" \
           DOCKER_TARGET="${{ inputs.target }}" \
           OLYMPIA_UID="$(id -u)" \
-          OLYMPIA_MOUNT="${{ inputs.mount }}" \
           OLYMPIA_DEPS="${{ inputs.deps }}" \
           DATA_BACKUP_SKIP="${{ inputs.data_backup_skip }}" \
           DOCKER_WAIT="true"

diff --git a/.github/workflows/_test_check.yml b/.github/workflows/_test_check.yml
@@ -46,7 +46,6 @@ jobs:
     name: |
       version: '${{ matrix.version }}' |
       target: '${{ matrix.target }}' |
-      mount: '${{ matrix.mount }}' |
       deps: '${{ matrix.deps }}'
     strategy:
       fail-fast: false
@@ -57,9 +56,6 @@ jobs:
         target:
           - development
           - production
-        mount:
-          - development
-          - production
         deps:
           - development
           - production
@@ -72,7 +68,6 @@ jobs:
             Values passed to the action:
             version: ${{ matrix.version }}
             target: ${{ matrix.target }}
-            mount: ${{ matrix.mount }}
             deps: ${{ matrix.deps }}
           EOF
       - name: ${{ matrix.version == 'local' && 'Uncached Build' || 'Pull' }} Check
@@ -84,7 +79,6 @@ jobs:
         with:
           version: ${{ matrix.version }}
           target: ${{ matrix.target }}
-          mount: ${{ matrix.mount }}
           deps: ${{ matrix.deps }}
           run: make check
       - name: Cached Build Check
@@ -93,7 +87,6 @@ jobs:
         with:
           version: ${{ matrix.version }}
           target: ${{ matrix.target }}
-          mount: ${{ matrix.mount }}
           deps: ${{ matrix.deps }}
           run: echo true
 

diff --git a/.github/workflows/_test_main.yml b/.github/workflows/_test_main.yml
@@ -88,7 +88,6 @@ jobs:
           services: ''
           digest: ${{ inputs.digest }}
           version: ${{ inputs.version }}
-          mount: development
           deps: development
           run: |
             split="--splits ${{ needs.test_config.outputs.splits }}"

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -87,7 +87,6 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          mount: development
           deps: development
           target: development
           run: |
@@ -140,7 +139,6 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          mount: development
           deps: development
           run: make extract_locales
 

diff --git a/.gitignore b/.gitignore
@@ -55,5 +55,3 @@ tmp/*
 # do not ignore the following files
 !docker-compose.private.yml
 !private/README.md
-!storage/.gitignore
-!deps/.gitkeep
diff --git a/Dockerfile b/Dockerfile
@@ -10,13 +10,24 @@ ENV BUILD_INFO=/build-info.json
 SHELL ["/bin/bash", "-xue", "-c"]
 
 ENV OLYMPIA_UID=9500
+# give olympia access to the HOME directory
+ENV HOME=/data/olympia
+ENV DEPS_DIR=${HOME}/deps
+ENV NPM_DEPS_DIR=${HOME}/node_modules
+
 RUN <<EOF
 groupadd -g ${OLYMPIA_UID} olympia
-useradd -u ${OLYMPIA_UID} -g ${OLYMPIA_UID} -s /sbin/nologin -d /data/olympia olympia
+useradd -u ${OLYMPIA_UID} -g ${OLYMPIA_UID} -s /sbin/nologin -d ${HOME} olympia
+
+# Create and chown olympia directories
+olympia_dirs=("${DEPS_DIR}" "${NPM_DEPS_DIR}" "${HOME}/storage")
+for dir in "${olympia_dirs[@]}"; do
+  mkdir -p ${dir}
+  chown -R olympia:olympia ${dir}
+done
 EOF
 
-# give olympia access to the HOME directory
-ENV HOME=/data/olympia
+
 WORKDIR ${HOME}
 RUN chown -R olympia:olympia ${HOME}
 
@@ -74,37 +85,28 @@ ENV LANG=en_US.UTF-8
 ENV LC_ALL=en_US.UTF-8
 
 RUN <<EOF
-# Create directory for dependencies
-mkdir /deps
-chown -R olympia:olympia /deps
-
-
 # For backwards-compatibility purposes, set up links to uwsgi. Note that
 # the target does not exist yet at this point, but it will later.
-ln -s /deps/bin/uwsgi /usr/bin/uwsgi
+ln -s ${DEPS_DIR}/bin/uwsgi /usr/bin/uwsgi
 ln -s /usr/bin/uwsgi /usr/sbin/uwsgi
 
-# Create the storage directory and the test file to verify nginx routing
-mkdir -p ${HOME}/storage
-chown -R olympia:olympia ${HOME}/storage
 EOF
 
 USER olympia:olympia
 
 ENV PIP_USER=true
-ENV PIP_BUILD=/deps/build/
-ENV PIP_CACHE_DIR=/deps/cache/
-ENV PIP_SRC=/deps/src/
-ENV PYTHONUSERBASE=/deps
+ENV PIP_BUILD=${DEPS_DIR}/build/
+ENV PIP_CACHE_DIR=${DEPS_DIR}/cache/
+ENV PIP_SRC=${DEPS_DIR}/src/
+ENV PYTHONUSERBASE=${DEPS_DIR}
 ENV PATH=$PYTHONUSERBASE/bin:$PATH
-ENV NPM_CONFIG_PREFIX=/deps/
-ENV NPM_CACHE_DIR=/deps/cache/npm
+ENV NPM_CACHE_DIR=${DEPS_DIR}/cache/npm
 ENV NPM_DEBUG=true
 # Set python path to the project root and src to resolve olympia modules correctly
 ENV PYTHONPATH=${HOME}:${HOME}/src
 
 ENV PIP_COMMAND="python3 -m pip"
-ENV NPM_ARGS="--prefix ${NPM_CONFIG_PREFIX} --cache ${NPM_CACHE_DIR} --loglevel verbose"
+ENV NPM_ARGS="--cache ${NPM_CACHE_DIR} --loglevel verbose"
 
 # All we need in "base" is pip to be installed
 #this let's other layers install packages using the correct version.
@@ -127,16 +129,16 @@ COPY docker/etc/mime.types /etc/mime.types
 
 # Define production dependencies as a single layer
 # let's the rest of the stages inherit prod dependencies
-# and makes copying the /deps dir to the final layer easy.
+# and makes copying the /data/olympia/deps dir to the final layer easy.
 FROM base AS pip_production
 
 RUN \
     --mount=type=bind,source=scripts/install_deps.py,target=${HOME}/scripts/install_deps.py \
     # Files required to install pip dependencies
     --mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
     # Files required to install npm dependencies
-    --mount=type=bind,source=package.json,target=/deps/package.json \
-    --mount=type=bind,source=package-lock.json,target=/deps/package-lock.json \
+    --mount=type=bind,source=package.json,target=${HOME}/package.json \
+    --mount=type=bind,source=package-lock.json,target=${HOME}/package-lock.json \
     # Mounts for caching dependencies
     --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
     --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
@@ -146,6 +148,9 @@ EOF
 
 FROM base AS development
 
+# Copy build info from info
+COPY --from=info ${BUILD_INFO} ${BUILD_INFO}
+
 FROM base AS locales
 ARG LOCALE_DIR=${HOME}/locale
 # Compile locales
@@ -155,7 +160,7 @@ COPY --chown=olympia:olympia locale ${LOCALE_DIR}
 RUN \
     --mount=type=bind,source=requirements/locale.txt,target=${HOME}/requirements/locale.txt \
     --mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \
-    --mount=type=bind,source=locale/compile-mo.sh,target=${HOME}/compile-mo.sh \
+    --mount=type=bind,source=scripts/compile_locales.py,target=${HOME}/scripts/compile_locales.py \
     make -f Makefile-docker compile_locales
 
 # More efficient caching by mounting the exact files we need
@@ -175,10 +180,10 @@ COPY --chown=olympia:olympia static/ ${HOME}/static/
 RUN \
     --mount=type=bind,src=src,target=${HOME}/src \
     --mount=type=bind,src=Makefile-docker,target=${HOME}/Makefile-docker \
+    --mount=type=bind,src=scripts/update_assets.py,target=${HOME}/scripts/update_assets.py \
     --mount=type=bind,src=manage.py,target=${HOME}/manage.py \
 <<EOF
-echo "from olympia.lib.settings_base import *" > settings_local.py
-DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets
+make -f Makefile-docker update_assets
 EOF
 
 FROM base AS production
@@ -194,4 +199,4 @@ COPY --from=info ${BUILD_INFO} ${BUILD_INFO}
 # Copy compiled locales from builder
 COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy dependencies from `pip_production`
-COPY --from=pip_production --chown=olympia:olympia /deps /deps
+COPY --from=pip_production --chown=olympia:olympia ${DEPS_DIR} ${DEPS_DIR}
diff --git a/Makefile-docker b/Makefile-docker
@@ -8,16 +8,6 @@ export PYTHON_COMMAND=python3
 export PIP_COMMAND=$(PYTHON_COMMAND) -m pip
 APP=src/olympia/
 
-NODE_MODULES := $(NPM_CONFIG_PREFIX)node_modules/
-
-REQUIRED_FILES := \
-	Makefile \
-	Makefile-os \
-	Makefile-docker \
-	/deps/package.json \
-	/deps/package-lock.json \
-	/addons-server-docker-container \
-
 # Build list of dependencies to install
 DEPS = pip prod
 # If we're running a development image, then we should install the development dependencies
@@ -39,29 +29,12 @@ check_pip_packages: ## check the existence of multiple python packages
 		./scripts/check_pip_packages.sh $$dep.txt; \
 	done
 
-.PHONY: check_files
-check_files: ## check the existence of multiple files
-	@for file in $(REQUIRED_FILES); do test -f "$$file" || (echo "$$file is missing." && exit 1); done
-	@echo "All required files are present."
-
-.PHONY: check_olympia_user
-check_olympia_user: ## check if the olympia user exists and is current user
-	@if [ "$$(id -u olympia)" != "$$(id -u)" ]; then echo "The current user is not the olympia user."; exit 1; fi
-	@echo "The current user is the olympia user."
-
 .PHONY: check_django
 check_django: ## check if the django app is configured properly
 	./manage.py check
 
-.PHONY: check_nginx
-check_nginx:  ## check if the nginx config for local development is configured properly
-	mkdir -p /data/olympia/storage/shared_storage/uploads
-	echo "OK" > /data/olympia/storage/shared_storage/uploads/.check
-	@if [ "$$(curl -sf http://nginx/user-media/.check)" != "OK" ]; then echo "Requesting http://nginx/user-media/.check failed"; exit 1; fi
-	@echo "Nginx user-media configuration looks correct."
-
 .PHONY: check
-check: check_nginx check_files check_olympia_user check_debian_packages check_pip_packages check_django
+check: check_debian_packages check_pip_packages check_django
 
 .PHONY: data_dump
 data_dump:
@@ -72,14 +45,8 @@ data_load:
 	./manage.py data_load $(ARGS)
 
 .PHONY: update_assets
-update_assets:
-	# Copy files required in compress_assets to the static folder
-	# If changing this here, make sure to adapt tests in amo/test_commands.py
-	$(PYTHON_COMMAND) manage.py compress_assets
-	$(PYTHON_COMMAND) manage.py generate_jsi18n_files
-	# Collect static files: This MUST be run last or files will be missing
-	$(PYTHON_COMMAND) manage.py collectstatic --noinput
-
+update_assets: ## Update the static assets
+	$(HOME)/scripts/update_assets.py
 
 .PHONY: update_deps
 update_deps: ## Update the dependencies
@@ -94,7 +61,7 @@ setup-ui-tests:
 lint: ## lint the code
 	ruff check .
 	ruff format --check .
-	NODE_PATH=$(NODE_MODULES) npm exec $(NPM_ARGS) -- prettier --check '**'
+	npm exec $(NPM_ARGS) -- prettier --check '**'
 	curlylint src/
 
 lint-codestyle: lint
@@ -199,15 +166,15 @@ test_failed: ## rerun the failed tests from the previous run
 
 .PHONY: run_js_tests
 run_js_tests: ## Run the JavaScript test suite (requires compiled/compressed assets).
-	NODE_PATH=$(NODE_MODULES) npm exec $(NPM_ARGS) -- jest tests/js
+	npm exec $(NPM_ARGS) -- jest tests/js
 
 .PHONY: watch_js_tests
 watch_js_tests: ## Run+watch the JavaScript test suite (requires compiled/compressed assets).
-	NODE_PATH=$(NODE_MODULES) npm exec $(NPM_ARGS) -- jest --watch
+	npm exec $(NPM_ARGS) -- jest --watch
 
 .PHONY: format
 format: ## Autoformat our codebase.
-	NODE_PATH=$(NODE_MODULES) npm exec $(NPM_ARGS) -- prettier --write '**'
+	npm exec $(NPM_ARGS) -- prettier --write '**'
 	ruff check --fix-only .
 	ruff format .
 
@@ -218,7 +185,7 @@ extract_locales: ## extracts and merges translation strings
 .PHONE: compile_locales
 compile_locales: ## compiles translation strings
 	$(PIP_COMMAND) install --progress-bar=off --no-deps -r requirements/locale.txt
-	./locale/compile-mo.sh ./locale/
+	$(HOME)/scripts/compile_locales.py
 
 .PHONY: help_submake
 help_submake: