Merge pull request #7249 from freedomofpress/noble-dev

Support noble dev environment and pass tests

.github/workflows/ci.yml

@@ -38,42 +38,60 @@ jobs:
 
   test-app:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ubuntu_version:
+          - focal
+          - noble
     steps:
       - uses: actions/checkout@v4
       - name: Build image
         run: |
-          DOCKER_BUILD_VERBOSE=true ./securedrop/bin/dev-shell true
+          UBUNTU_VERSION=${{ matrix.ubuntu_version }} DOCKER_BUILD_VERBOSE=true ./securedrop/bin/dev-shell true
       - name: Run application tests
         run: |
-          make test-app
+          UBUNTU_VERSION=${{ matrix.ubuntu_version }} make test-app
 
   test-functional:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ubuntu_version:
+          - focal
+          - noble
     steps:
       - uses: actions/checkout@v4
       - name: Build image
         run: |
-          DOCKER_BUILD_VERBOSE=true ./securedrop/bin/dev-shell true
+          UBUNTU_VERSION=${{ matrix.ubuntu_version }} DOCKER_BUILD_VERBOSE=true ./securedrop/bin/dev-shell true
       - name: Run functional tests
         run: |
-          make test-functional
+          UBUNTU_VERSION=${{ matrix.ubuntu_version }} make test-functional
 
   test-pageslayout:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ubuntu_version:
+          - focal
+          - noble
     steps:
       - uses: actions/checkout@v4
       - name: Build image
         run: |
-          DOCKER_BUILD_VERBOSE=true ./securedrop/bin/dev-shell true
+          UBUNTU_VERSION=${{ matrix.ubuntu_version }} DOCKER_BUILD_VERBOSE=true ./securedrop/bin/dev-shell true
       - name: Run page layout tests
         run: |
-          make test-pageslayout
+          UBUNTU_VERSION=${{ matrix.ubuntu_version }} make test-pageslayout
       - name: Validate HTML (informational)
         run: |
-          make validate-test-html || true
+          UBUNTU_VERSION=${{ matrix.ubuntu_version }} make validate-test-html || true
       - name: Validate accessibility (informational)
         run: |
-          make accessibility-summary || true
+          UBUNTU_VERSION=${{ matrix.ubuntu_version }} make accessibility-summary || true
 
   rust:
     runs-on: ubuntu-latest

admin/requirements-dev.txt

@@ -177,9 +177,9 @@ requests==2.26.0 \
     --hash=sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24 \
     --hash=sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7
     # via -r requirements-dev.in
-six==1.15.0 \
-    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
-    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
+six==1.16.0 \
+    --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
+    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
     # via
     #   mock
     #   tox

admin/requirements-testinfra.txt

@@ -324,9 +324,9 @@ resolvelib==0.8.1 \
     --hash=sha256:c6ea56732e9fb6fca1b2acc2ccc68a0b6b8c566d8f3e78e0443310ede61dbd37 \
     --hash=sha256:d9b7907f055c3b3a2cfc56c914ffd940122915826ff5fb5b1de0c99778f4de98
     # via ansible-core
-six==1.15.0 \
-    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
-    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
+six==1.16.0 \
+    --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
+    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
     # via
     #   -r requirements.in
     #   prompt-toolkit

admin/requirements.in

@@ -3,4 +3,4 @@ markupsafe>=1.1
 prompt_toolkit==2.0.9
 pyyaml>=6.0.1
 setuptools>=70.0.0
-six==1.15.0
+six>=1.16.0

admin/requirements.txt

@@ -238,9 +238,9 @@ resolvelib==0.8.1 \
     --hash=sha256:c6ea56732e9fb6fca1b2acc2ccc68a0b6b8c566d8f3e78e0443310ede61dbd37 \
     --hash=sha256:d9b7907f055c3b3a2cfc56c914ffd940122915826ff5fb5b1de0c99778f4de98
     # via ansible-core
-six==1.15.0 \
-    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
-    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
+six==1.16.0 \
+    --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
+    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
     # via
     #   -r requirements.in
     #   prompt-toolkit

securedrop/bin/dev-deps

@@ -175,17 +175,20 @@ function reset_demo() {
 }
 
 function build_redwood() {
+    source /etc/os-release
     # For the demo, we build the wheel at container build time instead of
     # during launch, so skip this part
     if [[ -z "${SKIP_REDWOOD_BUILD:-}" ]]; then
         # Create a debug build of redwood and install it in the app-code virtualenv
         # TODO: add live reload support
         # We tell cargo to put its registry/crate cache and build cache in the target/ folder so
-        # it persists instead of being recreated from scratch each time the container starts.
+        # it persists instead of being recreated from scratch each time the container starts. The
+        # cache needs to be split across OS versions because it can't tell that it's supposed to
+        # link to a different libssl version.
         # n.b. we can't re-use the host's caches because of permission differences
         PATH="$PATH:/opt/cargo/bin/" \
         CARGO_HOME="${REPOROOT}/target/cargo-dev" \
-            python3 "${REPOROOT}/redwood/build-wheel.py" --redwood "${REPOROOT}/redwood" --target "${REPOROOT}/target/dev"
+            python3 "${REPOROOT}/redwood/build-wheel.py" --redwood "${REPOROOT}/redwood" --target "${REPOROOT}/target/dev-${VERSION_CODENAME}"
         /opt/venvs/securedrop-app-code/bin/pip install "${REPOROOT}"/redwood/redwood-*.whl
     fi
 }

securedrop/bin/dev-shell

@@ -13,6 +13,7 @@ USE_TOR="${USE_TOR:-}"
 USE_PODMAN="${USE_PODMAN:-}"
 SLIM_BUILD="${SLIM_BUILD:-}"
 DOCKER_RUN_ARGUMENTS="${DOCKER_RUN_ARGUMENTS:-}"
+UBUNTU_VERSION="${UBUNTU_VERSION:-focal}"
 
 # Allow opting into using podman with USE_PODMAN=1
 if  [[ -n "${USE_PODMAN}" ]]; then
@@ -57,6 +58,7 @@ function docker_image() {
            ${DOCKER_BUILD_ARGUMENTS:-} \
            --build-arg=USER_ID="$(id -u)" \
            --build-arg=USER_NAME="${USER:-root}" \
+           --build-arg=UBUNTU_VERSION="${UBUNTU_VERSION}" \
            -t "${1}" \
            --file "${TOPLEVEL}/securedrop/dockerfiles/focal/python3/${2}" \
            "${TOPLEVEL}/securedrop" > $out
@@ -122,6 +124,7 @@ function docker_run() {
 
     # The --shm-size argument sets up dedicated shared memory for the
     # container. Our tests can fail with the default of 64m.
+    echo "Starting ${UBUNTU_VERSION} container..."
     $DOCKER_BIN run $ci_env \
            --shm-size 2g \
            --rm \
@@ -144,10 +147,10 @@ function docker_run() {
            $DOCKER_RUN_ARGUMENTS "${1}" "${@:2}"
 }
 
-image="securedrop-slim-focal-py3"
+image="securedrop-slim-${UBUNTU_VERSION}-py3"
 docker_image "$image" "SlimDockerfile"
 if  [[ -z "${SLIM_BUILD}" ]]; then
-    image="securedrop-test-focal-py3"
+    image="securedrop-test-${UBUNTU_VERSION}-py3"
     docker_image "$image" "Dockerfile"
 fi
 

securedrop/dockerfiles/focal/python3/Dockerfile

@@ -1,4 +1,5 @@
-FROM securedrop-slim-focal-py3
+ARG UBUNTU_VERSION=focal
+FROM securedrop-slim-${UBUNTU_VERSION}-py3
 
 RUN apt-get update  && apt-get install -y \
     libgtk2.0 devscripts xvfb x11vnc \
@@ -8,7 +9,7 @@ RUN apt-get update  && apt-get install -y \
     default-jdk \
     # For diffoscope:
     libarchive13 libmagic1 \
-    libasound2 libdbus-glib-1-2 libgtk2.0-0 libfontconfig1 libxrender1 \
+    libasound2-dev libdbus-glib-1-2 libgtk2.0-0 libfontconfig1 libxrender1 \
     libcairo-gobject2 libgtk-3-0 libstartup-notification0 npm && \
     npm install --global [email protected]
 
@@ -45,7 +46,7 @@ curl -LO https://archive.mozilla.org/pub/firefox/releases/${FF_VERSION}/linux-x8
     gpgv --keyring ./mozilla.keyring firefox-${FF_VERSION}.tar.bz2.asc firefox-${FF_VERSION}.tar.bz2 && \
     tar xjf firefox-*.tar.bz2 && \
     mv firefox /usr/bin && \
-    rm -f firefox-${FF_VERSION}.tar.bz2.asc firefox-${FF_VERSION}.tar.bz2 
+    rm -f firefox-${FF_VERSION}.tar.bz2.asc firefox-${FF_VERSION}.tar.bz2
 
 # Install geckodriver
 RUN wget https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz && \

securedrop/dockerfiles/focal/python3/SlimDockerfile

@@ -1,5 +1,5 @@
-# ubuntu 20.04 image from 2022-10-19
-FROM ubuntu@sha256:450e066588f42ebe1551f3b1a535034b6aa46cd936fe7f2c6b0d72997ec61dbd
+ARG UBUNTU_VERSION=focal
+FROM ubuntu:${UBUNTU_VERSION}
 ARG USER_NAME
 ENV USER_NAME ${USER_NAME:-root}
 ARG USER_ID
@@ -9,7 +9,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install
                        apache2-dev coreutils vim \
                        python3-pip python3-all python3-venv virtualenv python3-dev libssl-dev \
                        gnupg2 redis-server git curl wget jq \
-                       enchant libffi-dev sqlite3 gettext sudo tor basez pkg-config
+                       enchant-2 libffi-dev sqlite3 gettext sudo tor basez pkg-config
 
 # Install Rust using the same steps as <https://github.com/rust-lang/docker-rust>
 # 1) Download rustup-init and verify it matches hardcoded checksum
@@ -34,8 +34,11 @@ RUN python3 -m venv /opt/venvs/securedrop-app-code && \
     /opt/venvs/securedrop-app-code/bin/pip3 install --no-deps --require-hashes -r requirements/python3/test-requirements.txt && \
     /opt/venvs/securedrop-app-code/bin/pip3 install --no-deps --require-hashes -r requirements/python3/requirements.txt
 
+# noble ships with an `ubuntu` user that has id 1000, which conflicts with podman's ID mapping
+RUN if id ubuntu; then deluser ubuntu; fi
+
 RUN if test $USER_NAME != root ; then useradd --no-create-home --home-dir /tmp --uid $USER_ID $USER_NAME && echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers ; fi && \
-    chown -R $USER_NAME.$USER_NAME /opt/venvs/
+    chown -R $USER_NAME:$USER_NAME /opt/venvs/
 
 STOPSIGNAL SIGKILL
 

securedrop/pretty_bad_protocol/_meta.py

@@ -530,7 +530,8 @@ def _make_args(self, args, passphrase=False):  # type: ignore[no-untyped-def]
 
         if self.keyring:
             cmd.append("--no-default-keyring --keyring %s" % self.keyring)
-        if self.secring:
+        if self.secring and self.binary_version != "2.4.4":
+            # In GnuPG 2.4.4, --secret-keyring has no effect
             cmd.append("--secret-keyring %s" % self.secring)
 
         if passphrase:

securedrop/requirements/python3/develop-requirements.in

@@ -37,7 +37,7 @@ safety>2.2.0
 semgrep>=1.57.0
 setuptools>=70.0.0
 shellcheck-py
-six==1.15.0
+six>=1.16.0
 testinfra>=5.3.1
 translate-toolkit
 urllib3>=1.26.5

securedrop/requirements/python3/develop-requirements.txt

@@ -946,9 +946,9 @@ shellcheck-py==0.9.0.2 \
     --hash=sha256:bbaaba3044d266e793964e391c8caad4fcccafcd64e63de24e9b87a31a0c7aff \
     --hash=sha256:ddb9b9fd4750d726b9ac24df8b63599dc742c8749477bcdfd4ed639b22de21ae
     # via -r requirements/python3/develop-requirements.in
-six==1.15.0 \
-    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
-    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
+six==1.16.0 \
+    --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
+    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
     # via
     #   -r requirements/python3/develop-requirements.in
     #   ansible-lint

securedrop/requirements/python3/requirements.in

@@ -1,4 +1,4 @@
-alembic
+alembic>=1.1.0,<1.1.1
 argon2_cffi>=20.1.0
 babel>=2.9.1
 cffi==1.16.0

securedrop/requirements/python3/requirements.txt

@@ -1,7 +1,7 @@
 # This file was autogenerated by uv via the following command:
 #    ./securedrop/bin/update-requirements
-alembic==0.9.9 \
-    --hash=sha256:85bd3ea7633024e4930900bc64fb58f9742dedbc6ebb6ecf25be2ea9a3c1b32e
+alembic==1.1.0 \
+    --hash=sha256:4a4811119efbdc5259d1f4c8f6de977b36ad3bcc919f59a29c2960c5ef9149e4
     # via -r requirements/python3/requirements.in
 argon2-cffi==20.1.0 \
     --hash=sha256:05a8ac07c7026542377e38389638a8a1e9b78f1cd8439cd7493b39f08dd75fbf \
@@ -264,9 +264,9 @@ setuptools==70.3.0 \
     --hash=sha256:f171bab1dfbc86b132997f26a119f6056a57950d058587841a0082e8830f9dc5 \
     --hash=sha256:fe384da74336c398e0d956d1cae0669bc02eed936cdb1d49b57de1990dc11ffc
     # via -r requirements/python3/requirements.in
-six==1.11.0 \
-    --hash=sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9 \
-    --hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb
+six==1.16.0 \
+    --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
+    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
     # via
     #   argon2-cffi
     #   python-dateutil

securedrop/requirements/python3/test-requirements.in

@@ -1,4 +1,4 @@
-beautifulsoup4
+beautifulsoup4>=4.8.2,<4.9
 blinker
 coverage>=5.0  # #6091
 flaky
@@ -8,7 +8,7 @@ pathlib2
 pillow>=10.3.0  # Safety 67136 for CVE-2024-28219
 pytest>=7.2.0
 pytest-xdist>=3.0.2
-pytest-cov
+pytest-cov>=2.12,<3
 pytest-mock
 requests[socks]>=2.31.0
 setuptools>=70.0.0

securedrop/requirements/python3/test-requirements.txt

@@ -11,10 +11,10 @@ attrs==22.1.0 \
     #   outcome
     #   pytest
     #   trio
-beautifulsoup4==4.6.0 \
-    --hash=sha256:11a9a27b7d3bddc6d86f59fb76afb70e921a25ac2d6cc55b40d072bd68435a76 \
-    --hash=sha256:7015e76bf32f1f574636c4288399a6de66ce08fb7b2457f628a8d70c0fbabb11 \
-    --hash=sha256:808b6ac932dccb0a4126558f7dfdcf41710dd44a4ef497a0bb59a77f9f078e89
+beautifulsoup4==4.8.2 \
+    --hash=sha256:05fd825eb01c290877657a56df4c6e4c311b3965bda790c613a3d6fb01a5462a \
+    --hash=sha256:9fbb4d6e48ecd30bcacc5b63b94088192dcda178513b2ae3c394229f8911b887 \
+    --hash=sha256:e1505eeed31b0f4ce2dbb3bc8eb256c04cc2b3b72af7d551a4ab6efd5cbe5dae
     # via -r requirements/python3/test-requirements.in
 blinker==1.4 \
     --hash=sha256:471aee25f3992bd325afa3772f1063dbdbbca947a041b8b89466dc00d606f8b6
@@ -245,9 +245,9 @@ pytest==7.2.0 \
     #   pytest-cov
     #   pytest-mock
     #   pytest-xdist
-pytest-cov==2.5.1 \
-    --hash=sha256:03aa752cf11db41d281ea1d807d954c4eda35cfa1b21d6971966cc041bbf6e2d \
-    --hash=sha256:890fe5565400902b0c78b5357004aab1c814115894f4f21370e2433256a3eeec
+pytest-cov==2.12.1 \
+    --hash=sha256:261bb9e47e65bd099c89c3edf92972865210c36813f80ede5277dceb77a4a62a \
+    --hash=sha256:261ceeb8c227b726249b376b8526b600f38667ee314f910353fa318caa01f4d7
     # via -r requirements/python3/test-requirements.in
 pytest-mock==1.7.1 \
     --hash=sha256:03a2fea79d0a83a8de2e77e92afe5f0a5ca99a58cc68f843f9a74de34800a943 \
@@ -339,13 +339,21 @@ sortedcontainers==2.4.0 \
     --hash=sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88 \
     --hash=sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0
     # via trio
+soupsieve==2.6 \
+    --hash=sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb \
+    --hash=sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9
+    # via beautifulsoup4
 sqlalchemy-stubs==0.4 \
     --hash=sha256:5eec7aa110adf9b957b631799a72fef396b23ff99fe296df726645d01e312aa5 \
     --hash=sha256:c665d6dd4482ef642f01027fa06c3d5e91befabb219dc71fc2a09e7d7695f7ae
     # via -r requirements/python3/test-requirements.in
 tbselenium==0.8.1 \
     --hash=sha256:b40df4f339459d90e8c9e6fd66f7ebf9baabbc5b79d8ca7c94ebcb092f7e1726
     # via -r requirements/python3/test-requirements.in
+toml==0.10.2 \
+    --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
+    --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f
+    # via pytest-cov
 tomli==2.0.1 \
     --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \
     --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f

securedrop/tests/functional/pageslayout/test_source_static_pages.py

@@ -51,6 +51,6 @@ def test_instance_metadata(self, sd_servers):
 
         # Then it succeeds and the right information is returned
         returned_data = response.json()
-        assert returned_data["server_os"] == "20.04"
+        assert returned_data["server_os"] in ["20.04", "24.04"]
         assert returned_data["sd_version"] == __version__
         assert returned_data["gpg_fpr"]