Merge branch 'master' into PCF-634-new-stats-analysis

Author: misspran

.circleci/config.yml

@@ -1,7 +1,7 @@
 version: 2.1
 
 orbs:
-  node: circleci/[email protected].0
+  node: circleci/[email protected].1
   docker: circleci/[email protected]
   codecov: codecov/[email protected]
   gcp-cli: circleci/[email protected]
@@ -135,7 +135,7 @@ jobs:
           command: tox -e linters
           name: Run linters
 
-  python-tests-postgres:
+  python-tests-general:
     machine:
       image: default
     steps:
@@ -147,7 +147,37 @@ jobs:
           command: |
             pip install --upgrade pip
             pip install tox
-            tox -e docker-postgres
+            tox -e docker-general
+      - codecov/upload
+
+  python-tests-perf:
+    machine:
+      image: default
+    steps:
+      - checkout
+      - docker/install-docker-compose:
+          version: 1.29.2
+      - run:
+          name: Run tests and coverage within Docker container
+          command: |
+            pip install --upgrade pip
+            pip install tox
+            tox -e docker-perf
+      - codecov/upload
+
+  python-tests-frontend:
+    machine:
+      image: default
+    steps:
+      - checkout
+      - docker/install-docker-compose:
+          version: 1.29.2
+      - run:
+          name: Run tests and coverage within Docker container
+          command: |
+            pip install --upgrade pip
+            pip install tox
+            tox -e docker-frontend
       - codecov/upload
 
   test-docker-build:
@@ -173,7 +203,9 @@ workflows:
     jobs:
       - javascript-tests
       - builds
-      - python-tests-postgres
+      - python-tests-frontend
+      - python-tests-general
+      - python-tests-perf
       - test-docker-build
       - deploy:
           filters:

.gitignore

@@ -8,10 +8,15 @@ __pycache__
 .ruff_cache/
 .build/
 .venv/
+venv/
+.scratch/
 
 # vi
 *.swp
 
+# Database
+treeherder_backup_*
+
 # Caches
 .eslintcache
 # PyCharm

docker-compose.yml

@@ -67,7 +67,7 @@ services:
     volumes:
       - .:/app
     ports:
-      - '5000:5000'
+      - '${FRONTEND_PORT:-5000}:5000'
     platform: linux/amd64
 
   postgres:
@@ -78,10 +78,12 @@ services:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=mozilla1234
       - POSTGRES_DB=treeherder
+    labels:
+      - prune=False
     volumes:
       - postgres_data:/var/lib/postgresql/data
     ports:
-      - '5432:5432'
+      - '${POSTGRES_PORT:-5432}:5432'
 
   redis:
     container_name: redis
@@ -92,7 +94,7 @@ services:
     # Hide Redis `notice` log level startup output spam.
     command: redis-server --loglevel warning
     ports:
-      - '6379:6379'
+      - '${REDIS_PORT:-6379}:6379'
 
   rabbitmq:
     container_name: rabbitmq

docker/Dockerfile

@@ -12,7 +12,7 @@ RUN yarn build
 
 
 ## Backend stage
-FROM python:3.10.17-slim-bullseye
+FROM python:3.10.18-slim-bullseye
 
 WORKDIR /app
 

docker/dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10.17-bullseye
+FROM python:3.10.18-bullseye
 
 # Variables that are not specific to a particular environment.
 ENV NEW_RELIC_CONFIG_FILE newrelic.ini

pyproject.toml

@@ -6,7 +6,7 @@ description = "Defaut package, used for development or readthedocs"
 [project.optional-dependencies]
 docs = [
   "mkdocs==1.6.1",
-  "mkdocs-material==9.6.18",
+  "mkdocs-material==9.6.19",
   "mdx_truly_sane_lists==1.3",
 ]
 
@@ -31,6 +31,10 @@ target-version = "py310"
 # In addition to the standard set of exclusions, omit all tests, plus a specific file.
 extend-exclude = ["*/.*/",".*/","__pycache__","node_modules"]
 
+# Also lint/format pyi files
+extend-include = ["*.pyi"]
+
+[tool.ruff.lint]
 select = [
   # pycodestyle
   "E",
@@ -48,9 +52,6 @@ ignore = [
   "E501",
 ]
 
-# Also lint/format pyi files
-extend-include = ["*.pyi"]
-
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 # Ignore `module-import-not-at-top-of-file` rule of `pycodestyle`
 "treeherder/model/models.py" = ["E402"]

requirements/common.in

@@ -1,18 +1,18 @@
 # Packages that are shared between deployment and dev environments.
 gunicorn==23.0.0
-whitenoise[brotli]==6.9.0  # Used by Whitenoise to provide Brotli-compressed versions of static files.
-Django==5.1.11
+whitenoise[brotli]==6.10.0  # Used by Whitenoise to provide Brotli-compressed versions of static files.
+Django==5.1.12
 celery==5.5.3  # celery needed for data ingestion
 cached-property==2.0.1 # needed for kombu with --require-hashes
 simplejson==3.20.1  # import simplejson
-newrelic==10.16.0
+newrelic==10.17.0
 certifi==2025.8.3
 
 psycopg2-binary==2.9.9
 
 jsonschema==4.25.1  # import jsonschema
 djangorestframework==3.16.1  # Imported as rest_framework
-django-cors-headers==4.7.0  # Listed as 3rd party app on settings.py
+django-cors-headers==4.8.0  # Listed as 3rd party app on settings.py
 mozlog==8.0.0
 
 # Used directly and also by Django's YAML serializer.
@@ -24,7 +24,7 @@ python-dateutil==2.9.0.post0
 django-filter==25.1  # Listed in DEFAULT_FILTER_BACKENDS on settings.py
 django-redis==6.0.0  # Listed in CACHES on settings.py
 
-taskcluster==88.0.3  # import taskcluster
+taskcluster==88.0.5  # import taskcluster
 python-jose[pycryptodome]==3.5.0  # from jose import jwt
 
 furl==2.1.4  # Imported as furl

requirements/common.txt

@@ -421,9 +421,9 @@ distlib==0.4.0 \
     --hash=sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16 \
     --hash=sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d
     # via virtualenv
-django==5.1.11 \
-    --hash=sha256:3bcdbd40e4d4623b5e04f59c28834323f3086df583058e65ebce99f9982385ce \
-    --hash=sha256:e48091f364007068728aca938e7450fbfe3f2217079bfd2b8af45122585acf64
+django==5.1.12 \
+    --hash=sha256:8a8991b1ec052ef6a44fefd1ef336ab8daa221287bcb91a4a17d5e1abec5bbcc \
+    --hash=sha256:9eb695636cea3601b65690f1596993c042206729afb320ca0960b55f8ed4477b
     # via
     #   -r requirements/common.in
     #   django-cors-headers
@@ -434,9 +434,9 @@ django-cache-memoize==0.2.1 \
     --hash=sha256:025ff5d941420247b83452bb183bde172dde7def95501fca829adf8ea01b2b7b \
     --hash=sha256:07929d063a03557013875d453a13edc8e3359d3ba8a568b64ac3288578ed8be3
     # via -r requirements/common.in
-django-cors-headers==4.7.0 \
-    --hash=sha256:6fdf31bf9c6d6448ba09ef57157db2268d515d94fc5c89a0a1028e1fc03ee52b \
-    --hash=sha256:f1c125dcd58479fe7a67fe2499c16ee38b81b397463cf025f0e2c42937421070
+django-cors-headers==4.8.0 \
+    --hash=sha256:0a12a2efcd59a3cea741e44db8ab589e929949de5bc4cdf35a29c6ae77297686 \
+    --hash=sha256:3b883f4c6d07848673218456a5e070d8ab51f97341c1f27d0242ca167e7272ab
     # via -r requirements/common.in
 django-environ==0.11.2 \
     --hash=sha256:0ff95ab4344bfeff693836aa978e6840abef2e2f1145adff7735892711590c05 \
@@ -1096,36 +1096,36 @@ multidict==6.6.4 \
     # via
     #   aiohttp
     #   yarl
-newrelic==10.16.0 \
-    --hash=sha256:020e384fbb8543e5c54e0cc900b170876faa03fe27d00ba49b7a6ba6a2b0d5b4 \
-    --hash=sha256:053e73518cf0b2af412490723d3fa8338dab6ed48b748c0b7a0288160d1c33e2 \
-    --hash=sha256:072e54f72d57ab9b7891209ab85b3839a494fa6f89b1674da5ca7848a4b5b1ad \
-    --hash=sha256:07f65faed0a9aa40695e6e6ce4c943ca9c9643fc848c2344bb26665a65893a80 \
-    --hash=sha256:287e8fbab1c10e5fe2e20c81aec3fece01bf613ebf4985b6e36360fce2f5cd80 \
-    --hash=sha256:2f74301ed32ce625fb84c8bd0c079498dca603ff111d5baa2f59b6480ef7355b \
-    --hash=sha256:3efef3b10f02a086d9009a80b020db8f0b1db5072075779c9facd5d2fe3bb916 \
-    --hash=sha256:49a5f734847269ccba31962132fa570b49bcf04378d554aa6a871eda1a4abcf1 \
-    --hash=sha256:4beec39a1e290f808e35041b7728d61105737cddec9525d4e7dc2cf107c8904e \
-    --hash=sha256:5176be5123f987d6f9c12ebd189dabd739bba809225feac8c4ee5f9176580a0a \
-    --hash=sha256:5fe5c2f72f56f9670fabe2889b70d3d239aae715bb97c2460929605ffa8742fb \
-    --hash=sha256:6361b21416b529415208b551f9903a33c172489142df35c93fdad337c0309050 \
-    --hash=sha256:6e83e1341a72ac92118d1af40c2c3ba56069c589acceb1e89d236beab8d7891a \
-    --hash=sha256:71950c5e1f26bf1ed563d3641da97f6bb209f80697675888ee12808d8ac69452 \
-    --hash=sha256:8263ae16eb4745def12bace2e3e38e013e76d4e765333a77342b270702e3eaa9 \
-    --hash=sha256:8328f621f3683b28156cfa8e1648a56b894480bf4c4953cde7e659b1480ae1e9 \
-    --hash=sha256:833992875fda7aa3f1db9e9b8782a29d71094b7e6c0e7bcd7113534a652ba74d \
-    --hash=sha256:901212e07779fd437229b99fdc1f3d82509bf2d27afe77f80224409eeda51355 \
-    --hash=sha256:90478ee70f1e6520cf70271dbb41e7c0d88e3a968b8361e0d2d82872c70de6b7 \
-    --hash=sha256:952cc8f94bbb43a07f0f2982a251e6c973a2bb5db0febc5534586171a09a5619 \
-    --hash=sha256:9ddbf08714dc5d102dd5b4db87da08981c6390cb838f972387839f6031aad11c \
-    --hash=sha256:a0ee00809ca2b85ae049f2f7d93248f6e30bc005a3a41931a26bdfdf5d4e752c \
-    --hash=sha256:a6dbd306b72b0b3c6477b4ccfdc6fc7d7e62b793631f7856684733aad405ef50 \
-    --hash=sha256:a71c1034b47494179567341178f436804c6d33538fa25430580955725c847ef2 \
-    --hash=sha256:a7555b252e06848b98c0a0616d5c3487ad58903a1f21114f947ab91c4fac772a \
-    --hash=sha256:bfc9621afef69b089ee1ec6ab729220c2f5540b81e5666a3414797f48f5df3e6 \
-    --hash=sha256:caa7b9d026ec221bcfed35a6c3be65b7e9e0b50e68a1278ad5e38dab6d12d45f \
-    --hash=sha256:d20eb934380a88d787f93e037d2ccfd5a7c80e657db5bb2e645216eaafe32e26 \
-    --hash=sha256:e4b761b2a45bb5a76f22fa8d780e089a0dc6c61315441456d487f752b84da7d3
+newrelic==10.17.0 \
+    --hash=sha256:083777458f93d16ae9e605fd66a6298d856e32deea11cf3270ed237858dcbfe6 \
+    --hash=sha256:0b43fb53f8a63c1e792a9ba4439680265b481c44efd64f7d92069e974c38b199 \
+    --hash=sha256:165c7045da474342d4ceab67776e8aeb66e77674fab7484b4e2e4ea68d02ed4d \
+    --hash=sha256:214bd42ef8fb1c2046727bfe15c8f374ebf4ff2d5091ca1ac870abd2a7574ffe \
+    --hash=sha256:30b8b6a9f447a7276fd19a41ccc6dba264f9ef569e4bed92200a58b76e7f7754 \
+    --hash=sha256:328162a61e30f7b3e76241830989110d8f86d8392805a78034f8109e4f5434b4 \
+    --hash=sha256:36a14fd92b44324d9bb33e1c1698c2cd15ffa0e291d222365e929ee17cd9d6da \
+    --hash=sha256:3a34df04ff6315619a8d098c59751f766de57f14e5cd5c09427dde47bfbc5012 \
+    --hash=sha256:4144dd96fa55772456326b3a6032a53d54aa9245ffc5041b002ce9eb5dbd0992 \
+    --hash=sha256:5171f1020bc834b01faae8466b5375e8c2162fa219c98430e42af1c6e5077b14 \
+    --hash=sha256:54898db2b9eb81db9f3421302d2d03451630c033f2001fbe9f21d0e068f0418f \
+    --hash=sha256:5789f739e6ca1c4a49649e406b538d67c063ec33ab4658d01049303dfad9398b \
+    --hash=sha256:5ab9cbce3f8da5bfcd085b8a59591cfb653d75834b362e3eccb86bdf21eea917 \
+    --hash=sha256:61e81b726f3fab5fed155407f157ed67931d2d83403cc0ae473327e0b7b42bc5 \
+    --hash=sha256:6dd61153b82b1e265754820f3e91c96c16b1b75508b6c169e27b5c196f1b7b95 \
+    --hash=sha256:7669597193d8f6a3512fd5e8728a7b2b244db9694b3a03b3b99930ba9efc9e3b \
+    --hash=sha256:78a673440037ad61e0073da69f2549fac92271dadd7a2087c22a20ddc7f91260 \
+    --hash=sha256:889969407457a9344927b1a9e9d5afde77345857db4d2ea75b119d02c02d9174 \
+    --hash=sha256:91ebeaae460beaaeb3113474873bc5fc9c6c13ab6b0fd8936f584c508fec373d \
+    --hash=sha256:a1027039eae1b4d3ffee3077135900a41ffdc41bd2bee3162d6cf36bd990cf3b \
+    --hash=sha256:af28eb5e7f368f6f4fcde9593e7dd189b68e39a2e7409874a0cdea3a4bb4e904 \
+    --hash=sha256:c444e96918508decee6f532f04f0fa79027db0728770669ace4a7008c5dd52ca \
+    --hash=sha256:d65f7d08f90caf6dd3708068a0eda8328c49ec18e51d514a43afaa5a68180e9c \
+    --hash=sha256:d74c06863356b19c1fcf19900f85057ab33ef779aa3e0de3cb7e3d3f37ca8e20 \
+    --hash=sha256:d92b43c07cf137d8a6d8dc73c87fbef99ca449564c25385e66727f156c2c34d8 \
+    --hash=sha256:ed6f549b1ab42ad31d5ee34c8041f1484ab769e045ad3b9f321c9588339079f3 \
+    --hash=sha256:f092109ac024f9524fafdd06126924c0fbd2af54684571167c0ee1d1cc1bcb7d \
+    --hash=sha256:f62f70f1c01aa527de7e6ce2eb4c0285a7c40a89b42107be92bcc5dfd0fa8bb4 \
+    --hash=sha256:fc3c3c82c95b57fe30fc1f684bf920bd8b0ecae70a16cc11f55d0867ffb7520d
     # via -r requirements/common.in
 nodeenv==1.9.1 \
     --hash=sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f \
@@ -2129,9 +2129,9 @@ tabulate==0.9.0 \
     --hash=sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c \
     --hash=sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f
     # via mozci
-taskcluster==88.0.3 \
-    --hash=sha256:3879211116e17e1734057ce4d7cbe80b2d329852d15509b2d55cad9ae0c17e9d \
-    --hash=sha256:6b8aff2e7c099f903a312b77b2c2619bbd63f9d32b6c3b29d3f83b01794c5e36
+taskcluster==88.0.5 \
+    --hash=sha256:58f71976a95e7ff506f64e045b773f595573bb6c219cf4f89faa6893ba27e189 \
+    --hash=sha256:92910fe31d9b2e862093ac49fe5ead95ef794e11926d36b3ac8b36408f2c5c26
     # via
     #   -r requirements/common.in
     #   mozci
@@ -2164,9 +2164,10 @@ uritemplate==4.2.0 \
     --hash=sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e \
     --hash=sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686
     # via -r requirements/common.in
-urllib3==2.5.0 \
-    --hash=sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760 \
-    --hash=sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc
+
+urllib3==1.26.18 \
+    --hash=sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07 \
+    --hash=sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0
     # via
     #   botocore
     #   requests
@@ -2234,9 +2235,9 @@ wcwidth==0.2.13 \
     #   blessed
     #   prettytable
     #   prompt-toolkit
-whitenoise[brotli]==6.9.0 \
-    --hash=sha256:8c4a7c9d384694990c26f3047e118c691557481d624f069b7f7752a2f735d609 \
-    --hash=sha256:c8a489049b7ee9889617bb4c274a153f3d979e8f51d2efd0f5b403caf41c57df
+whitenoise[brotli]==6.10.0 \
+    --hash=sha256:7b7e53de65d749cb1ce4a7100e751d9742e323b52746f9f93944c0d348ea2d02 \
+    --hash=sha256:bad74a40b33b055ba59731b6048dd08d5647f273b72bef922aa43ddd287b02da
     # via -r requirements/common.in
 yarl==1.20.1 \
     --hash=sha256:03aa1e041727cb438ca762628109ef1333498b122e4c76dd858d186a37cec845 \

requirements/dev.in

@@ -1,8 +1,8 @@
 # Dependencies needed only for development/testing.
-pytest-cov==6.2.1
+pytest-cov==7.0.0
 django-debug-toolbar==5.2.0
 mock==5.2.0
-responses==0.25.7
+responses==0.25.8
 django-extensions==4.1
 PyPOM==2.2.4
 
@@ -15,12 +15,12 @@ pytest-watch==4.2.0
 
 # Required by django-extension's runserver_plus command.
 pytest-django==4.11.1
-pytest==8.4.1
+pytest==8.4.2
 black==25.1.0
 shellcheck-py==0.11.0.1
 
 # To test async code
-pytest-asyncio==1.0.0  # required to pass test_new_job_transformation
+pytest-asyncio==1.2.0  # required to pass test_new_job_transformation
 
 # To test code that's making system time calls
 # pytest-freezegun is not compatible with recent Django versions
@@ -34,7 +34,7 @@ betamax==0.9.0
 betamax-serializers==0.2.1
 
 # pip-compile for pinning versions
-pip-tools==7.4.1
+pip-tools==7.5.0
 
 requests==2.32.5
 urllib3==2.0.3