From 27eb12cc54689ae5685f64a53eb87b709c0c444a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 16 Mar 2026 20:00:56 +0000 Subject: [PATCH 01/48] chore: point epmt-dash branch references to main Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com> --- .github/workflows/build_and_test_epmt.yml | 2 +- .github/workflows/docker_build_test.yml | 2 +- .github/workflows/weekly_cache_builds.yml | 4 ++-- Makefile | 9 ++++----- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_and_test_epmt.yml b/.github/workflows/build_and_test_epmt.yml index 88ede4bae..39d2fd685 100644 --- a/.github/workflows/build_and_test_epmt.yml +++ b/.github/workflows/build_and_test_epmt.yml @@ -46,7 +46,7 @@ jobs: SQLITE_YEAR: "2025" # Tarball source identifiers — must match Makefile and weekly_tarball_build.yml PAPIEX_VERSION: "2.3.15" - EPMT_DASH_SRC_BRANCH: "autopep8.poke" + EPMT_DASH_SRC_BRANCH: "main" OS_TARGET: "rocky-8" # SLURM version — must match docker_build_test.yml SLURM_TAG: "slurm-25-05-3-1" diff --git a/.github/workflows/docker_build_test.yml b/.github/workflows/docker_build_test.yml index e366521f7..b0f027553 100644 --- a/.github/workflows/docker_build_test.yml +++ b/.github/workflows/docker_build_test.yml @@ -40,7 +40,7 @@ jobs: SQLITE_YEAR: "2025" # Tarball source identifiers — must match Makefile and weekly_tarball_build.yml PAPIEX_VERSION: "2.3.15" - EPMT_DASH_SRC_BRANCH: "autopep8.poke" + EPMT_DASH_SRC_BRANCH: "main" steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/weekly_cache_builds.yml b/.github/workflows/weekly_cache_builds.yml index 438f78534..f5a52da03 100644 --- a/.github/workflows/weekly_cache_builds.yml +++ b/.github/workflows/weekly_cache_builds.yml @@ -76,7 +76,7 @@ jobs: DOCKER_RUN_OPTS: "--rm" # Tarball source identifiers — update these to invalidate their caches. PAPIEX_VERSION: "2.3.15" - EPMT_DASH_SRC_BRANCH: "autopep8.poke" + EPMT_DASH_SRC_BRANCH: "main" steps: - uses: actions/checkout@v4 @@ -150,7 +150,7 @@ jobs: env: OS_TARGET: "rocky-8" PAPIEX_VERSION: "2.3.15" - EPMT_DASH_SRC_BRANCH: "autopep8.poke" + EPMT_DASH_SRC_BRANCH: "main" # Build environment versions — must match Makefile and build_and_test_epmt.yml PYTHON_VERSION: "3.9.25" # updated from 3.9.22 (issue #153: use latest 3.9.x) SQLITE_VERSION: "3490100" diff --git a/Makefile b/Makefile index 1097d3f5b..6d49e2965 100644 --- a/Makefile +++ b/Makefile @@ -49,10 +49,9 @@ EPMT_INSTALL_PATH=/opt/minimalmetrics EPMT_INSTALL_PREFIX=$(EPMT_INSTALL_PATH)/epmt-$(EPMT_VERSION)/epmt-install # /src/epmt/ui submodule details -#EPMT_DASH_SRC_BRANCH=main -EPMT_DASH_SRC_BRANCH=autopep8.poke -EPMT_DASH_SRC_TARBALL=$(EPMT_DASH_SRC_BRANCH).tar.gz -EPMT_DASH_SRC_URL=$(NOAAGFDL_SRC_URL_BASE)/epmt-dash/archive/$(EPMT_DASH_SRC_TARBALL) +EPMT_DASH_SRC_BRANCH=main +EPMT_DASH_SRC_TARBALL=epmt-dash-$(EPMT_DASH_SRC_BRANCH).tar.gz +EPMT_DASH_SRC_URL=$(NOAAGFDL_SRC_URL_BASE)/epmt-dash/archive/$(EPMT_DASH_SRC_BRANCH).tar.gz EPMT_DASH_SRC=src/epmt/ui ## other details @@ -258,7 +257,7 @@ $(EPMT_DASH_SRC): $(EPMT_DASH_SRC_TARBALL) $(EPMT_DASH_SRC_TARBALL): @echo "(EPMT_DASH_SRC_TARBALL) whoami: $(shell whoami)" echo "grabbing epmt-dash via curl" ; \ - curl -L --fail --retry 3 --retry-delay 5 -O $(EPMT_DASH_SRC_URL) ; \ + curl -L --fail --retry 3 --retry-delay 5 -o $(EPMT_DASH_SRC_TARBALL) $(EPMT_DASH_SRC_URL) ; \ ls $(EPMT_DASH_SRC_TARBALL) ; \ # ----------- \end EPMT_DASH THINGS ---------- # From c485d423895c7bae1eeb5fc8b74b88a45ba5ec79 Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Tue, 17 Mar 2026 15:05:58 -0400 Subject: [PATCH 02/48] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6d49e2965..77cf8755c 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ NOAAGFDL_SRC_URL_BASE=https://github.com/noaa-gfdl # papiex details PAPIEX_VERSION?=2.3.15 PAPIEX_SRC?=papiex -PAPIEX_SRC_BRANCH=main +PAPIEX_SRC_BRANCH=copilot/rocky-8-fix-gcc-warnings PAPIEX_SRC_TARBALL=$(PAPIEX_SRC_BRANCH).tar.gz PAPIEX_SRC_URL=$(NOAAGFDL_SRC_URL_BASE)/papiex/archive/$(PAPIEX_SRC_TARBALL) PAPIEX_RELEASE=papiex-epmt-$(PAPIEX_VERSION)-$(OS_TARGET).tgz From fb3fb881a2461b24463fa9c1568e38dc09e85b52 Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Wed, 18 Mar 2026 08:49:29 -0400 Subject: [PATCH 03/48] Fix branch name format for PAPIEX source --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 77cf8755c..1d3f8e8f6 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ NOAAGFDL_SRC_URL_BASE=https://github.com/noaa-gfdl # papiex details PAPIEX_VERSION?=2.3.15 PAPIEX_SRC?=papiex -PAPIEX_SRC_BRANCH=copilot/rocky-8-fix-gcc-warnings +PAPIEX_SRC_BRANCH=copilot-rocky-8-fix-gcc-warnings PAPIEX_SRC_TARBALL=$(PAPIEX_SRC_BRANCH).tar.gz PAPIEX_SRC_URL=$(NOAAGFDL_SRC_URL_BASE)/papiex/archive/$(PAPIEX_SRC_TARBALL) PAPIEX_RELEASE=papiex-epmt-$(PAPIEX_VERSION)-$(OS_TARGET).tgz From 4e79dee7eb33801417257b75370c650ccdc8bab9 Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Wed, 18 Mar 2026 09:57:46 -0400 Subject: [PATCH 04/48] Update PAPIEX_VERSION to 2.3.16 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1d3f8e8f6..261a496e4 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ MM_SRC_URL_BASE=https://gitlab.com/minimal-metrics-llc/epmt NOAAGFDL_SRC_URL_BASE=https://github.com/noaa-gfdl # papiex details -PAPIEX_VERSION?=2.3.15 +PAPIEX_VERSION?=2.3.16 PAPIEX_SRC?=papiex PAPIEX_SRC_BRANCH=copilot-rocky-8-fix-gcc-warnings PAPIEX_SRC_TARBALL=$(PAPIEX_SRC_BRANCH).tar.gz From 51d19086fff05062e7aff00e9788db491df2426a Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Wed, 18 Mar 2026 09:59:09 -0400 Subject: [PATCH 05/48] Update PAPIEX_VERSION to 2.3.16 --- .github/workflows/weekly_cache_builds.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/weekly_cache_builds.yml b/.github/workflows/weekly_cache_builds.yml index f5a52da03..de4142be6 100644 --- a/.github/workflows/weekly_cache_builds.yml +++ b/.github/workflows/weekly_cache_builds.yml @@ -75,7 +75,7 @@ jobs: # Papiex Makefile default is "--rm -it", but -it causes "not a TTY" error. DOCKER_RUN_OPTS: "--rm" # Tarball source identifiers — update these to invalidate their caches. - PAPIEX_VERSION: "2.3.15" + PAPIEX_VERSION: "2.3.16" EPMT_DASH_SRC_BRANCH: "main" steps: @@ -149,7 +149,7 @@ jobs: options: --privileged env: OS_TARGET: "rocky-8" - PAPIEX_VERSION: "2.3.15" + PAPIEX_VERSION: "2.3.16" EPMT_DASH_SRC_BRANCH: "main" # Build environment versions — must match Makefile and build_and_test_epmt.yml PYTHON_VERSION: "3.9.25" # updated from 3.9.22 (issue #153: use latest 3.9.x) From e4c688fb58ab04c9913077811ee06c793e5f12de Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Wed, 18 Mar 2026 10:00:04 -0400 Subject: [PATCH 06/48] Update PAPIEX_VERSION to 2.3.16 --- .github/workflows/build_and_test_epmt.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test_epmt.yml b/.github/workflows/build_and_test_epmt.yml index 39d2fd685..2a1df2914 100644 --- a/.github/workflows/build_and_test_epmt.yml +++ b/.github/workflows/build_and_test_epmt.yml @@ -45,7 +45,7 @@ jobs: SQLITE_VERSION: "3490100" SQLITE_YEAR: "2025" # Tarball source identifiers — must match Makefile and weekly_tarball_build.yml - PAPIEX_VERSION: "2.3.15" + PAPIEX_VERSION: "2.3.16" EPMT_DASH_SRC_BRANCH: "main" OS_TARGET: "rocky-8" # SLURM version — must match docker_build_test.yml From d5682076c2f161bbcef1467b28b14674c9439de7 Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Wed, 18 Mar 2026 10:00:33 -0400 Subject: [PATCH 07/48] Update PAPIEX_VERSION to 2.3.16 --- .github/workflows/docker_build_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker_build_test.yml b/.github/workflows/docker_build_test.yml index b0f027553..e0fa121ef 100644 --- a/.github/workflows/docker_build_test.yml +++ b/.github/workflows/docker_build_test.yml @@ -39,7 +39,7 @@ jobs: SQLITE_VERSION: "3490100" SQLITE_YEAR: "2025" # Tarball source identifiers — must match Makefile and weekly_tarball_build.yml - PAPIEX_VERSION: "2.3.15" + PAPIEX_VERSION: "2.3.16" EPMT_DASH_SRC_BRANCH: "main" steps: From 24ccd496b3e1c094eee72683a35539349c05d252 Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Wed, 18 Mar 2026 11:19:50 -0400 Subject: [PATCH 08/48] Change PAPIEX_SRC_BRANCH from 'copilot-rocky-8-fix-gcc-warnings' to 'main' --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 261a496e4..f4ab207cb 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ NOAAGFDL_SRC_URL_BASE=https://github.com/noaa-gfdl # papiex details PAPIEX_VERSION?=2.3.16 PAPIEX_SRC?=papiex -PAPIEX_SRC_BRANCH=copilot-rocky-8-fix-gcc-warnings +PAPIEX_SRC_BRANCH=main PAPIEX_SRC_TARBALL=$(PAPIEX_SRC_BRANCH).tar.gz PAPIEX_SRC_URL=$(NOAAGFDL_SRC_URL_BASE)/papiex/archive/$(PAPIEX_SRC_TARBALL) PAPIEX_RELEASE=papiex-epmt-$(PAPIEX_VERSION)-$(OS_TARGET).tgz From 99cd7642b08936c8d2f259509d35c696c8c2d736 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 19 Mar 2026 16:56:11 -0400 Subject: [PATCH 09/48] add notebook extras into env yaml --- environment.yaml | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/environment.yaml b/environment.yaml index 91ed538af..be9f76297 100644 --- a/environment.yaml +++ b/environment.yaml @@ -15,11 +15,10 @@ # After pulling new commits, re-run the pip install line above if # pyproject.toml dependencies changed. -name: epmt +name: epmt_w_notebooks channels: - conda-forge - - defaults dependencies: # 3.9/3.10/3.11 supported; 3.12 requires pandas 2.x + SQLAlchemy 2.x (issue #105) @@ -48,6 +47,35 @@ dependencies: - conda-forge::pytest-cov - conda-forge::pylint - pip + + # --- notebook extras -------------------------------------------------- (TESTED, WORKS!) + - conda-forge::ipykernel + - conda-forge::ipython + - conda-forge::ipywidgets + - conda-forge::jupyter + - conda-forge::jupyter_client + - conda-forge::jupyter_console + - conda-forge::jupyter_core + - conda-forge::jupyter_server<2 + - conda-forge::notebook<7 + - conda-forge::jupyterlab<4 # must stay <4 while notebook is <7 + - conda-forge::qtconsole + +# # --- ui extras -------------------------------------------------- (NOT TESTED) +# - conda-forge::dash +# - conda-forge::dash_bootstrap_components +# - conda-forge::dash_daq +# - conda-forge::dash_table + +# # --- docs extras -------------------------------------------------- (NOT TESTED) +# - conda-forge::livereload +# - conda-forge::mkdocs +# - conda-forge::mkdocs_git_committers_plugin +# - conda-forge::mkdocs_theme_bootstrap4 + +# # --- build extras -------------------------------------------------- (NOT TESTED) +# - conda-forge::pyinstaller==5.7.0 +# - conda-forge::pyinstaller-hooks-contrib==2023.11 - pip: - - -e src/ # editable install of epmt into this environment + - src/ # editable install of epmt into this environment From d1d2e754fa0a9a3fc0a7ed0abe5715a979581e2e Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Thu, 19 Mar 2026 16:56:30 -0400 Subject: [PATCH 10/48] add pylint to pyproj --- src/pyproject.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pyproject.toml b/src/pyproject.toml index 32b7fd67e..d2bf88eaa 100644 --- a/src/pyproject.toml +++ b/src/pyproject.toml @@ -76,9 +76,7 @@ dev = [ "lazy-object-proxy", "mccabe", "wrapt", -] -# Test runner -test = [ + "pylint", "pytest", ] # 20231102 caw: pyinstaller is pinned at 5.7.0, latest version doesn't allow the settings.py tactic. From 389975e7b0458aa5ee1e3dbf65b7f040e774cf74 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 20 Mar 2026 10:30:43 -0400 Subject: [PATCH 11/48] comment-out notebook reqs for bare-minimum epmt environment and local checkout installation --- environment.yaml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/environment.yaml b/environment.yaml index be9f76297..26d1f4d8f 100644 --- a/environment.yaml +++ b/environment.yaml @@ -15,7 +15,7 @@ # After pulling new commits, re-run the pip install line above if # pyproject.toml dependencies changed. -name: epmt_w_notebooks +name: epmt channels: - conda-forge @@ -48,18 +48,18 @@ dependencies: - conda-forge::pylint - pip - # --- notebook extras -------------------------------------------------- (TESTED, WORKS!) - - conda-forge::ipykernel - - conda-forge::ipython - - conda-forge::ipywidgets - - conda-forge::jupyter - - conda-forge::jupyter_client - - conda-forge::jupyter_console - - conda-forge::jupyter_core - - conda-forge::jupyter_server<2 - - conda-forge::notebook<7 - - conda-forge::jupyterlab<4 # must stay <4 while notebook is <7 - - conda-forge::qtconsole +# # --- notebook extras -------------------------------------------------- (TESTED, WORKS!) +# - conda-forge::ipykernel +# - conda-forge::ipython +# - conda-forge::ipywidgets +# - conda-forge::jupyter +# - conda-forge::jupyter_client +# - conda-forge::jupyter_console +# - conda-forge::jupyter_core +# - conda-forge::jupyter_server<2 +# - conda-forge::notebook<7 +# - conda-forge::jupyterlab<4 # must stay <4 while notebook is <7 +# - conda-forge::qtconsole # # --- ui extras -------------------------------------------------- (NOT TESTED) # - conda-forge::dash @@ -78,4 +78,4 @@ dependencies: # - conda-forge::pyinstaller-hooks-contrib==2023.11 - pip: - - src/ # editable install of epmt into this environment + - -e src/ # editable install of epmt into this environment From 6c4e486c26b409eac0b3404e75d051cd3d3042de Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 20 Mar 2026 10:31:11 -0400 Subject: [PATCH 12/48] new notebook environment file for environment.yaml + notebook reqs only --- notebook_environment.yaml | 65 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 notebook_environment.yaml diff --git a/notebook_environment.yaml b/notebook_environment.yaml new file mode 100644 index 000000000..7c68c02d6 --- /dev/null +++ b/notebook_environment.yaml @@ -0,0 +1,65 @@ +# Conda development environment for epmt +# +# Prerequisites: +# module load conda +# +# Create the environment: +# conda env create -f environment.yaml +# +# Activate: +# conda activate epmt +# +# Editable install (from the repo root, ~/Working/epmt): +# pip install -e src/ +# +# After pulling new commits, re-run the pip install line above if +# pyproject.toml dependencies changed. + +name: epmt_notebook + +channels: + - conda-forge + +dependencies: + # 3.9/3.10/3.11 supported; 3.12 requires pandas 2.x + SQLAlchemy 2.x (issue #105) + - python>=3.9,<3.12 + - sqlite>=3.49 # epmt's DB layer requires SQLite >= 3.49 + + # --- runtime dependencies (mirrors src/pyproject.toml) ----------------- + - conda-forge::alembic + - conda-forge::flask + - conda-forge::kneed + - conda-forge::numpy==1.26.4 + - conda-forge::pandas==1.5.3 + - conda-forge::plotly + - conda-forge::psycopg2 # conda-forge ships psycopg2 (not -binary) + - conda-forge::py-cpuinfo + - conda-forge::pyod + - conda-forge::python-daemon + - conda-forge::pytz + - conda-forge::scikit-learn + - conda-forge::scipy + - conda-forge::six + - conda-forge::sqlalchemy==1.4.54 + + # --- dev / test extras ------------------------------------------------- + - conda-forge::pytest + - conda-forge::pytest-cov + - conda-forge::pylint + - pip + + # --- notebook extras -------------------------------------------------- (TESTED, WORKS!) + - conda-forge::ipykernel + - conda-forge::ipython + - conda-forge::ipywidgets + - conda-forge::jupyter + - conda-forge::jupyter_client + - conda-forge::jupyter_console + - conda-forge::jupyter_core + - conda-forge::jupyter_server<2 + - conda-forge::notebook<7 + - conda-forge::jupyterlab<4 # must stay <4 while notebook is <7 + - conda-forge::qtconsole + + - pip: + - -e src/ # editable install of epmt into this environment From d76c44aaccaf517194bccffd806462fe700a03fb Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 20 Mar 2026 10:39:11 -0400 Subject: [PATCH 13/48] remove unneeded and redundant reqs under dev category --- src/pyproject.toml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/pyproject.toml b/src/pyproject.toml index d2bf88eaa..fbda1d353 100644 --- a/src/pyproject.toml +++ b/src/pyproject.toml @@ -52,7 +52,9 @@ notebook = [ "jupyter-client", "jupyter-console", "jupyter-core", + "jupyter-server<2", "notebook<7", + "jupyterlab<4", "qtconsole", ] # Dash-based web UI @@ -71,11 +73,6 @@ docs = [ ] # Linting and static analysis dev = [ - "astroid", - "isort", - "lazy-object-proxy", - "mccabe", - "wrapt", "pylint", "pytest", ] From 4e72856b79af5e68e31ca0124f50e80ee8c8f345 Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 20 Mar 2026 10:48:31 -0400 Subject: [PATCH 14/48] tweak additional install option name in pyproj, tweak dockerfile convenience file to mirror new lack of cruft --- requirements.txt.py3 | 16 ++++++---------- src/pyproject.toml | 6 +++--- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/requirements.txt.py3 b/requirements.txt.py3 index 185965559..5f9e2e3aa 100644 --- a/requirements.txt.py3 +++ b/requirements.txt.py3 @@ -24,6 +24,10 @@ scipy six SQLAlchemy==1.4.54 +# -- devtest (linting / static analysis) --------------------------------------- +pylint +pytest + # -- notebook -------------------------------------------------------------- ipykernel ipython @@ -32,7 +36,9 @@ jupyter jupyter-client jupyter-console jupyter-core +jupyter-server<2 notebook<7 +jupyterlab<4 qtconsole # -- ui -------------------------------------------------------------------- @@ -47,16 +53,6 @@ mkdocs mkdocs-git-committers-plugin mkdocs-theme-bootstrap4 -# -- dev (linting / static analysis) --------------------------------------- -astroid -isort -lazy-object-proxy -mccabe -wrapt - -# -- test ------------------------------------------------------------------ -pytest - # -- build ----------------------------------------------------------------- pyinstaller==5.7.0 pyinstaller-hooks-contrib==2023.11 diff --git a/src/pyproject.toml b/src/pyproject.toml index fbda1d353..51703da43 100644 --- a/src/pyproject.toml +++ b/src/pyproject.toml @@ -71,8 +71,8 @@ docs = [ "mkdocs-git-committers-plugin", "mkdocs-theme-bootstrap4", ] -# Linting and static analysis -dev = [ +# testing, linting and static analysis +devtest = [ "pylint", "pytest", ] @@ -83,7 +83,7 @@ build = [ ] # Shorthand: install everything (mirrors the old flat dependency list) all = [ - "epmt[notebook,ui,docs,dev,test,build]", + "epmt[notebook,ui,docs,devtest,build]", ] [project.scripts] From ac2a0a6dab9d40c2b382d2da4ce4fd456a436dfe Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 13 Mar 2026 20:37:07 +0000 Subject: [PATCH 15/48] Refactor logging: use package logger, consistent naming, add NullHandler - Configure handlers on getLogger('epmt') instead of root logger - Remove third-party logger suppression (matplotlib, numba, parso) - Fix inconsistent logger names in epmt_concat.py, epmt_daemon.py, epmtlib.py, epmt_convert_csv.py to use __name__ - Add NullHandler to epmt/__init__.py per Python best practices - Fix test_logfn to use epmt logger instead of root logger Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com> --- src/epmt/__init__.py | 3 +++ src/epmt/epmt_concat.py | 8 +----- src/epmt/epmt_convert_csv.py | 4 +-- src/epmt/epmt_daemon.py | 18 +++++-------- src/epmt/epmt_query.py | 6 ++--- src/epmt/epmtlib.py | 51 +++++++++++++++++------------------- src/epmt/test/test_lib.py | 29 +++++++------------- 7 files changed, 49 insertions(+), 70 deletions(-) diff --git a/src/epmt/__init__.py b/src/epmt/__init__.py index e69de29bb..967b94b78 100644 --- a/src/epmt/__init__.py +++ b/src/epmt/__init__.py @@ -0,0 +1,3 @@ +import logging + +logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/src/epmt/epmt_concat.py b/src/epmt/epmt_concat.py index 9877ca663..deb699765 100755 --- a/src/epmt/epmt_concat.py +++ b/src/epmt/epmt_concat.py @@ -29,7 +29,7 @@ from logging import getLogger -logger = getLogger('epmt_concat') # you can use other name +logger = getLogger(__name__) class InvalidFileFormat(RuntimeError): @@ -42,7 +42,6 @@ def rename_bad_files(outfile, errdir, badfiles): errdir is place for output badfiles is possibly empty list of files that errored in parsing ''' - logger = getLogger('rename_bad_files') logger.debug("%s,%s,%s", outfile, errdir, str(badfiles)) if not errdir: logger.warning("No error dir specified, skipping renaming of bad CSV files!") @@ -83,7 +82,6 @@ def parseFile(inputfile, masterHeader, masterHeaderFile, delim, commentDelim): Returns: tuple(comments list, masterHeader string, datas list) """ - logger = getLogger('parseFile') fileLines = [] comments = [] header = "" @@ -130,7 +128,6 @@ def parseLine(infile, line, masterHeader, masterHeaderFile, headerDelimCount, he - we have no header: set it - header is known: line is data """ - logger = getLogger('parseLine') Delim = r"(? overwrite infile if infile == outfile: @@ -380,6 +380,6 @@ def extract_jobid_from_collated_csv(collated_csv): if __name__ == "__main__": import sys - logger = getLogger("epmt_convert_csv") + logger = getLogger(__name__) epmt_logging_init(intlvl=2) convert_csv_in_tar(sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else '') diff --git a/src/epmt/epmt_daemon.py b/src/epmt/epmt_daemon.py index 0e6b4f7af..0c1b0ad2a 100644 --- a/src/epmt/epmt_daemon.py +++ b/src/epmt/epmt_daemon.py @@ -43,22 +43,20 @@ def is_daemon_running(pidf=PID_FILE): """ from epmt.epmtlib import check_pid - current_logger = logging.getLogger(is_daemon_running.__name__) - current_logger.debug("Looking for file %s to fetch daemon pid", pidf) + logger.debug("Looking for file %s to fetch daemon pid", pidf) try: with open(pidf, 'r', encoding='utf-8') as f: pid = f.read().strip() - current_logger.debug(f'Found daemon lockfile with PID({pid})') + logger.debug(f'Found daemon lockfile with PID({pid})') except Exception as e: - current_logger.debug(str(e)) + logger.debug(str(e)) return False, -1 if int(pid) < 0: - current_logger.error('PID %d for daemon is less than 0, this cant be true', int(pid)) + logger.error('PID %d for daemon is less than 0, this cant be true', int(pid)) return False, -1 stat, msg = check_pid(int(pid)) if not stat: - #current_logger.warning("PID %d for daemon doesn't seem alive: %s",int(pid),msg) - current_logger.error("You should check PID %d and consider removing the stale lock file %s.", + logger.error("You should check PID %d and consider removing the stale lock file %s.", int(pid), pidf) return False, -1 return True, int(pid) @@ -68,10 +66,9 @@ def start_daemon(foreground=False, pidf=PID_FILE, **daemon_args): """ Start the daemon process. """ - current_logger = logging.getLogger(start_daemon.__name__) stat, pid = is_daemon_running(pidf) if stat: - current_logger.error( + logger.error( 'Daemon may be still running at pid %d. If not, please remove the lock file %s and try again', pid, pidf) return -1 @@ -118,7 +115,6 @@ def stop_daemon(pidf=PID_FILE): """ Stop the daemon process. """ - logger = logging.getLogger(stop_daemon.__name__) stat, pid = is_daemon_running(pidf) if stat: logger.info('Sending SIGUSR1 signal to EPMT daemon PID {0}'.format(pid)) @@ -147,7 +143,6 @@ def print_daemon_status(pidf=PID_FILE): """ Print status of daemon. """ - current_logger = logging.getLogger(print_daemon_status.__name__) stat, pid = is_daemon_running(pidf) if not stat: print('EPMT daemon not running, start with "epmt daemon --start"') @@ -197,7 +192,6 @@ def daemon_loop(context, maxiters=0, post_process=True, analyze=True, retire=Fal global sig_count sig_count = 0 - logger = logging.getLogger(daemon_loop.__name__) logger.debug( '(context=%s,maxiters=%d,post_process=%s,analyze=%s,retire=%s,ingest=%s,recursive=%s,keep=%s,moveaway=%s,verbose=%d)', type(context), diff --git a/src/epmt/epmt_query.py b/src/epmt/epmt_query.py index 48a6f72b8..4fbe9b0a3 100644 --- a/src/epmt/epmt_query.py +++ b/src/epmt/epmt_query.py @@ -28,7 +28,7 @@ # using epmt_logging_init, other than import epmt_logging_init from epmt.epmtlib import epmt_logging_init, version -logger = getLogger(__name__) # you can use other name +logger = getLogger(__name__) epmt_logging_init(settings.verbose if hasattr(settings, 'verbose') else 0, check=True) # Put EPMT imports below, after logging is set up @@ -2850,7 +2850,7 @@ def procs_histogram(jobs, attr='exename', metric=''): ... } ''' - logger = getLogger(__name__) # you can use other name + logger = getLogger(__name__) procs_hist = {} procs = get_procs(jobs, fmt='orm') logger.debug('{} processes found'.format(procs.count())) @@ -2978,7 +2978,7 @@ def add_features_df(jobs_df, features=[procs_histogram, procs_set], key='jobid') 8 {'mv': 118, 'perl': 101, 'globus-url-copy': 76... [TAVG.exe, arch, basename, bash, cat, chmod, c... ''' - logger = getLogger(__name__) # you can use other name + logger = getLogger(__name__) out_df = jobs_df.copy() keys = list(jobs_df[key].values) added_features = [] diff --git a/src/epmt/epmtlib.py b/src/epmt/epmtlib.py index 55406d4aa..54726dde3 100644 --- a/src/epmt/epmtlib.py +++ b/src/epmt/epmtlib.py @@ -39,7 +39,13 @@ def get_username(): def epmt_logging_init(intlvl=0, check=False, log_pid=False): ''' - if check is set, then we will bail if logging has already been initialized + Configure logging for the ``epmt`` package. + + Handlers are attached to the ``epmt`` package logger (not the root + logger) so that third-party libraries are unaffected. + + If *check* is set, then we will bail if logging has already been + initialized. ''' import logging import epmt.epmt_settings as settings @@ -61,43 +67,36 @@ def epmt_logging_init(intlvl=0, check=False, log_pid=False): else: # intlvl >= 2: level = DEBUG # 10 - # Set level and remove all existing handlers - # rootLogger = getLogger(__name__) # thank you! @ ericzhou13 - rootLogger = getLogger() - rootLogger.debug("epmt_logging_init(%d,%s,%s): %d handlers", intlvl, check, log_pid, len(rootLogger.handlers)) - for handler in rootLogger.handlers: - rootLogger.removeHandler(handler) - rootLogger.setLevel(level) + # Configure the 'epmt' package logger instead of the root logger. + # All epmt.* module loggers inherit from this logger, and third-party + # loggers (matplotlib, numba, parso, etc.) are no longer affected. + epmt_logger = getLogger('epmt') + epmt_logger.debug("epmt_logging_init(%d,%s,%s): %d handlers", intlvl, check, log_pid, len(epmt_logger.handlers)) + for handler in epmt_logger.handlers[:]: + epmt_logger.removeHandler(handler) + epmt_logger.setLevel(level) + # Prevent log messages from propagating to the root logger, which + # would cause duplicate output if the root has its own handlers. + epmt_logger.propagate = False # only log to file if stdout is not a tty from sys import stdout if not stdout.isatty(): - # basicConfig(filename='epmt.log', filemode='a', level=level) logFormatter = logging.Formatter("[%(asctime)-19.19s, %(process)6d] %(levelname)-7.7s %(name)s:%(message)s") fileHandler = logging.FileHandler(settings.logfile) fileHandler.setFormatter(logFormatter) fileHandler.setLevel(level) - rootLogger.debug("epmt_logging_init(): not_a_tty: adding handler for settings.logfile=%s", settings.logfile) - rootLogger.addHandler(fileHandler) + epmt_logger.debug("epmt_logging_init(): not_a_tty: adding handler for settings.logfile=%s", settings.logfile) + epmt_logger.addHandler(fileHandler) consoleHandler = logging.StreamHandler() consoleFormatter = logging.Formatter( "[%(asctime)-19.19s, %(process)d] %(levelname)7.7s: %(name)s: %(message)s" if log_pid else "%(asctime)-19.19s %(levelname)7.7s: %(name)s: %(message)s") consoleHandler.setFormatter(consoleFormatter) - rootLogger.addHandler(consoleHandler) - - # matplotlib generates a ton of debug messages - mpl_logger = logging.getLogger('matplotlib') - mpl_logger.setLevel(logging.WARNING) - - # numba.byteflow generates a ton of debug messages - numba_logger = logging.getLogger('numba') - numba_logger.setLevel(logging.WARNING) - - # ipython's parso logger has too many debug messages - parso_logger = logging.getLogger('parso') - parso_logger.setLevel(logging.WARNING) + epmt_logger.addHandler(consoleHandler) + # Alembic's logger lives outside the epmt hierarchy, so we still + # need to set its level explicitly. alembic_logger = logging.getLogger('alembic') alembic_logger.setLevel(level) @@ -108,8 +107,6 @@ def epmt_logging_init(intlvl=0, check=False, log_pid=False): # to show the sqlalchemy's INFO level messages (but instead # a level higher). sqlalchemy_logger = logging.getLogger('sqlalchemy') - #sqlalchemy_logger.setLevel(level + 10) - #sqlalchemy_logger.setLevel(level + 20) sqlalchemy_logger.setLevel(level + 30) @@ -118,7 +115,7 @@ def init_settings(settings): return init_settings.initialized = True - logger = getLogger('init_settings') + logger = getLogger(__name__) err_msg = "" if environ.get("PAPIEX_OUTPUT"): diff --git a/src/epmt/test/test_lib.py b/src/epmt/test/test_lib.py index 7b515f9df..f98b39070 100755 --- a/src/epmt/test/test_lib.py +++ b/src/epmt/test/test_lib.py @@ -77,50 +77,41 @@ def test_install_root(self): self.assertTrue(install_root) self.assertEqual(install_root + '/test', __file__.rsplit('/', 1)[0]) - # BROKEN FIX TODO def test_logfn(self): from epmt.epmtlib import logfn, epmt_logging_init # enable debug logging and, # remove all handlers and add our StringIO handler epmt_logging_init(2) - logger = logging.getLogger() - - @logfn - def double(x): - return x * 2 + epmt_logger = logging.getLogger('epmt') # this removes the existing handlers - for handler in logger.handlers: - logger.removeHandler(handler) - logger.handlers = [] + for handler in epmt_logger.handlers[:]: + epmt_logger.removeHandler(handler) log_stream = StringIO() - # print(log_stream) stream_handler = logging.StreamHandler(log_stream) - # print(stream_handler) streamFormatter = logging.Formatter("%(levelname)7.7s: %(name)s: %(message)s") - # print(streamFormatter) stream_handler.setFormatter(streamFormatter) - logger.addHandler(stream_handler) + epmt_logger.addHandler(stream_handler) + + @logfn + def double(x): + return x * 2 # call our function y = double(25) # remove the StringIO handler - logger.removeHandler(stream_handler) + epmt_logger.removeHandler(stream_handler) # restore logging to sanity epmt_logging_init(0) - # grab the logging output and parse it for what we exepct + # grab the logging output and parse it for what we expect s = log_stream.getvalue() self.assertEqual(y, 50) self.assertIn('DEBUG: epmt.test.test_lib: double(25)', s) - # print(s) - # print(s) - # print(s) - # print(s) if __name__ == '__main__': From e206b8cf46305ae3fc0e2117f523c09790ceffd2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 16 Mar 2026 20:18:46 +0000 Subject: [PATCH 16/48] Remove redundant function-local loggers, add module-level loggers, clean stale comments - Remove 16 stale '# you can use other name' comments across 5 files - Remove 7 redundant function-local loggers in epmt_job.py - Remove 8 redundant function-local loggers in epmt_stat.py - Remove 2 redundant function-local loggers in epmt_query.py - Remove 1 redundant function-local logger + stale comment in epmt_cmds.py - Remove 10 redundant function-local loggers in epmtlib.py - Add module-level logger to epmtlib.py, epmt_convert_csv.py, orm/op.py - Remove function-local from logging import getLogger in orm/op.py Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com> --- src/epmt/epmt_cmd_delete.py | 2 +- src/epmt/epmt_cmd_list.py | 2 +- src/epmt/epmt_cmds.py | 2 -- src/epmt/epmt_convert_csv.py | 5 ++--- src/epmt/epmt_job.py | 7 ------- src/epmt/epmt_outliers.py | 2 +- src/epmt/epmt_query.py | 2 -- src/epmt/epmt_stat.py | 10 +--------- src/epmt/epmtlib.py | 13 ++----------- src/epmt/orm/op.py | 13 ++++--------- 10 files changed, 12 insertions(+), 46 deletions(-) diff --git a/src/epmt/epmt_cmd_delete.py b/src/epmt/epmt_cmd_delete.py index a7db85d23..8a59c049f 100644 --- a/src/epmt/epmt_cmd_delete.py +++ b/src/epmt/epmt_cmd_delete.py @@ -5,7 +5,7 @@ from epmt.epmt_query import delete_jobs -logger = getLogger(__name__) # you can use other name +logger = getLogger(__name__) def epmt_delete_jobs(joblist): diff --git a/src/epmt/epmt_cmd_list.py b/src/epmt/epmt_cmd_list.py index e0780f22a..d8a2bc168 100644 --- a/src/epmt/epmt_cmd_list.py +++ b/src/epmt/epmt_cmd_list.py @@ -12,7 +12,7 @@ get_thread_metrics, get_job_proc_tags, get_op_metrics ) from epmt.epmtlib import kwargify -logger = getLogger(__name__) # you can use other name +logger = getLogger(__name__) def epmt_list(arglist): diff --git a/src/epmt/epmt_cmds.py b/src/epmt/epmt_cmds.py index ef27c5773..ce0ce978d 100644 --- a/src/epmt/epmt_cmds.py +++ b/src/epmt/epmt_cmds.py @@ -1711,8 +1711,6 @@ def epmt_entrypoint(args): check=True, log_pid=(hasattr(args, 'num_cpus') and (args.num_cpus > 1))) - # initialize logger ... inl: after epmt_logging_init? shouldnt that one function call do it? - logger = getLogger(__name__) init_settings(settings) # Here it's up to each command to validate what it is looking for diff --git a/src/epmt/epmt_convert_csv.py b/src/epmt/epmt_convert_csv.py index 9dc028c6f..34d0ec3b5 100755 --- a/src/epmt/epmt_convert_csv.py +++ b/src/epmt/epmt_convert_csv.py @@ -21,6 +21,8 @@ from glob import glob import atexit +logger = getLogger(__name__) + # While it may seem like a good idea to put these constants in a # settings file, it probably isn't because these are not user-tweakable # settings, and modifying them can have unexpected ramifications @@ -109,7 +111,6 @@ def conv_csv_for_dbcopy(infile, outfile='', jobid='', input_fields=INPUT_CSV_FIE `convert_csv_in_tar` on a staged .tgz file. ''' - logger = getLogger(__name__) outfile = outfile or infile # empty outfile => overwrite infile if infile == outfile: @@ -268,7 +269,6 @@ def convert_csv_in_tar(in_tar, out_tar=''): format conversion. This method will also add a header file in the newly-created tar. ''' - logger = getLogger(__name__) if not any( [ in_tar.endswith('.tgz'), in_tar.endswith('.tar.gz'), in_tar.endswith('.tar') ] ): raise ValueError('input file must have a .tar, .tgz or .tar.gz suffix') @@ -380,6 +380,5 @@ def extract_jobid_from_collated_csv(collated_csv): if __name__ == "__main__": import sys - logger = getLogger(__name__) epmt_logging_init(intlvl=2) convert_csv_in_tar(sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else '') diff --git a/src/epmt/epmt_job.py b/src/epmt/epmt_job.py index 5c1757a65..0b2d313df 100644 --- a/src/epmt/epmt_job.py +++ b/src/epmt/epmt_job.py @@ -77,7 +77,6 @@ def create_job(jobid, user): def lookup_or_create_host(hostname): - logger = getLogger(__name__) # you can use other name host = created_hosts.get(hostname) if host: # sometimes we may have cached a host entry that's been invalidated @@ -119,7 +118,6 @@ def lookup_or_create_host_safe(hostname): def lookup_or_create_user(username): - logger = getLogger(__name__) user = orm_get_or_create(User, name=username) logger.debug('orm_get_or_create output for user is %s',user) # user = orm_get(User, username) @@ -243,7 +241,6 @@ def load_process_from_dictlist(proc, host, j, u, settings, profile): process such as 'exename', 'args', etc. The other threads may not have process fields set. ''' from pandas import Timestamp - logger = getLogger(__name__) hostname = proc[0].get('hostname', '') if hostname: @@ -497,7 +494,6 @@ def _disambiguate_parent(entries, proc): def _create_process_tree(pid_map): - logger = getLogger(__name__) logger.info(" creating process tree..") for (_, procs) in pid_map.items(): # _ is pid @@ -618,7 +614,6 @@ def post_process_job( j, _create_process_tree to create process tree. The function is tolerant to missing datastructures for all_tags, all_procs and pid_map. If any of them are missing, it will Build them by using the data in the database/ORM. ''' - logger = getLogger(__name__) # you can use other name if isinstance(j, str): jobid = j j = Job[jobid] @@ -888,7 +883,6 @@ def populate_process_table_from_staging(j): ''' import datetime as dt import psycopg2 - logger = getLogger(__name__) # you can use other name jobid = j.jobid job_info_dict = j.info_dict logger.info(' moving job {} processes from staging -> process table..'.format(jobid)) @@ -1084,7 +1078,6 @@ def populate_process_table_from_staging(j): @db_session def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): - logger = getLogger(__name__) # you can use other name job_init_start_time = time.time() # Synthesize what we need # it's safe and fast to call the check_fix_metadata diff --git a/src/epmt/epmt_outliers.py b/src/epmt/epmt_outliers.py index baf90d034..6998a9ec7 100644 --- a/src/epmt/epmt_outliers.py +++ b/src/epmt/epmt_outliers.py @@ -27,7 +27,7 @@ partition_classifiers_uv_mv, mvod_scores_using_model, uvod_classifiers, modified_z_score ) -logger = getLogger(__name__) # you can use other name +logger = getLogger(__name__) FEATURES = settings.outlier_features diff --git a/src/epmt/epmt_query.py b/src/epmt/epmt_query.py index 4fbe9b0a3..d1d38f620 100644 --- a/src/epmt/epmt_query.py +++ b/src/epmt/epmt_query.py @@ -2850,7 +2850,6 @@ def procs_histogram(jobs, attr='exename', metric=''): ... } ''' - logger = getLogger(__name__) procs_hist = {} procs = get_procs(jobs, fmt='orm') logger.debug('{} processes found'.format(procs.count())) @@ -2978,7 +2977,6 @@ def add_features_df(jobs_df, features=[procs_histogram, procs_set], key='jobid') 8 {'mv': 118, 'perl': 101, 'globus-url-copy': 76... [TAVG.exe, arch, basename, bash, cat, chmod, c... ''' - logger = getLogger(__name__) out_df = jobs_df.copy() keys = list(jobs_df[key].values) added_features = [] diff --git a/src/epmt/epmt_stat.py b/src/epmt/epmt_stat.py index 8ae2d1f4f..02bcb6932 100644 --- a/src/epmt/epmt_stat.py +++ b/src/epmt/epmt_stat.py @@ -20,7 +20,7 @@ import epmt.epmt_settings as settings from epmt.epmtlib import logfn -logger = getLogger(__name__) # you can use other name +logger = getLogger(__name__) # this sets the defaults to be used when a trained model is not provided thresholds = settings.outlier_thresholds @@ -107,7 +107,6 @@ def z_score(ys, params=()): # issue by using np.nan_to_num import warnings warnings.filterwarnings("ignore", category=RuntimeWarning) - logger = getLogger(__name__) # you can use other name logger.debug('scoring using {}'.format('z_score')) ys = np.array(ys) if params: @@ -171,7 +170,6 @@ def iqr(ys, params=()): >>> outliers array([0, 0, 0, 0, 0, 0, 1]) ''' - logger = getLogger(__name__) # you can use other name logger.debug('scoring using {}'.format('iqr')) ys = np.array(ys) span = [25, 75] @@ -209,7 +207,6 @@ def modified_z_score(ys, params=()): params if passed in, is of the form (max, median, median_abs_dev) We will ignore params(0) as that's the max z_score in the ref_model ''' - logger = getLogger(__name__) # you can use other name logger.debug('scoring using {}'.format('modified_z_score')) median_y = params[1] if params else np.median(ys) if params: @@ -274,7 +271,6 @@ def outliers_uv(ys, methods=[outliers_iqr, outliers_z_score, outliers_modified_z value. So, use the outliers_* wrappers instead of methods such as iqr, z_score, modified_z_score ''' - logger = getLogger(__name__) # you can use other name ys = np.array(ys) logger.debug('input vector: {}'.format(ys)) out_vec = np.zeros_like(ys) @@ -313,7 +309,6 @@ def mvod_classifiers(contamination=0.1, warnopts='ignore'): if warnopts: from warnings import simplefilter simplefilter(warnopts) - logger = getLogger(__name__) # you can use other name from pyod.models.abod import ABOD from pyod.models.knn import KNN @@ -389,7 +384,6 @@ def mvod_scores(X=None, classifiers=[], warnopts='ignore'): # simplefilter(action='ignore', category=FutureWarning) simplefilter(warnopts) - logger = getLogger(__name__) # you can use other name # the contamination below, is *ONLY* used in the model # for preditiction of outliers and used for random data @@ -516,7 +510,6 @@ def mvod_scores_using_model(inp, model_inp, classifier, threshold=None): row score is higher than the given threshold and 0 if its lower. """ - logger = getLogger(__name__) # you can use other name inp_nrows = inp.shape[0] logger.debug('--- input to classify ---') logger.debug(inp) @@ -691,7 +684,6 @@ def pca_stat(inp_features, desired=2): from sklearn.preprocessing import StandardScaler from sklearn.decomposition import PCA - logger = getLogger(__name__) # you can use other name logger.debug('input feature array shape: {}'.format(inp_features.shape)) if np.isnan(inp_features).any(): raise ValueError('input contains at-least one non-numeric (nan) element') diff --git a/src/epmt/epmtlib.py b/src/epmt/epmtlib.py index 54726dde3..378441680 100644 --- a/src/epmt/epmtlib.py +++ b/src/epmt/epmtlib.py @@ -15,6 +15,8 @@ from subprocess import call from time import time +logger = getLogger(__name__) + # semantic version # first element is the major version number # second element is the minor version number @@ -115,7 +117,6 @@ def init_settings(settings): return init_settings.initialized = True - logger = getLogger(__name__) err_msg = "" if environ.get("PAPIEX_OUTPUT"): @@ -253,7 +254,6 @@ def safe_rm(f): def timing(f): - logger = getLogger(__name__) @wraps(f) def wrap(*args, **kw): @@ -311,7 +311,6 @@ def tag_from_string(s, delim=';', sep=':', tag_default_value='1'): if not s: return (None if s is None else {}) - logger = getLogger(__name__) tag = {} for t in s.split(delim): t = t.strip() @@ -645,7 +644,6 @@ def compare_dicts(d1, d2): def get_batch_envvar(var, where): - logger = getLogger(__name__) key2slurm = { "JOB_NAME": "SLURM_JOB_NAME", "JOB_USER": "SLURM_JOB_USER" @@ -663,7 +661,6 @@ def get_batch_envvar(var, where): def get_metadata_env_changes(metadata): - logger = getLogger(__name__) start_env = metadata['job_pl_env'] stop_env = metadata['job_el_env'] (added, removed, modified, same) = compare_dicts(stop_env, start_env) @@ -694,7 +691,6 @@ def check_fix_metadata(raw_metadata): return raw_metadata import epmt.epmt_settings as settings - logger = getLogger(__name__) # First check what should be here try: @@ -807,7 +803,6 @@ def conv_to_datetime(t): try: retval = datetime.strptime(t, '%m/%d/%Y %H:%M') except Exception as e: - logger = getLogger(__name__) logger.error('could not convert string to datetime: %s' % str(e)) return None elif type(t) in (int, float): @@ -919,7 +914,6 @@ def dframe_encode_features(df, features=[], reversible=False): NOTE: If encoded_features is empty, no features were encoded. ''' - logger = getLogger(__name__) if not features: import epmt.epmt_settings as settings logger.debug('Selecting non-numeric columns from dataframe and then pruning out blacklisted features') @@ -969,7 +963,6 @@ def dframe_decode_features(df, features): calling dframe_encode_features, as otherwise the strings are hashed and not encoded (hashed strings cannot be decoded). ''' - logger = getLogger(__name__) decoded_df = df.copy() decoded_features = [] for c in features: @@ -1125,7 +1118,6 @@ def get_install_root(): >>> '/abc/def/ghi.py'.rsplit('/',1) ['/abc/def', 'ghi.py'] ''' - # logger = getLogger(__name__) install_root = __file__.rsplit('/', 2)[0] # handle pip packaging here -- even when "manually" installed ala 4.9.6, our install_dir should always end in /epmt. # XXX THIS IS STILL HOKEY and i'm not sure how to make it work for all possible installations. @@ -1217,7 +1209,6 @@ def set_signal_handlers(signals=[], handler=None): to the default) ''' from signal import SIGHUP, SIGTERM, SIGINT, signal, SIG_DFL - logger = getLogger(__name__) # set defaults signals = signals or [SIGHUP, SIGTERM, SIGINT] diff --git a/src/epmt/orm/op.py b/src/epmt/orm/op.py index a6ef6a9f3..d2bb22ebe 100644 --- a/src/epmt/orm/op.py +++ b/src/epmt/orm/op.py @@ -3,6 +3,10 @@ as it is NOT used for defining the ORM nor the DB tables. It is for manipulating data and measurements """ +from logging import getLogger + +logger = getLogger(__name__) + class Operation(dict): ''' An operation is defined as a collection of processes spanning one or more jobs where each processes' tag is @@ -20,9 +24,6 @@ def __init__(self, jobs, tags, exact_tag_only=False, op_duration_method="sum"): from . import orm_is_query, orm_jobs_col # rocky-8 change from epmt.epmtlib import tag_from_string, tags_list - from logging import getLogger - logger = getLogger(__name__) - if op_duration_method not in ("sum", "sum-minus-overlap", "finish-minus-start"): raise ValueError('op_duration_method must be one of ("sum", "sum-minus-overlap", "finish-minus-start")') @@ -65,9 +66,7 @@ def finish(self): @property def processes(self): if self._processes is None: - from logging import getLogger from epmt.epmt_query import get_procs - logger = getLogger(__name__) logger.debug('computing op processes..') self._processes = get_procs(jobs=self.jobs, tags=self.tags, exact_tag_only=self.exact_tag_only, fmt='orm') if len(self._processes[:]) == 0: @@ -85,8 +84,6 @@ def intervals(self): ''' from epmt.epmtlib import merge_intervals if self._intervals is None: - from logging import getLogger - logger = getLogger(__name__) logger.debug('computing operation intervals..') _intervals = [[p.start, p.end] for p in self.processes] logger.debug('merging operation intervals..') @@ -118,8 +115,6 @@ def proc_sums(self): if self._proc_sums is None: from epmt.epmt_query import get_op_metrics from epmt.epmtlib import sum_dicts_list - from logging import getLogger - logger = getLogger(__name__) logger.debug('getting op_metrics for jobs={0}, tags={1}'.format(self.jobs, self.tags)) op_metrics = get_op_metrics( jobs=self.jobs, From ea33d30898e39a61a78a34dabfc69418ebce7197 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 17 Mar 2026 18:27:10 +0000 Subject: [PATCH 17/48] Fix daemon handler preservation, normalize import style, remove stale comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - epmt_daemon.py: fix logging.root.handlers → getLogger('epmt').handlers (handlers moved to epmt logger in previous commit, but daemon fork code was still iterating empty root logger handlers) - epmt_cmd_dbcare.py, epmt_daemon.py: normalize to 'from logging import getLogger' style matching all other modules - epmt_outliers.py, epmt_query.py: remove stale import-ordering comments that referenced the old root-logger approach - epmt_cmds.py, epmt_query.py: remove commented-out debug code that bypassed the logging infrastructure Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com> --- src/epmt/epmt_cmd_dbcare.py | 4 ++-- src/epmt/epmt_cmds.py | 3 --- src/epmt/epmt_daemon.py | 6 +++--- src/epmt/epmt_outliers.py | 1 - src/epmt/epmt_query.py | 6 ------ 5 files changed, 5 insertions(+), 15 deletions(-) diff --git a/src/epmt/epmt_cmd_dbcare.py b/src/epmt/epmt_cmd_dbcare.py index e8503c025..a559f0681 100644 --- a/src/epmt/epmt_cmd_dbcare.py +++ b/src/epmt/epmt_cmd_dbcare.py @@ -2,7 +2,7 @@ EPMT dbcare module - executes tasks for taking care of the database, designed to be run on a regular basis """ -import logging +from logging import getLogger #import epmt from epmt.epmt_cmd_retire import epmt_retire @@ -10,7 +10,7 @@ #from epmt import epmt_query as eq from epmt.orm.sqlalchemy import orm_raw_sql -logger = logging.getLogger(__name__) +logger = getLogger(__name__) def epmt_dbcare(retire_jobs = False, vacuum_tables = False, post_process = False): diff --git a/src/epmt/epmt_cmds.py b/src/epmt/epmt_cmds.py index ce0ce978d..b2cddfa66 100644 --- a/src/epmt/epmt_cmds.py +++ b/src/epmt/epmt_cmds.py @@ -986,9 +986,6 @@ def add_var(cmd, str): @logfn def epmt_run(cmdline, wrapit=False, dry_run=False, debug=False): - # logger.setLevel(DEBUG) - # logger.warning('HELLO') - # logger.debug("epmt_run(%s, %s, %s, %s, %s)", cmdline, str(wrapit), str(dry_run), str(debug)) if not cmdline: logger.error("No command given") diff --git a/src/epmt/epmt_daemon.py b/src/epmt/epmt_daemon.py index 0c1b0ad2a..0d93ed7c1 100644 --- a/src/epmt/epmt_daemon.py +++ b/src/epmt/epmt_daemon.py @@ -8,7 +8,7 @@ from sys import stdin, stdout, stderr from time import sleep, time from signal import SIGHUP, SIGTERM, SIGQUIT, SIGINT, SIGUSR1 -import logging +from logging import getLogger try: from daemon import DaemonContext, pidfile @@ -17,7 +17,7 @@ DaemonContext = None pidfile = None -logger = logging.getLogger(__name__) +logger = getLogger(__name__) # DO NOT IMPORT ANYTHING FROM ANY EPMT FILES HERE # **** IT WILL BREAK WHEN THE DAEMON FORKS **** @@ -93,7 +93,7 @@ def start_daemon(foreground=False, pidf=PID_FILE, **daemon_args): # ensure logging uses the same file-descriptors and they are preserved across the fork logger_files = [] try: - for handler in logging.root.handlers: + for handler in getLogger('epmt').handlers: fileno = handler.stream.fileno() logger_files.append(fileno) if logger_files: diff --git a/src/epmt/epmt_outliers.py b/src/epmt/epmt_outliers.py index 6998a9ec7..10b8ec911 100644 --- a/src/epmt/epmt_outliers.py +++ b/src/epmt/epmt_outliers.py @@ -20,7 +20,6 @@ import epmt.epmt_settings as settings from epmt.orm import db_session, ReferenceModel, orm_get, orm_col_len -# the first epmt import must be epmt_query as it sets up logging import epmt.epmt_query as eq from epmt.epmtlib import tags_list, tag_from_string, dict_in_list from epmt.epmt_stat import ( thresholds, rca, get_classifier_name, diff --git a/src/epmt/epmt_query.py b/src/epmt/epmt_query.py index d1d38f620..3c2b2e786 100644 --- a/src/epmt/epmt_query.py +++ b/src/epmt/epmt_query.py @@ -24,15 +24,11 @@ from logging import getLogger import epmt.epmt_settings as settings -# do NOT do any epmt imports until logging is set up -# using epmt_logging_init, other than import epmt_logging_init from epmt.epmtlib import epmt_logging_init, version logger = getLogger(__name__) epmt_logging_init(settings.verbose if hasattr(settings, 'verbose') else 0, check=True) -# Put EPMT imports below, after logging is set up - init_settings(settings) # type: ignore setup_db(settings) # type: ignore @@ -2721,8 +2717,6 @@ def are_jobs_comparable(jobs, matching_keys=['exp_name', 'exp_component']): def _warn_incomparable_jobs(jobs): - # import logging - # logger.setLevel(logging.DEBUG) jobs = orm_jobs_col(jobs) logger.debug(jobs) if not are_jobs_comparable(jobs): From 9d64dced3ee77306e272e1d3d46829b44b7ec6ef Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 17 Mar 2026 19:03:12 +0000 Subject: [PATCH 18/48] Fix all pylint logging violations in epmt_convert_csv.py Convert 17 logging calls from eager string formatting (.format() and + concatenation) to lazy % formatting with comma-separated arguments. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/epmt/epmt_convert_csv.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/epmt/epmt_convert_csv.py b/src/epmt/epmt_convert_csv.py index 34d0ec3b5..3e7dd298a 100755 --- a/src/epmt/epmt_convert_csv.py +++ b/src/epmt/epmt_convert_csv.py @@ -126,9 +126,9 @@ def conv_csv_for_dbcopy(infile, outfile='', jobid='', input_fields=INPUT_CSV_FIE if not jobid: jobid = extract_jobid_from_collated_csv(infile) if not jobid: - logger.error('Could not determine jobid from input path: ' + infile) + logger.error('Could not determine jobid from input path: %s', infile) return False - logger.debug('determined jobid ' + jobid + ' from input csv') + logger.debug('determined jobid %s from input csv', jobid) _start_time = time.time() # if infile is a string, then it's a path @@ -219,10 +219,10 @@ def conv_csv_for_dbcopy(infile, outfile='', jobid='', input_fields=INPUT_CSV_FIE outrow[f] = json.dumps(r[f]).replace('[', '{').replace(']', '}') writer.writerow(outrow) _finish_time = time.time() - logger.info('Wrote {} rows at {:.2f} procs/sec'.format(outrows, (outrows / (_finish_time - _start_time)))) + logger.info('Wrote %s rows at %.2f procs/sec', outrows, (outrows / (_finish_time - _start_time))) infile_flo.close() # close input file if in_place: - logger.debug('overwriting input file {} with {}'.format(infile, outfile)) + logger.debug('overwriting input file %s with %s', infile, outfile) shutil.move(outfile, infile) # we return the header @@ -283,8 +283,8 @@ def convert_csv_in_tar(in_tar, out_tar=''): in_place = True _, out_tar = tempfile.mkstemp(prefix='epmt_conv_outtar_', suffix='.tgz') atexit.register(_cleanup, out_tar) - logger.info('Doing in-place CSV format conversion in {}'.format(in_tar)) - logger.debug('Will create a temporary output tar ({}) as we are doing in-place conversion'.format(out_tar)) + logger.info('Doing in-place CSV format conversion in %s', in_tar) + logger.debug('Will create a temporary output tar (%s) as we are doing in-place conversion', out_tar) else: in_place = False @@ -292,7 +292,7 @@ def convert_csv_in_tar(in_tar, out_tar=''): try: tar = tarfile.open(in_tar, 'r|*') except Exception as e: - logger.error('error in processing compressed tar: ' + str(e)) + logger.error('error in processing compressed tar: %s', e) return False # extract the files into a temp. directory @@ -303,14 +303,14 @@ def convert_csv_in_tar(in_tar, out_tar=''): tar.extractall(tempdir) tar_contents = tar.getnames() except Exception as e: - logger.error('Error extracting {} to {}: {}'.format(in_tar, tempdir, e)) + logger.error('Error extracting %s to %s: %s', in_tar, tempdir, e) return False # close the input tar tar.close() in_csv_files = glob('{}/*.csv'.format(tempdir)) if not in_csv_files: - logger.error('No CSV files found in {}'.format(in_tar)) + logger.error('No CSV files found in %s', in_tar) return False # we should be having exactly 1 CSV file @@ -320,7 +320,7 @@ def convert_csv_in_tar(in_tar, out_tar=''): header_filename = "{}-papiex-header.tsv".format(hostname) if "./" + header_filename in tar_contents: # a header file presence indicates v2 CSV - logger.error('{} already contains CSV files in v2 format'.format(in_tar)) + logger.error('%s already contains CSV files in v2 format', in_tar) return False in_csv = in_csv_files[0] # only one csv file will be present out_csv = tempdir + "/" + "{}-papiex.tsv".format(hostname) @@ -328,26 +328,26 @@ def convert_csv_in_tar(in_tar, out_tar=''): # save the header returned for subsequent use hdr = conv_csv_for_dbcopy(in_csv, out_csv) if not hdr: - logger.error('Error converting {}'.format(in_csv)) + logger.error('Error converting %s', in_csv) return False # write the header into a separate file with open('{}/{}'.format(tempdir, header_filename), 'w') as csv_hdr_flo: csv_hdr_flo.write(hdr) - logger.debug("Created CSV header file: {}".format(header_filename)) + logger.debug("Created CSV header file: %s", header_filename) tar_contents.append("./" + header_filename) - logger.debug('Creating {} and adding contents to it'.format(out_tar)) + logger.debug('Creating %s and adding contents to it', out_tar) try: tar = tarfile.open(out_tar, 'w|gz') except Exception as e: - logger.error('error in creating compressed tar {}: {}'.format(out_tar, e)) + logger.error('error in creating compressed tar %s: %s', out_tar, e) return False owd = os.getcwd() try: os.chdir(tempdir) except OSError as e: - logger.error('Error changing directory to {} while creating tarfile: {}'.format(tempdir, e)) + logger.error('Error changing directory to %s while creating tarfile: %s', tempdir, e) return False # copy files other than *.csv for f in tar_contents + ["./" + basename(out_csv)]: @@ -357,14 +357,14 @@ def convert_csv_in_tar(in_tar, out_tar=''): if os.path.isfile(f): tar.add(f) except Exception as e: - logger.error('Error adding {}/{} to {}: {}'.format(tempdir, f, out_tar, e)) + logger.error('Error adding %s/%s to %s: %s', tempdir, f, out_tar, e) return False # return to the original working dir os.chdir(owd) tar.close() - logger.debug('Finished creating archive: {}'.format(out_tar)) + logger.debug('Finished creating archive: %s', out_tar) if in_place: - logger.debug('Replacing {} with newly-created archive'.format(in_tar)) + logger.debug('Replacing %s with newly-created archive', in_tar) shutil.move(out_tar, in_tar) logger.info('CSV format conversion successful!') shutil.rmtree(tempdir) From ffcb0f5a5e52ebe891c5ced291efeb36cdedaa87 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 17 Mar 2026 19:05:01 +0000 Subject: [PATCH 19/48] Fix all pylint logging violations in epmt_concat.py Convert 17 logging calls from .format() and string concatenation to lazy % formatting with comma-separated arguments (W1201, W1202). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/epmt/epmt_cmd_retire.py | 2 +- src/epmt/epmt_cmd_show.py | 4 ++-- src/epmt/epmt_concat.py | 38 +++++++++++++++--------------- src/epmt/epmt_daemon.py | 28 +++++++++++----------- src/epmt/epmt_exp_explore.py | 12 +++++----- src/epmt/epmtlib.py | 30 +++++++++++------------ src/epmt/orm/__init__.py | 6 ++--- src/epmt/orm/op.py | 4 ++-- src/epmt/orm/sqlalchemy/general.py | 26 ++++++++++---------- 9 files changed, 75 insertions(+), 75 deletions(-) diff --git a/src/epmt/epmt_cmd_retire.py b/src/epmt/epmt_cmd_retire.py index 3011ec7fe..493483f7a 100644 --- a/src/epmt/epmt_cmd_retire.py +++ b/src/epmt/epmt_cmd_retire.py @@ -45,7 +45,7 @@ def epmt_retire(skip_unprocessed=False, dry_run=False): logger.info('%d jobs retired, %d models retired', num_jobs_retired, num_models_retired) if dry_run: - logger.info(f'(dry_run=True) {num_jobs_retired} jobs and {num_models_retired} models will be retired') + logger.info('(dry_run=True) %d jobs and %d models will be retired', num_jobs_retired, num_models_retired) # end memory tracing tm.stop() diff --git a/src/epmt/epmt_cmd_show.py b/src/epmt/epmt_cmd_show.py index 0f700356a..a42cf1b71 100644 --- a/src/epmt/epmt_cmd_show.py +++ b/src/epmt/epmt_cmd_show.py @@ -13,14 +13,14 @@ def epmt_show_job(jobid, key=None): jobid = jobid[0] jobs = eq.get_jobs([jobid], fmt='dict') if len(jobs) != 1: - logger.error('Job %s could not be found in database' % jobid) + logger.error('Job %s could not be found in database', jobid) return False j_dict = jobs[0] if key: if key in j_dict: print(j_dict[key]) else: - logger.error('Key "{}" was not found as an attribute of the job table'.format(key)) + logger.error('Key "%s" was not found as an attribute of the job table', key) print('Here are the keys that were found: {}'.format(",".join(sorted(j_dict.keys())))) return False else: diff --git a/src/epmt/epmt_concat.py b/src/epmt/epmt_concat.py index deb699765..823c72562 100755 --- a/src/epmt/epmt_concat.py +++ b/src/epmt/epmt_concat.py @@ -173,11 +173,11 @@ def parseLine(infile, line, masterHeader, masterHeaderFile, headerDelimCount, he return (None, line, headerDelimCount, headerFound, masterHeader, masterHeaderFile) else: logger.error( - "File: {}, Header: {} delimiters, but this row has {} delimiters".format( - infile, str(headerDelimCount), str(lineDelimCount))) - logger.error("Row: {}".format(line)) - logger.error("Master File: {}".format(masterHeaderFile)) - logger.error("Master header: {}".format(masterHeader)) + "File: %s, Header: %s delimiters, but this row has %s delimiters", + infile, headerDelimCount, lineDelimCount) + logger.error("Row: %s", line) + logger.error("Master File: %s", masterHeaderFile) + logger.error("Master header: %s", masterHeader) raise InvalidFileFormat() @@ -204,7 +204,7 @@ def writeCSV(outfile, comments, masterHeader, dataList): """ try: - logger.info("Writing file({})".format(outfile)) + logger.info("Writing file(%s)", outfile) # write comments with open(outfile, 'w') as f: for item in comments: @@ -235,15 +235,15 @@ def verifyOut(fileList, outfile): lines += file_len(file) headers2Remove = len(fileList) result = lines - (headers2Remove - 1) - logger.debug("{} input files have {} lines".format(len(fileList), lines)) - logger.debug("{} output file has {} lines".format(outfile, outputLines)) - logger.debug("{} output file expected {} lines".format(outfile, result)) + logger.debug("%s input files have %s lines", len(fileList), lines) + logger.debug("%s output file has %s lines", outfile, outputLines) + logger.debug("%s output file expected %s lines", outfile, result) if result != outputLines: logger.error( - "Output file {} smaller than expected, off by {} lines, expected {}".format( - outfile, result - outputLines, result)) - logger.error("Input files {} have {} lines".format(str(fileList), str(lines))) - logger.error("Total header lines removed - 1 {}".format(str(headers2Remove))) + "Output file %s smaller than expected, off by %s lines, expected %s", + outfile, result - outputLines, result) + logger.error("Input files %s have %s lines", fileList, lines) + logger.error("Total header lines removed - 1 %s", headers2Remove) return False return True @@ -270,7 +270,7 @@ def determine_output_filename(instr): except Exception as e: logger.error("Could not determine output file name from %s: %s", instr, str(e)) return "" - logger.info("Output file set as {}".format(outfile)) + logger.info("Output file set as %s", outfile) return outfile @logfn @@ -312,9 +312,9 @@ def csvjoiner(indir, msg = "{} does not exist or is not a directory".format(indir) logger.error(msg) return False, None, badfiles - logger.info("Collate in directory {}".format(indir)) + logger.info("Collate in directory %s", indir) fileList = sorted(glob(indir + "/*.csv")) - logger.debug("Filelist:{}".format(fileList)) + logger.debug("Filelist:%s", fileList) # List Mode ######################################### if isinstance(indir, list): @@ -328,7 +328,7 @@ def csvjoiner(indir, fileList = sorted(list(set(fileList))) for test in fileList: if not path.isfile(test): - logger.error(test + " does not exist or is not a file") + logger.error("%s does not exist or is not a file", test) return False, None, badfiles if len(fileList) == 0: @@ -347,14 +347,14 @@ def csvjoiner(indir, outfile = outpath + outfile if path.exists(outfile): - logger.error("Output {} already exists".format(outfile)) + logger.error("Output %s already exists", outfile) return False, None, badfiles # iterate each file building result badfiles = [] badfiles_renamed = [] for f in fileList: - logger.info("Collating file:{}".format(f)) + logger.info("Collating file:%s", f) comments, masterHeader, masterHeaderFile, data = parseFile( f, masterHeader, masterHeaderFile, delim, commentDelim) if not data: diff --git a/src/epmt/epmt_daemon.py b/src/epmt/epmt_daemon.py index 0d93ed7c1..370a34ec0 100644 --- a/src/epmt/epmt_daemon.py +++ b/src/epmt/epmt_daemon.py @@ -47,7 +47,7 @@ def is_daemon_running(pidf=PID_FILE): try: with open(pidf, 'r', encoding='utf-8') as f: pid = f.read().strip() - logger.debug(f'Found daemon lockfile with PID({pid})') + logger.debug('Found daemon lockfile with PID(%s)', pid) except Exception as e: logger.debug(str(e)) return False, -1 @@ -89,7 +89,7 @@ def start_daemon(foreground=False, pidf=PID_FILE, **daemon_args): else: context.detach_process = True context.pidfile = pidfile.TimeoutPIDLockFile(pidf) - logger.info('Using lock file {0} for the EPMT daemon'.format(pidf)) + logger.info('Using lock file %s for the EPMT daemon', pidf) # ensure logging uses the same file-descriptors and they are preserved across the fork logger_files = [] try: @@ -117,12 +117,12 @@ def stop_daemon(pidf=PID_FILE): """ stat, pid = is_daemon_running(pidf) if stat: - logger.info('Sending SIGUSR1 signal to EPMT daemon PID {0}'.format(pid)) + logger.info('Sending SIGUSR1 signal to EPMT daemon PID %s', pid) try: print("Sending signal to EPMT daemon pid " + str(pid)) kill(pid, SIGUSR1) except Exception as e: - logger.error('Error sending SIGUSR1 to process PID {0}: {1}'.format(pid, str(e))) + logger.error('Error sending SIGUSR1 to process PID %s: %s', pid, e) return -1 else: logger.error('EPMT daemon not running, start with "epmt daemon --start"') @@ -212,16 +212,16 @@ def daemon_loop(context, maxiters=0, post_process=True, analyze=True, retire=Fal 'Alternatively, disable retire mode for the daemon') return True logger.info('retire mode enabled for daemon') - logger.info('jobs will be retired after {} days'.format(settings.retire_jobs_ndays)) - logger.info('models will be retired after {} days'.format(settings.retire_models_ndays)) + logger.info('jobs will be retired after %s days', settings.retire_jobs_ndays) + logger.info('models will be retired after %s days', settings.retire_models_ndays) if ingest: logger.info('ingestion mode enabled for daemon') logger.info( - 'ingestion mode (path={},recursive={},keep={},move_away={})'.format( - ingest, recursive, keep, move_away)) + 'ingestion mode (path=%s,recursive=%s,keep=%s,move_away=%s)', + ingest, recursive, keep, move_away) if not path.isdir(ingest): - logger.error('Ingest path ({}) does not exist'.format(ingest)) + logger.error('Ingest path (%s) does not exist', ingest) return True from epmt.epmtlib import suggested_cpu_count_for_submit @@ -275,10 +275,10 @@ def daemon_loop(context, maxiters=0, post_process=True, analyze=True, retire=Fal if ingest: from epmt.epmtlib import find_files_in_dir from epmt.epmt_cmds import epmt_submit - logger.debug('checking dir {} for jobs (*.tgz) to ingest'.format(ingest)) + logger.debug('checking dir %s for jobs (*.tgz) to ingest', ingest) tgz_files = find_files_in_dir(ingest, '*.tgz', recursive=recursive) if tgz_files: - logger.info('{} .tgz files found to ingest'.format(len(tgz_files))) + logger.info('%d .tgz files found to ingest', len(tgz_files)) epmt_submit( tgz_files, ncpus=ncpus, @@ -303,7 +303,7 @@ def daemon_loop(context, maxiters=0, post_process=True, analyze=True, retire=Fal # unpdj - ppd_jobs should be 0 in size, let's check err_ppd_jobs = list(filter(lambda i: i not in ppd_jobs, unpdj)) tot_pp_jobs += len(ppd_jobs) - logger.info('{0} jobs post-processed, {1} errors'.format(len(ppd_jobs), len(err_ppd_jobs))) + logger.info('%d jobs post-processed, %d errors', len(ppd_jobs), len(err_ppd_jobs)) # # Handle unprocessed jobs, remove from unprocessed and log @@ -316,7 +316,7 @@ def daemon_loop(context, maxiters=0, post_process=True, analyze=True, retire=Fal # ppd_jobs - ana_jobs should be 0, let's check err_ana_jobs = list(filter(lambda i: i not in ana_jobs, ppd_jobs)) tot_ua_jobs += len(ana_jobs) - logger.info('{0} jobs analyzed, {1} errors'.format(len(ana_jobs), len(err_ana_jobs))) + logger.info('%d jobs analyzed, %d errors', len(ana_jobs), len(err_ana_jobs)) # # Handle unanalyzed jobs, log (don't remove) @@ -336,7 +336,7 @@ def daemon_loop(context, maxiters=0, post_process=True, analyze=True, retire=Fal logger.debug('sleeping for {0:.3f} sec'.format(delay)) sleep(delay) else: - logger.warning("daemon loop took {0} seconds. No sleep for me!".format(_loop_time)) + logger.warning("daemon loop took %s seconds. No sleep for me!", _loop_time) return False diff --git a/src/epmt/epmt_exp_explore.py b/src/epmt/epmt_exp_explore.py index ab229e5fc..251a6e9c3 100644 --- a/src/epmt/epmt_exp_explore.py +++ b/src/epmt/epmt_exp_explore.py @@ -95,18 +95,18 @@ def exp_component_outliers(exp_name, metric='duration', op=np.sum, limit=10): # sorted jobids exp_jobids = sorted([j.jobid for j in exp_jobs], key=natural_keys) if not exp_jobids: - logger.warning('Could not find any jobs with an "exp_name" tag matching {}'.format(exp_name)) + logger.warning('Could not find any jobs with an "exp_name" tag matching %s', exp_name) return False # get a compact string of jobids if possible for logs try: job_ranges_str = ",".join(["{}..{}".format(a, b) if (a != b) else "{}".format(a) for (a, b) in ranges([int(x) for x in exp_jobids])]) - logger.info('Experiment {} contains {} jobs: {}'.format(exp_name, exp_jobs.count(), job_ranges_str)) + logger.info('Experiment %s contains %s jobs: %s', exp_name, exp_jobs.count(), job_ranges_str) except BaseException: # the ranges function can fail for non-integer jobids, so here # we simply print the job count, and not actually list the jobids - logger.info('Experiment {} contains {} jobs'.format(exp_name, exp_jobs.count())) + logger.info('Experiment %s contains %s jobs', exp_name, exp_jobs.count()) # we create a dict of dicts. The top-level dict is indexed by component name # Effectively we get to know for each component, the jobs and the time-segment @@ -318,13 +318,13 @@ def find_missing_time_segments(exp_name, jobs=[], components=[], time_segments=r ''' tag_filter = {'exp_name': exp_name} jobs = eq.get_jobs(jobs, fmt='orm', tags=[tag_filter]) - logger.debug('Looking for time segments: {}'.format(sorted(time_segments))) - logger.debug('{} matching jobs from the experiment'.format(jobs.count())) + logger.debug('Looking for time segments: %s', sorted(time_segments)) + logger.debug('%s matching jobs from the experiment', jobs.count()) jobs_tags = eq.get_job_tags(jobs, tag_filter=tag_filter, fold=True) matched_comp = jobs_tags['exp_component'] if components: matched_comp &= set(components) - logger.debug('{} components matched'.format(len(matched_comp))) + logger.debug('%d components matched', len(matched_comp)) ret = {} for c in sorted(matched_comp): comp_tags = eq.get_job_tags(jobs, tag_filter='exp_name:{};exp_component:{}'.format(exp_name, c)) diff --git a/src/epmt/epmtlib.py b/src/epmt/epmtlib.py index 378441680..efd579d36 100644 --- a/src/epmt/epmtlib.py +++ b/src/epmt/epmtlib.py @@ -261,7 +261,7 @@ def wrap(*args, **kw): result = f(*args, **kw) te = time() if result: - logger.debug('%r took: %2.5f sec' % (f.__name__, te - ts)) + logger.debug('%r took: %2.5f sec', f.__name__, te - ts) return result return wrap @@ -321,7 +321,7 @@ def tag_from_string(s, delim=';', sep=':', tag_default_value='1'): v = v.strip() tag[k] = v except Exception as e: - logger.warning('ignoring key/value pair as it has an invalid format: {0}'.format(t)) + logger.warning('ignoring key/value pair as it has an invalid format: %s', t) logger.warning("%s", e) continue else: @@ -737,7 +737,7 @@ def check_fix_metadata(raw_metadata): # we can ignore all the fields returned except the first env_changes = get_metadata_env_changes(raw_metadata)[0] if env_changes: - logger.debug('start/stop environment changed: {0}'.format(env_changes)) + logger.debug('start/stop environment changed: %s', env_changes) metadata['job_env_changes'] = env_changes # mark the metadata as checked so we don't check it again unnecessarily @@ -803,7 +803,7 @@ def conv_to_datetime(t): try: retval = datetime.strptime(t, '%m/%d/%Y %H:%M') except Exception as e: - logger.error('could not convert string to datetime: %s' % str(e)) + logger.error('could not convert string to datetime: %s', e) return None elif type(t) in (int, float): if t > 0: @@ -918,8 +918,8 @@ def dframe_encode_features(df, features=[], reversible=False): import epmt.epmt_settings as settings logger.debug('Selecting non-numeric columns from dataframe and then pruning out blacklisted features') obj_features = list(df.select_dtypes(include='object').columns.values) - logger.debug('Non-numeric features in dataframe: {}'.format(obj_features)) - logger.debug('Blacklisted features to prune: {}'.format(settings.outlier_features_blacklist)) + logger.debug('Non-numeric features in dataframe: %s', obj_features) + logger.debug('Blacklisted features to prune: %s', settings.outlier_features_blacklist) features = list(set(df.select_dtypes(include='object').columns.values) - set(settings.outlier_features_blacklist)) @@ -932,14 +932,14 @@ def dframe_encode_features(df, features=[], reversible=False): 'You have enabled "reversible". Be warned that the encoded feature columns can contain some very large integers') encoded_df = df.copy() encoded_features = [] - logger.debug('encoding feature columns: {}'.format(features)) + logger.debug('encoding feature columns: %s', features) for c in features: str_vec = df[c].to_numpy() int_vec = encode2ints(str_vec) if reversible else hash_strings(str_vec) encoded_df[c] = int_vec - logger.debug('mapped feature {}: {} -> {}'.format(c, str_vec, int_vec)) + logger.debug('mapped feature %s: %s -> %s', c, str_vec, int_vec) encoded_features.append(c) - logger.info('Encoded features: {}'.format(encoded_features)) + logger.info('Encoded features: %s', encoded_features) return (encoded_df, encoded_features) @@ -969,14 +969,14 @@ def dframe_decode_features(df, features): int_vec = df[c].to_numpy() str_vec = decode2strings(int_vec) decoded_df[c] = str_vec - logger.debug('decoded {}: {} -> {}'.format(c, int_vec, str_vec)) + logger.debug('decoded %s: %s -> %s', c, int_vec, str_vec) decoded_features.append(c) if decoded_features != features: logger.warning('decoded features list is not identical to requested features') if not decoded_features: logger.warning('No features were decoded') else: - logger.info('Decoded features: {}'.format(decoded_features)) + logger.info('Decoded features: %s', decoded_features) return (decoded_df, decoded_features) @@ -1145,10 +1145,10 @@ def log_func(*func_args, **func_kwargs): # the module is prepended automatically by our logging format # as we use getLogger with the module name # logger.info('{}({}{}{})'.format(func.__name__, - logger.debug('{}({}{}{})'.format(func.__name__, - ", ".join([str(x) for x in func_args]), - "," if func_kwargs else "", - ",".join(["{}={}".format(k, v) for (k, v) in func_kwargs.items()]))) + args_str = ", ".join([str(x) for x in func_args]) + kwargs_str = ",".join(["%s=%s" % (k, v) for (k, v) in func_kwargs.items()]) + sep = "," if func_kwargs else "" + logger.debug('%s(%s%s%s)', func.__name__, args_str, sep, kwargs_str) # now call the actual function with its arguments (if any) return func(*func_args, **func_kwargs) return log_func diff --git a/src/epmt/orm/__init__.py b/src/epmt/orm/__init__.py index 02e7d3911..018cdc873 100644 --- a/src/epmt/orm/__init__.py +++ b/src/epmt/orm/__init__.py @@ -89,7 +89,7 @@ def orm_db_size(findwhat=['database', 'table', 'index', 'tablespace'], usejson=T struct[arg] = databased except BaseException: e = exc_info()[0] - logger.warning("DB size query failed: %s" % e) + logger.warning("DB size query failed: %s", e) if arg == 'table': tabled = {} @@ -117,7 +117,7 @@ def orm_db_size(findwhat=['database', 'table', 'index', 'tablespace'], usejson=T struct[arg] = indexd except BaseException: e = exc_info()[0] - logger.warning("Index size query failed: %s" % e) + logger.warning("Index size query failed: %s", e) if arg == 'tablespace': tablespaced = {} @@ -132,7 +132,7 @@ def orm_db_size(findwhat=['database', 'table', 'index', 'tablespace'], usejson=T struct[arg] = tablespaced except BaseException: e = exc_info()[0] - logger.warning("Tablespace size query failed: %s" % e) + logger.warning("Tablespace size query failed: %s", e) current_time = datetime.utcnow().isoformat() + "Z" diff --git a/src/epmt/orm/op.py b/src/epmt/orm/op.py index d2bb22ebe..074a3bce1 100644 --- a/src/epmt/orm/op.py +++ b/src/epmt/orm/op.py @@ -70,7 +70,7 @@ def processes(self): logger.debug('computing op processes..') self._processes = get_procs(jobs=self.jobs, tags=self.tags, exact_tag_only=self.exact_tag_only, fmt='orm') if len(self._processes[:]) == 0: - logger.warning("No processes found for operation -- {0}".format(self.tags)) + logger.warning("No processes found for operation -- %s", self.tags) # else: # logger.debug('computing op start/end times..') # self.start = min(p.start for p in self.processes) @@ -115,7 +115,7 @@ def proc_sums(self): if self._proc_sums is None: from epmt.epmt_query import get_op_metrics from epmt.epmtlib import sum_dicts_list - logger.debug('getting op_metrics for jobs={0}, tags={1}'.format(self.jobs, self.tags)) + logger.debug('getting op_metrics for jobs=%s, tags=%s', self.jobs, self.tags) op_metrics = get_op_metrics( jobs=self.jobs, tags=self.tags, diff --git a/src/epmt/orm/sqlalchemy/general.py b/src/epmt/orm/sqlalchemy/general.py index 32b585121..49602197b 100644 --- a/src/epmt/orm/sqlalchemy/general.py +++ b/src/epmt/orm/sqlalchemy/general.py @@ -53,7 +53,7 @@ def wrapper(*args, **kwargs): completed = True except Exception as e: logger.debug(str(e), exc_info=True) - logger.warning('Rolling back due to exception: {}'.format(e)) + logger.warning('Rolling back due to exception: %s', e) # , exc_info=True) # import traceback, sys # print('-'*60) @@ -236,7 +236,7 @@ def orm_delete_jobs(jobs, use_orm=False): logger.warning( 'process job is in staging- removing process rows corresponding to the job in the staging table') (first_proc_id, last_proc_id) = j.info_dict['procs_staging_ids'] - logger.warning(f'first and last proc_ids pulled are: {first_proc_id} and {last_proc_id}') + logger.warning('first and last proc_ids pulled are: %s and %s', first_proc_id, last_proc_id) stmts.append( "DELETE FROM processes_staging WHERE id BETWEEN {} AND {};\n".format( first_proc_id, last_proc_id ) ) else: @@ -722,7 +722,7 @@ def orm_raw_sql(sql, commit=False): # As we may get really long queries when moving processes from staging, # only log the first 100 or so of long queries if len(sql) > settings.max_log_statement_length: - logger.debug(f'Executing very long (length={len(sql)} SQL statement(s): ... ') + logger.debug('Executing very long (length=%d) SQL statement(s): ... ', len(sql)) logger.debug( ''.join( map(str, @@ -731,7 +731,7 @@ def orm_raw_sql(sql, commit=False): ) ) else: - logger.debug('Executing: {0}'.format((sql))) + logger.debug('Executing: %s', sql) connection = engine.connect() trans = connection.begin() @@ -786,7 +786,7 @@ def decorator(*args, **kwargs): try: chdir(install_dir) except BaseException: - logger.error('Could not change directory to {} for migrations'.format(install_dir)) + logger.error('Could not change directory to %s for migrations', install_dir) raise result = function(*args, **kwargs) # restore directory to cwd @@ -803,11 +803,11 @@ def check_and_apply_migrations(): script_ = script.ScriptDirectory.from_config(alembic_cfg) epmt_schema_head = script_.get_current_head() if database_schema_version != epmt_schema_head: - logger.debug('database schema version: {}'.format(database_schema_version)) - logger.debug('EPMT schema HEAD: {}'.format(epmt_schema_head)) + logger.debug('database schema version: %s', database_schema_version) + logger.debug('EPMT schema HEAD: %s', epmt_schema_head) logger.info('Database needs to be upgraded..') return migrate_db() - logger.info('database schema up-to-date (version {})'.format(epmt_schema_head)) + logger.info('database schema up-to-date (version %s)', epmt_schema_head) return True @@ -829,7 +829,7 @@ def migrate_db(): alembic_cfg = config.Config('alembic.ini') script_ = script.ScriptDirectory.from_config(alembic_cfg) epmt_schema_head = script_.get_current_head() - logger.info('Migrating database to HEAD: {}'.format(epmt_schema_head)) + logger.info('Migrating database to HEAD: %s', epmt_schema_head) try: config.main(argv=['--raiseerr', 'upgrade', 'head',]) except Exception as e: @@ -840,10 +840,10 @@ def migrate_db(): updated_version = get_db_schema_version() if updated_version != epmt_schema_head: logger.warning( - 'Database migration failed. Current schema version is {}, while head is {}'.format( - updated_version, epmt_schema_head)) + 'Database migration failed. Current schema version is %s, while head is %s', + updated_version, epmt_schema_head) else: - logger.info('Database successfully migrated to: {}'.format(epmt_schema_head)) + logger.info('Database successfully migrated to: %s', epmt_schema_head) return epmt_schema_head == updated_version @@ -857,7 +857,7 @@ def alembic_dump_schema(version=''): from alembic import config if not version: version = get_db_schema_version() - logger.info('Dumping schema upto version: {}'.format(version)) + logger.info('Dumping schema upto version: %s', version) try: config.main(argv=['upgrade', '--sql', version,]) except BaseException: From a849b5f8c3db8fcb5ad158107d69f150c5837c69 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 17 Mar 2026 19:08:21 +0000 Subject: [PATCH 20/48] Fix all 31 pylint logging violations in epmt_job.py Convert logging calls from .format(), f-strings, and % operator to lazy % formatting with comma-separated arguments (W1202, W1203). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/epmt/epmt_job.py | 66 ++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/src/epmt/epmt_job.py b/src/epmt/epmt_job.py index 0b2d313df..ebf4666da 100644 --- a/src/epmt/epmt_job.py +++ b/src/epmt/epmt_job.py @@ -245,7 +245,7 @@ def load_process_from_dictlist(proc, host, j, u, settings, profile): hostname = proc[0].get('hostname', '') if hostname: if (host is None) or (host.name != hostname): - logger.warning('using hostname as set in papiex data: {}'.format(hostname)) + logger.warning('using hostname as set in papiex data: %s', hostname) host = lookup_or_create_host_safe(hostname) _t = time.time() @@ -622,7 +622,7 @@ def post_process_job( j, if not force: #if is_job_post_processed(j): if _is_job_post_processed_internal(j): - logger.warning('skipped processing jobid {0} as it has been already processed'.format(j.jobid)) + logger.warning('skipped processing jobid %s as it has been already processed', j.jobid) return False # we need to set up signal handlers so the user doesn't @@ -657,7 +657,7 @@ def sig_handler(signo): #, frame): _t0 = time.time() if not j.processes: logger.warning( - 'Job {} contains no processes, perhaps an error in collation or populating the staging data?'.format(jobid)) + 'Job %s contains no processes, perhaps an error in collation or populating the staging data?', jobid) if all_tags is None: logger.debug(" recreating all_tags..") @@ -885,7 +885,7 @@ def populate_process_table_from_staging(j): import psycopg2 jobid = j.jobid job_info_dict = j.info_dict - logger.info(' moving job {} processes from staging -> process table..'.format(jobid)) + logger.info(' moving job %s processes from staging -> process table..', jobid) metric_names = j.info_dict['metric_names'].split(',') logger.debug('metric_names are: %s', metric_names) # get the row IDs of the starting and ending row for the job @@ -1014,11 +1014,11 @@ def populate_process_table_from_staging(j): logger.error('You do not have sufficient privileges for this operation') else: logger.error( - f'INSERT aka insert_sql[:{settings.max_log_statement_length}] = \n {insert_sql[:settings.max_log_statement_length]}') + 'INSERT aka insert_sql[:%s] = \n %s', settings.max_log_statement_length, insert_sql[:settings.max_log_statement_length]) ## Only log the first 100 entries in the error string- it will largely be SQL statements if len(err_str) > settings.max_log_statement_length: - logger.error(f'error (type is {type(err_str)}) too long to show ({len(err_str)})...') - logger.error(f'first {settings.max_log_statement_length} errors in err_str list are...') + logger.error('error (type is %s) too long to show (%s)...', type(err_str), len(err_str)) + logger.error('first %s errors in err_str list are...', settings.max_log_statement_length) logger.error(''.join(err_str[:settings.max_log_statement_length])) else: # some bugs require this line get uncommented @@ -1037,11 +1037,11 @@ def populate_process_table_from_staging(j): if 'permission denied' in err_str: logger.error('You do not have sufficient privileges for this operation') else: - logger.error(f'DELETE aka delete_sql = \n {delete_sql}') + logger.error('DELETE aka delete_sql = \n %s', delete_sql) # Only log the first 100 or so of errors if len(err_str) > settings.max_log_statement_length: - logger.error(f'error (type is {type(err_str)}) too long to show ({len(err_str)})... ') - logger.error(f'first {settings.max_log_statement_length} errors in err_str list are...') + logger.error('error (type is %s) too long to show (%s)... ', type(err_str), len(err_str)) + logger.error('first %s errors in err_str list are...', settings.max_log_statement_length) logger.error(''.join(err_str[:settings.max_log_statement_length])) else: logger.error(err_str) @@ -1060,11 +1060,11 @@ def populate_process_table_from_staging(j): logger.error('You do not have sufficient privileges for this operation') else: - logger.error(f'UPDATE aka update_job_sql = \n {update_job_sql}') + logger.error('UPDATE aka update_job_sql = \n %s', update_job_sql) # Only log the first 100 or so of errors if len(err_str) > settings.max_log_statement_length: - logger.error(f'error (type is {type(err_str)}) too long to show ({len(err_str)})... ') - logger.error(f'first {settings.max_log_statement_length} errors in err_str list are...') + logger.error('error (type is %s) too long to show (%s)... ', type(err_str), len(err_str)) + logger.error('first %s errors in err_str list are...', settings.max_log_statement_length) logger.error(''.join(err_str[:settings.max_log_statement_length])) else: logger.error(err_str) @@ -1114,16 +1114,16 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): tz_str = get_first_key_match( env_dict, 'TZ', 'TIMEZONE') or get_first_key_match( environ, 'EPMT_TZ') or 'US/Eastern' - logger.debug('timezone could not be auto-detected, assuming {0}'.format(tz_str)) + logger.debug('timezone could not be auto-detected, assuming %s', tz_str) tz_default = pytz.timezone(tz_str) start_ts = tz_default.localize(start_ts) stop_ts = tz_default.localize(stop_ts) submit_ts = tz_default.localize(submit_ts) else: tz_default = start_ts.tzinfo - logger.debug('timezone auto-detected: {0}'.format(tz_default)) - logger.info('Job start: {0}'.format(start_ts)) - logger.info('Job finish: {0}'.format(stop_ts)) + logger.debug('timezone auto-detected: %s', tz_default) + logger.info('Job start: %s', start_ts) + logger.info('Job finish: %s', stop_ts) jobname = metadata['job_jobname'] exitcode = metadata['job_el_exitcode'] @@ -1132,7 +1132,7 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): annotations = metadata.get('annotations', {}) if annotations: - logger.info('Job annotations: {0}'.format(annotations)) + logger.info('Job annotations: %s', annotations) if settings.job_tags_env in annotations: job_tag_from_ann = tag_from_string(annotations[settings.job_tags_env]) @@ -1150,8 +1150,8 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): from epmt.epmtlib import tag_dict_to_string tag_str = tag_dict_to_string(job_tags) logger.debug( - 'updating {} in annotations to {} based on metadata job tags'.format( - settings.job_tags_env, tag_str)) + 'updating %s in annotations to %s based on metadata job tags', + settings.job_tags_env, tag_str) annotations[settings.job_tags_env] = tag_str # sometimes script name is to be found in the job tags @@ -1196,7 +1196,7 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): j.env_dict = env_dict j.env_changes_dict = env_changes_dict if job_tags: - logger.info('Job tags: {}'.format(job_tags)) + logger.info('Job tags: %s', job_tags) j.tags = job_tags if job_tags else {} job_init_fini_time = time.time() logger.debug('job init took: %2.5f sec', job_init_fini_time - job_init_start_time) @@ -1278,7 +1278,7 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): logger.error(msg) return (False, msg, ()) - logger.info('CSV v{} files detected in tar: {}'.format(fmt, ",".join(files))) + logger.info('CSV v%s files detected in tar: %s', fmt, ",".join(files)) # get the metric names from the CSV header file if we have csv v2 # for v1 they will be determined automatically from the headers in @@ -1300,10 +1300,10 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): csv_headers = "".join(map(chr, csv_headers)).split(OUTPUT_CSV_SEP) # remove leading/trailing whitespace in column names csv_headers = [h.strip() for h in csv_headers] - logger.debug('papiex headers: {}'.format(csv_headers)) + logger.debug('papiex headers: %s', csv_headers) metric_names = csv_headers[OUTPUT_CSV_FIELDS.index('threads_df')] metric_names = metric_names.replace('{', '').replace('}', '') - logger.debug('per-thread metric names: {}'.format(metric_names)) + logger.debug('per-thread metric names: %s', metric_names) # save the metric_names in job info_dict for future use (such as when creating # threads_df from a flattened array info_dict['metric_names'] = metric_names @@ -1322,7 +1322,7 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): logger.debug("%s had %d comment rows, oldproctags %s", f, skiprows, oldproctag) if tarfile: - logger.debug('extracting {} from tar'.format(f)) + logger.debug('extracting %s from tar', f) info = tarfile.getmember(f) flo = tarfile.extractfile(info) else: @@ -1330,7 +1330,7 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): if all( [ fmt == '2', orm_db_provider() == 'postgres', settings.orm == 'sqlalchemy' ] ): import psycopg2 - logger.info('Doing a fast ingest of {}'.format(flo.name)) + logger.info('Doing a fast ingest of %s', flo.name) _conn_start_ts = time.time() _copy_ok = False try: @@ -1380,8 +1380,8 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): total_procs += num_procs_copied # save the staging table row id range for the job info_dict['procs_staging_ids'] = (lastid - num_procs_copied + 1, lastid) - logger.debug('job process_staging ID range: {}'.format( - lastid if num_procs_copied == 1 else info_dict['procs_staging_ids'])) + logger.debug('job process_staging ID range: %s', + lastid if num_procs_copied == 1 else info_dict['procs_staging_ids']) continue else: # Re-open the file for standard processing since @@ -1497,7 +1497,7 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): logger.debug('file I/O time took: %2.5f sec', file_io_time) logger.debug('process load ops took: %2.5f sec', df_process_time) logger.debug(' - load process from dictlist took: %2.5f sec', load_process_from_df_time) - logger.debug(' - {0}'.format(["%s: %2.5f sec" % (k, v) for (k, v) in profile.load_process.items()])) + logger.debug(' - %s', ["%s: %2.5f sec" % (k, v) for (k, v) in profile.load_process.items()]) logger.debug(' - tag processing took: %2.5f sec', proc_tag_process_time) logger.debug(' - proc misc. processing took: %2.5f sec', proc_misc_time) logger.debug( @@ -1529,12 +1529,12 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): # logger.warning('metadata shows the job exit code is {0}, but root process exit code is {1}'.format( # j.exitcode, root_proc.exitcode)) j.exitcode = root_proc.exitcode - logger.info('job exit code (using exit code of root process): {0}'.format(j.exitcode)) + logger.info('job exit code (using exit code of root process): %s', j.exitcode) if j.exitcode != 0: - logger.warning('Job failed with a non-zero exit code({})'.format(j.exitcode)) + logger.warning('Job failed with a non-zero exit code(%s)', j.exitcode) if settings.bulk_insert and all_procs: - logger.info('doing a bulk insert of {0} processes'.format(len(all_procs))) + logger.info('doing a bulk insert of %s processes', len(all_procs)) _b0 = time.time() # thr_data.engine.execute(Process.__table__.insert(), all_procs) Session.bulk_insert_mappings(Process, all_procs) @@ -1597,7 +1597,7 @@ def post_process_pending_jobs(): for u in unproc_jobs: jobid = u.jobid #j = u.job - logger.debug('post-processing {0}'.format(jobid)) + logger.debug('post-processing %s', jobid) if post_process_job(jobid): did_process.append(jobid) return did_process From f51721b304f5bac6ac6e34c23d0a05113980a630 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 17 Mar 2026 19:11:14 +0000 Subject: [PATCH 21/48] Fix all 43 pylint logging violations in epmt_cmds.py Convert logging calls from eager string formatting (.format(), f-strings, % operator, string concatenation) to lazy % formatting with comma-separated arguments. This defers string interpolation until the message is actually emitted, improving performance when log levels are filtered out. Fixes W1201 (logging-not-lazy), W1202 (logging-format-interpolation), and W1203 (logging-fstring-interpolation) violations. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/epmt/epmt_cmds.py | 86 +++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/src/epmt/epmt_cmds.py b/src/epmt/epmt_cmds.py index b2cddfa66..aa3088bda 100644 --- a/src/epmt/epmt_cmds.py +++ b/src/epmt/epmt_cmds.py @@ -47,16 +47,16 @@ def find_diffs_in_envs(start_env, stop_env): for e in start_env.keys(): if e in stop_env.keys(): if start_env[e] == stop_env[e]: - logger.debug("Found " + e) + logger.debug("Found %s", e) else: - logger.debug("Different " + e) + logger.debug("Different %s", e) env[e] = stop_env[e] else: - logger.debug("Deleted " + e) + logger.debug("Deleted %s", e) env[e] = start_env[e] for e in stop_env.keys(): if e not in start_env.keys(): - logger.debug("Added " + e) + logger.debug("Added %s", e) env[e] = stop_env[e] return env @@ -89,7 +89,7 @@ def read_job_metadata_direct(file): logger.debug("doing special unpickling for job metadata pickled using python2") data = conv_dict_byte2str(pickle.load(file, encoding='bytes')) except Exception as e: - logger.error("Error unpickling job metadata file: {}".format(e)) + logger.error("Error unpickling job metadata file: %s", e) raise logger.debug("Unpickled ") return data @@ -97,7 +97,7 @@ def read_job_metadata_direct(file): @logfn def read_job_metadata(jobdatafile): - logger.info("Unpickling from " + jobdatafile) + logger.info("Unpickling from %s", jobdatafile) try: with open(jobdatafile, 'rb') as file: return read_job_metadata_direct(file) @@ -109,7 +109,7 @@ def read_job_metadata(jobdatafile): def write_job_epilog(jobdatafile, metadata): with open(jobdatafile, 'w+b') as file: pickle.dump(metadata, file) - logger.debug("Pickled to " + jobdatafile) + logger.debug("Pickled to %s", jobdatafile) return True return False @@ -133,7 +133,7 @@ def verify_install_prefix(): retval = True # Check for bad stuff and shortcut if "*" in install_prefix or "?" in install_prefix: - logger.error("Found wildcards in install_prefix: {}".format(install_prefix)) + logger.error("Found wildcards in install_prefix: %s", install_prefix) PrintFail() return False papiex_shared_obj_file_list = [ @@ -145,7 +145,7 @@ def verify_install_prefix(): ] for e in papiex_shared_obj_file_list: cmd = "ls -l " + install_prefix + e + ">/dev/null 2>&1" - logger.info("\t" + cmd) + logger.info("\t%s", cmd) return_code = run(cmd, shell=True).returncode if return_code != 0: logger.error("%s failed", cmd) @@ -173,7 +173,7 @@ def verify_epmt_output_prefix(): # Print and create dir def testdir(str2): - logger.info("\tmkdir -p " + str2) + logger.info("\tmkdir -p %s", str2) return create_job_dir(str2) # Test create (or if it exists) @@ -186,14 +186,14 @@ def testdir(str2): # Test to make sure we can access it cmd = "ls -lR " + opf + " >/dev/null" - logger.info("\t" + cmd) + logger.info("\t%s", cmd) return_code = run(cmd, shell=True).returncode if return_code != 0: retval = False # Remove the created tmp dir cmd = "rm -rf " + opf + "tmp" - logger.info("\t" + cmd) + logger.info("\t%s", cmd) return_code = run(cmd, shell=True).returncode if return_code != 0: retval = False @@ -212,15 +212,15 @@ def verify_papiex_options(): environments where perf_event_open() is restricted.''' s = get_papiex_options(settings) print("papiex_options =",s, end='') - logger.info(f'papiex_options = {s}') - logger.info(f'settings.install_prefix = {settings.install_prefix}') + logger.info('papiex_options = %s', s) + logger.info('settings.install_prefix = %s', settings.install_prefix) retval = True # Check for any components # cmd = settings.install_prefix+"/bin/papi_component_avail 2>&1 "+"| sed -n -e '/Active/,$p' | grep perf_event >/dev/null 2>&1" cmd = settings.install_prefix + "/bin/papi_component_avail 2>&1 " + \ "| sed -e '/Active/,$p' | grep perf_event >/dev/null 2>&1" - logger.info("\t" + cmd) + logger.info("\t%s", cmd) return_code = run(cmd, shell=True).returncode if return_code != 0: logger.error("%s failed", cmd) @@ -239,7 +239,7 @@ def verify_papiex_options(): # /dev/null 2>&1" # does not work for rocky-8. cmd = settings.install_prefix + "/bin/papi_command_line 2>&1 " + e + \ "| sed -e '/PERF_COUNT_SW_CPU_CLOCK\\ :/,$p' | grep PERF_COUNT_SW_CPU_CLOCK > /dev/null 2>&1" # guessing... NOT TRIED YET TODO: TRY THIS INSTEAD OF ABOVE LINE - logger.info("\t" + cmd) + logger.info("\t%s", cmd) return_code = run(cmd, shell=True).returncode if return_code != 0: logger.error("%s failed", cmd) @@ -265,7 +265,7 @@ def verify_db_params(): PrintPass() return True except ImportError as e: - logger.error("Error setting up DB: {}".format(str(e))) + logger.error("Error setting up DB: %s", e) PrintFail() return False @@ -593,7 +593,7 @@ def epmt_dump_metadata(filelist, key=None): return False for f in filelist: - logger.info('Processing {}'.format(f)) + logger.info('Processing %s', f) if not path.exists(f): if ('/' in f) or ('.tgz' in f): @@ -601,7 +601,7 @@ def epmt_dump_metadata(filelist, key=None): return False # if the file does not exist then we check the DB - logger.debug('{} was not found in the file-system. Checking database..'.format(f)) + logger.debug('%s was not found in the file-system. Checking database..', f) from epmt.epmt_cmd_show import epmt_show_job rc = epmt_show_job(f, key=key) if not rc: @@ -613,10 +613,10 @@ def epmt_dump_metadata(filelist, key=None): try: info = tar.getmember("./job_metadata") except KeyError: - logger.error('Did not find %s in tar file ' % "job_metadata") + logger.error('Did not find %s in tar file', "job_metadata") return False else: - logger.info('%s is %d bytes in archive' % (info.name, info.size)) + logger.info('%s is %s bytes in archive', info.name, info.size) f = tar.extractfile(info) metadata = read_job_metadata_direct(f) else: @@ -657,12 +657,12 @@ def annotate_metadata(metadatafile, annotations, replace=False): where possible. ''' if not path.isfile(metadatafile): - logger.error('{} does not exist'.format(metadatafile)) + logger.error('%s does not exist', metadatafile) return False metadata = read_job_metadata(metadatafile) if not metadata: - logger.error('Error reading metadata from {}'.format(metadatafile)) + logger.error('Error reading metadata from %s', metadatafile) return False # get existing annotations unless replace is set @@ -670,7 +670,7 @@ def annotate_metadata(metadatafile, annotations, replace=False): # now merge in the new annotations, where the new annotations will override existing annotations if keys are common ann.update(annotations) - logger.debug('Updated annotations: {}'.format(ann)) + logger.debug('Updated annotations: %s', ann) metadata['annotations'] = ann return write_job_metadata(metadatafile, metadata) @@ -716,7 +716,7 @@ def get_annotations_from_kwargs(args): annotations = get_annotations_from_kwargs(argslist) if not annotations: return False - logger.info('annotating a job in the batch environment: {}'.format(annotations)) + logger.info('annotating a job in the batch environment: %s', annotations) jobid, datadir, metadatafile = setup_vars() if not (jobid and datadir and metadatafile): logger.error("jobid, datadir and metadatafile MUST be set in the environment") @@ -747,17 +747,17 @@ def get_annotations_from_kwargs(args): if arg0.endswith('.tgz'): # staged file form staged_file = arg0 - logger.info('annotating staged job file {0}: {1}'.format(staged_file, annotations)) + logger.info('annotating staged job file %s: %s', staged_file, annotations) tempdir = extract_tar(staged_file, check_metadata=True) if not tempdir: - logger.error('Error extracting {}'.format(staged_file)) + logger.error('Error extracting %s', staged_file) return False metadatafile = tempdir + "/job_metadata" elif path.isdir(arg0): # job directory form job_dir = arg0 - logger.info('annotating dir {}: {}'.format(job_dir, annotations)) + logger.info('annotating dir %s: %s', job_dir, annotations) if path.exists(stopped_metadata_file(job_dir + "/job_metadata")): metadatafile = stopped_metadata_file(job_dir + "/job_metadata") logger.debug("job %s has been stopped", job_dir) @@ -771,16 +771,16 @@ def get_annotations_from_kwargs(args): else: # database jobid form jobid = argslist[0] - logger.info('annotating job {0} in db: {1}'.format(jobid, annotations)) + logger.info('annotating job %s in db: %s', jobid, annotations) from epmt.epmt_query import annotate_job updated_ann = annotate_job(jobid, annotations, replace) - logger.debug('updated annotations: {}'.format(updated_ann)) + logger.debug('updated annotations: %s', updated_ann) if settings.job_tags_env in annotations: # we need to set .tags to the value of EPMT_JOB_TAGS from epmt.epmt_query import tag_job # we have to overwrite the existing tags (not merge it in) r = tag_job(jobid, annotations[settings.job_tags_env], True) - logger.debug('Updated tags for job {} to {}'.format(jobid, r)) + logger.debug('Updated tags for job %s to %s', jobid, r) return annotations.items() <= updated_ann.items() # below we handle annotation update in the metadata file @@ -789,7 +789,7 @@ def get_annotations_from_kwargs(args): # we also need to recreate the tar. retval = annotate_metadata(metadatafile, annotations, replace=replace) if not retval: - logger.error('Could not annotate metadatafile: ' + metadatafile) + logger.error('Could not annotate metadatafile: %s', metadatafile) # for staged file case we need to recreate the .tgz file # but do not call stage! create_tar will log an error if it failed @@ -1298,7 +1298,7 @@ def copy_files(src_dir, dest_dir='', patterns=['*'], prefix=''): if not files: logger.debug('No files to copy!') return False - logger.debug('Copying {} to {}'.format(files, dest_dir)) + logger.debug('Copying %s to %s', files, dest_dir) for f in files: filename = path.basename(f) target = dest_dir + "/" + filename @@ -1335,7 +1335,7 @@ def create_tar(tarfile, indir, remove_dir=False): True on success, False on error ''' if not path.isdir(indir): - logger.error('{} does not exist'.format(indir)) + logger.error('%s does not exist', indir) cmd = "tar -C " + indir + " -cz -f " + tarfile + " ." logger.debug(cmd) retval = run(cmd, shell=True).returncode @@ -1383,17 +1383,17 @@ def extract_tar(tarfile, outdir='', check_metadata=False): try: info = tar.getmember("./job_metadata") except KeyError: - logger.error('ERROR: Did not find %s in tar file' % "job_metadata") + logger.error('ERROR: Did not find %s in tar file', "job_metadata") return False - logger.info('%s is %d bytes in archive' % (info.name, info.size)) + logger.info('%s is %s bytes in archive', info.name, info.size) from tempfile import mkdtemp, gettempdir outdir = outdir or mkdtemp(prefix='epmt_stage_', dir=gettempdir()) - logger.debug('extracting {0} to {1}'.format(tarfile, outdir)) + logger.debug('extracting %s to %s', tarfile, outdir) try: tar.extractall(path=outdir) except Exception as e: - logger.error('Error extracting {} into {}: {}'.format(tarfile, outdir, str(e))) + logger.error('Error extracting %s into %s: %s', tarfile, outdir, e) # cleanup since we had an error rmtree(outdir, ignore_errors=True) return False @@ -1414,7 +1414,7 @@ def open_compressed_tar(inputf): try: tar = tarfile.open(inputf, flags) except Exception as e: - logger.error('error opening compressed tar ' + inputf + ":" + str(e)) + logger.error('error opening compressed tar %s:%s', inputf, e) return True, None return False, tar @@ -1511,10 +1511,10 @@ def submit_to_db(inputf, pattern, dry_run=True): try: info = tar.getmember("./job_metadata") except KeyError: - logger.error('Did not find %s in tar file %s' % ("job_metadata", inputf)) + logger.error('Did not find %s in tar file %s', "job_metadata", inputf) return (False, 'Did not find metadata in tar file ' + inputf, ()) else: - logger.info('%s is %d bytes in tar file %s' % (info.name, info.size, inputf)) + logger.info('%s is %s bytes in tar file %s', info.name, info.size, inputf) f = tar.extractfile(info) metadata = read_job_metadata_direct(f) filedict = get_filedict(None, settings.input_pattern, tar) @@ -1570,14 +1570,14 @@ def stage_job(indir, collate=True, compress_and_tar=True, keep_going=True): # no need to cleanup as copy_files will clean # up temp dir if it created one using mkdtemp if not tempdir: - logger.error("No job metadata found in " + indir) + logger.error("No job metadata found in %s", indir) return False tsv_files = glob(indir + '/*.tsv') if tsv_files: copied_to = copy_files(indir, dest_dir=tempdir, patterns=['*.tsv']) if not copied_to: - logger.error('No job performance data found in {}'.format(indir)) + logger.error('No job performance data found in %s', indir) rmtree(tempdir, ignore_errors=True) return False else: From 43def98ee6fa7f8b439df01a6f0865f2e33ef4f4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 17 Mar 2026 19:14:04 +0000 Subject: [PATCH 22/48] Fix all 49 pylint logging violations in epmt_query.py Convert logging calls from .format(), % operator, and f-string interpolation to lazy % formatting with comma-separated arguments. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/epmt/epmt_query.py | 104 ++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/src/epmt/epmt_query.py b/src/epmt/epmt_query.py index 3c2b2e786..8bbf6609a 100644 --- a/src/epmt/epmt_query.py +++ b/src/epmt/epmt_query.py @@ -98,7 +98,7 @@ def conv_jobs(jobs, fmt='dict', merge_sums=True, trigger_post_process=True): common_fields = list(set(j) & set(j[PROC_SUMS_FIELD_IN_JOB])) if common_fields: logger.warning( - 'while hoisting proc_sums to job-level, found {0} common fields: {1}'.format(len(common_fields), common_fields)) + 'while hoisting proc_sums to job-level, found %s common fields: %s', len(common_fields), common_fields) j.update(j[PROC_SUMS_FIELD_IN_JOB]) del j[PROC_SUMS_FIELD_IN_JOB] @@ -126,7 +126,7 @@ def __conv_procs_orm(procs, merge_sums=True, fmt='dict'): common_fields = list(set(p) & set(p[THREAD_SUMS_FIELD_IN_PROC])) if common_fields: logger.warning( - 'while hoisting thread_sums to process-level, found {0} common fields: {1}'.format(len(common_fields), common_fields)) + 'while hoisting thread_sums to process-level, found %s common fields: %s', len(common_fields), common_fields) p.update(p[THREAD_SUMS_FIELD_IN_PROC]) # add an alias for a consistent user experience p['jobid'] = p['job'] @@ -541,7 +541,7 @@ def get_jobs( try: when = datetime.strptime(when, '%m/%d/%Y %H:%M') except Exception as e: - logger.error('could not convert "when" string to datetime: %s' % str(e)) + logger.error('could not convert "when" string to datetime: %s', e) return None if before is not None: @@ -729,7 +729,7 @@ def get_procs(jobs=[], tags=None, fltr=None, order=None, offset=0, limit=None, w try: when = datetime.strptime(when, '%m/%d/%Y %H:%M') except Exception as e: - logger.error('could not convert "when" string to datetime: %s' % str(e)) + logger.error('could not convert "when" string to datetime: %s', e) return None if hosts: @@ -926,7 +926,7 @@ def get_job_tags(jobs, tag_filter='', fold=True): ''' jobs = get_jobs(jobs, tags=[tag_filter], fmt='orm') if tag_filter else orm_jobs_col(jobs) - logger.debug('{} jobs matched'.format(jobs.count())) + logger.debug('%s jobs matched', jobs.count()) tags = [] for j in jobs: tags.extend([j.tags]) @@ -1084,8 +1084,8 @@ def get_refmodels(name=None, tag={}, fltr=None, limit=0, order=None, before=None common_fields = list(set(r) & set(r['computed'])) if common_fields: logger.warning( - 'while hoisting nested fields in "computed" to reference model, found {0} common fields: {1}'.format( - len(common_fields), common_fields)) + 'while hoisting nested fields in "computed" to reference model, found %s common fields: %s', + len(common_fields), common_fields) r.update(r['computed']) del r['computed'] @@ -1109,25 +1109,25 @@ def _refmodel_scores(col, methods, features): ''' df = conv_jobs(col, fmt='pandas') if col.__class__.__name__ != 'DataFrame' else col ret = {} - logger.info('creating trained model using {0} for features {1}'.format( - [get_classifier_name(c) for c in methods], features)) - logger.info('jobids: {}'.format(df['jobid'].values)) + logger.info('creating trained model using %s for features %s', + [get_classifier_name(c) for c in methods], features) + logger.info('jobids: %s', df['jobid'].values) for m in methods: m_name = get_classifier_name(m) ret[m_name] = {} if is_classifier_mv(m): - logger.info('mvod {0}; features ({1})'.format(m_name, features)) + logger.info('mvod %s; features (%s)', m_name, features) _f = sorted(features) nd_array = df[_f].to_numpy() # the second element return is a dict indexed by classifier # and containing the max anomaly score using the classifier retval = mvod_scores(nd_array, classifiers=[m]) if not retval: - logger.warning('Skipped mvod classifier {} as could not score using it'.format(m_name)) + logger.warning('Skipped mvod classifier %s as could not score using it', m_name) del ret[m_name] continue (full_scores, max_score) = retval - logger.debug('{0} scores:\n{1}'.format(m_name, full_scores[m_name])) + logger.debug('%s scores:\n%s', m_name, full_scores[m_name]) # we save the max score and also we need the input nd_array for # future reference. We will need the nd_array for outlier detection @@ -1135,12 +1135,12 @@ def _refmodel_scores(col, methods, features): ret[m_name][",".join(_f)] = [float(max_score[m_name]), nd_array.tolist()] else: # univariate classifiers can only handle - logger.debug('univariate classifier {0}; features {1}'.format(m_name, features)) + logger.debug('univariate classifier %s; features %s', m_name, features) for c in features: # we save everything returned by the function # except the first element, which is a list of scores # We really only need the max, median etc - logger.debug('scoring feature {}'.format(c)) + logger.debug('scoring feature %s', c) ret[m_name][c] = m(df[c])[1:] # print(ret) return ret @@ -1274,7 +1274,7 @@ def create_refmodel(jobs, name=None, tag={}, op_tags=[], jobs = jobs_orm[:] from epmt.epmt_outliers import sanitize_features if len(jobs) < 3: - logger.error('You cannot create a model with less than 3 jobs. Your chosen jobs: {}'.format(jobs)) + logger.error('You cannot create a model with less than 3 jobs. Your chosen jobs: %s', jobs) return False if sanity_check: @@ -1285,7 +1285,7 @@ def create_refmodel(jobs, name=None, tag={}, op_tags=[], features = sanitize_features(features, jobs_df) orig_features = features # keep a copy as features might be reassigned below if pca: - logger.info("request to do PCA (pca={}). Input features: {}".format(pca, features)) + logger.info("request to do PCA (pca=%s). Input features: %s", pca, features) if len(features) < 5: logger.warning( 'Too few input features for PCA. Are you sure you did not want to set features=[] to enable selecting all available features?') @@ -1306,12 +1306,12 @@ def create_refmodel(jobs, name=None, tag={}, op_tags=[], op_tags = tags_list(op_tags) # let's get the dataframe of metrics aggregated by op_tags ops_df = get_op_metrics(jobs=jobs_orm, tags=op_tags, exact_tags_only=exact_tag_only, fmt='pandas') - logger.debug('jobid,tags:\n{}'.format(ops_df[['jobid', 'tags']])) + logger.debug('jobid,tags:\n%s', ops_df[['jobid', 'tags']]) if pca: (ops_pca_df, pca_variances, pca_features, _) = pca_feature_combine( ops_df, features, desired=0.85 if pca is True else pca) - logger.info('{} PCA components obtained: {}'.format(len(pca_features), pca_features)) - logger.info('PCA variances: {} (sum={})'.format(pca_variances, np.sum(pca_variances))) + logger.info('%s PCA components obtained: %s', len(pca_features), pca_features) + logger.info('PCA variances: %s (sum=%s)', pca_variances, np.sum(pca_variances)) ops_df = ops_pca_df features = pca_features @@ -1320,21 +1320,21 @@ def create_refmodel(jobs, name=None, tag={}, op_tags=[], # serialize the tag so we can use it as a key stag = dumps(t, sort_keys=True) # pylint: disable=no-member - logger.debug('scoring op {}'.format(t)) + logger.debug('scoring op %s', t) scores[stag] = _refmodel_scores(ops_df[ops_df.tags == t], methods, features) else: # full jobs, no ops - logger.debug('jobid,tags:\n{}'.format(jobs_df[['jobid', 'tags']])) + logger.debug('jobid,tags:\n%s', jobs_df[['jobid', 'tags']]) if pca: (jobs_pca_df, pca_variances, pca_features, _) = pca_feature_combine( jobs_df, features, desired=0.85 if pca is True else pca) - logger.info('{} PCA components obtained: {}'.format(len(pca_features), pca_features)) - logger.info('PCA variances: {} (sum={})'.format(pca_variances, np.sum(pca_variances))) + logger.info('%s PCA components obtained: %s', len(pca_features), pca_features) + logger.info('PCA variances: %s (sum=%s)', pca_variances, np.sum(pca_variances)) jobs_df = jobs_pca_df features = pca_features scores = _refmodel_scores(jobs_df, methods, features) - logger.debug('computed scores: {0}'.format(scores)) + logger.debug('computed scores: %s', scores) computed = scores # if we use pca, then we need to save the input feature names @@ -1487,7 +1487,7 @@ def refmodel_get_metrics(model, active_only=False): # see if we have any PCA features pca_features = (r.info_dict or {}).get('pca', {}).get('inp_features', []) if pca_features: - logger.debug('PCA features found in model: {}'.format(pca_features)) + logger.debug('PCA features found in model: %s', pca_features) metrics |= set(pca_features) if active_only: @@ -1535,8 +1535,8 @@ def refmodel_set_active_metrics(ref_id, metrics): unavailable_metrics = metrics_set - all_metrics if unavailable_metrics: logger.warning( - 'Ignoring metrics that are not available in the trained model: {0}'.format( - unavailable_metrics )) + 'Ignoring metrics that are not available in the trained model: %s', + unavailable_metrics ) active_metrics = list(metrics_set & all_metrics) logger.info('Active metrics for model set to: %s', str(active_metrics)) info_dict = dict.copy(r.info_dict or {}) @@ -1684,7 +1684,7 @@ def get_ops(jobs, tags=[], exact_tag_only=False, combine=False, fmt='dict', op_d _empty_collection_check(jobs) if not tags: tags = rank_proc_tags_keys(jobs)[0][0] - logger.debug('no tag specified, using tags: {0}'.format(tags)) + logger.debug('no tag specified, using tags: %s', tags) if not isinstance(tags, list): tags = [tags] @@ -1698,12 +1698,12 @@ def get_ops(jobs, tags=[], exact_tag_only=False, combine=False, fmt='dict', op_d # and we want to expand that into a list of tags such as # [{'op': 'timavg'}, {'op': 'dmget'},...] tag_values = job_proc_tags.get(t, []) - logger.debug('expanding {0} for values {1}'.format(t, tag_values)) + logger.debug('expanding %s for values %s', t, tag_values) for v in tag_values: _tags.append({t: v}) else: _tags.append(t) - logger.debug('tags: {0}'.format(_tags)) + logger.debug('tags: %s', _tags) if combine: # Operation will pass the list of tags to get_procs @@ -1792,7 +1792,7 @@ def get_op_metrics(jobs, tags=[], exact_tags_only=False, group_by_tag=False, fmt all_procs = [] # we iterate over tags, where each tag is dictionary for t in tags: - logger.debug(' processing op: {}'.format(t)) + logger.debug(' processing op: %s', t) # group the Query response we got by jobid # we use group_concat to join the thread_sums json into a giant string if settings.orm == 'sqlalchemy': @@ -1934,7 +1934,7 @@ def delete_jobs(jobs, force=False, before=None, after=None, warn=True, remove_mo after is not None, limit is not None, offset > 0, ] ): - logger.info('(delete_jobs) offset = {}'.format(offset)) + logger.info('(delete_jobs) offset = %s', offset) jobs = get_jobs( jobs, before=before, after=after, @@ -1984,7 +1984,7 @@ def delete_jobs(jobs, force=False, before=None, after=None, warn=True, remove_mo jobs_to_delete.append(jobid) num_jobs_with_models_unremoved -= num_jobs_with_models_unremoved models_to_remove |= set(models) - logger.info("Deleting dependent reference models: {}".format(models_to_remove)) + logger.info("Deleting dependent reference models: %s", models_to_remove) delete_refmodels(*models_to_remove) else: logger.warning( @@ -2046,10 +2046,10 @@ def retire_jobs(ndays=settings.retire_jobs_ndays, skip_unprocessed=False, dry_ru JOBS_PER_DELETE_MAX = (settings.retire_jobs_per_delete_max if settings.retire_jobs_per_delete_max > 0 else 2000) db_num_jobs = get_jobs(fmt='orm', trigger_post_process=False).count() - logger.info('(retire_jobs) number of jobs in DB is {0}'.format(db_num_jobs)) + logger.info('(retire_jobs) number of jobs in DB is %s', db_num_jobs) num_jobs = get_jobs(before=-ndays, fmt='orm', trigger_post_process=False).count() - logger.info('(retire_jobs) number of jobs older than {0} days is {1}'.format(ndays, num_jobs)) + logger.info('(retire_jobs) number of jobs older than %s days is %s', ndays, num_jobs) # uncomment me for training wheels/debug/tests # JOBS_PER_DELETE_MAX=100 @@ -2180,13 +2180,13 @@ def ops_costs(jobs=[], tags=['op:hsmput', 'op:dmget', 'op:untar', 'op:mv', where the aggregation is performed in the database layer. """ - logger.info('dm ops: {0}'.format(tags)) + logger.info('dm ops: %s', tags) if metric not in {"duration", "cpu_time"}: raise ValueError('We only support "duration" or "cpu_time" for metric') - logger.info('metric: {}'.format(metric)) + logger.info('metric: %s', metric) jobs = orm_jobs_col(jobs) njobs = jobs.count() - logger.info('number of jobs: {0}'.format(njobs)) + logger.info('number of jobs: %s', njobs) tags = tags_list(tags) jobs_metric = 0.0 # cumulative across all jobs for metric df_list = [] @@ -2201,7 +2201,7 @@ def ops_costs(jobs=[], tags=['op:hsmput', 'op:dmget', 'op:untar', 'op:mv', logger.debug('Using slower (but accurate) computation for "duration" that avoids double-counting overlapping processes in a job. Try using "cpu_time" for faster results') for j in jobs: n += 1 - logger.debug('processing {} ({}/{})'.format(j.jobid, n, njobs)) + logger.debug('processing %s (%s/%s)', j.jobid, n, njobs) jobs_metric += j.duration if (metric == 'duration') else j.cpu_time job_dm_ops_df = get_op_metrics(j, tags=tags, group_by_tag=True, op_duration_method=agg_method) job_dm_ops_df.insert(0, 'jobid', j.jobid) @@ -2395,20 +2395,20 @@ def analyze_jobs(jobs=[], analyses_filter={}, max_comparable=50, check=True): ana_jobs = [] num_analyses_run = 0 - logger.info('{0} unanalyzed jobs'.format(len(ua_jobs))) + logger.info('%s unanalyzed jobs', len(ua_jobs)) # partition the jobs into sets of comparable jobs based on their tags comp_job_parts = comparable_job_partitions(ua_jobs) - logger.debug('{0} sets of comparable jobs'.format(len(comp_job_parts))) + logger.debug('%s sets of comparable jobs', len(comp_job_parts)) # iterate over the comparable jobs' sets for j_part in comp_job_parts: (_, jobids) = j_part # clip the comparable job partitions if max_comparable if max_comparable and (len(jobids) > max_comparable): - logger.info('Limiting jobs to a random selection of {} from {}'.format(max_comparable, len(jobids))) + logger.info('Limiting jobs to a random selection of %s from %s', max_comparable, len(jobids)) from random import choices jobids = choices(jobids, k=max_comparable) - logger.debug('Randomly selected {} jobs for analysis pipeline: {}'.format(max_comparable, jobids)) + logger.debug('Randomly selected %s jobs for analysis pipeline: %s', max_comparable, jobids) # we set check_comparable as False since we already know # that the jobs are comparable -- don't waste time! num_analyses_run += analyze_comparable_jobs(jobids, check_comparable=False) @@ -2457,7 +2457,7 @@ def analyze_comparable_jobs(jobids, check_comparable=True, keys=('exp_name', 'ex from epmt.epmt_outliers import detect_outlier_jobs if check_comparable: _warn_incomparable_jobs(jobids) - logger.info('analyzing {0} jobs'.format(len(jobids))) + logger.info('analyzing %s jobs', len(jobids)) model_tag = {} for k in keys: model_tag[k] = Job[jobids[0]].tags.get(k, '') @@ -2466,14 +2466,14 @@ def analyze_comparable_jobs(jobids, check_comparable=True, keys=('exp_name', 'ex for j in jobids: v = jobids[j].tags.get(k, '') if k not in jobids[j].tags: - logger.warning('job {0} tags has no key -- {1}'.format(j, k)) + logger.warning('job %s tags has no key -- %s', j, k) assert jobids[j].tags.get(k, '') == model_tag[k] - logger.debug('Searching for trained models with tag: {0}'.format(model_tag)) + logger.debug('Searching for trained models with tag: %s', model_tag) trained_models = get_refmodels(tag=model_tag) outlier_results = [] # can we make the if/then more DNRY? if trained_models: - logger.debug('found {0} trained models for job set'.format(len(trained_models))) + logger.debug('found %s trained models for job set', len(trained_models)) for r in trained_models: model_id = r['id'] d = detect_outlier_jobs(jobids, trained_model=model_id)[1] @@ -2538,7 +2538,7 @@ def set_job_analyses(jobid, analyses, replace=False, size_limit=64 * 1024): size = len(dumps(full_analyses)) if size > size_limit: - logger.error("Analyses length({}) > max. analyses limit({}). Aborting..".format(size, size_limit)) + logger.error("Analyses length(%s) > max. analyses limit(%s). Aborting..", size, size_limit) return False j.analyses = full_analyses orm_commit() @@ -2846,7 +2846,7 @@ def procs_histogram(jobs, attr='exename', metric=''): ''' procs_hist = {} procs = get_procs(jobs, fmt='orm') - logger.debug('{} processes found'.format(procs.count())) + logger.debug('%s processes found', procs.count()) for p in procs: attr_val = getattr(p, attr) if metric: @@ -2977,7 +2977,7 @@ def add_features_df(jobs_df, features=[procs_histogram, procs_set], key='jobid') for c in features: out_df[c.__name__] = [c(k) for k in keys] added_features.append(c.__name__) - logger.info('Added features: {}'.format(added_features)) + logger.info('Added features: %s', added_features) return out_df, added_features @@ -3228,7 +3228,7 @@ def verify_collection(c, prefix='', ignore_empty=True): errors = {} for j in jobs: jobid = j['jobid'] - logger.debug('Verifying job {}..'.format(jobid)) + logger.debug('Verifying job %s..', jobid) # first get the errors in the job dict job_errors = verify_collection(j, ignore_empty=False) # now get the errors in the processes of this job From 69ae742a75a699771f45b3202cf91f606143defe Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 17 Mar 2026 19:17:00 +0000 Subject: [PATCH 23/48] Fix all 53 pylint logging violations in epmt_stat.py Convert .format(), % operator, and string concatenation in logging calls to use lazy % formatting with comma-separated arguments. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/epmt/epmt_stat.py | 110 +++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/src/epmt/epmt_stat.py b/src/epmt/epmt_stat.py index 02bcb6932..f8603d670 100644 --- a/src/epmt/epmt_stat.py +++ b/src/epmt/epmt_stat.py @@ -107,7 +107,7 @@ def z_score(ys, params=()): # issue by using np.nan_to_num import warnings warnings.filterwarnings("ignore", category=RuntimeWarning) - logger.debug('scoring using {}'.format('z_score')) + logger.debug('scoring using %s', 'z_score') ys = np.array(ys) if params: # if params is set, we use it to get the mean and stdev @@ -170,7 +170,7 @@ def iqr(ys, params=()): >>> outliers array([0, 0, 0, 0, 0, 0, 1]) ''' - logger.debug('scoring using {}'.format('iqr')) + logger.debug('scoring using %s', 'iqr') ys = np.array(ys) span = [25, 75] if not params: @@ -180,13 +180,13 @@ def iqr(ys, params=()): # we have the lower and upper quartiles from a model _, quartile_1, quartile_3 = params iqr = quartile_3 - quartile_1 - logger.debug('Q1, Q3, IQR: {}, {}, {}'.format(quartile_1, quartile_3, iqr)) + logger.debug('Q1, Q3, IQR: %s, %s, %s', quartile_1, quartile_3, iqr) lower_bound = quartile_1 - (iqr * 1.5) upper_bound = quartile_3 + (iqr * 1.5) - logger.debug('lower_bound, upper_bound: {}, {}'.format(lower_bound, upper_bound)) + logger.debug('lower_bound, upper_bound: %s, %s', lower_bound, upper_bound) # the + 0 below makes boolean array a numeric array of 0s and 1s outliers = ((ys > upper_bound) | (ys < lower_bound)) + 0 - logger.debug('outliers vec: {}'.format(outliers)) + logger.debug('outliers vec: %s', outliers) # If this vector were to be fitted, we can compute artifical # values of Q1 and Q3 based on the equation (see NOTES in the @@ -207,7 +207,7 @@ def modified_z_score(ys, params=()): params if passed in, is of the form (max, median, median_abs_dev) We will ignore params(0) as that's the max z_score in the ref_model ''' - logger.debug('scoring using {}'.format('modified_z_score')) + logger.debug('scoring using %s', 'modified_z_score') median_y = params[1] if params else np.median(ys) if params: median_absolute_deviation_y = params[2] @@ -221,10 +221,10 @@ def modified_z_score(ys, params=()): madz = [round(0.6745 * abs(y - median_y) / median_absolute_deviation_y, 4) for y in ys] else: madz = [float('inf') if abs((y - median_y)) > 0 else 0 for y in ys] - logger.debug('original vector: {}'.format(list(ys))) + logger.debug('original vector: %s', list(ys)) if params: - logger.debug('model params: {}'.format(params)) - logger.debug('madz scores: {}'.format(madz)) + logger.debug('model params: %s', params) + logger.debug('madz scores: %s', madz) return (madz, round(max(madz), 4), round(median_y, 4), round(median_absolute_deviation_y, 4)) @@ -272,16 +272,16 @@ def outliers_uv(ys, methods=[outliers_iqr, outliers_z_score, outliers_modified_z such as iqr, z_score, modified_z_score ''' ys = np.array(ys) - logger.debug('input vector: {}'.format(ys)) + logger.debug('input vector: %s', ys) out_vec = np.zeros_like(ys) if not methods: raise ValueError("'methods' needs to contain one or more univariate classifiers") - logger.debug('outlier detection using {} methods'.format(len(methods))) + logger.debug('outlier detection using %s methods', len(methods)) for m in methods: outliers = m(ys) - logger.debug('outliers using {}: {}'.format(m.__name__, outliers)) + logger.debug('outliers using %s: %s', m.__name__, outliers) out_vec += outliers - logger.info('outliers using {} classifiers: {}'.format(len(methods), out_vec)) + logger.info('outliers using %s classifiers: %s', len(methods), out_vec) return out_vec @@ -396,13 +396,13 @@ def mvod_scores(X=None, classifiers=[], warnopts='ignore'): if not classifiers: classifiers = mvod_classifiers(contamination, warnopts) - logger.debug('using classifiers: {}'.format([get_classifier_name(c) for c in classifiers])) + logger.debug('using classifiers: %s', [get_classifier_name(c) for c in classifiers]) Y = None # Y is only used to test predictor with random data if X is None: n_pts = 100 n_features = 16 - logger.warning('No input data for MVOD. Random data will be used with contamination {}'.format(contamination)) + logger.warning('No input data for MVOD. Random data will be used with contamination %s', contamination) from pyod.utils.data import generate_data, get_outliers_inliers from scipy import stats # generate random data with two features @@ -420,7 +420,7 @@ def mvod_scores(X=None, classifiers=[], warnopts='ignore'): n_outliers = len(x_outliers) (npts, ndim) = X.shape - logger.debug('mvod: input length {0}, dimensions {1}'.format(npts, ndim)) + logger.debug('mvod: input length %s, dimensions %s', npts, ndim) logger.debug(X) scores = {} @@ -438,11 +438,11 @@ def mvod_scores(X=None, classifiers=[], warnopts='ignore'): # predict raw anomaly score _clf_scores = clf.decision_function(X).round(4) except Exception as e: - logger.warning('Could not score using classifier {}: {}'.format(clf_name, e)) + logger.warning('Could not score using classifier %s: %s', clf_name, e) # logger.warning(e, exc_info=True) continue if not check_finite(_clf_scores): - logger.warning('Could not score using classifier {}: got NaN or Inf'.format(clf_name)) + logger.warning('Could not score using classifier %s: got NaN or Inf', clf_name) continue scores[clf_name] = _clf_scores max_score_for_cf[clf_name] = _clf_scores.max() @@ -460,7 +460,7 @@ def mvod_scores(X=None, classifiers=[], warnopts='ignore'): # threshold value to consider a datapoint inlier or outlier # 0.1 is the default outlier fraction in the generated data threshold = stats.scoreatpercentile(scores[clf_name], 100 * (1 - contamination)) - logger.debug('{0} threshold: {1}'.format(clf_name, threshold)) + logger.debug('%s threshold: %s', clf_name, threshold) # print(scores) if not scores: # some error occured and we didn't generate scores at all @@ -524,18 +524,18 @@ def mvod_scores_using_model(inp, model_inp, classifier, threshold=None): c_name = get_classifier_name(classifier) retval = mvod_scores(model_inp, [classifier]) if not retval: - logger.warning('could not score using {}'.format(c_name)) + logger.warning('could not score using %s', c_name) return False (model_scores, model_score_max) = retval model_score_max = model_score_max[c_name] - logger.debug('MVOD {0} (threshold={1})'.format(c_name, threshold)) + logger.debug('MVOD %s (threshold=%s)', c_name, threshold) from math import isclose if not isclose(model_score_max, threshold, rel_tol=1e-2): logger.warning( - 'MVOD {} is not stable. We computed a threshold {}, while the passed threshold from the saved model was {}'.format( + 'MVOD %s is not stable. We computed a threshold %s, while the passed threshold from the saved model was %s', c_name, model_score_max, - threshold)) + threshold) for i in range(inp_nrows): # pick the ith row row = inp[i] @@ -544,7 +544,7 @@ def mvod_scores_using_model(inp, model_inp, classifier, threshold=None): # now run the mvod scoring retval = mvod_scores(X, [classifier]) if not retval: - logger.warning('could not score using {}'.format(c_name)) + logger.warning('could not score using %s', c_name) return False (_scores, _) = retval # mvod_scores returns a dict indexed by classifier name @@ -552,7 +552,7 @@ def mvod_scores_using_model(inp, model_inp, classifier, threshold=None): _scores = list(_scores.values())[0] # pick the score of the appended row (last element) and save it score = _scores[-1] - logger.debug('MVOD {0} score for input index #{1}: {2}'.format(c_name, i, score)) + logger.debug('MVOD %s score for input index #%s: %s', c_name, i, score) scores.append(_scores[-1]) # make list into a numpy array @@ -561,7 +561,7 @@ def mvod_scores_using_model(inp, model_inp, classifier, threshold=None): # return scores if threshold is not set. Else return # a 0/1 vector of inlier / outliers # multiply by 1 to convert to a 0/1 vector - logger.debug('*** input scores (model threshold={}) ***'.format(threshold)) + logger.debug('*** input scores (model threshold=%s) ***', threshold) logger.debug(scores) return scores if (threshold is None) else (scores > threshold) * 1 @@ -594,7 +594,7 @@ def rca(ref, inp, features, methods=[modified_z_score]): # pick all the common numeric columns in the dataframe ref_cols_set = set(ref.columns.values) features = [f for f in list(inp.columns.values) if (isinstance(inp[f][0], Number) and (f in ref_cols_set))] - logger.debug('using following features for RCA analysis: ' + str(features)) + logger.debug('using following features for RCA analysis: %s', features) ref_computed = ref[features].describe() ref_computed.loc['input'] = inp.iloc[0] @@ -651,9 +651,9 @@ def check_finite(values): if isinf(v): n_infs += 1 if n_nans: - logger.debug('found {} NaN'.format(n_nans)) + logger.debug('found %s NaN', n_nans) if n_infs: - logger.debug('found {} Inf'.format(n_infs)) + logger.debug('found %s Inf', n_infs) return ((n_infs == 0) and (n_nans == 0)) @@ -684,37 +684,37 @@ def pca_stat(inp_features, desired=2): from sklearn.preprocessing import StandardScaler from sklearn.decomposition import PCA - logger.debug('input feature array shape: {}'.format(inp_features.shape)) + logger.debug('input feature array shape: %s', inp_features.shape) if np.isnan(inp_features).any(): raise ValueError('input contains at-least one non-numeric (nan) element') - logger.debug('input:\n{}'.format(inp_features)) + logger.debug('input:\n%s', inp_features) # the second paramer denotes the number of components usually # however if it is less than 1, then it denotes the desired variance. # In the latter case the number of components is automatically chosen # to achieve the desired variance. if desired >= 1: - logger.debug('desired num. PCA components: {}'.format(desired)) + logger.debug('desired num. PCA components: %s', desired) else: - logger.debug('desired variance ratio: {}'.format(desired)) + logger.debug('desired variance ratio: %s', desired) n_samples, n_dim = inp_features.shape assert n_dim > 1 x = StandardScaler().fit_transform(inp_features) - logger.debug('input after standard scaling:\n{}'.format(x)) + logger.debug('input after standard scaling:\n%s', x) pca = PCA(n_components=desired) if (desired >= 1) else PCA(desired) pc_array = pca.fit_transform(x) if desired < 1: - logger.debug('number of PCA components: {}'.format(pc_array.shape[1])) - logger.debug('PCA array:\n{}'.format(pc_array)) - logger.debug('PCA feature weights:\n{}'.format(abs(pca.components_))) + logger.debug('number of PCA components: %s', pc_array.shape[1]) + logger.debug('PCA array:\n%s', pc_array) + logger.debug('PCA feature weights:\n%s', abs(pca.components_)) sum_variance = sum(pca.explained_variance_ratio_) - logger.debug('PCA explained variance ratio: {}, sum({})'.format(pca.explained_variance_ratio_, sum_variance)) + logger.debug('PCA explained variance ratio: %s, sum(%s)', pca.explained_variance_ratio_, sum_variance) if sum_variance < 0.80: - logger.warning('cumulative variance for PCA ({}) < 0.80'.format(sum_variance)) + logger.warning('cumulative variance for PCA (%s) < 0.80', sum_variance) return (pc_array, pca) @@ -797,10 +797,10 @@ def check_dist(data=[], dist='norm', alpha=0.05): data = 5 * randn(100) + 50 else: data = np.asarray(data) - logger.debug('data array shape: {}'.format(data.shape)) + logger.debug('data array shape: %s', data.shape) (_min, _max, _mean, _std) = np.min(data), np.max(data), np.mean(data), np.std(data) - logger.debug('min=%.3f max=%.3f mean=%.3f std=%.3f' % (_min, _max, _mean, _std)) - logger.debug('alpha=%.2f' % alpha) + logger.debug('min=%.3f max=%.3f mean=%.3f std=%.3f', _min, _max, _mean, _std) + logger.debug('alpha=%.2f', alpha) passed = 0 failed = 0 @@ -818,21 +818,21 @@ def kstest_uniform(d): tests['norm'].append(('D\'Agostino', normaltest)) if dist not in tests: raise ValueError('We only support the following distributions: {}'.format(tests.keys())) - logger.debug('Testing for {} distribution'.format(dist)) + logger.debug('Testing for %s distribution', dist) for (test, f) in tests[dist]: # normality test - logger.debug('Doing {} test..'.format(test)) + logger.debug('Doing %s test..', test) stat, p = f(data) - logger.debug(' statistics=%.3f, p=%.3f' % (stat, p)) + logger.debug(' statistics=%.3f, p=%.3f', stat, p) if p > alpha: passed += 1 - logger.debug(' {} test: PASSED'.format(test)) + logger.debug(' %s test: PASSED', test) else: failed += 1 - logger.debug(' {} test: FAILED'.format(test)) + logger.debug(' %s test: FAILED', test) - logger.debug('check_dist: {} tests PASSED, {} tests FAILED'.format(passed, failed)) + logger.debug('check_dist: %s tests PASSED, %s tests FAILED', passed, failed) return (passed, failed) @@ -878,33 +878,33 @@ def get_modes(X, max_modes=10): km = KMeans(n_clusters=i, random_state=0).fit(X_scaled) preds = km.predict(X_scaled) - logger.debug("Score for number of cluster(s) {}: {}".format(i, km.score(X_scaled))) + logger.debug("Score for number of cluster(s) %s: %s", i, km.score(X_scaled)) km_scores.append(-km.score(X_scaled)) if i > 1: # silhouette method only works for n_clusters >= 2 silhouette = silhouette_score(X_scaled, preds) km_silhouette.append(silhouette) - logger.debug("Silhouette score for number of cluster(s) {}: {}".format(i, silhouette)) + logger.debug("Silhouette score for number of cluster(s) %s: %s", i, silhouette) # find optimal value according to elbow method diffs = np.abs(np.diff(km_scores)) - logger.debug('diffs of km_scores: {}'.format(diffs)) + logger.debug('diffs of km_scores: %s', diffs) from kneed import KneeLocator kneedle = KneeLocator(range(len(km_scores)), km_scores, S=1.0, curve='convex', direction='decreasing') modes_by_elbow_method = kneedle.elbow + 1 - logger.debug('optimal clustering according to elbow method: {}'.format(modes_by_elbow_method)) + logger.debug('optimal clustering according to elbow method: %s', modes_by_elbow_method) num_modes = modes_by_elbow_method if modes_by_elbow_method != 1: # the index of the peak value fo km_silhouette + 2 (since we start # from 2 to max_modes represents the number of modes modes_by_silhouette_method = np.argmax(km_silhouette) + 2 - logger.debug('optimal clustering according to silhouette method: {}'.format(modes_by_silhouette_method)) + logger.debug('optimal clustering according to silhouette method: %s', modes_by_silhouette_method) if modes_by_elbow_method != modes_by_silhouette_method: logger.warning( - 'Elbow and silhouette methods gave different mode counts -- {} and {}. Usually this means you might have a single mode or your data was not drawn from normal distributions'.format( + 'Elbow and silhouette methods gave different mode counts -- %s and %s. Usually this means you might have a single mode or your data was not drawn from normal distributions', modes_by_elbow_method, - modes_by_silhouette_method)) + modes_by_silhouette_method) num_modes = 1 km = KMeans(n_clusters=num_modes, random_state=0).fit(X_scaled) From dc9293f8e983d66e9e70c0b58f9f78bf9c3ed9ca Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 17 Mar 2026 19:19:51 +0000 Subject: [PATCH 24/48] Fix all 68 pylint logging-format-interpolation violations in epmt_outliers.py Convert all logging calls from .format() string interpolation to lazy % formatting with comma-separated arguments, as recommended by pylint W1202. This defers string formatting until the message is actually emitted, improving performance when log messages are filtered out. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/epmt/epmt_outliers.py | 144 +++++++++++++++++++------------------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/src/epmt/epmt_outliers.py b/src/epmt/epmt_outliers.py index 10b8ec911..cf82e9469 100644 --- a/src/epmt/epmt_outliers.py +++ b/src/epmt/epmt_outliers.py @@ -372,19 +372,19 @@ def detect_outlier_jobs(jobs, trained_model=None, features=FEATURES, methods=[], # skip over methods that do not exist in trained model methods = methods or uvod_classifiers() _methods = [] - logger.info('outlier detection provided {} classifiers'.format(len(methods))) + logger.info('outlier detection provided %s classifiers', len(methods)) for m in methods: c_name = get_classifier_name(m) if trained_model: if c_name not in trained_model.computed: - logger.warning("Skipping classifier {} -- could not find it in trained model".format(c_name)) + logger.warning("Skipping classifier %s -- could not find it in trained model", c_name) continue model_params[m] = trained_model.computed[c_name] else: model_params[m] = {} _methods.append(m) methods = _methods - logger.info('{} classifiers eligible'.format(len(methods))) + logger.info('%s classifiers eligible', len(methods)) if not methods: logger.error('No eligible classifiers available') return False @@ -406,9 +406,9 @@ def detect_outlier_jobs(jobs, trained_model=None, features=FEATURES, methods=[], raise ValueError(err_msg) logger.info( - 'outlier detection will be performed using {} univariate and {} multivariate classifiers'.format( + 'outlier detection will be performed using %s univariate and %s multivariate classifiers', len(uv_methods), - len(mv_methods))) + len(mv_methods)) # sanitize features list if pca and features and (features != '*'): @@ -416,7 +416,7 @@ def detect_outlier_jobs(jobs, trained_model=None, features=FEATURES, methods=[], features = sanitize_features(features, jobs, trained_model) if pca is not False: - logger.info("request to do PCA (pca={}). Input features: {}".format(pca, features)) + logger.info("request to do PCA (pca=%s). Input features: %s", pca, features) if len(features) < 5: logger.warning( 'Too few input features for PCA. Are you sure you did not want to set features=[] ' @@ -436,7 +436,7 @@ def detect_outlier_jobs(jobs, trained_model=None, features=FEATURES, methods=[], if mjob.jobid not in jobids_set: added_model_jobs.append(mjob.jobid) if added_model_jobs: - logger.debug('appending model jobs {} prior to PCA'.format(added_model_jobs)) + logger.debug('appending model jobs %s prior to PCA', added_model_jobs) added_model_jobs_df = eq.get_jobs(added_model_jobs, fmt='pandas')[['jobid'] + features] jobs = pd.concat([jobs[['jobid'] + features], added_model_jobs_df], axis=0, ignore_index=True) (jobs_pca_df, pca_variances, pca_features, _) = pca_feature_combine( @@ -446,22 +446,22 @@ def detect_outlier_jobs(jobs, trained_model=None, features=FEATURES, methods=[], # if trained_model and added_model_jobs: # jobs_pca_df = jobs_pca_df[~jobs_pca_df.jobid.isin(added_model_jobs)].reset_index(drop=True) - logger.info('{} PCA components obtained: {}'.format(len(pca_features), pca_features)) - logger.info('PCA variances: {} (sum={})'.format(pca_variances, np.sum(pca_variances))) + logger.info('%s PCA components obtained: %s', len(pca_features), pca_features) + logger.info('PCA variances: %s (sum=%s)', pca_variances, np.sum(pca_variances)) jobs = jobs_pca_df features = pca_features # list of stuff to return from this fn retlist = [] - logger.debug('doing outlier detection on:\n{}'.format(jobs[['jobid'] + features])) + logger.debug('doing outlier detection on:\n%s', jobs[['jobid'] + features]) # unfortunately we cannot leverage the same code for # univariate and multivariate classifiers, since the # univariate code needs to iterate over the features # while the multivariate code takes them in one go if uv_methods: # initialize a df with all values set to False - logger.debug('OD using UV classifiers: {}'.format([m.__name__ for m in uv_methods])) + logger.debug('OD using UV classifiers: %s', [m.__name__ for m in uv_methods]) retval = pd.DataFrame(0, columns=features, index=jobs.index) for c in features: # print('data-type for feature column {0} is {1}'.format(c, jobs[c].dtype)) @@ -470,17 +470,17 @@ def detect_outlier_jobs(jobs, trained_model=None, features=FEATURES, methods=[], # We ignore params for PCA, as the underlying PCA vector is not stable params = model_params[m].get(c, ()) if not pca else () if params: - logger.debug('params[{0}][{1}]: {2}'.format(m_name, c, params)) + logger.debug('params[%s][%s]: %s', m_name, c, params) scores = m(jobs[c], params)[0] - logger.debug('scores: {}'.format(scores)) + logger.debug('scores: %s', scores) if pca and trained_model: # when using PCA with trained model, we need to figure the threshold # from the rows comprising of the model jobs model_indices = jobs[jobs.jobid.isin(trained_model_jobs)].index.values - logger.debug('trained model job indices: {}'.format(list(model_indices))) + logger.debug('trained model job indices: %s', list(model_indices)) trained_model_scores = np.asarray(scores).take(model_indices) threshold = trained_model_scores.max() - logger.debug('trained model scores: {}, max: {}'.format(trained_model_scores, threshold)) + logger.debug('trained model scores: %s, max: %s', trained_model_scores, threshold) else: # use the max score in the refmodel if we have a trained model # otherwise use the default threshold for the method @@ -488,9 +488,9 @@ def detect_outlier_jobs(jobs, trained_model=None, features=FEATURES, methods=[], # have thresholds, instead those methods return a mask. # So for such methods that return a mask we use a threshold of 0 threshold = params[0] if params else thresholds.get(m_name, 0) - logger.debug('threshold: {}'.format(threshold)) + logger.debug('threshold: %s', threshold) outlier_rows = np.where(np.abs(scores) > threshold)[0] - logger.debug('outliers for [{}][{}] -> {}'.format(m_name, c, outlier_rows)) + logger.debug('outliers for [%s][%s] -> %s', m_name, c, outlier_rows) retval.loc[outlier_rows, c] += 1 # add a jobid column to the output dataframe @@ -507,7 +507,7 @@ def detect_outlier_jobs(jobs, trained_model=None, features=FEATURES, methods=[], retlist.append(parts) if mv_methods: - logger.debug('OD using UV classifiers: {}'.format(mv_methods)) + logger.debug('OD using UV classifiers: %s', mv_methods) # initialize a df with all values set to False features_str = ",".join(sorted(features)) mvod_outliers = None @@ -516,16 +516,16 @@ def detect_outlier_jobs(jobs, trained_model=None, features=FEATURES, methods=[], m_name = get_classifier_name(m) if features_str not in model_params[m]: logger.warning( - 'Skipping classifier {}, as could not find model threshold for the feature set'.format(m_name)) + 'Skipping classifier %s, as could not find model threshold for the feature set', m_name) continue (model_score, model_inp) = model_params[m].get(features_str) model_ndarray = np.asarray(model_inp) - logger.info('classifier {0} model threshold: {1}'.format(m_name, model_score)) + logger.info('classifier %s model threshold: %s', m_name, model_score) outliers_vec = mvod_scores_using_model(jobs[features].to_numpy(), model_ndarray, m, model_score) if outliers_vec is False: - logger.warning('Could not score using {}, skipping it'.format(m_name)) + logger.warning('Could not score using %s, skipping it', m_name) continue - logger.info('outliers vector using {0}: {1}'.format(m_name, outliers_vec)) + logger.info('outliers vector using %s: %s', m_name, outliers_vec) classfiers_od_dict[m_name] = list(outliers_vec) # sum the bitmap vectors - the value for the ith row in the result # shows the number of mvod classifiers that considered the row (job) to @@ -796,7 +796,7 @@ def detect_outlier_ops(jobs, tags=[], trained_model=None, features=FEATURES, "Trained model is disabled. You will need to enable it using 'refmodel_set_enabled' and try again") # logger.debug('Ref. model scores: {0}'.format(trained_model.computed)) # logger.debug('Ref. model op_tags: {0}'.format(trained_model.op_tags)) - logger.debug('Ref model contains {0} op_tags'.format(len(trained_model.op_tags))) + logger.debug('Ref model contains %s op_tags', len(trained_model.op_tags)) for t in trained_model.op_tags: model_tags_set.add(dumps(t, sort_keys=True)) if not tags: @@ -805,12 +805,12 @@ def detect_outlier_ops(jobs, tags=[], trained_model=None, features=FEATURES, logger.warning('Set of unique tags are different from the model') if jobs_tags_set - model_tags_set: logger.warning( - 'Jobs have the following tags, not found in the model: {0}'.format( - jobs_tags_set - model_tags_set)) + 'Jobs have the following tags, not found in the model: %s', + jobs_tags_set - model_tags_set) if model_tags_set - jobs_tags_set: logger.warning( - 'Model has the following tags, not found in the jobs: {0}'.format( - model_tags_set - jobs_tags_set)) + 'Model has the following tags, not found in the jobs: %s', + model_tags_set - jobs_tags_set) else: _err_col_len(jobs, 4, 'Too few jobs to do outlier detection. Need at least 4!') if not tags: @@ -842,7 +842,7 @@ def detect_outlier_ops(jobs, tags=[], trained_model=None, features=FEATURES, _methods.add(m) methods = list(_methods) - logger.info('{} classifiers eligible'.format(len(methods))) + logger.info('%s classifiers eligible', len(methods)) if not methods: logger.error('No eligible classifiers available') return False @@ -878,7 +878,7 @@ def detect_outlier_ops(jobs, tags=[], trained_model=None, features=FEATURES, if mjob.jobid not in jobids_set: added_model_jobs.append(mjob.jobid) if added_model_jobs: - logger.debug('appending model jobs {} prior to PCA'.format(added_model_jobs)) + logger.debug('appending model jobs %s prior to PCA', added_model_jobs) jobs = jobids + added_model_jobs # get the dataframe of aggregate metrics, where each row @@ -886,29 +886,29 @@ def detect_outlier_ops(jobs, tags=[], trained_model=None, features=FEATURES, # jobid and tag ops = eq.get_op_metrics(jobs=jobs, tags=tags_to_use) if len(ops) == 0: - logger.warning('no matching tags found in the tag set: {0}'.format(tags_to_use)) + logger.warning('no matching tags found in the tag set: %s', tags_to_use) return False if pca and features and (features != '*'): logger.warning('It is strongly recommended to set features=[] when doing PCA') features = sanitize_features(features, ops, trained_model) if pca: - logger.info("request to do PCA (pca={}). Input features: {}".format(pca, features)) + logger.info("request to do PCA (pca=%s). Input features: %s", pca, features) if len(features) < 5: logger.warning('Too few input features for PCA. Are you sure you did not want to set ' 'features=[] to enable selecting all available features?' ) - logger.debug('jobid,tags:\n{}'.format(ops[['jobid', 'tags']])) + logger.debug('jobid,tags:\n%s', ops[['jobid', 'tags']]) (ops_pca_df, pca_variances, pca_features, _) = pca_feature_combine( ops, features, desired=0.85 if pca is True else pca) - logger.info('{} PCA components obtained: {}'.format(len(pca_features), pca_features)) - logger.info('PCA variances: {} (sum={})'.format(pca_variances, np.sum(pca_variances))) + logger.info('%s PCA components obtained: %s', len(pca_features), pca_features) + logger.info('PCA variances: %s (sum=%s)', pca_variances, np.sum(pca_variances)) ops = ops_pca_df features = pca_features - logger.debug('jobid,tags:\n{}'.format(ops[['jobid', 'tags']])) + logger.debug('jobid,tags:\n%s', ops[['jobid', 'tags']]) if uv_methods: - logger.debug('OD using UV classifiers: {}'.format(uv_methods)) + logger.debug('OD using UV classifiers: %s', uv_methods) retval = pd.DataFrame(0, columns=features, index=ops.index) # the dict below will be indexed by tag, and will store @@ -931,8 +931,8 @@ def detect_outlier_ops(jobs, tags=[], trained_model=None, features=FEATURES, t = dumps(tag, sort_keys=True) # select only those rows with matching tag rows = ops[ops.tags == tag] # pylint: disable=no-member - logger.debug('Processing tag: {}, rows index: {}'.format(tag, rows.index.values)) - logger.debug('\n:{}'.format(rows[['jobid', 'tags']])) + logger.debug('Processing tag: %s, rows index: %s', tag, rows.index.values) + logger.debug('\n:%s', rows[['jobid', 'tags']]) # logger.debug('input: \n{0}\n'.format(rows[['tags']+features])) tags_max[t] = [] for c in features: @@ -943,26 +943,26 @@ def detect_outlier_ops(jobs, tags=[], trained_model=None, features=FEATURES, m_name = get_classifier_name(m) logger.debug(rows[c]) scores = m(rows[c], params)[0] - logger.debug('scores: {}'.format(scores)) + logger.debug('scores: %s', scores) if pca and trained_model: # when using PCA with trained model, we need to figure the threshold # from the rows comprising of the model jobs _r = rows.reset_index(drop=True) model_indices = _r[_r.jobid.isin(trained_model_jobs)].index.values - logger.debug('trained model job indices: {}'.format(list(model_indices))) + logger.debug('trained model job indices: %s', list(model_indices)) trained_model_scores = np.asarray(scores).take(model_indices) threshold = trained_model_scores.max() - logger.debug('trained model scores: {}, max: {}'.format(trained_model_scores, threshold)) + logger.debug('trained model scores: %s, max: %s', trained_model_scores, threshold) else: # use the max score in the refmodel if we have a trained model # otherwise use the default threshold for the method threshold = params[0] if params else thresholds.get(m_name, 0) - logger.debug('threshold: {}'.format(threshold)) + logger.debug('threshold: %s', threshold) outlier_rows = np.where(np.abs(scores) > threshold)[0] score_diff += max(max(scores) - threshold, 0) # map to the outlier rows indices to the indices in the original df outlier_rows = rows.index[outlier_rows].values - logger.debug('outliers for [{}][{}][{}] -> {}'.format(t, m_name, c, outlier_rows)) + logger.debug('outliers for [%s][%s][%s] -> %s', t, m_name, c, outlier_rows) retval.loc[outlier_rows, c] += 1 tags_max[t].append(round(score_diff, 3)) retval['jobid'] = ops['job'] @@ -1019,7 +1019,7 @@ def detect_outlier_ops(jobs, tags=[], trained_model=None, features=FEATURES, return (final_df, parts, final_scores_df, sorted_tags, sorted_features) else: # MVOD - logger.debug('OD using UV classifiers: {}'.format(mv_methods)) + logger.debug('OD using UV classifiers: %s', mv_methods) # initialize a df with all values set to False features_str = ",".join(sorted(features)) classfiers_od_dict = {} # will store outlier vectors indexed by tag and then by classifier @@ -1030,33 +1030,33 @@ def detect_outlier_ops(jobs, tags=[], trained_model=None, features=FEATURES, classfiers_od_dict[t] = {} # select only those rows with matching tag rows = ops[ops.tags == tag] # pylint: disable=no-member - logger.debug('Processing tag: {}, rows index: {}'.format(tag, rows.index.values)) - logger.debug('\n:{}'.format(rows[['jobid', 'tags']])) + logger.debug('Processing tag: %s, rows index: %s', tag, rows.index.values) + logger.debug('\n:%s', rows[['jobid', 'tags']]) for m in mv_methods: m_name = get_classifier_name(m) if features_str not in model_params[t][m]: logger.warning( - 'Skipping classifier {}, could not find model threshold for the feature set for tag {}'.format( - m_name, t)) + 'Skipping classifier %s, could not find model threshold for the feature set for tag %s', + m_name, t) continue (model_score, model_inp) = model_params[t][m].get(features_str) model_ndarray = np.asarray(model_inp) - logger.debug('classifier {} model threshold for tag [{}]: {}'.format(m_name, t, model_score)) + logger.debug('classifier %s model threshold for tag [%s]: %s', m_name, t, model_score) outliers_vec = mvod_scores_using_model(rows[features].to_numpy(), model_ndarray, m, model_score) if outliers_vec is False: - logger.warning('Could not score using {}, skipping it'.format(m_name)) + logger.warning('Could not score using %s, skipping it', m_name) continue - logger.debug('outliers vector using {} for tag [{}]: {}'.format(m_name, t, outliers_vec)) + logger.debug('outliers vector using %s for tag [%s]: %s', m_name, t, outliers_vec) classfiers_od_dict[t][m_name] = list(outliers_vec) # sum the bitmap vectors - the value for the ith row in the result # shows the number of mvod classifiers that considered the row (job) to # be an outlier mvod_outliers = outliers_vec if (mvod_outliers is None) else mvod_outliers + outliers_vec - logger.info('Outlier vector for tag [{}]: {}'.format(t, mvod_outliers)) + logger.info('Outlier vector for tag [%s]: %s', t, mvod_outliers) outlier_indices = np.where(mvod_outliers > 0)[0] # map to the outlier rows indices to the indices in the original df outlier_rows = rows.index[outlier_indices].values - logger.debug('Outlier indices (in original df) for tag [{}] -> {}'.format(t, outlier_rows)) + logger.debug('Outlier indices (in original df) for tag [%s] -> %s', t, outlier_rows) mvod_df.loc[outlier_rows, 'outlier'] = mvod_outliers[mvod_outliers > 0] # add a jobid column to the output dataframe mvod_df['jobid'] = ops['jobid'] @@ -1097,17 +1097,17 @@ def detect_outliers(df, features=[], methods=[]): features = features or list(df.columns.values) retval = pd.DataFrame(0, columns=features, index=df.index) methods = methods or uvod_classifiers() - logger.debug("Doing outlier detection using: {}".format(features)) - logger.debug("Using the following classifiers: {}".format([f.__name__ for f in methods])) + logger.debug("Doing outlier detection using: %s", features) + logger.debug("Using the following classifiers: %s", [f.__name__ for f in methods]) for c in features: for m in methods: m_name = get_classifier_name(m) scores = m(df[c])[0] - logger.debug('classifier[{}], feature[{}], scores: {}'.format(m_name, c, scores)) + logger.debug('classifier[%s], feature[%s], scores: %s', m_name, c, scores) threshold = thresholds.get(m_name, 0) - logger.debug('threshold: {}'.format(threshold)) + logger.debug('threshold: %s', threshold) outlier_rows = np.where(np.abs(scores) > threshold)[0] - logger.debug('outliers for [{}][{}] -> {}'.format(m_name, c, outlier_rows)) + logger.debug('outliers for [%s][%s] -> %s', m_name, c, outlier_rows) retval.loc[outlier_rows, c] += 1 return retval @@ -1456,7 +1456,7 @@ def pca_feature_combine(inp_df, inp_features=[], desired=2, retain_features=Fals return False features = sanitize_features(inp_features, inp_df) - logger.debug('PCA input features: {}'.format(features)) + logger.debug('PCA input features: %s', features) # check if df contains nans, if so print out the columns nan_cols = inp_df.columns[inp_df.isnull().any()].tolist() if nan_cols: @@ -1614,7 +1614,7 @@ def pca_feature_rank(jobs, inp_features=[]): jobs_df = eq.get_jobs(jobs, fmt='pandas') (_, variances, _, features_df) = pca_feature_combine(jobs_df, inp_features) weights = variances / variances.sum() - logger.debug('normalized weights: {}'.format(list(weights))) + logger.debug('normalized weights: %s', list(weights)) # sort dataframe returned by dframe_append_weighted_row based on the values # in the last row, breaking ties by column name for deterministic ordering df_with_weights = dframe_append_weighted_row( @@ -1665,14 +1665,14 @@ def feature_scatter_plot(jobs, features=[], outfile='', annotate=False): features = sanitize_features(features, jobs_df) pca_variances = None if len(features) > 2: - logger.info('Performing 2-component PCA as input features({}) more than 2'.format(features)) + logger.info('Performing 2-component PCA as input features(%s) more than 2', features) (jobs_pca_df, pca_variances, pca_features, _) = pca_feature_combine(jobs_df, features, desired=2) - logger.info('{} PCA components obtained: {}'.format(len(pca_features), pca_features)) - logger.info('PCA variances: {}, sum={})'.format(pca_variances, np.sum(pca_variances))) + logger.info('%s PCA components obtained: %s', len(pca_features), pca_features) + logger.info('PCA variances: %s, sum=%s)', pca_variances, np.sum(pca_variances)) jobs_df = jobs_pca_df features = pca_features if len(features) != 2: - logger.error('Cannot generate scatter plot as requested features ({}) < 2'.format(features)) + logger.error('Cannot generate scatter plot as requested features (%s) < 2', features) return False import plotly.express as px @@ -1728,20 +1728,20 @@ def sanitize_features(f, df, model=None): logger.debug('using all available features in dataframe') f = set(df.columns.values) else: - logger.debug('Choosing common features between: {} and {}'.format(f, df.columns.values)) + logger.debug('Choosing common features between: %s and %s', f, df.columns.values) f = set(f) & set(df.columns.values) if model is not None: model_id = model.id if (not isinstance(model, int)) else model model_metrics = eq.refmodel_get_metrics(model_id, active_only=True) # take the intersection set - logger.debug('Choosing intersection of features with model metrics ({})'.format(model_metrics)) + logger.debug('Choosing intersection of features with model metrics (%s)', model_metrics) f &= model_metrics # remove blacklisted features _blacklisted_features = f & set(settings.outlier_features_blacklist) if _blacklisted_features: - logger.debug('Pruning blacklisted features: {}'.format(_blacklisted_features)) + logger.debug('Pruning blacklisted features: %s', _blacklisted_features) f -= set(settings.outlier_features_blacklist) features = [] @@ -1750,12 +1750,12 @@ def sanitize_features(f, df, model=None): if df[c].dtype in ('int64', 'float64'): features.append(c) else: - logger.debug('skipping feature({0}) as type is not int/float'.format(c)) + logger.debug('skipping feature(%s) as type is not int/float', c) if not features: raise RuntimeError("Need a non-empty list of features for outlier detection") features = sorted(features) - logger.debug('input features: {0}'.format(features)) + logger.debug('input features: %s', features) return features @@ -1807,9 +1807,9 @@ def get_feature_distributions(jobs, features=[]): dist_dict = {} from epmt.epmt_stat import check_dist for c in features: - logger.debug('determining distribution of feature {}'.format(c)) + logger.debug('determining distribution of feature %s', c) v = jobs[c].to_numpy() - logger.debug('feature vector: {}'.format(v)) + logger.debug('feature vector: %s', v) v_dist = 'unknown' for dist in ['norm', 'uniform']: (passed, failed) = check_dist(v, dist) @@ -1817,7 +1817,7 @@ def get_feature_distributions(jobs, features=[]): v_dist = dist break dist_dict[c] = v_dist - logger.debug('{} -> {}'.format(c, v_dist)) + logger.debug('%s -> %s', c, v_dist) return dist_dict From 9897bde4c9102e092cab5758663fcc1a516ad80a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 17 Mar 2026 19:21:25 +0000 Subject: [PATCH 25/48] Fix 329 pylint logging violations, bump --fail-under to 7.4 Convert all logging calls from .format(), f-string, %, and string concatenation to lazy % formatting per pylint rules: - logging-format-interpolation (W1202): 277 occurrences - logging-not-lazy (W1201): 37 occurrences - logging-fstring-interpolation (W1203): 15 occurrences Files fixed: epmt_outliers.py (68), epmt_stat.py (53), epmt_query.py (49), epmt_cmds.py (43), epmt_job.py (31), epmt_convert_csv.py (17), epmt_concat.py (17), epmt_daemon.py (13), general.py (12), epmtlib.py (12), epmt_exp_explore.py (6), orm/__init__.py (3), op.py (2), epmt_cmd_show.py (2), epmt_cmd_retire.py (1) Bump pylint --fail-under from 7.1 to 7.4 in build_and_test_epmt.yml. Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com> --- .github/workflows/build_and_test_epmt.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test_epmt.yml b/.github/workflows/build_and_test_epmt.yml index 2a1df2914..d42edd75b 100644 --- a/.github/workflows/build_and_test_epmt.yml +++ b/.github/workflows/build_and_test_epmt.yml @@ -514,7 +514,7 @@ jobs: - name: run pylint on epmt module, ignoring ui (epmt-dash) run: | - pylint --rcfile pylintrc --fail-under 7.1 --ignore-paths src/epmt/ui src/epmt + pylint --rcfile pylintrc --fail-under 7.4 --ignore-paths src/epmt/ui src/epmt - name: run pylint on ui submodule (epmt-dash) run: | From 2a203c379fb80d3a49df41291554ca5b61a28f74 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 17 Mar 2026 19:23:45 +0000 Subject: [PATCH 26/48] Normalize %d to %s in logging calls for consistency Code review identified 5 logging calls using %d format specifiers where the codebase convention is %s. Normalized for consistency. Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com> --- src/epmt/epmt_cmd_retire.py | 2 +- src/epmt/epmt_daemon.py | 6 +++--- src/epmt/epmt_exp_explore.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/epmt/epmt_cmd_retire.py b/src/epmt/epmt_cmd_retire.py index 493483f7a..2b586e398 100644 --- a/src/epmt/epmt_cmd_retire.py +++ b/src/epmt/epmt_cmd_retire.py @@ -45,7 +45,7 @@ def epmt_retire(skip_unprocessed=False, dry_run=False): logger.info('%d jobs retired, %d models retired', num_jobs_retired, num_models_retired) if dry_run: - logger.info('(dry_run=True) %d jobs and %d models will be retired', num_jobs_retired, num_models_retired) + logger.info('(dry_run=True) %s jobs and %s models will be retired', num_jobs_retired, num_models_retired) # end memory tracing tm.stop() diff --git a/src/epmt/epmt_daemon.py b/src/epmt/epmt_daemon.py index 370a34ec0..2341759a9 100644 --- a/src/epmt/epmt_daemon.py +++ b/src/epmt/epmt_daemon.py @@ -278,7 +278,7 @@ def daemon_loop(context, maxiters=0, post_process=True, analyze=True, retire=Fal logger.debug('checking dir %s for jobs (*.tgz) to ingest', ingest) tgz_files = find_files_in_dir(ingest, '*.tgz', recursive=recursive) if tgz_files: - logger.info('%d .tgz files found to ingest', len(tgz_files)) + logger.info('%s .tgz files found to ingest', len(tgz_files)) epmt_submit( tgz_files, ncpus=ncpus, @@ -303,7 +303,7 @@ def daemon_loop(context, maxiters=0, post_process=True, analyze=True, retire=Fal # unpdj - ppd_jobs should be 0 in size, let's check err_ppd_jobs = list(filter(lambda i: i not in ppd_jobs, unpdj)) tot_pp_jobs += len(ppd_jobs) - logger.info('%d jobs post-processed, %d errors', len(ppd_jobs), len(err_ppd_jobs)) + logger.info('%s jobs post-processed, %s errors', len(ppd_jobs), len(err_ppd_jobs)) # # Handle unprocessed jobs, remove from unprocessed and log @@ -316,7 +316,7 @@ def daemon_loop(context, maxiters=0, post_process=True, analyze=True, retire=Fal # ppd_jobs - ana_jobs should be 0, let's check err_ana_jobs = list(filter(lambda i: i not in ana_jobs, ppd_jobs)) tot_ua_jobs += len(ana_jobs) - logger.info('%d jobs analyzed, %d errors', len(ana_jobs), len(err_ana_jobs)) + logger.info('%s jobs analyzed, %s errors', len(ana_jobs), len(err_ana_jobs)) # # Handle unanalyzed jobs, log (don't remove) diff --git a/src/epmt/epmt_exp_explore.py b/src/epmt/epmt_exp_explore.py index 251a6e9c3..efa2ae7e7 100644 --- a/src/epmt/epmt_exp_explore.py +++ b/src/epmt/epmt_exp_explore.py @@ -324,7 +324,7 @@ def find_missing_time_segments(exp_name, jobs=[], components=[], time_segments=r matched_comp = jobs_tags['exp_component'] if components: matched_comp &= set(components) - logger.debug('%d components matched', len(matched_comp)) + logger.debug('%s components matched', len(matched_comp)) ret = {} for c in sorted(matched_comp): comp_tags = eq.get_job_tags(jobs, tag_filter='exp_name:{};exp_component:{}'.format(exp_name, c)) From e0242ac9849e348b7ac51a1ca764d5dd605466f6 Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Fri, 20 Mar 2026 12:32:24 -0400 Subject: [PATCH 27/48] Add frame parameter to signal handler function --- src/epmt/epmt_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/epmt/epmt_job.py b/src/epmt/epmt_job.py index ebf4666da..02f61ba6e 100644 --- a/src/epmt/epmt_job.py +++ b/src/epmt/epmt_job.py @@ -627,7 +627,7 @@ def post_process_job( j, # we need to set up signal handlers so the user doesn't # abort the post-processing midway. - def sig_handler(signo): #, frame): + def sig_handler(signo, frame): if hasattr(sig_handler, 'interrupted'): sys.exit(signo) sig_handler.interrupted = True From 85e828e759032c871a1f519a6984a5cc2809009c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 20 Mar 2026 16:34:51 +0000 Subject: [PATCH 28/48] fix: optimize retire_jobs loop by pre-filtering model-associated jobs and fix inverted warn parameter - Remove deprecated spurious warning in delete_jobs() that fired when warn=False - Add fltr parameter to delete_jobs() to pass through to get_jobs() - In retire_jobs(), use ~Job.ref_models.any() filter to exclude model-associated jobs from both the count and each chunk query - Remove deprecated warn=False from internal retire_jobs calls Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com> --- src/epmt/epmt_query.py | 43 ++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/src/epmt/epmt_query.py b/src/epmt/epmt_query.py index 8bbf6609a..c70b57c76 100644 --- a/src/epmt/epmt_query.py +++ b/src/epmt/epmt_query.py @@ -1870,7 +1870,7 @@ def get_op_metrics(jobs, tags=[], exact_tags_only=False, group_by_tag=False, fmt @db_session def delete_jobs(jobs, force=False, before=None, after=None, warn=True, remove_models=False, - limit=None, offset=0, skip_unprocessed=False, dry_run=False): + limit=None, offset=0, skip_unprocessed=False, dry_run=False, fltr=None): """ Deletes one or more jobs and returns the number of jobs deleted::Jobs @@ -1890,18 +1890,17 @@ def delete_jobs(jobs, force=False, before=None, after=None, warn=True, remove_mo these options. warn : boolean, optional - THIS ARGUMENT IS DEPRECATED - This option is only useful in daemon mode where we want to - disable unnecessary copious warnings in logs. - Default True. When disabled, no warnings will be given about attempting - to delete jobs that have models associated with them. Instead - those jobs will be skipped. + THIS ARGUMENT IS DEPRECATED AND IGNORED. remove_models : boolean, optional By default False. If set to True, dependent reference models will removed prior to removal of the job. If False, jobs with dependent models will not be deleted. + fltr : SQLAlchemy filter expression, optional + An optional SQLAlchemy filter to apply when querying jobs + for deletion. Passed through to get_jobs. + Notes ----- The function will either delete all requested jobs or none. The delete @@ -1925,21 +1924,21 @@ def delete_jobs(jobs, force=False, before=None, after=None, warn=True, remove_mo >>> delete_jobs([], force=True, after=-7) """ - if not warn: - logger.warning('verbosity is controlled elsewhere, this argument is now impotent as evidenced by this message.') logger.debug("Jobs sent in %s", str(jobs) ) jobs = orm_jobs_col(jobs) if any( [ before is not None, after is not None, limit is not None, - offset > 0, ] ): + offset > 0, + fltr is not None, ] ): logger.info('(delete_jobs) offset = %s', offset) jobs = get_jobs( jobs, before=before, after=after, limit=limit, offset=offset, + fltr=fltr, fmt='orm', trigger_post_process=not skip_unprocessed ) @@ -2048,23 +2047,25 @@ def retire_jobs(ndays=settings.retire_jobs_ndays, skip_unprocessed=False, dry_ru db_num_jobs = get_jobs(fmt='orm', trigger_post_process=False).count() logger.info('(retire_jobs) number of jobs in DB is %s', db_num_jobs) - num_jobs = get_jobs(before=-ndays, fmt='orm', trigger_post_process=False).count() - logger.info('(retire_jobs) number of jobs older than %s days is %s', ndays, num_jobs) + # Pre-filter: exclude model-associated jobs from the candidate count + # so we don't waste iterations on jobs that will never be deleted. + no_model_fltr = ~Job.ref_models.any() + num_jobs_total = get_jobs(before=-ndays, fmt='orm', trigger_post_process=False).count() + num_jobs = get_jobs(before=-ndays, fltr=no_model_fltr, fmt='orm', trigger_post_process=False).count() + logger.info('(retire_jobs) number of jobs older than %s days is %s (%s excluding model-associated)', + ndays, num_jobs_total, num_jobs) - # uncomment me for training wheels/debug/tests - # JOBS_PER_DELETE_MAX=100 - # num_jobs=get_jobs(before=-ndays, limit=400, fmt='orm', trigger_post_process = False).count() if num_jobs <= JOBS_PER_DELETE_MAX: # can delete in one swoop, when less than max. - return delete_jobs([], force=True, before=-ndays, warn=False, + return delete_jobs([], force=True, before=-ndays, + fltr=no_model_fltr, skip_unprocessed=skip_unprocessed, dry_run=dry_run) - #if num_jobs > JOBS_PER_DELETE_MAX: logger.warning('(retire_jobs) will be deleting jobs in chunks of %d', JOBS_PER_DELETE_MAX) tot_num_deleted = 0 num_delete_attempts = 0 # keep track of num we attempt to delete - offset = 0 # if jobs spared via ref-model-assoc, stop targeting those jobs. + offset = 0 # if jobs spared via skip_unprocessed, stop targeting those jobs. while num_delete_attempts < num_jobs: logger.info('%d jobs to go', num_jobs - num_delete_attempts) @@ -2077,8 +2078,10 @@ def retire_jobs(ndays=settings.retire_jobs_ndays, skip_unprocessed=False, dry_ru limit = num_jobs - num_delete_attempts logger.info('attempting to delete %d jobs now...', limit) - num_deleted = delete_jobs(jobs=[], force=True, before=-ndays, warn=False, - limit=limit, offset=offset, skip_unprocessed=skip_unprocessed, + num_deleted = delete_jobs(jobs=[], force=True, before=-ndays, + limit=limit, offset=offset, + fltr=no_model_fltr, + skip_unprocessed=skip_unprocessed, dry_run=dry_run) tot_num_deleted += num_deleted From 149f13531e7cf0a1a307247a7f0aa12076e3dc55 Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Fri, 20 Mar 2026 14:00:18 -0400 Subject: [PATCH 29/48] Copilot `pylint` complaints (#188) * pylint: enable consider-using-f-string check Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in epmt_convert_csv.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in test/test_submit.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in epmt_exp_explore.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in test/test_query.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in test/test_cmds.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in epmt_daemon.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in epmt_cmds.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in orm/sqlalchemy/general.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in orm/sqlalchemy/models.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in epmt_cmd_help.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in epmt_cmd_show.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in orm/__init__.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in orm/op.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in test/test_explore.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in epmtlib.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in test/test_db_migration.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in epmt_rootcause.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in epmt_default_settings.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in test/test_outliers.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in test/test_lib.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in epmt_concat.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in epmt_stat.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in epmt_job.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in epmt_outliers.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix consider-using-f-string in epmt_query.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix line-too-long in epmt_job.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix line-too-long in epmt_cmds.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix line-too-long in epmt_exp_explore.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix line-too-long in epmt_convert_csv.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix line-too-long in orm/sqlalchemy/models.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix line-too-long in epmtlib.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix line-too-long in epmt_stat.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix line-too-long in test/test_lib.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix line-too-long in test/test_anysh.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix line-too-long in epmt_outliers.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix line-too-long in epmt_daemon.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix line-too-long in test/test_query.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix line-too-long in epmt_query.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix line-too-long in test/test_submit.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: fix line-too-long in test/test_outliers.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * pylint: bump fail-under threshold to 7.2, update README badge Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix: repair indentation/syntax errors introduced by f-string conversion Fix IndentationError in orm/op.py line 74 (logger.warning under if block) and SyntaxError in orm/sqlalchemy/general.py line 612 (ternary expression parenthesization inside text() call). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix: wrap numpy floats with int() for :d format specifiers in epmt_exp_explore.py The f-string conversion changed '%12d' to ':12d' format specs, but numpy operations (np.sum, np.min, np.max) return float64 values. Unlike %-formatting, f-string ':d' requires actual integers. Wrap with int() to fix ValueError on float values. Fixes test_integration_explore CI failure. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix: repair indentation of logger.warning in orm/__init__.py except block The logger.warning at line 92 was outside the except block, causing UnboundLocalError when the try block succeeded (variable 'e' only assigned in the except branch). Re-indented under except. Fixes test_dbsize_json CI failure on postgres. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Increase pylint fail-under threshold for epmt module Updated pylint minimum score requirement for epmt module. * Update pylint badge version in README Updated pylint badge to reflect new version. --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build_and_test_epmt.yml | 2 +- README.md | 4 +- pylintrc | 2 +- src/epmt/epmt_cmd_help.py | 10 +- src/epmt/epmt_cmd_show.py | 4 +- src/epmt/epmt_cmds.py | 46 +++-- src/epmt/epmt_concat.py | 29 ++- src/epmt/epmt_convert_csv.py | 21 ++- src/epmt/epmt_daemon.py | 12 +- src/epmt/epmt_default_settings.py | 2 +- src/epmt/epmt_exp_explore.py | 64 ++++--- src/epmt/epmt_job.py | 90 ++++----- src/epmt/epmt_outliers.py | 46 ++--- src/epmt/epmt_query.py | 109 ++++++----- src/epmt/epmt_rootcause.py | 2 +- src/epmt/epmt_stat.py | 24 +-- src/epmt/epmtlib.py | 32 ++-- src/epmt/orm/op.py | 4 +- src/epmt/orm/sqlalchemy/general.py | 38 ++-- src/epmt/orm/sqlalchemy/models.py | 20 +- src/epmt/test/test_anysh.py | 3 +- src/epmt/test/test_cmds.py | 18 +- src/epmt/test/test_db_migration.py | 4 +- src/epmt/test/test_explore.py | 4 +- src/epmt/test/test_lib.py | 4 +- src/epmt/test/test_outliers.py | 211 +++++++++++++--------- src/epmt/test/test_query.py | 45 +++-- src/epmt/test/test_submit.py | 64 ++++--- 28 files changed, 519 insertions(+), 395 deletions(-) diff --git a/.github/workflows/build_and_test_epmt.yml b/.github/workflows/build_and_test_epmt.yml index d42edd75b..8f1556f07 100644 --- a/.github/workflows/build_and_test_epmt.yml +++ b/.github/workflows/build_and_test_epmt.yml @@ -514,7 +514,7 @@ jobs: - name: run pylint on epmt module, ignoring ui (epmt-dash) run: | - pylint --rcfile pylintrc --fail-under 7.4 --ignore-paths src/epmt/ui src/epmt + pylint --rcfile pylintrc --fail-under 8.1 --ignore-paths src/epmt/ui src/epmt - name: run pylint on ui submodule (epmt-dash) run: | diff --git a/README.md b/README.md index 67226eeb1..bc71131c2 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![build_and_test_epmt](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml) [![docker_build_test](https://github.com/NOAA-GFDL/epmt/actions/workflows/docker_build_test.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/docker_build_test.yml) [![codecov](https://codecov.io/gh/NOAA-GFDL/epmt/branch/main/graph/badge.svg)](https://codecov.io/gh/NOAA-GFDL/epmt) -[![pylint](https://img.shields.io/badge/pylint-%E2%89%A57.0-brightgreen)](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml) +[![pylint](https://img.shields.io/badge/pylint-%E2%89%A58.1-brightgreen)](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml) **Experiment Performance Management Tool** aka **WorkflowDB** aka @@ -567,4 +567,4 @@ or `requirements.txt.py3` is modified — no manual version bump is needed. respective caches before the working week begins. `docker_build_test.yml` then restores those caches on pull request and push runs, skipping the expensive Docker compile steps. If the cache is missing (first run, eviction, or new key), `docker_build_test.yml` falls back to building the -artifact inline so the pipeline never silently skips a required build step. \ No newline at end of file +artifact inline so the pipeline never silently skips a required build step. diff --git a/pylintrc b/pylintrc index 2c8738566..52dfc3c87 100644 --- a/pylintrc +++ b/pylintrc @@ -444,7 +444,7 @@ disable=raw-checker-failed, # either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). See also the "--disable" option for examples. -enable= +enable=consider-using-f-string [METHOD_ARGS] diff --git a/src/epmt/epmt_cmd_help.py b/src/epmt/epmt_cmd_help.py index 0b8215cb8..93168aba4 100644 --- a/src/epmt/epmt_cmd_help.py +++ b/src/epmt/epmt_cmd_help.py @@ -25,18 +25,18 @@ def epmt_help_api(funcs=[]): func = getattr(m, fname) break if func: - print("from {} import {}\n".format(m.__name__, fname)) + print(f"from {m.__name__} import {fname}\n") section = el.docs_func_section(func) - print("{}{}".format(func.__name__, signature(func))) + print(f"{func.__name__}{signature(func)}") doc = func.__doc__ if section: # add the section name with suitable indent - print('\n Section::{}'.format(section)) + print(f'\n Section::{section}') # remove the ugly section suffix from the summary string - doc = doc.replace('::{}'.format(section), '') + doc = doc.replace(f'::{section}', '') print(doc, '\n\n') else: - print('Could not find function {} in any module'.format(fname), file=stderr) + print(f'Could not find function {fname} in any module', file=stderr) else: for m in (eq, eod, exp, es): print(m.__doc__) diff --git a/src/epmt/epmt_cmd_show.py b/src/epmt/epmt_cmd_show.py index a42cf1b71..2b95c45bc 100644 --- a/src/epmt/epmt_cmd_show.py +++ b/src/epmt/epmt_cmd_show.py @@ -21,9 +21,9 @@ def epmt_show_job(jobid, key=None): print(j_dict[key]) else: logger.error('Key "%s" was not found as an attribute of the job table', key) - print('Here are the keys that were found: {}'.format(",".join(sorted(j_dict.keys())))) + print(f'Here are the keys that were found: {",".join(sorted(j_dict.keys()))}') return False else: for k in sorted(j_dict.keys()): - print("%-20s %-20s" % (k, j_dict[k])) + print(f"{k:<20} {j_dict[k]!s:<20}") return True diff --git a/src/epmt/epmt_cmds.py b/src/epmt/epmt_cmds.py index aa3088bda..7c3ae9ac0 100644 --- a/src/epmt/epmt_cmds.py +++ b/src/epmt/epmt_cmds.py @@ -66,7 +66,7 @@ def dump_config(outf, sep=":"): for key, value in sorted(settings.__dict__.items()): if not (key.startswith('__') or key.startswith('_') or key == 'ERROR'): if type(value) in [str, int, float, list, dict, bool]: - print("%s%s%s" % (key, sep, str(value)), file=outf) + print(f"{key}{sep}{str(value)}", file=outf) print("\nenvironment variables (overrides settings.py):", file=outf) env_var_list = [ "PAPIEX_OSS_PATH", "PAPIEX_OUTPUT", @@ -77,7 +77,7 @@ def dump_config(outf, sep=":"): # ]: for v in env_var_list: if v in environ: - print("%s%s%s" % (v, sep, environ[v]), file=outf) + print(f"{v}{sep}{environ[v]}", file=outf) @logfn @@ -217,7 +217,8 @@ def verify_papiex_options(): retval = True # Check for any components - # cmd = settings.install_prefix+"/bin/papi_component_avail 2>&1 "+"| sed -n -e '/Active/,$p' | grep perf_event >/dev/null 2>&1" + # cmd = settings.install_prefix+"/bin/papi_component_avail 2>&1 " + # +"| sed -n -e '/Active/,$p' | grep perf_event >/dev/null 2>&1" cmd = settings.install_prefix + "/bin/papi_component_avail 2>&1 " + \ "| sed -e '/Active/,$p' | grep perf_event >/dev/null 2>&1" logger.info("\t%s", cmd) @@ -237,9 +238,10 @@ def verify_papiex_options(): # cmd = settings.install_prefix+"/bin/papi_command_line 2>&1 "+e+"| sed -n # -e '/PERF_COUNT_SW_CPU_CLOCK\ :/,$p' | grep PERF_COUNT_SW_CPU_CLOCK > # /dev/null 2>&1" # does not work for rocky-8. + # guessing... NOT TRIED YET TODO: TRY THIS INSTEAD OF ABOVE LINE cmd = settings.install_prefix + "/bin/papi_command_line 2>&1 " + e + \ - "| sed -e '/PERF_COUNT_SW_CPU_CLOCK\\ :/,$p' | grep PERF_COUNT_SW_CPU_CLOCK > /dev/null 2>&1" # guessing... NOT TRIED YET TODO: TRY THIS INSTEAD OF ABOVE LINE - logger.info("\t%s", cmd) + "| sed -e '/PERF_COUNT_SW_CPU_CLOCK\\ :/,$p' | grep PERF_COUNT_SW_CPU_CLOCK > /dev/null 2>&1" + logger.info("\t" + cmd) return_code = run(cmd, shell=True).returncode if return_code != 0: logger.error("%s failed", cmd) @@ -265,7 +267,7 @@ def verify_db_params(): PrintPass() return True except ImportError as e: - logger.error("Error setting up DB: %s", e) + logger.error("Error setting up DB: %s", str(e)) PrintFail() return False @@ -310,8 +312,8 @@ def verify_stage_command(): tmp = environ.get('TMPDIR', '/tmp') tmpfile = 'test_stage_cmd' - inp = '/{0}/{1}'.format(tmp, tmpfile) - target = '{0}/{1}'.format(dest, tmpfile) + inp = f'/{tmp}/{tmpfile}' + target = f'{dest}/{tmpfile}' try: safe_rm(target) open(inp, 'a').close() @@ -320,7 +322,7 @@ def verify_stage_command(): print(str(e), file=stderr) PrintFail() if not path.exists(target): - raise FileNotFoundError("could not create output in {0}".format(dest)) from e + raise FileNotFoundError(f"could not create output in {dest}") from e return False finally: safe_rm(inp) @@ -616,7 +618,7 @@ def epmt_dump_metadata(filelist, key=None): logger.error('Did not find %s in tar file', "job_metadata") return False else: - logger.info('%s is %s bytes in archive', info.name, info.size) + logger.info('%s is %d bytes in archive', info.name, info.size) f = tar.extractfile(info) metadata = read_job_metadata_direct(f) else: @@ -628,7 +630,7 @@ def epmt_dump_metadata(filelist, key=None): print(metadata[key]) else: for d in sorted(metadata.keys()): - print("%-24s%-56s" % (d, str(metadata[d]))) + print(f"{d:<24}{str(metadata[d]):<56}") return rc_final @@ -971,13 +973,17 @@ def add_var(cmd, str): elif not run_cmd: # set up functions - cmd += "epmt_push_preload ()\n{\nif [ -z \"$PAPIEX_OLD_LD_PRELOAD\" ]; then export LD_PRELOAD=$PAPIEX_LD_PRELOAD ; else export LD_PRELOAD=$PAPIEX_LD_PRELOAD:$PAPIEX_OLD_LD_PRELOAD ; fi\n};\n" - cmd += "epmt_pop_preload ()\n{\nif [ -z \"$PAPIEX_OLD_LD_PRELOAD\" ]; then export -n LD_PRELOAD ; else export LD_PRELOAD=$PAPIEX_OLD_LD_PRELOAD ; fi\n};\n" + cmd += ("epmt_push_preload ()\n{\nif [ -z \"$PAPIEX_OLD_LD_PRELOAD\" ]; then" + " export LD_PRELOAD=$PAPIEX_LD_PRELOAD ; else" + " export LD_PRELOAD=$PAPIEX_LD_PRELOAD:$PAPIEX_OLD_LD_PRELOAD ; fi\n};\n") + cmd += ("epmt_pop_preload ()\n{\nif [ -z \"$PAPIEX_OLD_LD_PRELOAD\" ]; then" + " export -n LD_PRELOAD ; else export LD_PRELOAD=$PAPIEX_OLD_LD_PRELOAD ; fi\n};\n") cmd += "epmt_instrument () {\nexport MONITOR_DEBUG PAPIEX_OUTPUT PAPIEX_DEBUG PAPIEX_OPTIONS LD_PRELOAD;\n" cmd += "epmt_push_preload;\n};\n" cmd += "epmt_uninstrument () {\nexport -n MONITOR_DEBUG PAPIEX_OUTPUT PAPIEX_DEBUG PAPIEX_OPTIONS;\n" cmd += "epmt_pop_preload;\n};\n" - #cmd += "epmt () {\nepmt_pop_preload;\n cmd=`command epmt`;\nif [ $? -eq 0 ]; then $cmd $* ; else \"epmt not in \$PATH\"; fi\n;epmt_push_preload;\n};\n" + #cmd += "epmt () {\nepmt_pop_preload;\n cmd=`command epmt`;\nif [ $? -eq 0 ]; then" + # " $cmd $* ; else \"epmt not in $PATH\"; fi\n;epmt_push_preload;\n};\n" # Now enable instrumentation cmd += "epmt_instrument;\n" @@ -1385,7 +1391,7 @@ def extract_tar(tarfile, outdir='', check_metadata=False): except KeyError: logger.error('ERROR: Did not find %s in tar file', "job_metadata") return False - logger.info('%s is %s bytes in archive', info.name, info.size) + logger.info('%s is %d bytes in archive', info.name, info.size) from tempfile import mkdtemp, gettempdir outdir = outdir or mkdtemp(prefix='epmt_stage_', dir=gettempdir()) @@ -1393,7 +1399,7 @@ def extract_tar(tarfile, outdir='', check_metadata=False): try: tar.extractall(path=outdir) except Exception as e: - logger.error('Error extracting %s into %s: %s', tarfile, outdir, e) + logger.error('Error extracting %s into %s: %s', tarfile, outdir, str(e)) # cleanup since we had an error rmtree(outdir, ignore_errors=True) return False @@ -1464,7 +1470,7 @@ def goodpath(from_path): status = False exc = None r = None - msg = "submit_dir_or_tgz_to_db({}): ".format(inputf) + msg = f"submit_dir_or_tgz_to_db({inputf}): " try: r = submit_to_db(inputf, pattern, dry_run=dry_run) @@ -1514,7 +1520,7 @@ def submit_to_db(inputf, pattern, dry_run=True): logger.error('Did not find %s in tar file %s', "job_metadata", inputf) return (False, 'Did not find metadata in tar file ' + inputf, ()) else: - logger.info('%s is %s bytes in tar file %s', info.name, info.size, inputf) + logger.info('%s is %d bytes in tar file %s', info.name, info.size, inputf) f = tar.extractfile(info) metadata = read_job_metadata_direct(f) filedict = get_filedict(None, settings.input_pattern, tar) @@ -1823,7 +1829,9 @@ def epmt_entrypoint(args): # submit does the drop on its own, so here we handle... Ian: why was this sentence never finished?? if args.command == 'drop': if not args.force: - confirm = input("This will drop the entire database. This action cannot be reversed. Are you sure (yes/NO): ") + confirm = input( + "This will drop the entire database. This action cannot be reversed." + " Are you sure (yes/NO): ") if confirm.upper() not in ('Y', 'YES'): return 0 diff --git a/src/epmt/epmt_concat.py b/src/epmt/epmt_concat.py index 823c72562..958c627f8 100755 --- a/src/epmt/epmt_concat.py +++ b/src/epmt/epmt_concat.py @@ -149,12 +149,9 @@ def parseLine(infile, line, masterHeader, masterHeaderFile, headerDelimCount, he logger.info("Master header set: %s", masterHeader) return (line, None, headerDelimCount, headerFound, masterHeader, masterHeaderFile) elif line != masterHeader: - msg = "Header mismatch: File {} does not match master file {}".format(infile, masterHeaderFile) - logger.error(msg) - msg = "Header: {}".format(line) - logger.error(msg) - msg = "Master: {}".format(masterHeader) - logger.error(msg) + logger.error("Header mismatch: File %s does not match master file %s", infile, masterHeaderFile) + logger.error("Header: %s", line) + logger.error("Master: %s", masterHeader) raise InvalidFileFormat() else: return (None, None, headerDelimCount, headerFound, masterHeader, masterHeaderFile) @@ -174,7 +171,7 @@ def parseLine(infile, line, masterHeader, masterHeaderFile, headerDelimCount, he else: logger.error( "File: %s, Header: %s delimiters, but this row has %s delimiters", - infile, headerDelimCount, lineDelimCount) + infile, str(headerDelimCount), str(lineDelimCount)) logger.error("Row: %s", line) logger.error("Master File: %s", masterHeaderFile) logger.error("Master header: %s", masterHeader) @@ -208,13 +205,13 @@ def writeCSV(outfile, comments, masterHeader, dataList): # write comments with open(outfile, 'w') as f: for item in comments: - f.write("%s\n" % item) + f.write(f"{item}\n") # write header - f.write("%s" % masterHeader) + f.write(f"{masterHeader}") f.write("\n") # write data for item in dataList: - f.write("%s\n" % item) + f.write(f"{item}\n") except Exception as e: # parent of IOError, OSError logger.error("Error writing output file %s, removing...: %s", outfile, str(e)) remove(outfile) @@ -241,9 +238,9 @@ def verifyOut(fileList, outfile): if result != outputLines: logger.error( "Output file %s smaller than expected, off by %s lines, expected %s", - outfile, result - outputLines, result) - logger.error("Input files %s have %s lines", fileList, lines) - logger.error("Total header lines removed - 1 %s", headers2Remove) + outfile, result - outputLines, result) + logger.error("Input files %s have %s lines", str(fileList), str(lines)) + logger.error("Total header lines removed - 1 %s", str(headers2Remove)) return False return True @@ -309,8 +306,7 @@ def csvjoiner(indir, # String (Directory) Mode ################################## if isinstance(indir, str): if not path.isdir(indir): - msg = "{} does not exist or is not a directory".format(indir) - logger.error(msg) + logger.error("%s does not exist or is not a directory", indir) return False, None, badfiles logger.info("Collate in directory %s", indir) fileList = sorted(glob(indir + "/*.csv")) @@ -332,8 +328,7 @@ def csvjoiner(indir, return False, None, badfiles if len(fileList) == 0: - msg = "{} has no CSV files to concatenate".format(indir) - logger.warning(msg) + logger.warning("%s has no CSV files to concatenate", indir) return True, None, badfiles if any(("collated" in FL for FL in fileList)): diff --git a/src/epmt/epmt_convert_csv.py b/src/epmt/epmt_convert_csv.py index 3e7dd298a..1966a9552 100755 --- a/src/epmt/epmt_convert_csv.py +++ b/src/epmt/epmt_convert_csv.py @@ -69,8 +69,14 @@ OUTPUT_CSV_SEP = '\t' # Expected input format -# tags,hostname,exename,path,args,exitcode,pid,generation,ppid,pgid,sid,numtids,tid,start,end,usertime,systemtime,rssmax,minflt,majflt,inblock,outblock,vol_ctxsw,invol_ctxsw,num_threads,starttime,processor,delayacct_blkio_time,guest_time,rchar,wchar,syscr,syscw,read_bytes,write_bytes,cancelled_write_bytes,time_oncpu,time_waiting,timeslices,rdtsc_duration,PERF_COUNT_SW_CPU_CLOCK -# ,pp208,tcsh,/bin/tcsh,-f /home/Jeffrey.Durachta/ESM4/DECK/ESM4_historical_D151/gfdl.ncrc4-intel16-prod-openmp/scripts/postProcess/ESM4_historical_D151_ocean_annual_rho2_1x1deg_18840101.tags,0,6099,0,6098,6089,6084,1,6099,1560599524133795,1560599524134048,2999,0,2852,387,0,0,0,0,0,0,1296261120000,0,0,0,17618,0,40,0,0,0,0,3604195,47138,1,846248,246094 +# tags,hostname,exename,path,args,exitcode,pid,generation,ppid,pgid,sid,numtids,tid,start,end,usertime,systemtime, +# rssmax,minflt,majflt,inblock,outblock,vol_ctxsw,invol_ctxsw,num_threads,starttime,processor,delayacct_blkio_time, +# guest_time,rchar,wchar,syscr,syscw,read_bytes,write_bytes,cancelled_write_bytes,time_oncpu,time_waiting,timeslices, +# rdtsc_duration,PERF_COUNT_SW_CPU_CLOCK +# ,pp208,tcsh,/bin/tcsh,-f /home/Jeffrey.Durachta/ESM4/DECK/ESM4_historical_D151/gfdl.ncrc4-intel16-prod-openmp/ +# scripts/postProcess/ESM4_historical_D151_ocean_annual_rho2_1x1deg_18840101.tags,0,6099,0,6098,6089,6084,1,6099, +# 1560599524133795,1560599524134048,2999,0,2852,387,0,0,0,0,0,0,1296261120000,0,0,0,17618,0,40,0,0,0,0, +# 3604195,47138,1,846248,246094 def conv_csv_for_dbcopy(infile, outfile='', jobid='', input_fields=INPUT_CSV_FIELDS): @@ -144,7 +150,8 @@ def conv_csv_for_dbcopy(infile, outfile='', jobid='', input_fields=INPUT_CSV_FIE if row_num == 1: if input_fields and set(r.keys()) < input_fields: # sanity check to make sure our input file has the correct format - logger.error('Input CSV format is not correct. Likely missing header row. Is it already in v2 format?') + logger.error('Input CSV format is not correct. ' + 'Likely missing header row. Is it already in v2 format?') return False thr_fields = sorted(set(r.keys()) - set(settings.skip_for_thread_sums) - @@ -308,7 +315,7 @@ def convert_csv_in_tar(in_tar, out_tar=''): # close the input tar tar.close() - in_csv_files = glob('{}/*.csv'.format(tempdir)) + in_csv_files = glob(f'{tempdir}/*.csv') if not in_csv_files: logger.error('No CSV files found in %s', in_tar) return False @@ -317,13 +324,13 @@ def convert_csv_in_tar(in_tar, out_tar=''): assert len(in_csv_files) == 1 hostname = basename(in_csv_files[0]).split('-')[0] - header_filename = "{}-papiex-header.tsv".format(hostname) + header_filename = f"{hostname}-papiex-header.tsv" if "./" + header_filename in tar_contents: # a header file presence indicates v2 CSV logger.error('%s already contains CSV files in v2 format', in_tar) return False in_csv = in_csv_files[0] # only one csv file will be present - out_csv = tempdir + "/" + "{}-papiex.tsv".format(hostname) + out_csv = f"{tempdir}/{hostname}-papiex.tsv" logger.info('Starting CSV conversion..') # save the header returned for subsequent use hdr = conv_csv_for_dbcopy(in_csv, out_csv) @@ -332,7 +339,7 @@ def convert_csv_in_tar(in_tar, out_tar=''): return False # write the header into a separate file - with open('{}/{}'.format(tempdir, header_filename), 'w') as csv_hdr_flo: + with open(f'{tempdir}/{header_filename}', 'w') as csv_hdr_flo: csv_hdr_flo.write(hdr) logger.debug("Created CSV header file: %s", header_filename) tar_contents.append("./" + header_filename) diff --git a/src/epmt/epmt_daemon.py b/src/epmt/epmt_daemon.py index 2341759a9..6374185e8 100644 --- a/src/epmt/epmt_daemon.py +++ b/src/epmt/epmt_daemon.py @@ -193,7 +193,8 @@ def daemon_loop(context, maxiters=0, post_process=True, analyze=True, retire=Fal sig_count = 0 logger.debug( - '(context=%s,maxiters=%d,post_process=%s,analyze=%s,retire=%s,ingest=%s,recursive=%s,keep=%s,moveaway=%s,verbose=%d)', + '(context=%s,maxiters=%d,post_process=%s,analyze=%s,' + 'retire=%s,ingest=%s,recursive=%s,keep=%s,moveaway=%s,verbose=%d)', type(context), maxiters, post_process, @@ -217,9 +218,8 @@ def daemon_loop(context, maxiters=0, post_process=True, analyze=True, retire=Fal if ingest: logger.info('ingestion mode enabled for daemon') - logger.info( - 'ingestion mode (path=%s,recursive=%s,keep=%s,move_away=%s)', - ingest, recursive, keep, move_away) + logger.info('ingestion mode (path=%s,recursive=%s,keep=%s,move_away=%s)', + ingest, recursive, keep, move_away) if not path.isdir(ingest): logger.error('Ingest path (%s) does not exist', ingest) return True @@ -333,10 +333,10 @@ def daemon_loop(context, maxiters=0, post_process=True, analyze=True, retire=Fal logger.debug('ending daemon loop, as requested %d iterations completed', maxiters) break if delay > 0: - logger.debug('sleeping for {0:.3f} sec'.format(delay)) + logger.debug('sleeping for %.3f sec', delay) sleep(delay) else: - logger.warning("daemon loop took %s seconds. No sleep for me!", _loop_time) + logger.warning('daemon loop took %s seconds. No sleep for me!', _loop_time) return False diff --git a/src/epmt/epmt_default_settings.py b/src/epmt/epmt_default_settings.py index 9255bdc25..3b76d9963 100644 --- a/src/epmt/epmt_default_settings.py +++ b/src/epmt/epmt_default_settings.py @@ -47,7 +47,7 @@ # place for error'd CSV files error_dest = "/tmp" # when we are not attached to a terminal we log to the file below -logfile = path.expandvars("/tmp/epmt_{}.log".format(getuser() or "unknown")) +logfile = path.expandvars(f"/tmp/epmt_{getuser() or 'unknown'}.log") # blacklist for environment filter (in addition to all keys with # leading underscores) diff --git a/src/epmt/epmt_exp_explore.py b/src/epmt/epmt_exp_explore.py index efa2ae7e7..e5e16ca18 100644 --- a/src/epmt/epmt_exp_explore.py +++ b/src/epmt/epmt_exp_explore.py @@ -72,12 +72,15 @@ def exp_component_outliers(exp_name, metric='duration', op=np.sum, limit=10): # jobs 625151 (time-segment 18540101) and 691209 (time-segment 18890101) # are likely outliers; one on the higher side, and one on the lower. >>> exp_component_outliers('ESM4_historical_D151', 'duration') - INFO: epmt_query: Experiment ESM4_historical_D151 contains 13 jobs: 625151,627907,629322,633114,675992, 680163,685000..685001,685003,685016,691209,692500,693129 + INFO: epmt_query: Experiment ESM4_historical_D151 contains 13 jobs: + 625151,627907,629322,633114,675992, 680163,685000..685001,685003,685016,691209,692500,693129 [{ 'exp_component': 'ocean_annual_z_1x1deg', - 'exp_times': ['18540101','18590101','18640101','18690101','18740101','18790101','18840101','18890101','18940101'], + 'exp_times': ['18540101','18590101','18640101','18690101','18740101', + '18790101','18840101','18890101','18940101'], 'jobids': ['625151','627907','629322','633114','675992','680163','685001','691209','693129'], - 'metrics': array([1.04256232e+10, 6.58917488e+09, 7.28633175e+09, 6.03672005e+09, 9.11415052e+09, 6.15619201e+09, 6.81571048e+09, 8.60163243e+08, 3.61932477e+09]), + 'metrics': array([1.04256232e+10, 6.58917488e+09, 7.28633175e+09, 6.03672005e+09, + 9.11415052e+09, 6.15619201e+09, 6.81571048e+09, 8.60163243e+08, 3.61932477e+09]), 'outlier_scores': array([2., 0., 0., 0., 0., 0., 0., 2., 1.]) }, { @@ -100,7 +103,7 @@ def exp_component_outliers(exp_name, metric='duration', op=np.sum, limit=10): # get a compact string of jobids if possible for logs try: - job_ranges_str = ",".join(["{}..{}".format(a, b) if (a != b) else "{}".format(a) + job_ranges_str = ",".join([f"{a}..{b}" if (a != b) else f"{a}" for (a, b) in ranges([int(x) for x in exp_jobids])]) logger.info('Experiment %s contains %s jobs: %s', exp_name, exp_jobs.count(), job_ranges_str) except BaseException: @@ -243,23 +246,23 @@ def exp_explore(exp_name, metric='duration', op=np.sum, limit=10): agg_metric = np.sum([np.array(d['metrics']).sum() for d in ordered_comp_list]) - print('\ntop {} components by {}({}):'.format(limit, op.__name__, metric)) - print("%16s %12s %12s %12s %4s" % ("component", "sum", "min", "max", "cv")) + print(f'\ntop {limit} components by {op.__name__}({metric}):') + print(f"{'component':>16} {'sum':>12} {'min':>12} {'max':>12} {'cv':>4}") for v in ordered_comp_list: - print("%16.16s: %12d [%4.1f%%] %12d %12d %4.1f" % - (v['exp_component'], op(v['metrics']), 100 * - np.sum(v['metrics']) / - agg_metric, np.min(v['metrics']), np.max(v['metrics']), variation(v['metrics']))) + print(f"{v['exp_component']:>16.16}: {int(op(v['metrics'])):12d}" + f" [{100 * np.sum(v['metrics']) / agg_metric:4.1f}%]" + f" {int(np.min(v['metrics'])):12d} {int(np.max(v['metrics'])):12d} {variation(v['metrics']):4.1f}") # now let's the variations within a component across different time segments print('\nvariations across time segments (by component):') - print("%16s %12s %12s %16s" % ("component", "exp_time", "jobid", metric)) + print(f"{'component':>16} {'exp_time':>12} {'jobid':>12} {metric:>16}") for v in ordered_comp_list: outliers = v['outlier_scores'] for idx in range(len(v['metrics'])): - print("%16.16s %12s %12s %16d %6s" % (v['exp_component'], v['exp_times'] - [idx], v['jobids'][idx], v['metrics'][idx], "**" * int(outliers[idx]))) + print(f"{v['exp_component']:>16.16} {v['exp_times'][idx]:>12}" + f" {v['jobids'][idx]:>12} {int(v['metrics'][idx]):16d}" + f" {'**' * int(outliers[idx]):>6}") print() # finally let's see if by summing the metric across all the jobs in a @@ -268,9 +271,9 @@ def exp_explore(exp_name, metric='duration', op=np.sum, limit=10): time_segments = list(od.keys()) metric_sums = [np.sum(od[t]['metrics']) for t in time_segments] outlier_scores = es.outliers_uv(metric_sums) - print('{} by time segment:'.format(metric)) + print(f'{metric} by time segment:') for idx in range(len(time_segments)): - print("%12s %16d %6s" % (time_segments[idx], metric_sums[idx], "**" * int(outlier_scores[idx]))) + print(f"{time_segments[idx]:>12} {int(metric_sums[idx]):16d} {'**' * int(outlier_scores[idx]):>6}") return True @@ -309,11 +312,23 @@ def find_missing_time_segments(exp_name, jobs=[], components=[], time_segments=r Examples -------- >>> d = exp.find_missing_time_segments('ESM4_historical_D151', time_segments=range(18540101, 20140101, 50000)) - ocean_annual_rho2_1x1deg is missing [18540101, 18590101, 18640101, 18690101, 18740101, 18790101, 18890101, 18940101, 18990101, 19040101, 19090101, 19140101, 19190101, 19240101, 19290101, 19340101, 19390101, 19440101, 19490101, 19540101, 19590101, 19640101, 19690101, 19740101, 19790101, 19840101, 19890101, 19940101, 19990101, 20040101, 20090101] - ocean_cobalt_fdet_100 is missing [18540101, 18590101, 18640101, 18690101, 18740101, 18790101, 18890101, 18940101, 18990101, 19040101, 19090101, 19140101, 19190101, 19240101, 19290101, 19340101, 19390101, 19440101, 19490101, 19540101, 19590101, 19640101, 19690101, 19740101, 19790101, 19840101, 19890101, 19940101, 19990101, 20040101, 20090101] + ocean_annual_rho2_1x1deg is missing [18540101, 18590101, 18640101, 18690101, 18740101, + 18790101, 18890101, 18940101, 18990101, 19040101, 19090101, 19140101, 19190101, 19240101, + 19290101, 19340101, 19390101, 19440101, 19490101, 19540101, 19590101, 19640101, 19690101, + 19740101, 19790101, 19840101, 19890101, 19940101, 19990101, 20040101, 20090101] + ocean_cobalt_fdet_100 is missing [18540101, 18590101, 18640101, 18690101, 18740101, + 18790101, 18890101, 18940101, 18990101, 19040101, 19090101, 19140101, 19190101, 19240101, + 19290101, 19340101, 19390101, 19440101, 19490101, 19540101, 19590101, 19640101, 19690101, + 19740101, 19790101, 19840101, 19890101, 19940101, 19990101, 20040101, 20090101] >>> d - { 'ocean_annual_rho2_1x1deg' : [18540101, 18590101, 18640101, 18690101, 18740101, 18790101, 18890101, 18940101, 18990101, 19040101, 19090101, 19140101, 19190101, 19240101, 19290101, 19340101, 19390101, 19440101, 19490101, 19540101, 19590101, 19640101, 19690101, 19740101, 19790101, 19840101, 19890101, 19940101, 19990101, 20040101, 20090101], - 'ocean_cobalt_fdet_100': [18540101, 18590101, 18640101, 18690101, 18740101, 18790101, 18890101, 18940101, 18990101, 19040101, 19090101, 19140101, 19190101, 19240101, 19290101, 19340101, 19390101, 19440101, 19490101, 19540101, 19590101, 19640101, 19690101, 19740101, 19790101, 19840101, 19890101, 19940101, 19990101, 20040101, 20090101] + { 'ocean_annual_rho2_1x1deg' : [18540101, 18590101, 18640101, 18690101, 18740101, + 18790101, 18890101, 18940101, 18990101, 19040101, 19090101, 19140101, 19190101, 19240101, + 19290101, 19340101, 19390101, 19440101, 19490101, 19540101, 19590101, 19640101, 19690101, + 19740101, 19790101, 19840101, 19890101, 19940101, 19990101, 20040101, 20090101], + 'ocean_cobalt_fdet_100': [18540101, 18590101, 18640101, 18690101, 18740101, + 18790101, 18890101, 18940101, 18990101, 19040101, 19090101, 19140101, 19190101, 19240101, + 19290101, 19340101, 19390101, 19440101, 19490101, 19540101, 19590101, 19640101, 19690101, + 19740101, 19790101, 19840101, 19890101, 19940101, 19990101, 20040101, 20090101] } ''' tag_filter = {'exp_name': exp_name} @@ -327,14 +342,14 @@ def find_missing_time_segments(exp_name, jobs=[], components=[], time_segments=r logger.debug('%s components matched', len(matched_comp)) ret = {} for c in sorted(matched_comp): - comp_tags = eq.get_job_tags(jobs, tag_filter='exp_name:{};exp_component:{}'.format(exp_name, c)) + comp_tags = eq.get_job_tags(jobs, tag_filter=f'exp_name:{exp_name};exp_component:{c}') exp_times = comp_tags['exp_time'] if isinstance(exp_times, str): exp_times = [exp_times] exp_times = set([int(t) for t in exp_times]) missing_times = set(time_segments) - exp_times if missing_times: - print('{} is missing {}'.format(c, sorted(missing_times))) + print(f'{c} is missing {sorted(missing_times)}') ret[c] = sorted(missing_times) return ret @@ -383,11 +398,14 @@ def exp_find_jobs(exp_name, components=[], exp_times=[], failed=None, **kwargs): Examples -------- - >>> exp.exp_find_jobs('ESM4_historical_D151', components=['ocean_annual_rho2_1x1deg', 'ocean_cobalt_fdet_100'], exp_times=['18540101', '18840101'], failed = False) + >>> exp.exp_find_jobs('ESM4_historical_D151', + ... components=['ocean_annual_rho2_1x1deg', 'ocean_cobalt_fdet_100'], + ... exp_times=['18540101', '18840101'], failed = False) ['685000', '685003'] ''' if 'tags' in kwargs: - logger.warning('If you use the "tags" option then "exp_name", "components" and "exp_time" options will be ignored') + logger.warning('If you use the "tags" option then "exp_name", ' + '"components" and "exp_time" options will be ignored') else: # create tags using exp_name and components tags = [{'exp_name': exp_name, 'exp_component': c} diff --git a/src/epmt/epmt_job.py b/src/epmt/epmt_job.py index 02f61ba6e..59c1d0465 100644 --- a/src/epmt/epmt_job.py +++ b/src/epmt/epmt_job.py @@ -158,7 +158,7 @@ def get_proc_rows(csvfile, skiprows=0, fmt='1', metric_names=[]): from epmt.epmt_convert_csv import OUTPUT_CSV_FIELDS, OUTPUT_CSV_SEP # we only support two formats at present if fmt not in ('1', '2'): - raise ValueError('CSV format ({}), not recognized'.format(fmt)) + raise ValueError(f'CSV format ({fmt}), not recognized') if skiprows > 0: err_msg = 'Do not know how to handle a non-zero value for skiprows while reading CSV file' @@ -341,7 +341,8 @@ def load_process_from_dictlist(proc, host, j, u, settings, profile): # # is needed in queries, and Pony doesn't allow operations on json fields # # in a Query # # TODO: can this be removed? - # thread_metric_sums['user+system'] = thread_metric_sums.get('usertime', 0) + thread_metric_sums.get('systemtime', 0) + # thread_metric_sums['user+system'] = thread_metric_sums.get('usertime', 0) + + # thread_metric_sums.get('systemtime', 0) _t = time.time() fields = set(proc[0].keys()) - set(settings.skip_for_thread_sums) - set(settings.per_process_fields) @@ -370,7 +371,7 @@ def extract_tags_from_comment_line(jobdatafile, comment="#", tarfile=None): try: info = tarfile.getmember(jobdatafile) except KeyError as e: - err_msg = 'BUG: Did not find %s in tar archive' % str(tarfile) + err_msg = f'BUG: Did not find {str(tarfile)} in tar archive' logger.error(err_msg) logger.error(e) raise LookupError(err_msg) from e @@ -729,8 +730,12 @@ def sig_handler(signo, frame): papiex_err_pids.add(proc.pid) logger.debug(' rdtsc_duration for PID (%d) < 0 (database ID %s)', proc.pid, str(proc.id if proc.id is not None else "not set yet")) - papiex_err = 'papiex / PAPI library could not be preloaded (rdtsc_duration = 0).' if ( - rdtsc == 0) else 'PAPI failed or misbehaved process closed a descriptor it did not own (rdtsc_duration < 0).' + papiex_err = ( + 'papiex / PAPI library could not be preloaded (rdtsc_duration = 0).' + if rdtsc == 0 else + 'PAPI failed or misbehaved process closed a descriptor it did not own' + ' (rdtsc_duration < 0).' + ) # Set rdtsc_duration to -1 in errant process and threads # we need to clone the ORM object as the ORM skips update # at times if you just do an in-place field change @@ -773,7 +778,7 @@ def sig_handler(signo, frame): # If we have a errors, we need to annotate the job if num_errs: - papiex_err += ' {} processes have potentially erroneous PAPI metric counts'.format(num_errs) + papiex_err += f' {num_errs} processes have potentially erroneous PAPI metric counts' logger.warning('papiex error: %s. Setting rdtsc_duration to -1 for job %s', papiex_err, jobid) proc_sums['rdtsc_duration'] = -1 # the current sum is wrong, so use -1 # use a dict copy so we force an ORM update of this field @@ -899,9 +904,9 @@ def populate_process_table_from_staging(j): logger.warning('Moving staged processes for job %s will take approx. %2.0f sec..', jobid, num_procs / 3000) staged_procs = orm_raw_sql( - "SELECT id, threads_df, start, finish, tags, hostname, numtids, exename, path, args, pid, ppid, pgid, sid, generation, exitcode, exitsignal FROM processes_staging WHERE id BETWEEN {} AND {}".format( - first_proc_id, - last_proc_id)) + f"SELECT id, threads_df, start, finish, tags, hostname, numtids, exename, path, args," + f" pid, ppid, pgid, sid, generation, exitcode, exitsignal" + f" FROM processes_staging WHERE id BETWEEN {first_proc_id} AND {last_proc_id}") proc_ids = [] nprocs = 0 insert_sql = "" @@ -965,29 +970,15 @@ def populate_process_table_from_staging(j): # threads_df is to be saved as JSON threads_df = dumps(_thr_dict_list) - insert_sql += prefix_insert_sql + """('{jobid}',{duration},{tags},'{host_id}','{threads_df}','{threads_sums}',{numtids},{cpu_time},{exename},{path},{args},{pid},{ppid},{pgid},{sid},{gen},{exitcode},'{start}','{end}');\n""".format( - jobid=jobid, - start=start, - end=end, - duration=duration, - tags=tags, - host_id=host_id, - threads_df=threads_df, - threads_sums=threads_sums, - numtids=numtids, - cpu_time=cpu_time, - exename=exename, - path=path, - args=args, - pid=pid, - ppid=ppid, - pgid=pgid, - sid=sid, - gen=gen, - exitcode=exitcode) + insert_sql += ( + prefix_insert_sql + + f"('{jobid}',{duration},{tags},'{host_id}','{threads_df}','{threads_sums}'," + f"{numtids},{cpu_time},{exename},{path},{args},{pid},{ppid},{pgid}," + f"{sid},{gen},{exitcode},'{start}','{end}');\n" + ) # sql to delete the rows from the staging table - delete_sql = "DELETE FROM processes_staging WHERE id BETWEEN {} AND {};\n".format(first_proc_id, last_proc_id) + delete_sql = f"DELETE FROM processes_staging WHERE id BETWEEN {first_proc_id} AND {last_proc_id};\n" job_info_dict['procs_in_process_table'] = 1 @@ -998,7 +989,7 @@ def populate_process_table_from_staging(j): # We want to retain the metric_names in the job info_dict, so don't remove it below, anymore # del job_info_dict['metric_names'] - update_job_sql = "UPDATE jobs SET info_dict = '{}' WHERE jobid = '{}'".format(dumps(job_info_dict), jobid) + update_job_sql = f"UPDATE jobs SET info_dict = '{dumps(job_info_dict)}' WHERE jobid = '{jobid}'" # INSERT SQL transaction try: @@ -1014,7 +1005,8 @@ def populate_process_table_from_staging(j): logger.error('You do not have sufficient privileges for this operation') else: logger.error( - 'INSERT aka insert_sql[:%s] = \n %s', settings.max_log_statement_length, insert_sql[:settings.max_log_statement_length]) + f'INSERT aka insert_sql[:{settings.max_log_statement_length}] = \n' + f' {insert_sql[:settings.max_log_statement_length]}') ## Only log the first 100 entries in the error string- it will largely be SQL statements if len(err_str) > settings.max_log_statement_length: logger.error('error (type is %s) too long to show (%s)...', type(err_str), len(err_str)) @@ -1138,8 +1130,10 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): job_tag_from_ann = tag_from_string(annotations[settings.job_tags_env]) if all( [ job_tags, job_tag_from_ann, job_tags != job_tag_from_ann ] ): - err_msg = 'Metadata and annotations contain different job tags:\n{} (metadata),\n{} (annotations)'.format( - job_tags, job_tag_from_ann) + err_msg = ( + f'Metadata and annotations contain different job tags:\n' + f'{job_tags} (metadata),\n{job_tag_from_ann} (annotations)' + ) return ( False, err_msg, () ) logger.warning('Both metadata and annotations have the same job tags') @@ -1151,7 +1145,7 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): tag_str = tag_dict_to_string(job_tags) logger.debug( 'updating %s in annotations to %s based on metadata job tags', - settings.job_tags_env, tag_str) + settings.job_tags_env, tag_str) annotations[settings.job_tags_env] = tag_str # sometimes script name is to be found in the job tags @@ -1186,7 +1180,7 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): j = create_job(jobid, u) if j is None: - return (False, 'Assuming job {} is already in database'.format(str(jobid)), ()) + return (False, f'Assuming job {str(jobid)} is already in database', ()) j.jobname = jobname j.exitcode = exitcode @@ -1249,7 +1243,7 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): fileno = 0 csv = datetime.now() fmt = '1' # default csv format - header_filename = "{}-papiex-header.tsv".format(hostname) + header_filename = f"{hostname}-papiex-header.tsv" if tarfile: logger.debug('checking if tarfile contains CSV v2 files') try: @@ -1274,7 +1268,7 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): try: csv_hdr_flo = open(full_hdr_path, 'r', encoding='utf-8') except Exception as e: - msg = 'Could not open {} for reading: {}'.format(full_hdr_path, str(e)) + msg = f'Could not open {full_hdr_path} for reading: {str(e)}' logger.error(msg) return (False, msg, ()) @@ -1336,15 +1330,17 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): try: conn = psycopg2.connect(settings.db_params['url']) except Exception as e: - msg = 'Error establishing connection to PostgreSQL database: {}'.format(str(e)) + msg = f'Error establishing connection to PostgreSQL database: {str(e)}' logger.error(msg) return (False, msg, ()) cur = conn.cursor() _copy_start_ts = time.time() logger.debug('establishing connection to DB took: %2.5f sec', _copy_start_ts - _conn_start_ts) - copy_sql = "COPY processes_staging({}) FROM STDIN DELIMITER '{}' CSV QUOTE E'\b'".format( - ",".join(OUTPUT_CSV_FIELDS), OUTPUT_CSV_SEP) + copy_sql = ( + f"COPY processes_staging({','.join(OUTPUT_CSV_FIELDS)})" + f" FROM STDIN DELIMITER '{OUTPUT_CSV_SEP}' CSV QUOTE E'\b'" + ) logger.debug('Issuing direct-copy SQL: %s' , copy_sql) try: # copy_from is deprecated and copy_expert is recommended @@ -1363,7 +1359,7 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): conn.commit() _copy_ok = True except Exception as e: - msg = 'copy_expert to processes_staging {}'.format(str(e)) + msg = f'copy_expert to processes_staging {str(e)}' logger.warning('%s; falling back to standard processing for file %s', msg, f) conn.rollback() @@ -1459,8 +1455,12 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): p.start = p.start.replace(tzinfo=pytz.utc).astimezone(tz=tz_default) p.end = p.end.replace(tzinfo=pytz.utc).astimezone(tz=tz_default) if p.start < start_ts or p.end > stop_ts: - msg = 'Corrupted CSV detected: Process ({0}, pid {1}) start/finish times ({2}, {3}) do not fall within job interval ({4}, {5}). Bailing on job ingest..'.format( - p.exename, p.pid, p.start, p.end, start_ts, stop_ts) + msg = ( + f'Corrupted CSV detected: Process ({p.exename}, pid {p.pid})' + f' start/finish times ({p.start}, {p.end})' + f' do not fall within job interval ({start_ts}, {stop_ts}).' + f' Bailing on job ingest..' + ) logger.error(msg) raise ValueError(msg) @@ -1497,7 +1497,7 @@ def ETL_job_dict(raw_metadata, filedict, settings, tarfile=None): logger.debug('file I/O time took: %2.5f sec', file_io_time) logger.debug('process load ops took: %2.5f sec', df_process_time) logger.debug(' - load process from dictlist took: %2.5f sec', load_process_from_df_time) - logger.debug(' - %s', ["%s: %2.5f sec" % (k, v) for (k, v) in profile.load_process.items()]) + logger.debug(' - %s', [f"{k}: {v:2.5f} sec" for (k, v) in profile.load_process.items()]) logger.debug(' - tag processing took: %2.5f sec', proc_tag_process_time) logger.debug(' - proc misc. processing took: %2.5f sec', proc_misc_time) logger.debug( diff --git a/src/epmt/epmt_outliers.py b/src/epmt/epmt_outliers.py index cf82e9469..6cd77b538 100644 --- a/src/epmt/epmt_outliers.py +++ b/src/epmt/epmt_outliers.py @@ -348,7 +348,7 @@ def detect_outlier_jobs(jobs, trained_model=None, features=FEATURES, methods=[], # check if any columns contain nans nan_columns = jobs.columns[jobs.isnull().any()].tolist() if nan_columns: - raise ValueError('dataframe columns ({}) contain atlleast one NaN each'.format(nan_columns)) + raise ValueError(f'dataframe columns ({nan_columns}) contain atlleast one NaN each') model_params = {} if trained_model: @@ -377,7 +377,7 @@ def detect_outlier_jobs(jobs, trained_model=None, features=FEATURES, methods=[], c_name = get_classifier_name(m) if trained_model: if c_name not in trained_model.computed: - logger.warning("Skipping classifier %s -- could not find it in trained model", c_name) + logger.warning('Skipping classifier %s -- could not find it in trained model', c_name) continue model_params[m] = trained_model.computed[c_name] else: @@ -406,9 +406,9 @@ def detect_outlier_jobs(jobs, trained_model=None, features=FEATURES, methods=[], raise ValueError(err_msg) logger.info( - 'outlier detection will be performed using %s univariate and %s multivariate classifiers', - len(uv_methods), - len(mv_methods)) + 'outlier detection will be performed using %s univariate and %s multivariate classifiers', + len(uv_methods), + len(mv_methods)) # sanitize features list if pca and features and (features != '*'): @@ -416,7 +416,7 @@ def detect_outlier_jobs(jobs, trained_model=None, features=FEATURES, methods=[], features = sanitize_features(features, jobs, trained_model) if pca is not False: - logger.info("request to do PCA (pca=%s). Input features: %s", pca, features) + logger.info('request to do PCA (pca=%s). Input features: %s', pca, features) if len(features) < 5: logger.warning( 'Too few input features for PCA. Are you sure you did not want to set features=[] ' @@ -805,12 +805,12 @@ def detect_outlier_ops(jobs, tags=[], trained_model=None, features=FEATURES, logger.warning('Set of unique tags are different from the model') if jobs_tags_set - model_tags_set: logger.warning( - 'Jobs have the following tags, not found in the model: %s', - jobs_tags_set - model_tags_set) + 'Jobs have the following tags, not found in the model: %s', + jobs_tags_set - model_tags_set) if model_tags_set - jobs_tags_set: logger.warning( - 'Model has the following tags, not found in the jobs: %s', - model_tags_set - jobs_tags_set) + 'Model has the following tags, not found in the jobs: %s', + model_tags_set - jobs_tags_set) else: _err_col_len(jobs, 4, 'Too few jobs to do outlier detection. Need at least 4!') if not tags: @@ -893,7 +893,7 @@ def detect_outlier_ops(jobs, tags=[], trained_model=None, features=FEATURES, features = sanitize_features(features, ops, trained_model) if pca: - logger.info("request to do PCA (pca=%s). Input features: %s", pca, features) + logger.info('request to do PCA (pca=%s). Input features: %s', pca, features) if len(features) < 5: logger.warning('Too few input features for PCA. Are you sure you did not want to set ' 'features=[] to enable selecting all available features?' ) @@ -1007,9 +1007,9 @@ def detect_outlier_ops(jobs, tags=[], trained_model=None, features=FEATURES, parts = {} for tag in tags_to_use: dft = retval[retval.tags == tag] - q_ref = "&".join(["{0} == 0".format(f) for f in features]) + q_ref = "&".join([f"{f} == 0" for f in features]) dft_ref = dft.query(q_ref).reset_index(drop=True) - q_outlier = "|".join(["{0} > 0".format(f) for f in features]) + q_outlier = "|".join([f"{f} > 0" for f in features]) dft_outlier = dft.query(q_outlier).reset_index(drop=True) parts[dumps(tag)] = (set(dft_ref['jobid'].values), (set(dft_outlier['jobid'].values))) @@ -1036,8 +1036,8 @@ def detect_outlier_ops(jobs, tags=[], trained_model=None, features=FEATURES, m_name = get_classifier_name(m) if features_str not in model_params[t][m]: logger.warning( - 'Skipping classifier %s, could not find model threshold for the feature set for tag %s', - m_name, t) + 'Skipping classifier %s, could not find model threshold for the feature set for tag %s', + m_name, t) continue (model_score, model_inp) = model_params[t][m].get(features_str) model_ndarray = np.asarray(model_inp) @@ -1097,8 +1097,8 @@ def detect_outliers(df, features=[], methods=[]): features = features or list(df.columns.values) retval = pd.DataFrame(0, columns=features, index=df.index) methods = methods or uvod_classifiers() - logger.debug("Doing outlier detection using: %s", features) - logger.debug("Using the following classifiers: %s", [f.__name__ for f in methods]) + logger.debug('Doing outlier detection using: %s', features) + logger.debug('Using the following classifiers: %s', [f.__name__ for f in methods]) for c in features: for m in methods: m_name = get_classifier_name(m) @@ -1276,8 +1276,8 @@ def detect_rootcause_op(jobs, inp, tag, features=FEATURES, methods=[modified_z_s # this is just a sanity check to make sure we only compare # rows that have the same tag. Ordinarily this code won't be # triggered as eq.get_op_metrics will only return rows that match 'tag' - print('ref jobs have multiple distinct ' + \ - 'tags({0}) that are a superset of specified tag. Please specify an exact tag match'.format(unique_tags) ) + print(f'ref jobs have multiple distinct tags({unique_tags}) that are a superset of ' + 'specified tag. Please specify an exact tag match') return (False, None, None) return rca(ref_ops_df, inp_ops_df, features, methods) @@ -1460,13 +1460,13 @@ def pca_feature_combine(inp_df, inp_features=[], desired=2, retain_features=Fals # check if df contains nans, if so print out the columns nan_cols = inp_df.columns[inp_df.isnull().any()].tolist() if nan_cols: - raise ValueError('PCA input dataframe contains nans in columns: {}'.format(nan_cols)) + raise ValueError(f'PCA input dataframe contains nans in columns: {nan_cols}') inp_data = inp_df[features].to_numpy() (pca_data, pca_) = pca_stat(inp_data, desired) pca_feature_names = [] features_df = pd.DataFrame(data=pca_.components_, columns=features) for i in range(len(pca_.explained_variance_ratio_)): - pca_feature_names.append('pca_{:02d}'.format(i + 1)) + pca_feature_names.append(f'pca_{i + 1:02d}') out_df = pd.DataFrame(data=pca_data, columns=pca_feature_names, index=inp_df.index) inp_features_set = set(features) @@ -1679,7 +1679,7 @@ def feature_scatter_plot(jobs, features=[], outfile='', annotate=False): x_label_ext = '' if pca_variances is not None: pca_01_weight = round(pca_variances[0] / pca_variances[1], 1) - x_label_ext = ' (weight: {})'.format(pca_01_weight) + x_label_ext = f' (weight: {pca_01_weight})' plt = px.scatter( jobs_df, x=features[0], @@ -1827,6 +1827,6 @@ def _err_col_len(c, min_length=1, msg=None): ''' l = orm_col_len(c) if l < min_length: - msg = msg or "length of collection is less than the minimum ({0})".format(min_length) + msg = msg or f"length of collection is less than the minimum ({min_length})" logger.warning(msg) raise RuntimeError(msg) diff --git a/src/epmt/epmt_query.py b/src/epmt/epmt_query.py index c70b57c76..bb5d1dd32 100644 --- a/src/epmt/epmt_query.py +++ b/src/epmt/epmt_query.py @@ -98,7 +98,8 @@ def conv_jobs(jobs, fmt='dict', merge_sums=True, trigger_post_process=True): common_fields = list(set(j) & set(j[PROC_SUMS_FIELD_IN_JOB])) if common_fields: logger.warning( - 'while hoisting proc_sums to job-level, found %s common fields: %s', len(common_fields), common_fields) + 'while hoisting proc_sums to job-level, found %s common fields: %s', + len(common_fields), common_fields) j.update(j[PROC_SUMS_FIELD_IN_JOB]) del j[PROC_SUMS_FIELD_IN_JOB] @@ -126,7 +127,8 @@ def __conv_procs_orm(procs, merge_sums=True, fmt='dict'): common_fields = list(set(p) & set(p[THREAD_SUMS_FIELD_IN_PROC])) if common_fields: logger.warning( - 'while hoisting thread_sums to process-level, found %s common fields: %s', len(common_fields), common_fields) + 'while hoisting thread_sums to process-level, found %s common fields: %s', + len(common_fields), common_fields) p.update(p[THREAD_SUMS_FIELD_IN_PROC]) # add an alias for a consistent user experience p['jobid'] = p['job'] @@ -215,7 +217,8 @@ def timeline(jobs, limit=0, fltr='', when=None, hosts=[], fmt='pandas'): 3 685000 modulecmd 2019-06-15 11:52:04.176020 3416 4 685000 test 2019-06-15 11:52:04.192758 3417 - >>> eq.timeline([u'685000', u'685016'], limit=5, hosts=[Host[u'pp313'], Host[u'pp208']])[['job', 'exename', 'start', 'host']] + >>> eq.timeline([u'685000', u'685016'], limit=5, + ... hosts=[Host[u'pp313'], Host[u'pp208']])[['job', 'exename', 'start', 'host']] job exename start host 0 685000 tcsh 2019-06-15 11:52:04.126892 pp208 1 685000 tcsh 2019-06-15 11:52:04.133795 pp208 @@ -541,7 +544,7 @@ def get_jobs( try: when = datetime.strptime(when, '%m/%d/%Y %H:%M') except Exception as e: - logger.error('could not convert "when" string to datetime: %s', e) + logger.error('could not convert "when" string to datetime: %s', str(e)) return None if before is not None: @@ -729,7 +732,7 @@ def get_procs(jobs=[], tags=None, fltr=None, order=None, offset=0, limit=None, w try: when = datetime.strptime(when, '%m/%d/%Y %H:%M') except Exception as e: - logger.error('could not convert "when" string to datetime: %s', e) + logger.error('could not convert "when" string to datetime: %s', str(e)) return None if hosts: @@ -890,7 +893,8 @@ def get_job_tags(jobs, tag_filter='', fold=True): 'ocn_res': '0.5l75', 'exp_name': 'ESM4_historical_D151', 'exp_component': 'ocean_annual_z_1x1deg', - 'exp_time': {'18890101', '18640101', '18940101', '18690101', '18590101', '18790101', '18540101', '18740101', '18840101'}, + 'exp_time': {'18890101', '18640101', '18940101', '18690101', '18590101', + '18790101', '18540101', '18740101', '18840101'}, 'script_name': {'ESM4_historical_D151_ocean_annual_z_1x1deg_18540101', 'ESM4_historical_D151_ocean_annual_z_1x1deg_18840101', 'ESM4_historical_D151_ocean_annual_z_1x1deg_18640101', @@ -971,9 +975,17 @@ def rank_proc_tags_keys(jobs, order='cardinality', exclude=[]): Examples -------- >>> eq.rank_proc_tags_keys(['685000']) - [('op', {'ncatted', 'ncrcat', 'dmput', 'fregrid', 'rm', 'timavg', 'hsmget', 'mv', 'cp', 'splitvars', 'untar'}), ('op_instance', {'9', '19', '6', '4', '20', '12', '8', '16', '2', '15', '5', '13', '10', '3', '11', '7', '14', '1', '18'}), ('op_sequence', {'83', '9', '67', '82', '60', '89', '85', '79', '20', '72', '8', '12', '27', '2', '51', '55', '87', '17', '48', '61', '40', '14', '7', '53', '26', '56', '37', '35', '4', '18', '36', '54', '62', '84', '70', '24', '50', '63', '58', '5', '13', '64', '57', '76', '44', '34', '1', '39', '21', '29', '81', '78', '42', '46', '19', '66', '43', '16', '28', '49', '30', '15', '10', '22', '73', '86', '77', '33', '47', '68', '31', '75', '6', '45', '32', '71', '41', '65', '80', '25', '74', '3', '11', '69', '52', '23', '59', '88', '38'})] + [('op', {'ncatted', 'ncrcat', 'dmput', 'fregrid', 'rm', 'timavg', + 'hsmget', 'mv', 'cp', 'splitvars', 'untar'}), + ('op_instance', + {'9', '19', '6', '4', '20', '12', '8', '16', '2', '15', '5', '13', '10', '3', '11', '7', '14', '1', '18'}), + ('op_sequence', {'83', '9', '67', '82', '60', '89', '85', '79', '20', '72', '8', '12', '27', '2', ...})] >>> eq.rank_proc_tags_keys(['685000'], order = 'frequency') - [('op', {'ncatted', 'ncrcat', 'dmput', 'fregrid', 'rm', 'timavg', 'hsmget', 'mv', 'cp', 'splitvars', 'untar'}), ('op_instance', {'9', '19', '6', '4', '20', '12', '8', '16', '2', '15', '5', '13', '10', '3', '11', '7', '14', '1', '18'}), ('op_sequence', {'83', '9', '67', '82', '60', '89', '85', '79', '20', '72', '8', '12', '27', '2', '51', '55', '87', '17', '48', '61', '40', '14', '7', '53', '26', '56', '37', '35', '4', '18', '36', '54', '62', '84', '70', '24', '50', '63', '58', '5', '13', '64', '57', '76', '44', '34', '1', '39', '21', '29', '81', '78', '42', '46', '19', '66', '43', '16', '28', '49', '30', '15', '10', '22', '73', '86', '77', '33', '47', '68', '31', '75', '6', '45', '32', '71', '41', '65', '80', '25', '74', '3', '11', '69', '52', '23', '59', '88', '38'})] + [('op', {'ncatted', 'ncrcat', 'dmput', 'fregrid', 'rm', 'timavg', + 'hsmget', 'mv', 'cp', 'splitvars', 'untar'}), + ('op_instance', + {'9', '19', '6', '4', '20', '12', '8', '16', '2', '15', '5', '13', '10', '3', '11', '7', '14', '1', '18'}), + ('op_sequence', {'83', '9', '67', '82', '60', '89', '85', '79', '20', '72', '8', '12', '27', '2', ...})] ''' _empty_collection_check(jobs) if order.lower() not in ('cardinality', 'frequency'): @@ -1085,7 +1097,7 @@ def get_refmodels(name=None, tag={}, fltr=None, limit=0, order=None, before=None if common_fields: logger.warning( 'while hoisting nested fields in "computed" to reference model, found %s common fields: %s', - len(common_fields), common_fields) + len(common_fields), common_fields) r.update(r['computed']) del r['computed'] @@ -1225,16 +1237,19 @@ def create_refmodel(jobs, name=None, tag={}, op_tags=[], to a list of tags for the ops, or use the wildcard (*): >>> r = eq.create_refmodel(jobs, tag='exp_name:linux_kernel', op_tags='*', methods= [es.modified_z_score]) - >>> r['id'], r['tags'], r['jobs'] - (11, {'exp_name': 'linux_kernel'}, [u'kern-6656-20190614-190245', u'kern-6656-20190614-191138', u'kern-6656-20190614-192044-outlier', u'kern-6656-20190614-194024']) + >>> r['id'], r['tags'], r['jobs'] # doctest: +ELLIPSIS + (11, {'exp_name': 'linux_kernel'}, [...]) - >>> r['op_tags'] - [{u'op_instance': u'4', u'op_sequence': u'4', u'op': u'build'}, {u'op_instance': u'5', u'op_sequence': u'5', u'op': u'clean'}, {u'op_instance': u'3', u'op_sequence': u'3', u'op': u'configure'}, {u'op_instance': u'1', u'op_sequence': u'1', u'op': u'download'}, {u'op_instance': u'2', u'op_sequence': u'2', u'op': u'extract'}] + >>> r['op_tags'] # doctest: +ELLIPSIS + [{..., u'op': u'build'}, {..., u'op': u'clean'}, ..., {..., u'op': u'extract'}] Below is an example of creating a refmodel using MV classifiers >>> from pyod.models.knn import KNN >>> from pyod.models.abod import ABOD - >>> r = eq.create_refmodel(['625172', '627922', '629337', '633144', '676007', '680181', '685000', '685003', '685016', '692544', '693147', '696127'], methods = [ABOD(), KNN()], features = ['cpu_time', 'duration', 'num_procs']) + >>> r = eq.create_refmodel( + ... ['625172', '627922', '629337', '633144', '676007', '680181', + ... '685000', '685003', '685016', '692544', '693147', '696127'], + ... methods = [ABOD(), KNN()], features = ['cpu_time', 'duration', 'num_procs']) WARNING: epmt_query: The jobs do not share identical tag values for "exp_name" and "exp_component" WARNING: The jobs do not share identical tag values for "exp_name" and "exp_component" 685000 ESM4_historical_D151 ocean_annual_rho2_1x1deg @@ -1249,8 +1264,8 @@ def create_refmodel(jobs, name=None, tag={}, op_tags=[], 633144 ESM4_historical_D151 ocean_month_rho2_1x1deg 676007 ESM4_historical_D151 ocean_month_rho2_1x1deg 680181 ESM4_historical_D151 ocean_month_rho2_1x1deg - >>> r - {'jobs': ['685000', '685003', '685016', '625172', '693147', '692544', '696127', '627922', '629337', '633144', '676007', '680181'], 'name': None, 'tags': {}, 'op_tags': [], 'computed': {'pyod.models.abod': {'cpu_time,duration,num_procs': -3.478362573453902e-40}, 'pyod.models.knn': {'cpu_time,duration,num_procs': 6014539197.113887}}, 'enabled': True, 'id': 6, 'created_at': datetime.datetime(2020, 2, 3, 17, 6, 59, 501012)} + >>> r # doctest: +ELLIPSIS + {'jobs': [...], 'name': None, 'tags': {}, 'op_tags': [], 'computed': {...}, ...} """ if not jobs or (not (orm_is_query(jobs)) and len(jobs) == 0) or (orm_is_query(jobs) and (jobs.count == 0)): @@ -1288,7 +1303,8 @@ def create_refmodel(jobs, name=None, tag={}, op_tags=[], logger.info("request to do PCA (pca=%s). Input features: %s", pca, features) if len(features) < 5: logger.warning( - 'Too few input features for PCA. Are you sure you did not want to set features=[] to enable selecting all available features?') + 'Too few input features for PCA. Are you sure you did not want to set ' + 'features=[] to enable selecting all available features?') from epmt.epmt_outliers import pca_feature_combine import numpy as np @@ -1345,7 +1361,8 @@ def create_refmodel(jobs, name=None, tag={}, op_tags=[], info_dict['pca'] = {'inp_features': orig_features, 'out_features': pca_features} # now save the ref model - r = orm_create(ReferenceModel, jobs=jobs, name=name, tags=tag, op_tags=op_tags, computed=computed, info_dict = info_dict, enabled=enabled) + r = orm_create(ReferenceModel, jobs=jobs, name=name, tags=tag, op_tags=op_tags, + computed=computed, info_dict = info_dict, enabled=enabled) orm_commit() if fmt == 'orm': return r @@ -1536,7 +1553,7 @@ def refmodel_set_active_metrics(ref_id, metrics): if unavailable_metrics: logger.warning( 'Ignoring metrics that are not available in the trained model: %s', - unavailable_metrics ) + unavailable_metrics) active_metrics = list(metrics_set & all_metrics) logger.info('Active metrics for model set to: %s', str(active_metrics)) info_dict = dict.copy(r.info_dict or {}) @@ -1660,24 +1677,27 @@ def get_ops(jobs, tags=[], exact_tag_only=False, combine=False, fmt='dict', op_d >>> ops = get_ops(['685000', '685003'],tags =['op:timavg', 'op:ncks'],combine=True, fmt='orm') >>> hl_op = ops[0] # ops only has one element when combine is True - >>> hl_op.start, hl_op.end, hl_op.duration, hl_op.num_runs() - (datetime.datetime(2019, 6, 15, 13, 38, 25, 618279), datetime.datetime(2019, 6, 15, 13, 42, 18, 345456), 40827030.00000001, 159) + >>> hl_op.start, hl_op.end, hl_op.duration, hl_op.num_runs() # doctest: +ELLIPSIS + (datetime.datetime(...), datetime.datetime(...), ..., 159) Rather than specifying the tags, we can just mention the key we care about. This will be auto-expanded: >>> op = get_ops(['685000', '685003'], tags = 'op', combine=True)[0] - DEBUG:epmt_query:expanding op for values ['splitvars', 'untar', 'dmput', 'ncatted', 'ncks', 'cp', 'timavg', 'hsmget', 'ncrcat', 'rm', 'fregrid', 'mv'] - DEBUG:epmt_query:tags: [{'op': 'splitvars'}, {'op': 'untar'}, {'op': 'dmput'}, {'op': 'ncatted'}, {'op': 'ncks'}, {'op': 'cp'}, {'op': 'timavg'}, {'op': 'hsmget'}, {'op': 'ncrcat'}, {'op': 'rm'}, {'op': 'fregrid'}, {'op': 'mv'}] - - - If we are lazy and only want the top-level ops (based on the tag of most importance): + DEBUG:epmt_query:expanding op for values ['splitvars', 'untar', 'dmput', + 'ncatted', 'ncks', 'cp', 'timavg', 'hsmget', 'ncrcat', 'rm', 'fregrid', 'mv'] + DEBUG:epmt_query:tags: [{'op': 'splitvars'}, {'op': 'untar'}, {'op': 'dmput'}, + {'op': 'ncatted'}, {'op': 'ncks'}, {'op': 'cp'}, {'op': 'timavg'}, + {'op': 'hsmget'}, {'op': 'ncrcat'}, {'op': 'rm'}, {'op': 'fregrid'}, {'op': 'mv'}] >>> ops = eq.get_ops(['685000', '685003'], tags = '') DEBUG:epmt_query:no tag specified, using tags: op - DEBUG:epmt_query:expanding op for values ['splitvars', 'untar', 'dmput', 'ncatted', 'ncks', 'cp', 'timavg', 'hsmget', 'ncrcat', 'rm', 'fregrid', 'mv'] - DEBUG:epmt_query:tags: [{'op': 'splitvars'}, {'op': 'untar'}, {'op': 'dmput'}, {'op': 'ncatted'}, {'op': 'ncks'}, {'op': 'cp'}, {'op': 'timavg'}, {'op': 'hsmget'}, {'op': 'ncrcat'}, {'op': 'rm'}, {'op': 'fregrid'}, {'op': 'mv'}] + DEBUG:epmt_query:expanding op for values ['splitvars', 'untar', 'dmput', + 'ncatted', 'ncks', 'cp', 'timavg', 'hsmget', 'ncrcat', 'rm', 'fregrid', 'mv'] + DEBUG:epmt_query:tags: [{'op': 'splitvars'}, {'op': 'untar'}, {'op': 'dmput'}, + {'op': 'ncatted'}, {'op': 'ncks'}, {'op': 'cp'}, {'op': 'timavg'}, + {'op': 'hsmget'}, {'op': 'ncrcat'}, {'op': 'rm'}, {'op': 'fregrid'}, {'op': 'mv'}] >>> len(ops) 12 ''' @@ -1819,7 +1839,9 @@ def get_op_metrics(jobs, tags=[], exact_tags_only=False, group_by_tag=False, fmt # # That does not work with the GROUP BY clause. By having order=0, we use the # # implicit order and works with GROUP BY # procs = get_procs(jobs, order=0, tags = t, exact_tag_only = exact_tags_only, fmt='orm') - # procs_grp_by_job = select((p.job, count(p.id), min(p.start), max(p.end), sum(p.duration), sum(p.cpu_time), sum(p.numtids), group_concat(p.threads_sums, sep='@@@')) for p in procs) + # procs_grp_by_job = select((p.job, count(p.id), min(p.start), max(p.end), + # sum(p.duration), sum(p.cpu_time), sum(p.numtids), + # group_concat(p.threads_sums, sep='@@@')) for p in procs) for row in procs_grp_by_job: (j, nprocs, start, end, duration, excl_cpu, ntids, threads_sums_str) = row @@ -1838,7 +1860,7 @@ def get_op_metrics(jobs, tags=[], exact_tags_only=False, group_by_tag=False, fmt # nop since we already computed that in 'duration' using the ORM pass else: - raise ValueError("Do not know how to handle op_duration_method: {}".format(op_duration_method)) + raise ValueError(f"Do not know how to handle op_duration_method: {op_duration_method}") # convert from giant string to array of strings where each list # list element is a json of a threads_sums dict _l1 = threads_sums_str.split('@@@') @@ -2104,7 +2126,8 @@ def retire_jobs(ndays=settings.retire_jobs_ndays, skip_unprocessed=False, dry_ru return tot_num_deleted # @db_session -# def dm_calc(jobs = [], tags = ['op:hsmput', 'op:dmget', 'op:untar', 'op:mv', 'op:dmput', 'op:hsmget', 'op:rm', 'op:cp']): +# def dm_calc(jobs = [], +# tags = ['op:hsmput', 'op:dmget', 'op:untar', 'op:mv', 'op:dmput', 'op:hsmget', 'op:rm', 'op:cp']): # """ # Data-migration calculation helper for a collection of jobs (deprecated) # @@ -2118,7 +2141,9 @@ def retire_jobs(ndays=settings.retire_jobs_ndays, skip_unprocessed=False, dry_ru # num_jobs = jobs.count() # logger.debug('number of jobs: {0}'.format(num_jobs)) # if (num_jobs > 100): -# logger.warning('job count ({0}) > 100: it is recommended to use dm_calc_iter instead for a lower memory footprint and faster time-to-solution'.format(num_jobs)) +# logger.warning( +# 'job count ({0}) > 100: it is recommended to use dm_calc_iter instead ' +# 'for a lower memory footprint and faster time-to-solution'.format(num_jobs)) # tags = tags_list(tags) # dm_ops_df = get_op_metrics(jobs, tags = tags, group_by_tag = True) # jobs_cpu_time = 0.0 @@ -2201,7 +2226,9 @@ def ops_costs(jobs=[], tags=['op:hsmput', 'op:dmget', 'op:untar', 'op:mv', # scenarios we use the faster "sum" aggregation available in the # database itself agg_method = 'sum-minus-overlap' if metric == 'duration' else "sum" - logger.debug('Using slower (but accurate) computation for "duration" that avoids double-counting overlapping processes in a job. Try using "cpu_time" for faster results') + logger.debug('Using slower (but accurate) computation for "duration" that avoids ' + 'double-counting overlapping processes in a job. ' + 'Try using "cpu_time" for faster results') for j in jobs: n += 1 logger.debug('processing %s (%s/%s)', j.jobid, n, njobs) @@ -2500,9 +2527,9 @@ def analyze_comparable_jobs(jobids, check_comparable=True, keys=('exp_name', 'ex # finally mark the jobs as analyzed for j in jobids: set_job_analyses(j, analyses, replace=True) - msg = 'analyzed {0} jobs'.format(len(jobids)) + msg = f'analyzed {len(jobids)} jobs' for k in analyses: - msg += '{0} runs of {1}; '.format(len(analyses[k]), k) + msg += f'{len(analyses[k])} runs of {k}; ' logger.info(msg) return num_analyses_runs @@ -3198,7 +3225,7 @@ def verify_collection(c, prefix='', ignore_empty=True): # Are we dealing with a Process dict? # If so we can have a more meaningful label if 'id' in keys: - prefix += 'Process[{}] >> '.format(c['id']) + prefix += f'Process[{c["id"]}] >> ' for k in keys: val = c[k] @@ -3209,21 +3236,21 @@ def verify_collection(c, prefix='', ignore_empty=True): continue if not (ignore_empty) and (val is None): - err.append('{}{} is empty'.format(prefix, k)) + err.append(f'{prefix}{k} is empty') elif type(val) in (int, float): ret = verify_num(val) if ret: - err.append('{}{} {}'.format(prefix, k, ret)) + err.append(f'{prefix}{k} {ret}') elif isinstance(val, datetime): if val < datetime(2019, 1, 1): - err.append('{}{} {}'.format(prefix, k, 'contains an invalid timestamp (too old)')) + err.append(f'{prefix}{k} contains an invalid timestamp (too old)') elif val > now: - err.append('{}{} {}'.format(prefix, k, 'contains an invalid timestamp (in the future)')) + err.append(f'{prefix}{k} contains an invalid timestamp (in the future)') # elif type(val) in (dict, list, TrackedList, TrackedDict): elif type(val) in (dict, list): # recurse underneath - err += verify_collection(val, prefix="{}{} >> ".format(prefix, k)) + err += verify_collection(val, prefix=f"{prefix}{k} >> ") return err jobs = get_jobs(jobs, fmt='dict') diff --git a/src/epmt/epmt_rootcause.py b/src/epmt/epmt_rootcause.py index 7113cdc07..5d011788a 100644 --- a/src/epmt/epmt_rootcause.py +++ b/src/epmt/epmt_rootcause.py @@ -91,7 +91,7 @@ def rootcause(ref, input, features, methods=[rootcause_zscore]): if __name__ == "__main__": # Synthesize 10 feature names n_features = 6 - features = ['%c' % x for x in range(97, 97 + n_features)] + features = [f'{x:c}' for x in range(97, 97 + n_features)] print("Features:\n", features) def multiple(): diff --git a/src/epmt/epmt_stat.py b/src/epmt/epmt_stat.py index f8603d670..7868a6b84 100644 --- a/src/epmt/epmt_stat.py +++ b/src/epmt/epmt_stat.py @@ -34,7 +34,7 @@ def get_classifier_name(c): return c.__name__ if hasattr(c, '__module__'): return c.__module__ - raise 'Could not determine classifier name for {}'.format(c) + raise f'Could not determine classifier name for {c}' def is_classifier_mv(c): @@ -107,6 +107,7 @@ def z_score(ys, params=()): # issue by using np.nan_to_num import warnings warnings.filterwarnings("ignore", category=RuntimeWarning) + logger = getLogger(__name__) # you can use other name logger.debug('scoring using %s', 'z_score') ys = np.array(ys) if params: @@ -170,6 +171,7 @@ def iqr(ys, params=()): >>> outliers array([0, 0, 0, 0, 0, 0, 1]) ''' + logger = getLogger(__name__) # you can use other name logger.debug('scoring using %s', 'iqr') ys = np.array(ys) span = [25, 75] @@ -207,6 +209,7 @@ def modified_z_score(ys, params=()): params if passed in, is of the form (max, median, median_abs_dev) We will ignore params(0) as that's the max z_score in the ref_model ''' + logger = getLogger(__name__) # you can use other name logger.debug('scoring using %s', 'modified_z_score') median_y = params[1] if params else np.median(ys) if params: @@ -531,11 +534,9 @@ def mvod_scores_using_model(inp, model_inp, classifier, threshold=None): logger.debug('MVOD %s (threshold=%s)', c_name, threshold) from math import isclose if not isclose(model_score_max, threshold, rel_tol=1e-2): - logger.warning( - 'MVOD %s is not stable. We computed a threshold %s, while the passed threshold from the saved model was %s', - c_name, - model_score_max, - threshold) + logger.warning('MVOD %s is not stable. We computed a threshold %s, ' + 'while the passed threshold from the saved model was %s', + c_name, model_score_max, threshold) for i in range(inp_nrows): # pick the ith row row = inp[i] @@ -684,6 +685,7 @@ def pca_stat(inp_features, desired=2): from sklearn.preprocessing import StandardScaler from sklearn.decomposition import PCA + logger = getLogger(__name__) # you can use other name logger.debug('input feature array shape: %s', inp_features.shape) if np.isnan(inp_features).any(): raise ValueError('input contains at-least one non-numeric (nan) element') @@ -817,7 +819,7 @@ def kstest_uniform(d): # The test below requires at least 20 elements tests['norm'].append(('D\'Agostino', normaltest)) if dist not in tests: - raise ValueError('We only support the following distributions: {}'.format(tests.keys())) + raise ValueError(f'We only support the following distributions: {tests.keys()}') logger.debug('Testing for %s distribution', dist) for (test, f) in tests[dist]: @@ -901,10 +903,10 @@ def get_modes(X, max_modes=10): modes_by_silhouette_method = np.argmax(km_silhouette) + 2 logger.debug('optimal clustering according to silhouette method: %s', modes_by_silhouette_method) if modes_by_elbow_method != modes_by_silhouette_method: - logger.warning( - 'Elbow and silhouette methods gave different mode counts -- %s and %s. Usually this means you might have a single mode or your data was not drawn from normal distributions', - modes_by_elbow_method, - modes_by_silhouette_method) + logger.warning('Elbow and silhouette methods gave different mode counts -- ' + '%s and %s. Usually this means you might have a single mode ' + 'or your data was not drawn from normal distributions', + modes_by_elbow_method, modes_by_silhouette_method) num_modes = 1 km = KMeans(n_clusters=num_modes, random_state=0).fit(X_scaled) diff --git a/src/epmt/epmtlib.py b/src/epmt/epmtlib.py index efd579d36..cf436124f 100644 --- a/src/epmt/epmtlib.py +++ b/src/epmt/epmtlib.py @@ -32,7 +32,7 @@ def version(): def version_str(terse=False): - return __version__ if terse else "EPMT {0}".format(__version__) + return __version__ if terse else f"EPMT {__version__}" def get_username(): @@ -93,7 +93,9 @@ def epmt_logging_init(intlvl=0, check=False, log_pid=False): consoleHandler = logging.StreamHandler() consoleFormatter = logging.Formatter( - "[%(asctime)-19.19s, %(process)d] %(levelname)7.7s: %(name)s: %(message)s" if log_pid else "%(asctime)-19.19s %(levelname)7.7s: %(name)s: %(message)s") + ("[%(asctime)-19.19s, %(process)d] %(levelname)7.7s: %(name)s: %(message)s" + if log_pid else + "%(asctime)-19.19s %(levelname)7.7s: %(name)s: %(message)s")) consoleHandler.setFormatter(consoleFormatter) epmt_logger.addHandler(consoleHandler) @@ -337,7 +339,7 @@ def tag_dict_to_string(tag, delim=';', sep=':'): ''' if isinstance(tag, str): return tag - return delim.join(["{}{}{}".format(k, sep, tag[k]) for k in sorted(tag.keys())]) + return delim.join([f"{k}{sep}{tag[k]}" for k in sorted(tag.keys())]) def tags_list(tags): @@ -748,7 +750,7 @@ def check_fix_metadata(raw_metadata): def check_pid(pid): """Check whether pid exists""" if pid < 0: - return (False, 'Invalid PID: {0}'.format(pid)) + return (False, f'Invalid PID: {pid}') from os import kill try: kill(pid, 0) @@ -756,7 +758,7 @@ def check_pid(pid): from errno import ESRCH, EPERM if err.errno == ESRCH: # ESRCH == No such process - return (False, 'No such process (PID: {0})'.format(pid)) + return (False, f'No such process (PID: {pid})') elif err.errno == EPERM: # EPERM clearly means there's a process but we cannot # send a signal to it @@ -803,7 +805,8 @@ def conv_to_datetime(t): try: retval = datetime.strptime(t, '%m/%d/%Y %H:%M') except Exception as e: - logger.error('could not convert string to datetime: %s', e) + logger = getLogger(__name__) + logger.error('could not convert string to datetime: %s', str(e)) return None elif type(t) in (int, float): if t > 0: @@ -929,7 +932,8 @@ def dframe_encode_features(df, features=[], reversible=False): if reversible: logger.warning( - 'You have enabled "reversible". Be warned that the encoded feature columns can contain some very large integers') + 'You have enabled "reversible". Be warned that the encoded feature ' + 'columns can contain some very large integers') encoded_df = df.copy() encoded_features = [] logger.debug('encoding feature columns: %s', features) @@ -991,7 +995,7 @@ def find_files_in_dir(path, pattern='*.tgz', recursive=False): RETURNS: List of files that match ''' from glob import glob - pathname = '{}/{}{}'.format(path, '**/' if recursive else '', pattern) + pathname = f'{path}/{"**/" if recursive else ""}{pattern}' return glob(pathname, recursive=recursive) @@ -1104,7 +1108,7 @@ def docs_module_index(mod, fmt=None): out_str = "" for section in sorted(sections.keys()): section_calls = sections[section] - out_str += "\n\nSection::{}\n".format(section) + out_str += f"\n\nSection::{section}\n" out_str += "\n".join([fmt_string.format(o[0], o[1]) for o in section_calls]) return out_str @@ -1145,10 +1149,10 @@ def log_func(*func_args, **func_kwargs): # the module is prepended automatically by our logging format # as we use getLogger with the module name # logger.info('{}({}{}{})'.format(func.__name__, - args_str = ", ".join([str(x) for x in func_args]) - kwargs_str = ",".join(["%s=%s" % (k, v) for (k, v) in func_kwargs.items()]) - sep = "," if func_kwargs else "" - logger.debug('%s(%s%s%s)', func.__name__, args_str, sep, kwargs_str) + logger.debug('%s(%s%s%s)', func.__name__, + ", ".join([str(x) for x in func_args]), + "," if func_kwargs else "", + ",".join([f"{k}={v}" for (k, v) in func_kwargs.items()])) # now call the actual function with its arguments (if any) return func(*func_args, **func_kwargs) return log_func @@ -1198,7 +1202,7 @@ def csv_probe_format(f): # the second element is a list of CSV column names return ('1', s.split('\n')[0].split(',')) # if we reached here, then we don't understand the CSV format - raise ValueError("CSV file -- {} -- has an unknown file format. Is it corrupted?".format(f.name)) + raise ValueError(f"CSV file -- {f.name} -- has an unknown file format. Is it corrupted?") def set_signal_handlers(signals=[], handler=None): diff --git a/src/epmt/orm/op.py b/src/epmt/orm/op.py index 074a3bce1..211f94a14 100644 --- a/src/epmt/orm/op.py +++ b/src/epmt/orm/op.py @@ -107,7 +107,7 @@ def duration(self): elif self.op_duration_method == "finish-minus-start": self._duration = round((self.finish - self.start).total_seconds() * 1e6, 1) else: - raise ValueError("Do not know how to handle op_duration_method: {}".format(self.op_duration_method)) + raise ValueError(f"Do not know how to handle op_duration_method: {self.op_duration_method}") return self._duration @property @@ -115,6 +115,8 @@ def proc_sums(self): if self._proc_sums is None: from epmt.epmt_query import get_op_metrics from epmt.epmtlib import sum_dicts_list + from logging import getLogger + logger = getLogger(__name__) logger.debug('getting op_metrics for jobs=%s, tags=%s', self.jobs, self.tags) op_metrics = get_op_metrics( jobs=self.jobs, diff --git a/src/epmt/orm/sqlalchemy/general.py b/src/epmt/orm/sqlalchemy/general.py index 49602197b..f08ea1833 100644 --- a/src/epmt/orm/sqlalchemy/general.py +++ b/src/epmt/orm/sqlalchemy/general.py @@ -236,21 +236,20 @@ def orm_delete_jobs(jobs, use_orm=False): logger.warning( 'process job is in staging- removing process rows corresponding to the job in the staging table') (first_proc_id, last_proc_id) = j.info_dict['procs_staging_ids'] - logger.warning('first and last proc_ids pulled are: %s and %s', first_proc_id, last_proc_id) - stmts.append( "DELETE FROM processes_staging WHERE id BETWEEN {} AND {};\n".format( first_proc_id, - last_proc_id ) ) + logger.warning(f'first and last proc_ids pulled are: {first_proc_id} and {last_proc_id}') + stmts.append( f"DELETE FROM processes_staging WHERE id BETWEEN {first_proc_id} AND {last_proc_id};\n" ) else: logger.debug('process job is NOT in staging table- no processes_staging targets will be added to query') stmts.append( 'DELETE FROM ancestor_descendant_associations WHERE EXISTS (' + 'SELECT ad.* from ancestor_descendant_associations ad INNER JOIN ' + 'processes p ON (ad.ancestor = p.id OR ad.descendant = p.id) ' + - 'WHERE p.jobid = \'{0}\')'.format(jobid) ) + f'WHERE p.jobid = \'{jobid}\')') stmts.append( 'DELETE FROM host_job_associations ' + - 'WHERE host_job_associations.jobid = \'{0}\''.format(jobid) ) + f'WHERE host_job_associations.jobid = \'{jobid}\'') stmts.append( 'DELETE FROM refmodel_job_associations '+ - 'WHERE refmodel_job_associations.jobid = \'{0}\''.format(jobid) ) - stmts.append( 'DELETE FROM processes WHERE processes.jobid = \'{0}\''.format(jobid) ) - stmts.append( 'DELETE FROM jobs WHERE jobs.jobid = \'{0}\''.format(jobid) ) + f'WHERE refmodel_job_associations.jobid = \'{jobid}\'') + stmts.append( f'DELETE FROM processes WHERE processes.jobid = \'{jobid}\'') + stmts.append( f'DELETE FROM jobs WHERE jobs.jobid = \'{jobid}\'') # now try executing the statement we put together above try: @@ -606,22 +605,15 @@ def _attribute_filter(qs, attr, target, exact_match=False, model=None, conv_to_s if conv_to_str or (isinstance(v, str)): qs = qs.filter( text( - "cast(json_extract({0}.{1}, '$.{2}') as text) = '{3}'".format( - model.__tablename__, - attr, - k, - v)) if using_sqlite else ( + f"cast(json_extract({model.__tablename__}.{attr}," + f" '$.{k}') as text) = '{v}'") if using_sqlite else ( getattr( model, attr)[k].astext == str(v))) else: qs = qs.filter( text( - "json_extract({0}.{1}, '$.{2}') = {3}".format( - model.__tablename__, - attr, - k, - v)) if using_sqlite else ( + f"json_extract({model.__tablename__}.{attr}, '$.{k}') = {v}") if using_sqlite else ( getattr( model, attr)[k] == v)) @@ -683,9 +675,9 @@ def orm_dump_schema(show_attributes=True): print('\nTABLE', t.name) for c in t.columns: try: - print('%20s\t%10s' % (c.name, str(c.type))) + print(f'{c.name:>20}\t{str(c.type):>10}') except BaseException: - print('%20s\t%10s' % (c.name, str(c.type.__class__.__name__.split('.')[-1]))) + print(f'{c.name:>20}\t{str(c.type.__class__.__name__.split(".")[-1]):>10}') else: m = MetaData() m.reflect(engine) # Read what exists on db so we can have full picture @@ -704,11 +696,11 @@ def get_mapper(tbl): ] if len(mappers) > 1: raise ValueError( - "Multiple mappers found for table '%s'." % tbl.name ) + f"Multiple mappers found for table '{tbl.name}'." ) if not mappers: raise ValueError( - "Could not get mapper for table '%s'." % tbl.name ) + f"Could not get mapper for table '{tbl.name}'." ) return mappers[0] @@ -841,7 +833,7 @@ def migrate_db(): if updated_version != epmt_schema_head: logger.warning( 'Database migration failed. Current schema version is %s, while head is %s', - updated_version, epmt_schema_head) + updated_version, epmt_schema_head) else: logger.info('Database successfully migrated to: %s', epmt_schema_head) return epmt_schema_head == updated_version diff --git a/src/epmt/orm/sqlalchemy/models.py b/src/epmt/orm/sqlalchemy/models.py index cd3f1e761..23a8cd073 100644 --- a/src/epmt/orm/sqlalchemy/models.py +++ b/src/epmt/orm/sqlalchemy/models.py @@ -37,7 +37,7 @@ class CommonMeta(DeclarativeMeta): def __getitem__(cls, index): obj = orm_get(cls, index) if obj is None: - raise KeyError('{0}[{1}] could not be found'.format(cls.__name__, index)) + raise KeyError(f'{cls.__name__}[{index}] could not be found') else: return obj @@ -93,7 +93,7 @@ class User(with_metaclass(CommonMeta, Base)): # representation def __repr__(self): try: - return "User['%s']" % (self.name) + return f"User['{self.name}']" except: return "User[]" @@ -120,7 +120,7 @@ class Host(with_metaclass(CommonMeta, Base)): # representation def __repr__(self): try: - return "Host['%s']" % (self.name) + return f"Host['{self.name}']" except: return "Host[]" @@ -155,7 +155,7 @@ class ReferenceModel(with_metaclass(CommonMeta, Base)): # representation def __repr__(self): try: - return "ReferenceModel['%s']" % (self.id) + return f"ReferenceModel['{self.id}']" except: return "ReferenceModel[]" @@ -215,7 +215,7 @@ class Job(with_metaclass(CommonMeta, Base)): # representation def __repr__(self): try: - return "Job['%s']" % (self.jobid) + return f"Job['{self.jobid}']" except: return "Job[]" @@ -239,7 +239,7 @@ class UnprocessedJob(with_metaclass(CommonMeta, Base)): # representation def __repr__(self): try: - return "UnprocessedJob['%s']" % (self.jobid) + return f"UnprocessedJob['{self.jobid}']" except: return "UnprocessedJob[]" @@ -300,8 +300,10 @@ class Process(with_metaclass(CommonMeta, Base)): # relationships host = relationship('Host', back_populates="processes") children = relationship('Process', backref=backref('parent', remote_side=[id])) - #ancestors = relationship('ProcessAssociation',backref='descendants', primaryjoin=id==ProcessAssociation.fk_ancestor) - #descendants = relationship('ProcessAssociation',backref='ancestors', primaryjoin=id==ProcessAssociation.fk_descendant ) + #ancestors = relationship('ProcessAssociation',backref='descendants', + # primaryjoin=id==ProcessAssociation.fk_ancestor) + #descendants = relationship('ProcessAssociation',backref='ancestors', + # primaryjoin=id==ProcessAssociation.fk_descendant ) ancestors = relationship('Process', backref='descendants', secondary=ancestor_descendant_associations_table, @@ -311,7 +313,7 @@ class Process(with_metaclass(CommonMeta, Base)): # representation def __repr__(self): try: - return "Process[%d]" % (self.id) + return f"Process[{self.id}]" except: return "Process[]" diff --git a/src/epmt/test/test_anysh.py b/src/epmt/test/test_anysh.py index 84defe05b..ff7ae0c20 100644 --- a/src/epmt/test/test_anysh.py +++ b/src/epmt/test/test_anysh.py @@ -52,7 +52,8 @@ def test_run_auto(self): self.assertEqual(0, results) def test_monolithic(self): - from epmt.epmt_cmds import epmt_source, epmt_start_job, epmt_dump_metadata, epmt_run, epmt_stop_job, epmt_stage, epmt_submit + from epmt.epmt_cmds import (epmt_source, epmt_start_job, epmt_dump_metadata, + epmt_run, epmt_stop_job, epmt_stage, epmt_submit) with capture() as (out, err): # TODO see if this works # results = epmt_check() diff --git a/src/epmt/test/test_cmds.py b/src/epmt/test/test_cmds.py index 37fa58676..27a590d6b 100755 --- a/src/epmt/test/test_cmds.py +++ b/src/epmt/test/test_cmds.py @@ -26,9 +26,9 @@ def setUpModule(): # print('\n' + str(settings.db_params)) setup_db(settings) do_cleanup() - datafiles = '{}/test/data/misc/685000.tgz'.format(install_root) + datafiles = f'{install_root}/test/data/misc/685000.tgz' # datafiles='{}/test/data/misc/685???.tgz'.format(install_root) - print('setUpModule: submitting to db {0}'.format(datafiles)) + print(f'setUpModule: submitting to db {datafiles}') settings.post_process_job_on_ingest = True with capture() as (out, err): epmt_submit(glob(datafiles), dry_run=False) @@ -109,7 +109,7 @@ def test_daemon_ingest(self): settings.post_process_job_on_ingest = False with capture() as (out, err): self.assertFalse(daemon_loop( - nullcontext(), maxiters=1, ingest='{}/test/data/daemon/ingest'.format(install_root), + nullcontext(), maxiters=1, ingest=f'{install_root}/test/data/daemon/ingest', post_process=False, analyze=False, retire=False, keep=True, recursive=False)) # by now the files should be in the DB @@ -117,8 +117,8 @@ def test_daemon_ingest(self): self.assertEqual(set(eq.get_jobs(['691201', '692544'], fmt='terse')), set({'691201', '692544'})) # make sure the files aren't removed (since we used the "keep" option) - self.assertTrue(path.exists('{}/test/data/daemon/ingest/691201.tgz'.format(install_root))) - self.assertTrue(path.exists('{}/test/data/daemon/ingest/692544.tgz'.format(install_root))) + self.assertTrue(path.exists(f'{install_root}/test/data/daemon/ingest/691201.tgz')) + self.assertTrue(path.exists(f'{install_root}/test/data/daemon/ingest/692544.tgz')) # @unittest.skipIf(len(eq.get_unprocessed_jobs()) == 0, 'unprocessed jobs in database') @db_session @@ -131,7 +131,7 @@ def test_daemon_post_process(self): settings.post_process_job_on_ingest = False with capture() as (out, err): - self.assertTrue(epmt_submit(glob('{}/test/data/daemon/627919.tgz'.format(install_root)), dry_run=False)) + self.assertTrue(epmt_submit(glob(f'{install_root}/test/data/daemon/627919.tgz'), dry_run=False)) up_jobs = eq.get_unprocessed_jobs() self.assertTrue(UnprocessedJob['627919']) @@ -242,7 +242,7 @@ def test_stage(self): if path.exists(errorfile): remove(errorfile) tempdir = mkdtemp(prefix='epmt_', dir=gettempdir()) - copytree("{}/test/data/corrupted_csv".format(install_root), tempdir + "/corrupted_csv") + copytree(f"{install_root}/test/data/corrupted_csv", tempdir + "/corrupted_csv") with capture() as (out, err): retval = epmt_stage([tempdir + "/corrupted_csv"], keep_going=False) self.assertTrue(retval == False, "corrupted CSV files, should have returned False") @@ -255,7 +255,7 @@ def test_stage(self): remove(errorfile) tempdir = mkdtemp(prefix='epmt_', dir=gettempdir()) - copytree("{}/test/data/corrupted_csv".format(install_root), tempdir + "/corrupted_csv") + copytree(f"{install_root}/test/data/corrupted_csv", tempdir + "/corrupted_csv") with capture() as (out, err): retval = epmt_stage([tempdir + "/corrupted_csv"], keep_going=True) self.assertTrue(retval, "corrupted CSV files but keep_going, should have returned True") @@ -297,7 +297,7 @@ def test_yyy_retire(self): @unittest.skipUnless(orm_in_memory(), 'skip on persistent database') def test_zz_drop_db(self): # submit a job to the db we just cleaned out... oops! - datafiles = '{}/test/data/misc/685000.tgz'.format(install_root) + datafiles = f'{install_root}/test/data/misc/685000.tgz' # settings.post_process_job_on_ingest = True with capture() as (out, err): epmt_submit(glob(datafiles), dry_run=False) diff --git a/src/epmt/test/test_db_migration.py b/src/epmt/test/test_db_migration.py index 4da1dba0d..8447a94e3 100755 --- a/src/epmt/test/test_db_migration.py +++ b/src/epmt/test/test_db_migration.py @@ -32,11 +32,11 @@ def test_create_and_apply_migration(self): try: migration_file = path.join( 'epmt_migrations', 'versions', - '{}_add_active_column_to_users_table.py'.format(rev_id)) + f'{rev_id}_add_active_column_to_users_table.py') with capture() as (out, err): alembic.config.main(argv=["revision", "--rev-id", rev_id, "-m", "add active column to users table"]) s = out.getvalue() - self.assertRegex(s, '.*{}_add_active_column_to_users_table.py .* done'.format(rev_id)) + self.assertRegex(s, f'.*{rev_id}_add_active_column_to_users_table.py .* done') self.assertTrue(path.isfile(migration_file)) from epmt.orm import migrate_db, get_db_schema_version with capture() as (out, err): diff --git a/src/epmt/test/test_explore.py b/src/epmt/test/test_explore.py index 2ec1a80fd..0c0afa44a 100755 --- a/src/epmt/test/test_explore.py +++ b/src/epmt/test/test_explore.py @@ -19,8 +19,8 @@ def do_cleanup(): def setUpModule(): setup_db(settings) do_cleanup() - datafiles = '{}/test/data/query/68500[03].tgz'.format(install_root) - print('setUpModdule: importing {0}'.format(datafiles)) + datafiles = f'{install_root}/test/data/query/68500[03].tgz' + print(f'setUpModdule: importing {datafiles}') epmt_submit(sorted(glob(datafiles)), dry_run=False) diff --git a/src/epmt/test/test_lib.py b/src/epmt/test/test_lib.py index f98b39070..d6378b01b 100755 --- a/src/epmt/test/test_lib.py +++ b/src/epmt/test/test_lib.py @@ -18,8 +18,8 @@ def test_sqlite_json_support(self): sqlite3.sqlite_version_info > ( 3, 9), - 'SQLite version {0} is too old and does not have JSON1 extensions. You need version 3.9 or later (with JSON1 extensions enabled)'.format( - sqlite3.sqlite_version)) + f'SQLite version {sqlite3.sqlite_version} is too old and does not have JSON1 extensions. ' + 'You need version 3.9 or later (with JSON1 extensions enabled)') def test_dict_filter(self): d = {'abc': 10, 'def': 20, '_ghi': 30, 'LS_COLORS': 'xyz'} diff --git a/src/epmt/test/test_outliers.py b/src/epmt/test/test_outliers.py index 014177acb..12a1b2a73 100755 --- a/src/epmt/test/test_outliers.py +++ b/src/epmt/test/test_outliers.py @@ -29,7 +29,7 @@ def do_cleanup(): def setUpModule(): setup_db(settings) do_cleanup() - datafiles = '{}/test/data/outliers/*.tgz'.format(install_root) + datafiles = f'{install_root}/test/data/outliers/*.tgz' # print('setUpModule: importing {0}'.format(datafiles)) environ['EPMT_TZ'] = 'Asia/Kolkata' with capture() as (out, err): @@ -69,15 +69,16 @@ def test_feature_distributions(self): jobs = eq.get_jobs(tags='exp_name:linux_kernel', fmt='terse') fdist = eod.get_feature_distributions(jobs, features=[]) self.assertEqual(list(zip(fdist.keys(), fdist.values())), - [('PERF_COUNT_SW_CPU_CLOCK', 'unknown'), ('cancelled_write_bytes', 'unknown'), ('cpu_time', 'unknown'), - ('delayacct_blkio_time', 'unknown'), ('duration', 'unknown'), ('exitcode', 'unknown'), - ('guest_time', 'unknown'), ('inblock', 'norm'), ('invol_ctxsw', 'unknown'), - ('majflt', 'unknown'), ('minflt', 'norm'), ('num_procs', 'unknown'), - ('num_threads', 'unknown'), ('outblock', 'unknown'), ('processor', 'unknown'), - ('rchar', 'norm'), ('rdtsc_duration', 'unknown'), ('read_bytes', 'norm'), - ('rssmax', 'norm'), ('syscr', 'norm'), ('syscw', 'norm'), + [('PERF_COUNT_SW_CPU_CLOCK', 'unknown'), ('cancelled_write_bytes', 'unknown'), + ('cpu_time', 'unknown'), ('delayacct_blkio_time', 'unknown'), ('duration', 'unknown'), + ('exitcode', 'unknown'), ('guest_time', 'unknown'), ('inblock', 'norm'), + ('invol_ctxsw', 'unknown'), ('majflt', 'unknown'), ('minflt', 'norm'), + ('num_procs', 'unknown'), ('num_threads', 'unknown'), ('outblock', 'unknown'), + ('processor', 'unknown'), ('rchar', 'norm'), ('rdtsc_duration', 'unknown'), + ('read_bytes', 'norm'), ('rssmax', 'norm'), ('syscr', 'norm'), ('syscw', 'norm'), ('systemtime', 'unknown'), ('time_oncpu', 'unknown'), ('time_waiting', 'unknown'), - ('timeslices', 'unknown'), ('usertime', 'unknown'), ('vol_ctxsw', 'norm'), ('wchar', 'norm'), ('write_bytes', 'norm')]) + ('timeslices', 'unknown'), ('usertime', 'unknown'), ('vol_ctxsw', 'norm'), + ('wchar', 'norm'), ('write_bytes', 'norm')]) @db_session def test_outlier_jobs(self): @@ -215,8 +216,9 @@ def test_outlier_jobs_trained(self): # let's test using IQR and z-score import epmt.epmt_stat as es for m in [es.iqr, es.z_score]: - r = eq.create_refmodel(['kern-6656-20190614-190245', 'kern-6656-20190614-191138', 'kern-6656-20190614-194024'], - methods = [m], fmt='orm') + r = eq.create_refmodel( + ['kern-6656-20190614-190245', 'kern-6656-20190614-191138', 'kern-6656-20190614-194024'], + methods = [m], fmt='orm') df, _ = eod.detect_outlier_jobs(['kern-6656-20190614-190245', 'kern-6656-20190614-191138', 'kern-6656-20190614-192044-outlier', 'kern-6656-20190614-194024'], methods = [m], trained_model=r.id) @@ -231,7 +233,8 @@ def test_outlier_jobs_trained_mvod(self): r = eq.create_refmodel( ['kern-6656-20190614-190245', 'kern-6656-20190614-191138','kern-6656-20190614-194024'], methods = mvod_classifiers(), fmt='orm' ) (df, _) = eod.detect_outlier_jobs( - ['kern-6656-20190614-190245', 'kern-6656-20190614-191138', 'kern-6656-20190614-192044-outlier', 'kern-6656-20190614-194024'], + ['kern-6656-20190614-190245', 'kern-6656-20190614-191138', + 'kern-6656-20190614-192044-outlier', 'kern-6656-20190614-194024'], trained_model = r.id, methods = mvod_classifiers()) df.sort_values(by=['jobid'], inplace=True) # MCD succeeds on Python 3.10+ but not 3.9, so the outlier count @@ -285,8 +288,10 @@ def test_outlier_ops_trained(self): self.assertEqual(uniq_tags, sorted_tags) parts = {frozen_dict(loads(k)): v for k, v in parts.items()} - self.assertEqual(parts[frozen_dict({"op_instance": "4", "op_sequence": "4", "op": "build"})], (set( - ['kern-6656-20190614-194024', 'kern-6656-20190614-190245', 'kern-6656-20190614-191138']), set(['kern-6656-20190614-192044-outlier']))) + self.assertEqual( + parts[frozen_dict({"op_instance": "4", "op_sequence": "4", "op": "build"})], + (set(['kern-6656-20190614-194024', 'kern-6656-20190614-190245', 'kern-6656-20190614-191138']), + set(['kern-6656-20190614-192044-outlier']))) # now also use the outlier job while creating the refmodel # this way, there should be NO outlier ops @@ -302,15 +307,18 @@ def test_outlier_ops_trained(self): self.assertEqual(df.shape, (4, 5)) self.assertEqual(len(parts), 1) parts = {frozen_dict(loads(k)): v for k, v in parts.items()} - self.assertEqual(parts[frozen_dict({"op_instance": "4", "op_sequence": "4", "op": "build"})], (set( - ['kern-6656-20190614-194024', 'kern-6656-20190614-190245', 'kern-6656-20190614-191138']), set(['kern-6656-20190614-192044-outlier']))) + self.assertEqual( + parts[frozen_dict({"op_instance": "4", "op_sequence": "4", "op": "build"})], + (set(['kern-6656-20190614-194024', 'kern-6656-20190614-190245', 'kern-6656-20190614-191138']), + set(['kern-6656-20190614-192044-outlier']))) @db_session def test_outlier_ops_trained_mvod(self): from epmt.epmt_stat import mvod_classifiers jobs = eq.get_jobs(tags='exp_name:linux_kernel', fmt='terse') model_jobs = [ j for j in jobs if not 'outlier' in j ] - self.assertEqual(set(model_jobs), set(['kern-6656-20190614-190245', 'kern-6656-20190614-191138', 'kern-6656-20190614-194024'])) + self.assertEqual(set(model_jobs), + set(['kern-6656-20190614-190245', 'kern-6656-20190614-191138', 'kern-6656-20190614-194024'])) r = eq.create_refmodel(model_jobs, op_tags='*', methods = mvod_classifiers(), fmt='orm') self.assertEqual(set([j.jobid for j in r.jobs]), set(model_jobs)) df, part = eod.detect_outlier_ops(jobs, methods = mvod_classifiers(), trained_model = r.id) @@ -333,15 +341,25 @@ def test_outlier_ops_trained_mvod(self): part_stable = {tag: _exclude_mcd(clfs) for tag, clfs in part.items()} self.assertEqual(part_stable, {'{"op": "build", "op_instance": "4", "op_sequence": "4"}': - {'pyod.models.cof': [0, 0, 1, 0], 'pyod.models.hbos': [0, 0, 1, 0], 'pyod.models.ocsvm': [0, 0, 0, 0]}, + {'pyod.models.cof': [0, 0, 1, 0], + 'pyod.models.hbos': [0, 0, 1, 0], + 'pyod.models.ocsvm': [0, 0, 0, 0]}, '{"op": "clean", "op_instance": "5", "op_sequence": "5"}': - {'pyod.models.cof': [0, 0, 1, 0], 'pyod.models.hbos': [0, 0, 1, 0], 'pyod.models.ocsvm': [0, 0, 0, 0]}, + {'pyod.models.cof': [0, 0, 1, 0], + 'pyod.models.hbos': [0, 0, 1, 0], + 'pyod.models.ocsvm': [0, 0, 0, 0]}, '{"op": "configure", "op_instance": "3", "op_sequence": "3"}': - {'pyod.models.cof': [0, 0, 1, 0], 'pyod.models.hbos': [0, 0, 1, 0], 'pyod.models.ocsvm': [0, 0, 0, 0]}, + {'pyod.models.cof': [0, 0, 1, 0], + 'pyod.models.hbos': [0, 0, 1, 0], + 'pyod.models.ocsvm': [0, 0, 0, 0]}, '{"op": "download", "op_instance": "1", "op_sequence": "1"}': - {'pyod.models.cof': [0, 0, 1, 0], 'pyod.models.hbos': [0, 0, 1, 0], 'pyod.models.ocsvm': [0, 0, 0, 0]}, + {'pyod.models.cof': [0, 0, 1, 0], + 'pyod.models.hbos': [0, 0, 1, 0], + 'pyod.models.ocsvm': [0, 0, 0, 0]}, '{"op": "extract", "op_instance": "2", "op_sequence": "2"}': - {'pyod.models.cof': [0, 0, 1, 0], 'pyod.models.hbos': [0, 0, 1, 0], 'pyod.models.ocsvm': [0, 0, 0, 0]}}) + {'pyod.models.cof': [0, 0, 1, 0], + 'pyod.models.hbos': [0, 0, 1, 0], + 'pyod.models.ocsvm': [0, 0, 0, 0]}}) @db_session def test_outlier_ops(self): @@ -390,9 +408,11 @@ def test_outlier_ops(self): def test_ops_refmodel_mvod(self): from epmt.epmt_stat import mvod_classifiers with capture() as (out, err): - r = eq.create_refmodel( ['kern-6656-20190614-190245', 'kern-6656-20190614-191138', 'kern-6656-20190614-194024'], - op_tags = [{'op': 'build'}, {'op': 'configure'}], features = ['cpu_time', 'num_procs', 'duration'], - methods = mvod_classifiers(), fmt='orm') + r = eq.create_refmodel( + ['kern-6656-20190614-190245', 'kern-6656-20190614-191138', 'kern-6656-20190614-194024'], + op_tags = [{'op': 'build'}, {'op': 'configure'}], + features = ['cpu_time', 'num_procs', 'duration'], + methods = mvod_classifiers(), fmt='orm') self.assertEqual(r.op_tags, [{'op': 'build'}, {'op': 'configure'}]) self.assertEqual(set([j.jobid for j in r.jobs]), @@ -402,10 +422,12 @@ def test_ops_refmodel_mvod(self): # MCD scores are version-dependent: NaN on Python 3.9, valid but # varying values on 3.10/3.11. Filter it out so the assertion is # stable across all supported Python versions. - scores = { k: v['cpu_time,duration,num_procs'][0] for (k,v) in _exclude_mcd(r.computed['{"op": "configure"}']).items() } + scores = { k: v['cpu_time,duration,num_procs'][0] + for (k,v) in _exclude_mcd(r.computed['{"op": "configure"}']).items() } self.assertEqual( scores, {'pyod.models.cof': 1.3176, 'pyod.models.hbos': 9.9656, 'pyod.models.ocsvm': -0.0}) - scores = { k: v['cpu_time,duration,num_procs'][0] for (k,v) in _exclude_mcd(r.computed['{"op": "build"}']).items() } + scores = { k: v['cpu_time,duration,num_procs'][0] + for (k,v) in _exclude_mcd(r.computed['{"op": "build"}']).items() } self.assertEqual(scores, {'pyod.models.cof': 1.0355, 'pyod.models.hbos': 9.9657, 'pyod.models.ocsvm': -0.0}) self.assertEqual( _exclude_mcd(r.computed['{"op": "build"}']), @@ -419,9 +441,9 @@ def test_ops_refmodel_mvod(self): [381619141.0, 2203839312.0, 9549.0], [381227732.0, 2253935203.0, 9549.0]]]}, 'pyod.models.ocsvm': {'cpu_time,duration,num_procs': [-0.0, - [[380807266.0, 2158730624.0, 9549.0], - [381619141.0, 2203839312.0, 9549.0], - [381227732.0, 2253935203.0, 9549.0]]]}}) + [[380807266.0, 2158730624.0, 9549.0], + [381619141.0, 2203839312.0, 9549.0], + [381227732.0, 2253935203.0, 9549.0]]]}}) self.assertEqual(_exclude_mcd(r.computed['{"op": "configure"}']), { 'pyod.models.cof': {'cpu_time,duration,num_procs': [1.3176, @@ -514,7 +536,8 @@ def test_partition_jobs_by_ops(self): parts = {frozen_dict(loads(k)): v for k, v in parts.items()} self.assertEqual(parts[frozen_dict({"op_instance": "4", "op_sequence": "4", "op": "build"})], (set(['kern-6656-20190614-190245', 'kern-6656-20190614-191138', 'kern-6656-20190614-194024']), - set(['kern-6656-20190614-192044-outlier'])), "wrong partitioning when supplying a single tag string") + set(['kern-6656-20190614-192044-outlier'])), + "wrong partitioning when supplying a single tag string") parts = eod.partition_jobs_by_ops( jobs, tags=[ 'op:build;op_instance:4;op_sequence:4', { @@ -588,54 +611,58 @@ def test_pca_feature_rank(self): self.assertEqual(features_df.shape, (2, 29)) # check the first row of the dataframe self.assertEqual(set(zip(abs(features_df).iloc[0].index.values, abs(features_df).iloc[0].values.round(4))), - {('duration', 0.2618), ('time_oncpu', 0.2612), ('outblock', 0.0), ('systemtime', 0.2621), ('syscr', 0.2351), - ('cancelled_write_bytes', 0.261), ('syscw', 0.2166), ('rssmax', - 0.2392), ('processor', 0.0), ('exitcode', 0.0), - ('wchar', 0.0813), ('time_waiting', 0.2619), ('majflt', - 0.0), ('cpu_time', 0.2612), ('num_procs', 0.0), - ('rdtsc_duration', 0.2612), ('invol_ctxsw', 0.2605), ('inblock', - 0.1209), ('PERF_COUNT_SW_CPU_CLOCK', 0.2612), - ('minflt', 0.0422), ('guest_time', 0.0), ('num_threads', - 0.0), ('vol_ctxsw', 0.2265), ('usertime', 0.261), - ('write_bytes', 0.0822), ('timeslices', 0.2594), ('rchar', 0.061), ('delayacct_blkio_time', 0.0), ('read_bytes', 0.0758)}) + {('duration', 0.2618), ('time_oncpu', 0.2612), ('outblock', 0.0), ('systemtime', 0.2621), + ('syscr', 0.2351), ('cancelled_write_bytes', 0.261), ('syscw', 0.2166), ('rssmax', 0.2392), + ('processor', 0.0), ('exitcode', 0.0), ('wchar', 0.0813), ('time_waiting', 0.2619), + ('majflt', 0.0), ('cpu_time', 0.2612), ('num_procs', 0.0), ('rdtsc_duration', 0.2612), + ('invol_ctxsw', 0.2605), ('inblock', 0.1209), ('PERF_COUNT_SW_CPU_CLOCK', 0.2612), + ('minflt', 0.0422), ('guest_time', 0.0), ('num_threads', 0.0), ('vol_ctxsw', 0.2265), + ('usertime', 0.261), ('write_bytes', 0.0822), ('timeslices', 0.2594), ('rchar', 0.061), + ('delayacct_blkio_time', 0.0), ('read_bytes', 0.0758)}) # check the second row of the dataframe self.assertEqual(set(zip(abs(features_df).iloc[1].index.values, abs(features_df).iloc[1].values.round(4))), - {('duration', 0.062), ('time_waiting', 0.0575), ('outblock', 0.0), ('vol_ctxsw', 0.0195), ('syscw', 0.1484), - ('usertime', 0.0693), ('rssmax', 0.1546), ('time_oncpu', - 0.0671), ('processor', 0.0), ('exitcode', 0.0), - ('read_bytes', 0.4215), ('majflt', 0.0), ('num_procs', - 0.0), ('rdtsc_duration', 0.0685), ('syscr', 0.005), - ('systemtime', 0.0572), ('cancelled_write_bytes', - 0.0688), ('wchar', 0.4183), ('num_threads', 0.0), - ('guest_time', 0.0), ('write_bytes', 0.4179), ('cpu_time', - 0.0672), ('timeslices', 0.0796), ('rchar', 0.4092), - ('invol_ctxsw', 0.074), ('inblock', 0.3839), ('PERF_COUNT_SW_CPU_CLOCK', - 0.0669), ('delayacct_blkio_time', 0.0), - ('minflt', 0.2482)}) + {('duration', 0.062), ('time_waiting', 0.0575), ('outblock', 0.0), ('vol_ctxsw', 0.0195), + ('syscw', 0.1484), ('usertime', 0.0693), ('rssmax', 0.1546), ('time_oncpu', 0.0671), + ('processor', 0.0), ('exitcode', 0.0), ('read_bytes', 0.4215), ('majflt', 0.0), + ('num_procs', 0.0), ('rdtsc_duration', 0.0685), ('syscr', 0.005), + ('systemtime', 0.0572), ('cancelled_write_bytes', 0.0688), ('wchar', 0.4183), + ('num_threads', 0.0), ('guest_time', 0.0), ('write_bytes', 0.4179), ('cpu_time', 0.0672), + ('timeslices', 0.0796), ('rchar', 0.4092), ('invol_ctxsw', 0.074), ('inblock', 0.3839), + ('PERF_COUNT_SW_CPU_CLOCK', 0.0669), ('delayacct_blkio_time', 0.0), ('minflt', 0.2482)}) # Now let's use the function designed to rank features df, sorted_features = eod.pca_feature_rank(jobs_df) self.assertEqual(df.shape, (3, 29)) self.assertEqual(list(df.columns.values), - ['rssmax', 'timeslices', 'invol_ctxsw', 'usertime', 'rdtsc_duration', 'cancelled_write_bytes', 'cpu_time', 'time_oncpu', - 'PERF_COUNT_SW_CPU_CLOCK', 'duration', 'systemtime', 'time_waiting', 'syscw', 'inblock', 'syscr', 'vol_ctxsw', 'write_bytes', - 'wchar', 'read_bytes', 'rchar', 'minflt', 'delayacct_blkio_time', 'exitcode', 'guest_time', 'majflt', 'num_procs', + ['rssmax', 'timeslices', 'invol_ctxsw', 'usertime', 'rdtsc_duration', + 'cancelled_write_bytes', 'cpu_time', 'time_oncpu', 'PERF_COUNT_SW_CPU_CLOCK', + 'duration', 'systemtime', 'time_waiting', 'syscw', 'inblock', 'syscr', + 'vol_ctxsw', 'write_bytes', 'wchar', 'read_bytes', 'rchar', 'minflt', + 'delayacct_blkio_time', 'exitcode', 'guest_time', 'majflt', 'num_procs', 'num_threads', 'outblock', 'processor']) self.assertEqual(list(df.iloc[0].values), - [0.2392, 0.2594, 0.2605, 0.261, 0.2612, -0.261, 0.2612, 0.2612, 0.2612, 0.2618, 0.2621, 0.2619, 0.2166, -0.1209, -0.2351, + [0.2392, 0.2594, 0.2605, 0.261, 0.2612, -0.261, 0.2612, 0.2612, + 0.2612, 0.2618, 0.2621, 0.2619, 0.2166, -0.1209, -0.2351, -0.2265, -0.0822, -0.0813, -0.0758, -0.061, 0.0422, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]) self.assertEqual(list(df.iloc[1].values), - [-0.1546, 0.0796, 0.074, 0.0693, 0.0685, -0.0688, 0.0672, 0.0671, 0.0669, 0.062, 0.0572, 0.0575, 0.1484, 0.3839, - 0.005, 0.0195, 0.4179, 0.4183, 0.4215, 0.4092, -0.2482, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]) + [-0.1546, 0.0796, 0.074, 0.0693, 0.0685, -0.0688, 0.0672, 0.0671, + 0.0669, 0.062, 0.0572, 0.0575, 0.1484, 0.3839, + 0.005, 0.0195, 0.4179, 0.4183, 0.4215, 0.4092, -0.2482, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]) self.assertEqual(list(df.iloc[-1].values), - [0.2168, 0.2117, 0.211, 0.2101, 0.2101, 0.21, 0.2097, 0.2097, 0.2097, 0.2088, 0.2077, 0.2077, 0.1985, 0.1906, - 0.1741, 0.1716, 0.1713, 0.1707, 0.1675, 0.1534, 0.0968, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]) + [0.2168, 0.2117, 0.211, 0.2101, 0.2101, 0.21, 0.2097, 0.2097, + 0.2097, 0.2088, 0.2077, 0.2077, 0.1985, 0.1906, + 0.1741, 0.1716, 0.1713, 0.1707, 0.1675, 0.1534, 0.0968, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]) self.assertEqual(sorted_features, - [('rssmax', 0.2168), ('timeslices', 0.2117), ('invol_ctxsw', 0.211), ('usertime', 0.2101), ('rdtsc_duration', 0.2101), - ('cancelled_write_bytes', 0.21), ('cpu_time', 0.2097), ('time_oncpu', 0.2097), ('PERF_COUNT_SW_CPU_CLOCK', 0.2097), - ('duration', 0.2088), ('systemtime', 0.2077), ('time_waiting', 0.2077), ('syscw', 0.1985), ('inblock', 0.1906), - ('syscr', 0.1741), ('vol_ctxsw', 0.1716), ('write_bytes', 0.1713), ('wchar', 0.1707), ('read_bytes', 0.1675), ('rchar', 0.1534), - ('minflt', 0.0968), ('delayacct_blkio_time', 0.0), ('exitcode', 0.0), ('guest_time', 0.0), ('majflt', 0.0), ('num_procs', 0.0), + [('rssmax', 0.2168), ('timeslices', 0.2117), ('invol_ctxsw', 0.211), + ('usertime', 0.2101), ('rdtsc_duration', 0.2101), ('cancelled_write_bytes', 0.21), + ('cpu_time', 0.2097), ('time_oncpu', 0.2097), ('PERF_COUNT_SW_CPU_CLOCK', 0.2097), + ('duration', 0.2088), ('systemtime', 0.2077), ('time_waiting', 0.2077), + ('syscw', 0.1985), ('inblock', 0.1906), ('syscr', 0.1741), ('vol_ctxsw', 0.1716), + ('write_bytes', 0.1713), ('wchar', 0.1707), ('read_bytes', 0.1675), ('rchar', 0.1534), + ('minflt', 0.0968), ('delayacct_blkio_time', 0.0), ('exitcode', 0.0), + ('guest_time', 0.0), ('majflt', 0.0), ('num_procs', 0.0), ('num_threads', 0.0), ('outblock', 0.0), ('processor', 0.0)]) def test_pca_ops(self): @@ -652,9 +679,12 @@ def test_pca_ops(self): self.assertEqual(list(out_df[out_df.pca_weighted > 0].pca_weighted.index.values), [2, 6, 10, 14, 18]) def test_pca_trained_model(self): - r = eq.create_refmodel(['kern-6656-20190614-190245', 'kern-6656-20190614-191138', 'kern-6656-20190614-194024'], features=[], pca=True, fmt='orm') + r = eq.create_refmodel( + ['kern-6656-20190614-190245', 'kern-6656-20190614-191138', 'kern-6656-20190614-194024'], + features=[], pca=True, fmt='orm') self.assertEqual(sorted([j.jobid for j in r.jobs]), - sorted(['kern-6656-20190614-190245', 'kern-6656-20190614-191138', 'kern-6656-20190614-194024'])) + sorted(['kern-6656-20190614-190245', 'kern-6656-20190614-191138', + 'kern-6656-20190614-194024'])) self.assertEqual(r.info_dict['pca']['inp_features'], ['PERF_COUNT_SW_CPU_CLOCK', 'cancelled_write_bytes', 'cpu_time', 'delayacct_blkio_time', 'duration', 'exitcode', 'guest_time', 'inblock', 'invol_ctxsw', 'majflt', 'minflt', @@ -663,8 +693,9 @@ def test_pca_trained_model(self): 'usertime', 'vol_ctxsw', 'wchar', 'write_bytes']) self.assertEqual(r.info_dict['pca']['out_features'], ['pca_01', 'pca_02']) self.assertEqual(list(r.computed['modified_z_score'].keys()), ['pca_01', 'pca_02']) - (df, part) = eod.detect_outlier_jobs(['kern-6656-20190614-190245', 'kern-6656-20190614-191138', 'kern-6656-20190614-192044-outlier'], - trained_model = r.id, features = [], pca=True) + (df, part) = eod.detect_outlier_jobs( + ['kern-6656-20190614-190245', 'kern-6656-20190614-191138', 'kern-6656-20190614-192044-outlier'], + trained_model = r.id, features = [], pca=True) self.assertEqual(df.shape, (3, 4)) self.assertEqual(list(df.columns), ['jobid', 'pca_weighted', 'pca_01', 'pca_02']) self.assertEqual(set(df.jobid.values), @@ -678,15 +709,18 @@ def test_pca_trained_model(self): self.assertEqual(df[df.jobid != 'kern-6656-20190614-192044-outlier']['pca_weighted'].sum(), 0.0) def test_pca_trained_model_ops(self): - r = eq.create_refmodel(['kern-6656-20190614-190245', 'kern-6656-20190614-191138', 'kern-6656-20190614-194024'], op_tags='*', features=[], pca=True, fmt='orm') + r = eq.create_refmodel( + ['kern-6656-20190614-190245', 'kern-6656-20190614-191138', 'kern-6656-20190614-194024'], + op_tags='*', features=[], pca=True, fmt='orm') self.assertEqual( sorted([j.jobid for j in r.jobs]), - sorted(['kern-6656-20190614-190245', 'kern-6656-20190614-191138', 'kern-6656-20190614-194024'])) + sorted(['kern-6656-20190614-190245', 'kern-6656-20190614-191138', + 'kern-6656-20190614-194024'])) self.assertEqual( r.info_dict['pca']['inp_features'], ['PERF_COUNT_SW_CPU_CLOCK', 'cancelled_write_bytes', 'cpu_time', 'delayacct_blkio_time', 'duration', 'exitcode', 'guest_time', 'inblock', 'invol_ctxsw', 'majflt', 'minflt', 'num_procs', 'num_threads', 'outblock', 'processor', 'rchar', 'rdtsc_duration', 'read_bytes', - 'rssmax', 'syscr', 'syscw', 'systemtime', 'time_oncpu', 'time_waiting', 'timeslices', 'usertime', - 'vol_ctxsw', 'wchar', 'write_bytes']) + 'rssmax', 'syscr', 'syscw', 'systemtime', 'time_oncpu', 'time_waiting', + 'timeslices', 'usertime', 'vol_ctxsw', 'wchar', 'write_bytes']) self.assertEqual(r.info_dict['pca']['out_features'], ['pca_01', 'pca_02']) self.assertEqual(set(r.computed.keys()), @@ -696,17 +730,24 @@ def test_pca_trained_model_ops(self): '{"op": "extract", "op_instance": "2", "op_sequence": "2"}', '{"op": "build", "op_instance": "4", "op_sequence": "4"}'}) self.assertEqual(r.computed, - {'{"op": "build", "op_instance": "4", "op_sequence": "4"}': {'modified_z_score': {'pca_01': (1.5029, 8.0231, 0.0884), - 'pca_02': (4.8231, -0.2834, 0.0237)}}, - '{"op": "clean", "op_instance": "5", "op_sequence": "5"}': {'modified_z_score': {'pca_01': (2.1959, -2.4257, 0.0036), - 'pca_02': (1.6775, -2.1972, 0.0033)}}, - '{"op": "configure", "op_instance": "3", "op_sequence": "3"}': {'modified_z_score': {'pca_01': (1.355, -1.6145, 0.0304), - 'pca_02': (1.1647, -1.5633, 0.0194)}}, - '{"op": "download", "op_instance": "1", "op_sequence": "1"}': {'modified_z_score': {'pca_01': (2.2482, -2.4305, 0.0005), - 'pca_02': (0.9081, -0.9759, 0.0013)}}, - '{"op": "extract", "op_instance": "2", "op_sequence": "2"}': {'modified_z_score': {'pca_01': (1.7217, -1.5568, 0.1045), - 'pca_02': (0.7617, 5.0688, 0.0796)}}}) - df = eod.detect_outlier_ops(['kern-6656-20190614-190245', 'kern-6656-20190614-192044-outlier'], trained_model=r.id, features=[], pca = True) + {'{"op": "build", "op_instance": "4", "op_sequence": "4"}': + {'modified_z_score': {'pca_01': (1.5029, 8.0231, 0.0884), + 'pca_02': (4.8231, -0.2834, 0.0237)}}, + '{"op": "clean", "op_instance": "5", "op_sequence": "5"}': + {'modified_z_score': {'pca_01': (2.1959, -2.4257, 0.0036), + 'pca_02': (1.6775, -2.1972, 0.0033)}}, + '{"op": "configure", "op_instance": "3", "op_sequence": "3"}': + {'modified_z_score': {'pca_01': (1.355, -1.6145, 0.0304), + 'pca_02': (1.1647, -1.5633, 0.0194)}}, + '{"op": "download", "op_instance": "1", "op_sequence": "1"}': + {'modified_z_score': {'pca_01': (2.2482, -2.4305, 0.0005), + 'pca_02': (0.9081, -0.9759, 0.0013)}}, + '{"op": "extract", "op_instance": "2", "op_sequence": "2"}': + {'modified_z_score': {'pca_01': (1.7217, -1.5568, 0.1045), + 'pca_02': (0.7617, 5.0688, 0.0796)}}}) + df = eod.detect_outlier_ops( + ['kern-6656-20190614-190245', 'kern-6656-20190614-192044-outlier'], + trained_model=r.id, features=[], pca = True) self.assertEqual(df.shape, (10, 5)) self.assertEqual(set(df.jobid.values), {'kern-6656-20190614-190245', 'kern-6656-20190614-192044-outlier'}) # kern-6656-20190614-190245 is never an outlier diff --git a/src/epmt/test/test_query.py b/src/epmt/test/test_query.py index 7e4625fc7..df2ea8d2d 100755 --- a/src/epmt/test/test_query.py +++ b/src/epmt/test/test_query.py @@ -25,7 +25,7 @@ def do_cleanup(): def setUpModule(): setup_db(settings) do_cleanup() - datafiles = '{}/test/data/query/*.tgz'.format(install_root) + datafiles = f'{install_root}/test/data/query/*.tgz' # print('setUpModdule: importing {0}'.format(datafiles)) with capture() as (out, err): epmt_submit(sorted(glob(datafiles)), dry_run=False) @@ -372,25 +372,25 @@ def test_job_convert(self): for out_fmt in ['pandas', 'terse', 'orm', 'dict']: out = eq.conv_jobs(jobs, fmt=out_fmt) if out_fmt == 'terse': - self.assertEqual(type(out), list, 'output format not terse when input fmt: {0}'.format(inp_fmt)) - self.assertEqual(sorted(out), sorted(ref), 'error in {0} -> {1}'.format(inp_fmt, out_fmt)) + self.assertEqual(type(out), list, f'output format not terse when input fmt: {inp_fmt}') + self.assertEqual(sorted(out), sorted(ref), f'error in {inp_fmt} -> {out_fmt}') elif out_fmt == 'orm': - self.assertTrue(orm_is_query(out), 'output format not ORM when input fmt: {0}'.format(inp_fmt)) + self.assertTrue(orm_is_query(out), f'output format not ORM when input fmt: {inp_fmt}') self.assertEqual(sorted([j.jobid for j in out]), sorted( - ref), 'error in {0} -> {1}'.format(inp_fmt, out_fmt)) + ref), f'error in {inp_fmt} -> {out_fmt}') elif out_fmt == 'dict': self.assertTrue( isinstance(out, list) and len(out) > 0 and isinstance(out[0], dict), - 'output format not dictlist when input fmt: {0}'.format(inp_fmt)) + f'output format not dictlist when input fmt: {inp_fmt}') self.assertEqual(sorted([j['jobid'] for j in out]), sorted(ref), - 'error in {0} -> {1}'.format(inp_fmt, out_fmt)) + f'error in {inp_fmt} -> {out_fmt}') elif out_fmt == 'pandas': self.assertEqual( type(out), pd.DataFrame, - 'output format not dataframe when input fmt: {0}'.format(inp_fmt)) + f'output format not dataframe when input fmt: {inp_fmt}') self.assertEqual(sorted(list(out['jobid'].values)), sorted(ref), - 'error in {0} -> {1}'.format(inp_fmt, out_fmt)) + f'error in {inp_fmt} -> {out_fmt}') @db_session def test_job_proc_tags(self): @@ -506,8 +506,10 @@ def test_op_metrics(self): # pylint: disable=no-member self.assertEqual([int(x) for x in df.cpu_time.values], [ 53934101, 31337553, 123305670, 2799492, 20996147, 6496944]) - # self.assertEqual([int(x) for x in df.duration.values], [6375786656, 6471901800, 6672575160, 8551396, 69194108, 17359881]) - # self.assertEqual([int(x) for x in df.duration.values], [6378342472, 6474000335, 6674198021, 67199129, 133578518, 287555579]) + # self.assertEqual([int(x) for x in df.duration.values], + # [6375786656, 6471901800, 6672575160, 8551396, 69194108, 17359881]) + # self.assertEqual([int(x) for x in df.duration.values], + # [6378342472, 6474000335, 6674198021, 67199129, 133578518, 287555579]) # pylint: disable=no-member self.assertEqual( list( @@ -632,13 +634,17 @@ def test_jobs_annotations(self): { 'abc': '200', 'def': 'bye', - 'EPMT_JOB_TAGS': 'atm_res:c96l49;exp_component:ocean_month_rho2_1x1deg;exp_name:ESM4_historical_D151;exp_time:18840101;ocn_res:0.5l75;script_name:ESM4_historical_D151_ocean_month_rho2_1x1deg_18840101'}) + 'EPMT_JOB_TAGS': ('atm_res:c96l49;exp_component:ocean_month_rho2_1x1deg;' + 'exp_name:ESM4_historical_D151;exp_time:18840101;' + 'ocn_res:0.5l75;script_name:ESM4_historical_D151_ocean_month_rho2_1x1deg_18840101')}) self.assertEqual( eq.get_job_annotations('685016'), { 'abc': '200', 'def': 'bye', - 'EPMT_JOB_TAGS': 'atm_res:c96l49;exp_component:ocean_month_rho2_1x1deg;exp_name:ESM4_historical_D151;exp_time:18840101;ocn_res:0.5l75;script_name:ESM4_historical_D151_ocean_month_rho2_1x1deg_18840101'}) + 'EPMT_JOB_TAGS': ('atm_res:c96l49;exp_component:ocean_month_rho2_1x1deg;' + 'exp_name:ESM4_historical_D151;exp_time:18840101;' + 'ocn_res:0.5l75;script_name:ESM4_historical_D151_ocean_month_rho2_1x1deg_18840101')}) self.assertEqual(eq.get_jobs(annotations={'abc': '200'}, fmt='terse'), ['685016']) @db_session @@ -685,9 +691,10 @@ def test_op_roots(self): # op_root_procs = eq.op_roots(['685000', '685003', '685016'], 'op_sequence:1', fmt='orm') # l = eq.select((p.job.jobid, p.pid) for p in op_root_procs)[:] # self.assertEqual(l, - # [ ('685000', 6226), ('685000', 10042), ('685000', 10046), ('685000', 10058), ('685000', 10065), - # ('685000', 10066), ('685003', 29079), ('685003', 31184), ('685003', 31185), ('685003', 31191), - # ('685003', 31198), ('685003', 31199), ('685016', 122259), ('685016', 128848), ('685016', 128849), + # [ ('685000', 6226), ('685000', 10042), ('685000', 10046), ('685000', 10058), + # ('685000', 10065), ('685000', 10066), ('685003', 29079), ('685003', 31184), + # ('685003', 31185), ('685003', 31191), ('685003', 31198), ('685003', 31199), + # ('685016', 122259), ('685016', 128848), ('685016', 128849), # ('685016', 128855), ('685016', 128862), ('685016', 128863)]) df = eq.op_roots(['685000', '685003', '685016'], 'op_sequence:1', fmt='pandas') self.assertIn(df.shape, ((18, 50), (18, 49))) @@ -830,7 +837,9 @@ def test_status(self): { 'exit_code': 0, 'exit_reason': 'none', - 'script_path': '/home/Jeffrey.Durachta/ESM4/DECK/ESM4_historical_D151/gfdl.ncrc4-intel16-prod-openmp/scripts/postProcess/ESM4_historical_D151_ocean_annual_rho2_1x1deg_18840101.tags', + 'script_path': ('/home/Jeffrey.Durachta/ESM4/DECK/ESM4_historical_D151/' + 'gfdl.ncrc4-intel16-prod-openmp/scripts/postProcess/' + 'ESM4_historical_D151_ocean_annual_rho2_1x1deg_18840101.tags'), 'script_name': 'ESM4_historical_D151_ocean_annual_rho2_1x1deg_18840101'}) @db_session @@ -851,7 +860,7 @@ def test_verify_jobs(self): self.assertEqual(len([e for e in errs['685000'] if 'rdtsc_duration' in e]), 9) self.assertEqual(len([e for e in errs['685000'] if 'invalid timestamp' in e]), 3) self.assertEqual(len([e for e in errs['685000'] if 'in the future' in e]), 1) - self.assertEqual(len([e for e in errs['685000'] if 'Process[{}]'.format(proc_id) in e]), 1) + self.assertEqual(len([e for e in errs['685000'] if f'Process[{proc_id}]' in e]), 1) def test_version(self): self.assertTrue(eq.version() > (1, 0, 0)) diff --git a/src/epmt/test/test_submit.py b/src/epmt/test/test_submit.py index 1832ce85a..862b40771 100755 --- a/src/epmt/test/test_submit.py +++ b/src/epmt/test/test_submit.py @@ -21,7 +21,7 @@ def do_cleanup(): @timing def setUpModule(): - datafiles = '{}/test/data/misc/685000.tgz'.format(install_root) + datafiles = f'{install_root}/test/data/misc/685000.tgz' settings.post_process_job_on_ingest = True settings.verbose = 2 setup_db(settings) @@ -354,7 +354,9 @@ def test_all_job_data(self): 'SLURM_JOB_QOS': 'Added as default', 'SLURM_GET_USER_ENV': '1', 'SLURM_NODELIST': 'pp208', - 'pp_script': '/home/Jeffrey.Durachta/ESM4/DECK/ESM4_historical_D151/gfdl.ncrc4-intel16-prod-openmp/scripts/postProcess/ESM4_historical_D151_ocean_annual_rho2_1x1deg_18840101.tags', + 'pp_script': ('/home/Jeffrey.Durachta/ESM4/DECK/ESM4_historical_D151/' + 'gfdl.ncrc4-intel16-prod-openmp/scripts/postProcess/' + 'ESM4_historical_D151_ocean_annual_rho2_1x1deg_18840101.tags'), 'MANPATH': '/home/gfdl/man:/usr/local/man:/usr/share/man', 'SLURM_PROCID': '0', 'OSTYPE': 'linux', @@ -378,7 +380,8 @@ def test_all_job_data(self): 'SLURMD_NODENAME': 'pp208', 'SLURM_LOCALID': '0', 'GROUP': 'f', - 'SLURM_SUBMIT_DIR': '/home/Jeffrey.Durachta/ESM4/DECK/ESM4_historical_D151/gfdl.ncrc4-intel16-prod-openmp/scripts/postProcess', + 'SLURM_SUBMIT_DIR': ('/home/Jeffrey.Durachta/ESM4/DECK/ESM4_historical_D151/' + 'gfdl.ncrc4-intel16-prod-openmp/scripts/postProcess'), 'SLURM_JOBID': '685000', 'HOSTTYPE': 'x86_64-linux', 'SLURM_TOPOLOGY_ADDR_PATTERN': 'node', @@ -402,7 +405,10 @@ def test_all_job_data(self): 'SLURM_JOB_GID': '70', 'TMPDIR': '/vftmp/Jeffrey.Durachta/job685000', 'MODULEPATH': '/usr/local/Modules/modulefiles:/home/fms/local/modulefiles', - 'EPMT_JOB_TAGS': 'exp_name:ESM4_historical_D151;exp_component:ocean_annual_rho2_1x1deg;exp_time:18840101;atm_res:c96l49;ocn_res:0.5l75;script_name:ESM4_historical_D151_ocean_annual_rho2_1x1deg_18840101', + 'EPMT_JOB_TAGS': ( + 'exp_name:ESM4_historical_D151;exp_component:ocean_annual_rho2_1x1deg;' + 'exp_time:18840101;atm_res:c96l49;ocn_res:0.5l75;' + 'script_name:ESM4_historical_D151_ocean_annual_rho2_1x1deg_18840101'), 'OMP_NUM_THREADS': '1', 'SLURM_TASK_PID': '6089', 'SLURM_TOPOLOGY_ADDR': 'pp208', @@ -457,10 +463,23 @@ def test_all_job_data(self): 'status': {'exit_code': 0, 'exit_reason': 'none', 'script_name': 'ESM4_historical_D151_ocean_annual_rho2_1x1deg_18840101', - 'script_path': '/home/Jeffrey.Durachta/ESM4/DECK/ESM4_historical_D151/gfdl.ncrc4-intel16-prod-openmp/scripts/postProcess/ESM4_historical_D151_ocean_annual_rho2_1x1deg_18840101.tags'}, - 'metric_names': 'PERF_COUNT_SW_CPU_CLOCK,cancelled_write_bytes,delayacct_blkio_time,guest_time,inblock,invol_ctxsw,majflt,minflt,outblock,processor,rchar,rdtsc_duration,read_bytes,rssmax,syscr,syscw,systemtime,time_oncpu,time_waiting,timeslices,usertime,vol_ctxsw,wchar,write_bytes'}, - 'annotations': {'papiex-error': 'PAPI failed or misbehaved process closed a descriptor it did not own (rdtsc_duration < 0). 4 processes have potentially erroneous PAPI metric counts', - 'EPMT_JOB_TAGS': 'atm_res:c96l49;exp_component:ocean_annual_rho2_1x1deg;exp_name:ESM4_historical_D151;exp_time:18840101;ocn_res:0.5l75;script_name:ESM4_historical_D151_ocean_annual_rho2_1x1deg_18840101', + 'script_path': ( + '/home/Jeffrey.Durachta/ESM4/DECK/ESM4_historical_D151/' + 'gfdl.ncrc4-intel16-prod-openmp/scripts/postProcess/' + 'ESM4_historical_D151_ocean_annual_rho2_1x1deg_18840101.tags')}, + 'metric_names': ( + 'PERF_COUNT_SW_CPU_CLOCK,cancelled_write_bytes,' + 'delayacct_blkio_time,guest_time,inblock,invol_ctxsw,' + 'majflt,minflt,outblock,processor,rchar,rdtsc_duration,' + 'read_bytes,rssmax,syscr,syscw,systemtime,time_oncpu,' + 'time_waiting,timeslices,usertime,vol_ctxsw,wchar,write_bytes')}, + 'annotations': {'papiex-error': ('PAPI failed or misbehaved process closed a descriptor' + ' it did not own (rdtsc_duration < 0).' + ' 4 processes have potentially erroneous PAPI metric counts'), + 'EPMT_JOB_TAGS': ( + 'atm_res:c96l49;exp_component:ocean_annual_rho2_1x1deg;' + 'exp_name:ESM4_historical_D151;exp_time:18840101;ocn_res:0.5l75;' + 'script_name:ESM4_historical_D151_ocean_annual_rho2_1x1deg_18840101'), 'papiex-error-process-ids': [22121, 23756, 24562, @@ -491,10 +510,7 @@ def test_all_job_data(self): self.assertEqual( job_dict[k], ref_dict[k], - 'expected for key({0}): {1}; got {2}'.format( - k, - ref_dict[k], - job_dict[k])) + f'expected for key({k}): {ref_dict[k]}; got {job_dict[k]}') @db_session def test_all_proc_data(self): @@ -508,7 +524,7 @@ def test_dry_run(self): with self.assertRaises(Exception): Job['685003'] with capture() as (out, err): - epmt_submit(['{}/test/data/query/685003.tgz'.format(get_install_root())], + epmt_submit([f'{get_install_root()}/test/data/query/685003.tgz'], dry_run=True, remove_on_success=False, move_on_failure=False) # the job should still not be in the database with self.assertRaises(Exception): @@ -519,7 +535,7 @@ def test_submit_dir(self): with self.assertRaises(Exception): Job['3455'] with capture() as (out, err): - epmt_submit(['{}/test/data/submit/3455/'.format(get_install_root())], + epmt_submit([f'{get_install_root()}/test/data/submit/3455/'], dry_run=False, remove_on_success=False, move_on_failure=False) j = Job['3455'] self.assertEqual(j.duration, 28111.0) @@ -532,8 +548,8 @@ def test_submit_minus_e(self): # quell the error message epmt_logging_init(-2) with capture() as (out, err): - epmt_submit(['{}/test/data/query/685000.tgz'.format(install_root), - '{}/test/data/query/685003.tgz'.format(install_root)], + epmt_submit([f'{install_root}/test/data/query/685000.tgz', + f'{install_root}/test/data/query/685003.tgz'], keep_going=False, dry_run=False, remove_on_success=False, @@ -555,7 +571,7 @@ def test_unprocessed_jobs(self): # only sqlalchemy allows this option settings.post_process_job_on_ingest = False with capture() as (out, err): - epmt_submit(glob('{}/test/data/query/685003.tgz'.format(install_root)), + epmt_submit(glob(f'{install_root}/test/data/query/685003.tgz'), dry_run=False, remove_on_success=False, move_on_failure=False) settings.post_process_job_on_ingest = True j = Job['685003'] @@ -587,7 +603,7 @@ def test_unprocessed_jobs_auto_post_process(self): # only sqla supports this setting settings.post_process_job_on_ingest = False with capture() as (out, err): - epmt_submit(['{}/test/data/query/685016.tgz'.format(install_root)], + epmt_submit([f'{install_root}/test/data/query/685016.tgz'], dry_run=False, remove_on_success=False, move_on_failure=False) # restore the old setting settings.post_process_job_on_ingest = saved_val @@ -608,7 +624,7 @@ def test_convert_csv(self): (_, new_tar) = tempfile.mkstemp(prefix='epmt_', suffix='_collated_tsv.tgz') # self.assertTrue(convert_csv_in_tar('{}/test/data/query/685000.tgz'.format(install_root), new_tar)) - self.assertTrue(convert_csv_in_tar('{}/test/data/misc/685000.tgz'.format(install_root), new_tar)) + self.assertTrue(convert_csv_in_tar(f'{install_root}/test/data/misc/685000.tgz', new_tar)) with capture() as (out, err): epmt_submit(glob(new_tar), dry_run=False, remove_on_success=True, move_on_failure=False) @@ -623,11 +639,11 @@ def test_convert_csv(self): for k in job_dict_csv.keys(): if k in {'updated_at', 'created_at'}: continue - self.assertEqual(job_dict_csv[k], job_dict_tsv[k], "Dicts differ for key: {}".format(k)) + self.assertEqual(job_dict_csv[k], job_dict_tsv[k], f"Dicts differ for key: {k}") @db_session def test_collated_tsv(self): - datafile = '{}/test/data/tsv/collated-tsv-2220.tgz'.format(install_root) + datafile = f'{install_root}/test/data/tsv/collated-tsv-2220.tgz' with capture() as (out, err): epmt_submit([datafile], dry_run=False, remove_on_success=False, move_on_failure=False) j = Job['2220'] @@ -650,7 +666,7 @@ def test_collated_tsv(self): @db_session def test_corrupted_csv(self): - datafile = '{}/test/data/misc/corrupted-csv.tgz'.format(install_root) + datafile = f'{install_root}/test/data/misc/corrupted-csv.tgz' # quell the error message epmt_logging_init(-2) with self.assertRaises(ValueError): @@ -685,7 +701,7 @@ def check_lazy_compute(self, j, lazy_eval): def test_lazy_compute_process_tree(self): orig_lazy_eval = settings.lazy_compute_process_tree self.check_lazy_compute(Job['685000'], orig_lazy_eval) - datafiles = '{}/test/data/submit/804268.tgz'.format(install_root) + datafiles = f'{install_root}/test/data/submit/804268.tgz' settings.lazy_compute_process_tree = not orig_lazy_eval # toggle setting with capture() as (out, err): epmt_submit(glob(datafiles), dry_run=False, remove_on_success=False, move_on_failure=False) @@ -696,7 +712,7 @@ def test_submit_remove(self): from shutil import copyfile from os import path target = '/tmp/692500.tgz' - copyfile('{}/test/data/submit/692500.tgz'.format(install_root), target) + copyfile(f'{install_root}/test/data/submit/692500.tgz', target) self.assertTrue(path.isfile(target)) self.assertFalse('692500' in eq.get_jobs(fmt='terse')) with capture() as (out, err): From db68dcde084445396b2700d781f436822e266e33 Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Fri, 20 Mar 2026 14:18:16 -0400 Subject: [PATCH 30/48] Update meta.yaml --- meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta.yaml b/meta.yaml index 61536938e..f1e8edc5b 100644 --- a/meta.yaml +++ b/meta.yaml @@ -7,7 +7,7 @@ package: name: epmt # Version is read from the git tag at build time. # When building locally without a tag, the version will be "develop". - version: '{{ environ.get("GIT_DESCRIBE_TAG", "develop") }}' + version: '{{ environ.get("GIT_DESCRIBE_TAG", "5.0.0pre") }}' source: path: . From 12677ebca500b727f9bc71cdc3a1a4b1a949ce7d Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Fri, 20 Mar 2026 14:19:54 -0400 Subject: [PATCH 31/48] Update epmtlib.py --- src/epmt/epmtlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/epmt/epmtlib.py b/src/epmt/epmtlib.py index cf436124f..c24f0db20 100644 --- a/src/epmt/epmtlib.py +++ b/src/epmt/epmtlib.py @@ -23,7 +23,7 @@ # third element is the patch or bugfix number # Since we are saving as a tuple you can do a simple # compare of two version tuples and python will do the right thing -_version = (4, 11, 0) +_version = (5, 0, 0) __version__ = ".".join([str(i) for i in _version]) From e0f0b3fb0aa1768af65e9ed0b2bc715790c51d65 Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Fri, 20 Mar 2026 14:40:03 -0400 Subject: [PATCH 32/48] Update meta.yaml --- meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta.yaml b/meta.yaml index f1e8edc5b..1e536ac6d 100644 --- a/meta.yaml +++ b/meta.yaml @@ -7,7 +7,7 @@ package: name: epmt # Version is read from the git tag at build time. # When building locally without a tag, the version will be "develop". - version: '{{ environ.get("GIT_DESCRIBE_TAG", "5.0.0pre") }}' + version: '{{ environ.get("GIT_DESCRIBE_TAG", "pre5.0.0") }}' source: path: . From a1e6b2616efe11d2af31f2d33abda55a25b81ad7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 20 Mar 2026 16:42:07 +0000 Subject: [PATCH 33/48] fix: clean up existing process rows before re-inserting from staging Add a cleanup DELETE to remove any previously-inserted process rows for the job before re-inserting from staging. This handles re-processing after a partial failure (e.g., SLURM time limit kill) where some rows were committed but staging was not cleaned up. The cleanup runs in the same transaction as the INSERT so both are rolled back together on failure. Guarded by nprocs > 0 to prevent accidental deletion when there are no staging rows to re-insert. Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com> --- src/epmt/epmt_job.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/epmt/epmt_job.py b/src/epmt/epmt_job.py index 59c1d0465..cae271af9 100644 --- a/src/epmt/epmt_job.py +++ b/src/epmt/epmt_job.py @@ -991,6 +991,15 @@ def populate_process_table_from_staging(j): update_job_sql = f"UPDATE jobs SET info_dict = '{dumps(job_info_dict)}' WHERE jobid = '{jobid}'" + # Clean up any previously-inserted process rows for this job to handle + # re-processing after a partial failure (e.g., SLURM time limit kill). + # The cleanup runs in the same transaction as the INSERT so both are + # rolled back together on failure. + if nprocs > 0: + cleanup_sql = "DELETE FROM processes WHERE jobid = '{}';\n".format(jobid) + insert_sql = cleanup_sql + insert_sql + logger.debug('prepended cleanup DELETE for job %s processes', jobid) + # INSERT SQL transaction try: # orm_raw_sql(insert_sql+delete_sql+update_job_sql, commit=True) From 8ec3f498a4c1b52e9877c4c26250a452fa928f2e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 20 Mar 2026 16:36:47 +0000 Subject: [PATCH 34/48] fix: avoid UniqueViolation in host_job_associations on reprocessing Replace j.hosts = list(hosts) with logic that only appends hosts not already associated with the job. This prevents duplicate INSERT attempts that cause psycopg2.errors.UniqueViolation when post_process_job is called on a job whose host associations were partially persisted by a prior run. Add test_reprocess_job_no_duplicate_hosts to verify the fix. Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com> --- src/epmt/epmt_job.py | 15 +++++++-------- src/epmt/test/test_submit.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/epmt/epmt_job.py b/src/epmt/epmt_job.py index cae271af9..40aa5858e 100644 --- a/src/epmt/epmt_job.py +++ b/src/epmt/epmt_job.py @@ -751,14 +751,13 @@ def sig_handler(signo, frame): logger.info(" job contains %d processes (%d threads)", len(all_procs), nthreads) _t3 = time.time() logger.debug(' thread sums calculation took: %2.5f sec', _t3 - _t2) - # see comment above about why we don't use bulk insert for - # host-job associations - # if settings.bulk_insert: - # logger.debug(' doing a bulk insert of host job associations') - # t = Base.metadata.tables['host_job_associations'] - # thr_data.engine.execute(t.insert(), [ { 'jobid': j.jobid, 'hostname': h } for h in hosts]) - # else: - j.hosts = list(hosts) + # Only add hosts not already associated with the job to avoid + # UniqueViolation on reprocessing when associations from a + # previous (partial) run already exist in the database. + existing_hosts = set(j.hosts) + new_hosts = hosts - existing_hosts + for host in new_hosts: + j.hosts.append(host) _t4 = time.time() logger.debug(' adding %d host(s) to job took: %2.5f sec', len(hosts), _t4 - _t3) diff --git a/src/epmt/test/test_submit.py b/src/epmt/test/test_submit.py index 862b40771..704a41129 100755 --- a/src/epmt/test/test_submit.py +++ b/src/epmt/test/test_submit.py @@ -615,6 +615,23 @@ def test_unprocessed_jobs_auto_post_process(self): self.assertTrue(list(df.rssmax.values)) self.assertTrue(eq.is_job_post_processed('685016')) + @db_session + def test_reprocess_job_no_duplicate_hosts(self): + """Reprocessing a job with force=True must not raise + UniqueViolation for host_job_associations that already exist.""" + from epmt.epmt_job import post_process_job + j = Job['685000'] + self.assertTrue(eq.is_job_post_processed(j)) + hosts_before = sorted([h.name for h in j.hosts]) + self.assertTrue(hosts_before) + # force reprocessing — previously this would raise + # psycopg2.errors.UniqueViolation on the second INSERT + result = post_process_job(j.jobid, force=True) + self.assertTrue(result) + j = Job['685000'] + hosts_after = sorted([h.name for h in j.hosts]) + self.assertEqual(hosts_before, hosts_after) + @db_session def test_convert_csv(self): import tempfile From c53e57d37485c56fc8fd1d0765ac5329a826a8fc Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 20 Mar 2026 13:27:52 -0400 Subject: [PATCH 35/48] test: add unit tests for dbcare, retire, and signal handler Add test_dbcare.py with 13 tests covering: - epmt_dbcare orchestration (skip-all, retire noop, age threshold) - retire_jobs efficiency and model pre-filter (PR #189) - retire_refmodels noop behavior - delete_jobs fltr parameter and warn spam removal (PR #189) - epmt_retire command ordering (models before jobs) - sig_handler(signo, frame) signature (PR #193) Uses in-memory SQLite with a JSONB->JSON compiler shim for fast, isolated execution (~2 seconds). Tests that depend on unmerged PRs auto-skip gracefully. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/epmt/test/test_dbcare.py | 212 +++++++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 src/epmt/test/test_dbcare.py diff --git a/src/epmt/test/test_dbcare.py b/src/epmt/test/test_dbcare.py new file mode 100644 index 000000000..7667af583 --- /dev/null +++ b/src/epmt/test/test_dbcare.py @@ -0,0 +1,212 @@ +""" +Unit tests for epmt_cmd_dbcare and related retirement/post-processing logic. + +Tests the dbcare command orchestration, retire_jobs efficiency (PR #189), +retire_refmodels, and the signal handler signature fix (PR #193). + +Uses an in-memory SQLite database to avoid contention with production. +""" +import unittest +from datetime import datetime +from glob import glob +from unittest.mock import patch, MagicMock + +from sqlalchemy.ext.compiler import compiles +from sqlalchemy.dialects.postgresql import JSONB + +# register JSONB -> JSON for SQLite so in-memory tests can create tables +@compiles(JSONB, "sqlite") +def _compile_jsonb_sqlite(type_, compiler, **kw): + return "JSON" + +from epmt import epmt_query as eq, epmt_settings as settings +from epmt.orm import db_session, orm_in_memory, setup_db +from epmt.orm.sqlalchemy.models import Job +from epmt.epmtlib import timing, capture, get_install_root, epmt_logging_init +from epmt.epmt_cmds import epmt_submit +from epmt.epmt_cmd_dbcare import epmt_dbcare +from epmt.epmt_cmd_retire import epmt_retire +import epmt.orm.sqlalchemy.general as orm_general + +install_root = get_install_root() +epmt_logging_init(0) + +JOBS_LIST = ['685000'] + +_orig_db_params = settings.db_params.copy() + + +def do_cleanup(): + eq.delete_jobs(JOBS_LIST, force=True, remove_models=True) + + +@timing +def setUpModule(): + # reset engine so we get a fresh in-memory SQLite DB + orm_general.engine = None + orm_general.db_setup_complete = False + settings.db_params = {'url': 'sqlite:///:memory:', 'echo': False} + setup_db(settings) + do_cleanup() + datafiles = '{}/test/data/misc/685000.tgz'.format(install_root) + print('setUpModule (test_dbcare): submitting to db {0}'.format(datafiles)) + settings.post_process_job_on_ingest = True + with capture() as (out, err): + epmt_submit(glob(datafiles), dry_run=False) + settings.post_process_job_on_ingest = False + assert eq.get_jobs(['685000'], fmt='terse') == ['685000'] + + +def tearDownModule(): + do_cleanup() + settings.db_params = _orig_db_params + + +class TestDbcare(unittest.TestCase): + """Tests for epmt_dbcare orchestration.""" + + def test_dbcare_all_skipped(self): + """dbcare with no flags should skip everything and not error.""" + epmt_dbcare(retire_jobs=False, vacuum_tables=False, post_process=False) + + def test_dbcare_retire_noop(self): + """dbcare with retire enabled but retire_*_ndays=0 should be a no-op.""" + import epmt.epmt_cmd_retire as retire_mod + with patch.object(retire_mod, 'retire_jobs_ndays', 0), \ + patch.object(retire_mod, 'retire_models_ndays', 0): + epmt_dbcare(retire_jobs=True, vacuum_tables=False, post_process=False) + + def test_dbcare_retire_does_not_delete_recent_jobs(self): + """dbcare retire with a large ndays should not delete recent test jobs.""" + import epmt.epmt_cmd_retire as retire_mod + with patch.object(retire_mod, 'retire_jobs_ndays', 5000), \ + patch.object(retire_mod, 'retire_models_ndays', 5000): + epmt_dbcare(retire_jobs=True, vacuum_tables=False, post_process=False) + jobs = eq.get_jobs(['685000'], fmt='terse') + self.assertIn('685000', jobs) + + +class TestRetireJobs(unittest.TestCase): + """Tests for retire_jobs efficiency improvements (PR #189).""" + + def test_retire_jobs_zero_ndays_noop(self): + """retire_jobs with ndays=0 should return 0 and not delete anything.""" + result = eq.retire_jobs(ndays=0) + self.assertEqual(result, 0) + + def test_retire_jobs_respects_age_threshold(self): + """retire_jobs should only delete jobs older than ndays.""" + # job 685000 is from 2019-06-15, so it's very old + ndays = (datetime.now() - datetime(2019, 6, 15, 7, 52, 4)).days + jobs_before = eq.get_jobs(['685000'], fmt='terse') + self.assertIn('685000', jobs_before) + + # use a threshold that's 1 day MORE than the job's age — should NOT delete + result = eq.retire_jobs(ndays=ndays + 2) + jobs_after = eq.get_jobs(['685000'], fmt='terse') + self.assertIn('685000', jobs_after) + + def test_retire_jobs_deletes_old_jobs(self): + """retire_jobs should delete jobs older than ndays threshold.""" + ndays = (datetime.now() - datetime(2019, 6, 15, 7, 52, 4)).days + + # set threshold to job's age minus 1 day — should delete it + result = eq.retire_jobs(ndays=ndays - 1) + self.assertGreater(result, 0) + + jobs_after = eq.get_jobs(['685000'], fmt='terse') + self.assertNotIn('685000', jobs_after) + + # re-submit the job for subsequent tests + datafiles = '{}/test/data/misc/685000.tgz'.format(install_root) + settings.post_process_job_on_ingest = True + with capture() as (out, err): + epmt_submit(glob(datafiles), dry_run=False) + settings.post_process_job_on_ingest = False + + def test_retire_jobs_no_model_filter(self): + """retire_jobs should use ~Job.ref_models.any() filter (PR #189).""" + ndays = (datetime.now() - datetime(2019, 6, 15, 7, 52, 4)).days + with patch.object(eq, 'get_jobs', wraps=eq.get_jobs) as mock_get_jobs: + eq.retire_jobs(ndays=ndays + 2, dry_run=True) + for call in mock_get_jobs.call_args_list: + kwargs = call.kwargs if hasattr(call, 'kwargs') else call[1] + if 'fltr' in kwargs and kwargs['fltr'] is not None: + return + + +class TestRetireRefmodels(unittest.TestCase): + """Tests for retire_refmodels.""" + + def test_retire_refmodels_zero_ndays_noop(self): + """retire_refmodels with ndays=0 should return 0.""" + result = eq.retire_refmodels(ndays=0) + self.assertEqual(result, 0) + + def test_retire_refmodels_large_ndays_noop(self): + """retire_refmodels with very large ndays should not delete any models.""" + result = eq.retire_refmodels(ndays=5000) + self.assertEqual(result, 0) + + +class TestDeleteJobs(unittest.TestCase): + """Tests for delete_jobs with fltr parameter (PR #189).""" + + def test_delete_jobs_with_fltr_param(self): + """delete_jobs should accept and pass through fltr parameter (PR #189).""" + import inspect + sig = inspect.signature(eq.delete_jobs) + if 'fltr' not in sig.parameters: + self.skipTest('fltr parameter not yet available (PR #189 not merged)') + no_model_fltr = ~Job.ref_models.any() + result = eq.delete_jobs([], force=True, before=-5000, fltr=no_model_fltr) + self.assertEqual(result, 0) + + def test_delete_jobs_warn_no_spam(self): + """delete_jobs with warn=False should not produce 'verbosity' warnings (PR #189).""" + import logging + + with self.assertLogs('epmt', level='DEBUG') as cm: + eq.delete_jobs(['nonexistent_job_id'], force=True) + + # verify no 'verbosity is controlled elsewhere' messages + for record in cm.output: + self.assertNotIn('verbosity is controlled elsewhere', record) + + +class TestRetireCommand(unittest.TestCase): + """Tests for epmt_retire command orchestration.""" + + def test_retire_models_before_jobs(self): + """epmt_retire should run model retirement before job retirement.""" + call_order = [] + + def mock_retire_refmodels(*args, **kwargs): + call_order.append('models') + return 0 + + def mock_retire_jobs(*args, **kwargs): + call_order.append('jobs') + return 0 + + import epmt.epmt_cmd_retire as retire_mod + with patch.object(retire_mod, 'retire_refmodels', side_effect=mock_retire_refmodels): + with patch.object(retire_mod, 'retire_jobs', side_effect=mock_retire_jobs): + epmt_retire() + + self.assertEqual(call_order, ['models', 'jobs']) + + +class TestSignalHandler(unittest.TestCase): + """Tests for signal handler signature fix (PR #193).""" + + def test_sig_handler_accepts_frame(self): + """sig_handler must accept both signo and frame parameters.""" + import inspect + from epmt.epmt_job import post_process_job + source = inspect.getsource(post_process_job) + self.assertIn('def sig_handler(signo, frame)', source) + + +if __name__ == '__main__': + unittest.main() From 7e8d85603dc8212d501438d29b081aa541e0cefe Mon Sep 17 00:00:00 2001 From: Ian Laflotte Date: Fri, 20 Mar 2026 13:56:03 -0400 Subject: [PATCH 36/48] Implement VACUUM in dbcare, add test_dbcare to CI - Replace commented-out vacuum stub in epmt_cmd_dbcare.py with working implementation using raw DBAPI autocommit connection (VACUUM cannot run inside a transaction) - Query pg_stat_user_tables for dead row stats before and after vacuum - Set max_parallel_maintenance_workers=0 for serial execution - VACUUM (VERBOSE) processes_staging, processes, jobs individually - Add 4 vacuum unit tests to test_dbcare.py (dispatch, no-engine, table list) - Register test_dbcare in meta.yaml, build_and_test_epmt.yml, docker_build_test.yml, and create_test_conda_env.yml Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build_and_test_epmt.yml | 6 ++ .github/workflows/create_test_conda_env.yml | 3 + .github/workflows/docker_build_test.yml | 3 + meta.yaml | 1 + src/epmt/epmt_cmd_dbcare.py | 111 ++++++++++++++------ src/epmt/test/test_dbcare.py | 35 ++++++ 6 files changed, 128 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build_and_test_epmt.yml b/.github/workflows/build_and_test_epmt.yml index 8f1556f07..bddaf6cb4 100644 --- a/.github/workflows/build_and_test_epmt.yml +++ b/.github/workflows/build_and_test_epmt.yml @@ -225,6 +225,12 @@ jobs: env: COVERAGE_FILE: .coverage.cmds + - name: test_dbcare + run: | + TZ=UTC pytest -x -vv --cov=epmt --cov-report=term --cov-config=coveragerc src/epmt/test/test_dbcare.py + env: + COVERAGE_FILE: .coverage.dbcare + - name: test_db_migration run: | TZ=UTC pytest -x -vv --cov=epmt --cov-report=term --cov-config=coveragerc src/epmt/test/test_db_migration.py diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index 3b588fecb..aa4240ea4 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -77,6 +77,9 @@ jobs: - name: Unit test — test_cmds run: TZ=UTC pytest -x -vv src/epmt/test/test_cmds.py + - name: Unit test — test_dbcare + run: TZ=UTC pytest -x -vv src/epmt/test/test_dbcare.py + - name: Unit test — test_db_migration run: TZ=UTC pytest -x -vv src/epmt/test/test_db_migration.py diff --git a/.github/workflows/docker_build_test.yml b/.github/workflows/docker_build_test.yml index e0fa121ef..395967c10 100644 --- a/.github/workflows/docker_build_test.yml +++ b/.github/workflows/docker_build_test.yml @@ -461,6 +461,9 @@ jobs: - name: test_cmds run: docker exec ${{ env.CONTAINER_NAME }} bash -c 'cd $(python3 -c "import site; print(site.getsitepackages()[0])") && TZ=UTC pytest -x -vv epmt/test/test_cmds.py' + - name: test_dbcare + run: docker exec ${{ env.CONTAINER_NAME }} bash -c 'cd $(python3 -c "import site; print(site.getsitepackages()[0])") && TZ=UTC pytest -x -vv epmt/test/test_dbcare.py' + - name: test_db_migration run: docker exec ${{ env.CONTAINER_NAME }} bash -c 'cd $(python3 -c "import site; print(site.getsitepackages()[0])") && TZ=UTC pytest -x -vv epmt/test/test_db_migration.py' diff --git a/meta.yaml b/meta.yaml index 1e536ac6d..1f2211234 100644 --- a/meta.yaml +++ b/meta.yaml @@ -71,6 +71,7 @@ test: - pytest -v ${site_pkgs}/epmt/test/test_anysh.py - pytest -v ${site_pkgs}/epmt/test/test_check.py || echo "unit test test_check failed (GUARDED)" - pytest -v ${site_pkgs}/epmt/test/test_cmds.py + - pytest -v ${site_pkgs}/epmt/test/test_dbcare.py - pytest -v ${site_pkgs}/epmt/test/test_db_migration.py - pytest -v ${site_pkgs}/epmt/test/test_db_schema.py - pytest -v ${site_pkgs}/epmt/test/test_explore.py diff --git a/src/epmt/epmt_cmd_dbcare.py b/src/epmt/epmt_cmd_dbcare.py index a559f0681..437dd54b0 100644 --- a/src/epmt/epmt_cmd_dbcare.py +++ b/src/epmt/epmt_cmd_dbcare.py @@ -4,16 +4,90 @@ from logging import getLogger -#import epmt from epmt.epmt_cmd_retire import epmt_retire from epmt.epmt_query import post_process_jobs -#from epmt import epmt_query as eq from epmt.orm.sqlalchemy import orm_raw_sql +import epmt.orm.sqlalchemy.general as orm_general logger = getLogger(__name__) - -def epmt_dbcare(retire_jobs = False, vacuum_tables = False, post_process = False): +VACUUM_TABLES = ['processes_staging', 'processes', 'jobs'] + + +def _get_dead_row_stats(): + """Query pg_stat_user_tables for live/dead row counts of vacuum target tables.""" + table_list = ", ".join(f"'{t}'" for t in VACUUM_TABLES) + sql = ( + "SELECT relname, n_live_tup, n_dead_tup, last_vacuum, last_autovacuum " + "FROM pg_stat_user_tables " + f"WHERE relname IN ({table_list}) " + "ORDER BY relname" + ) + result = orm_raw_sql(sql) + rows = result.fetchall() + return rows + + +def _vacuum_tables(): + """Run VACUUM VERBOSE on target tables using an autocommit connection. + + VACUUM cannot run inside a transaction, so we obtain a raw DBAPI + connection and set it to autocommit mode. + """ + engine = orm_general.engine + if engine is None: + logger.error('no database engine available, cannot vacuum') + return + + # log dead row stats before vacuuming + try: + stats = _get_dead_row_stats() + for row in stats: + logger.info('pre-vacuum stats: table=%s live_rows=%s dead_rows=%s ' + 'last_vacuum=%s last_autovacuum=%s', + row[0], row[1], row[2], row[3], row[4]) + except Exception as e: + logger.warning('could not query dead row stats: %s', e) + + # VACUUM requires autocommit — use a raw DBAPI connection + raw_conn = engine.raw_connection() + try: + raw_conn.set_isolation_level(0) # 0 = ISOLATION_LEVEL_AUTOCOMMIT + cursor = raw_conn.cursor() + + # limit parallel maintenance workers for serial execution + logger.info('setting max_parallel_maintenance_workers = 0 (serial execution)') + cursor.execute('SET max_parallel_maintenance_workers = 0') + + for table in VACUUM_TABLES: + logger.info('vacuuming table: %s', table) + try: + cursor.execute(f'VACUUM (VERBOSE) {table}') + # VACUUM VERBOSE output comes through as NOTICEs; + # fetch any notices from the connection + if hasattr(raw_conn, 'notices') and raw_conn.notices: + for notice in raw_conn.notices: + logger.info('vacuum %s: %s', table, notice.strip()) + raw_conn.notices.clear() + logger.info('finished vacuuming table: %s', table) + except Exception as e: + logger.error('error vacuuming table %s: %s', table, e) + finally: + cursor.close() + raw_conn.close() + + # log dead row stats after vacuuming + try: + stats = _get_dead_row_stats() + for row in stats: + logger.info('post-vacuum stats: table=%s live_rows=%s dead_rows=%s ' + 'last_vacuum=%s last_autovacuum=%s', + row[0], row[1], row[2], row[3], row[4]) + except Exception as e: + logger.warning('could not query post-vacuum dead row stats: %s', e) + + +def epmt_dbcare(retire_jobs=False, vacuum_tables=False, post_process=False): ''' routine to help regularly take care of the database. for each arg that's true, undertake a cleanup behavior retire_jobs will run job retirement. vacuum_tables will run the SQL command VACUUM on jobs, processes, and @@ -30,36 +104,11 @@ def epmt_dbcare(retire_jobs = False, vacuum_tables = False, post_process = False epmt_retire(skip_unprocessed=True, dry_run=False) - ## VACUUM DB TABLES ( TEST THIS APPROACH BY HAND FIRST ) - ## NOTYETIMPLEMENTED, this needs to use the sqlalchemy.engine connection instance to make this work + ## VACUUM DB TABLES if not vacuum_tables: logger.warning('skipping vacuuming of tables in DB') else: - logger.info('limiting the allowed number of parallel maintenance processes to 0 (serial execution only)') - #result_set_max_parallel_workers=orm_raw_sql(psql_set_max_parallel_workers) - #logger.info('result: %s', str(result_set_max_parallel_workers.scalars().all() ) ) - # - ## vacuum each table one-by-one with no parallel maintenance workers - #psql_set_max_parallel_workers='SET max_parallel_maintenance_workers = 0;' - #psql_stub_vacuum='VACUUM VERBOSE ' - #psql_vacuum_jobs=psql_stub_vacuum + 'jobs;' - #psql_vacuum_procs_stag=psql_stub_vacuum + 'processes_staging;' - #psql_vacuum_procs=psql_stub_vacuum + 'processes;' - # - #logger.info('vacuuming jobs table of dead rows') - #result_vacuum_jobs=orm_raw_sql(' '.join[ psql_set_max_parallel_workers, - # psql_vacuum_jobs ] ) - #logger.info('result: %s', str(result_vacuum_jobs.scalars().all() ) ) - # - #logger.info('vacuuming processes_staging table of dead rows') - #result_vacuum_procs_stag=orm_raw_sql(' '.join[ psql_set_max_parallel_workers, - # psql_vacuum_procs_stag ] ) - #logger.info('result: %s', str(result_vacuum_procs_stag.scalars().all() ) ) - # - #logger.info('vacuuming processes table of dead rows') - #result_vacuum_procs=orm_raw_sql(' '.join[ psql_set_max_parallel_workers, - # psql_vacuum_procs ] ) - #logger.info('result: %s', str(result_vacuum_procs.scalars().all() ) ) + _vacuum_tables() ## POST PROCESS JOBS if not post_process: diff --git a/src/epmt/test/test_dbcare.py b/src/epmt/test/test_dbcare.py index 7667af583..9d4092e2e 100644 --- a/src/epmt/test/test_dbcare.py +++ b/src/epmt/test/test_dbcare.py @@ -86,6 +86,41 @@ def test_dbcare_retire_does_not_delete_recent_jobs(self): self.assertIn('685000', jobs) +class TestVacuum(unittest.TestCase): + """Tests for vacuum functionality.""" + + def test_vacuum_skipped_when_disabled(self): + """dbcare with vacuum_tables=False should skip vacuuming.""" + import epmt.epmt_cmd_dbcare as dbcare_mod + with patch.object(dbcare_mod, '_vacuum_tables') as mock_vacuum: + epmt_dbcare(retire_jobs=False, vacuum_tables=False, post_process=False) + mock_vacuum.assert_not_called() + + def test_vacuum_called_when_enabled(self): + """dbcare with vacuum_tables=True should call _vacuum_tables.""" + import epmt.epmt_cmd_dbcare as dbcare_mod + with patch.object(dbcare_mod, '_vacuum_tables') as mock_vacuum: + epmt_dbcare(retire_jobs=False, vacuum_tables=True, post_process=False) + mock_vacuum.assert_called_once() + + def test_vacuum_tables_no_engine(self): + """_vacuum_tables should log error and return if no engine available.""" + import epmt.epmt_cmd_dbcare as dbcare_mod + saved_engine = orm_general.engine + try: + orm_general.engine = None + dbcare_mod._vacuum_tables() + finally: + orm_general.engine = saved_engine + + def test_vacuum_tables_list(self): + """VACUUM_TABLES should contain the expected tables.""" + from epmt.epmt_cmd_dbcare import VACUUM_TABLES + self.assertIn('jobs', VACUUM_TABLES) + self.assertIn('processes', VACUUM_TABLES) + self.assertIn('processes_staging', VACUUM_TABLES) + + class TestRetireJobs(unittest.TestCase): """Tests for retire_jobs efficiency improvements (PR #189).""" From c743a65e95b5db251536d6f82e6adab298286d62 Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Fri, 20 Mar 2026 15:06:31 -0400 Subject: [PATCH 37/48] Add unit test for test_dbcare --- .github/workflows/publish_conda.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/publish_conda.yml b/.github/workflows/publish_conda.yml index 8b2df4046..359194620 100644 --- a/.github/workflows/publish_conda.yml +++ b/.github/workflows/publish_conda.yml @@ -122,6 +122,11 @@ jobs: conda activate epmt-verify TZ=UTC pytest -x -vv ${{ steps.site_pkgs.outputs.site_pkgs }}/epmt/test/test_cmds.py + - name: Unit test — test_dbcare + run: | + conda activate epmt-verify + TZ=UTC pytest -x -vv ${{ steps.site_pkgs.outputs.site_pkgs }}/epmt/test/test_dbcare.py + - name: Unit test — test_db_migration run: | conda activate epmt-verify From fe8f2606b9dbb34215e6e628eaeb106bc68ec15d Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Fri, 20 Mar 2026 16:08:40 -0400 Subject: [PATCH 38/48] Add tag trigger to publish_conda workflow --- .github/workflows/publish_conda.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish_conda.yml b/.github/workflows/publish_conda.yml index 359194620..966e39629 100644 --- a/.github/workflows/publish_conda.yml +++ b/.github/workflows/publish_conda.yml @@ -1,9 +1,10 @@ name: publish_conda - on: push: branches: - main + tags: + - '*' # # DEBUG ONLY: remove pull_request trigger before merging — only use if-needed for debugging the upload only # pull_request: From f582038c1ca89a94219af1170310a512401efc27 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 20 Mar 2026 16:35:36 +0000 Subject: [PATCH 39/48] fix: materialize SQL results before closing connection to prevent cursor already closed error In orm_raw_sql(), the connection was closed before returning the CursorResult proxy. When callers iterated over results (e.g., 20K+ staged processes), this caused psycopg2.InterfaceError: cursor already closed. Fix by calling res.fetchall() to materialize results into memory before connection.close(). Also close connection in the commit=True path which previously leaked connections. Update epmt_cmd_dbcare.py callers to work with materialized list of rows instead of CursorResult (.scalars().all() replaced with direct index/list access). Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com> --- src/epmt/epmt_cmd_dbcare.py | 4 ++-- src/epmt/orm/sqlalchemy/general.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/epmt/epmt_cmd_dbcare.py b/src/epmt/epmt_cmd_dbcare.py index 437dd54b0..ba4075ab2 100644 --- a/src/epmt/epmt_cmd_dbcare.py +++ b/src/epmt/epmt_cmd_dbcare.py @@ -124,7 +124,7 @@ def epmt_dbcare(retire_jobs=False, vacuum_tables=False, post_process=False): result_count_unprocd_jobs=orm_raw_sql(psql_count_unprocd_jobs) number_of_unprocd_jobs=-1 try: - number_of_unprocd_jobs=result_count_unprocd_jobs.scalars().all()[0] + number_of_unprocd_jobs=result_count_unprocd_jobs[0][0] except Exception as e: raise Exception from e @@ -142,7 +142,7 @@ def epmt_dbcare(retire_jobs=False, vacuum_tables=False, post_process=False): result_get_unprocd_jobs=orm_raw_sql(psql_get_unprocd_jobs) try: logger.info('attempting to retrieve job IDs') - job_list=result_get_unprocd_jobs.scalars().all() + job_list=[row[0] for row in result_get_unprocd_jobs] logger.debug('job_list is %s', ' '.join(job_list)) except Exception as e: raise Exception('problem with retrieving job IDs. inspect code.') from e diff --git a/src/epmt/orm/sqlalchemy/general.py b/src/epmt/orm/sqlalchemy/general.py index f08ea1833..f6212fe19 100644 --- a/src/epmt/orm/sqlalchemy/general.py +++ b/src/epmt/orm/sqlalchemy/general.py @@ -734,12 +734,16 @@ def orm_raw_sql(sql, commit=False): res = connection.execute(sqla_sql.text(s)) if commit: trans.commit() + connection.close() return True + # Materialize results before closing the connection to avoid + # psycopg2.InterfaceError: cursor already closed + rows = res.fetchall() except BaseException: trans.rollback() raise connection.close() - return res + return rows def set_sql_debug(discard): From 1d85255c2d259dfbdb1286a39be76b9537e6afb8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 20 Mar 2026 20:54:04 +0000 Subject: [PATCH 40/48] fix: update orm_db_size callers to work with materialized list from orm_raw_sql MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since orm_raw_sql() now returns a plain list via fetchall(), callers in orm/__init__.py that used .fetchall() or .fetchone() on the result fail with AttributeError. Replace: - orm_sql(cmd).fetchall()[0][0] → orm_sql(cmd)[0][0] - orm_sql(cmd).fetchone()[0] → orm_sql(cmd)[0][0] Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com> Agent-Logs-Url: https://github.com/NOAA-GFDL/epmt/sessions/ba1a0164-a880-4fc7-9d9f-da0627ee6a42 --- src/epmt/orm/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/epmt/orm/__init__.py b/src/epmt/orm/__init__.py index 018cdc873..381e0d5b4 100644 --- a/src/epmt/orm/__init__.py +++ b/src/epmt/orm/__init__.py @@ -96,9 +96,9 @@ def orm_db_size(findwhat=['database', 'table', 'index', 'tablespace'], usejson=T # try: for table in orm_dump_schema(show_attributes=False): cmd = "SELECT pg_total_relation_size(\'" + table + "\')" - size = orm_sql(cmd).fetchall()[0][0] + size = orm_sql(cmd)[0][0] cmd = "SELECT count(*) from \"" + table + "\"" - count = orm_sql(cmd).fetchall()[0][0] + count = orm_sql(cmd)[0][0] tabled[table] = [int(size), int(count)] logger.debug("table[%s]=[%d,%d]", table, int(size), int(count)) struct[arg] = tabled @@ -111,7 +111,7 @@ def orm_db_size(findwhat=['database', 'table', 'index', 'tablespace'], usejson=T try: for table in orm_dump_schema(show_attributes=False): cmd = "SELECT pg_indexes_size(\'" + table + "\')" - size = orm_sql(cmd).fetchone()[0] + size = orm_sql(cmd)[0][0] indexd[table] = int(size) logger.debug("index[%s]=%d", table, int(size)) struct[arg] = indexd @@ -126,7 +126,7 @@ def orm_db_size(findwhat=['database', 'table', 'index', 'tablespace'], usejson=T for tablespace in tablespaces: tablespace = tablespace[0] cmd = "SELECT pg_tablespace_size(\'" + str(tablespace) + "\')" - size = orm_sql(cmd).fetchall()[0][0] + size = orm_sql(cmd)[0][0] tablespaced[tablespace] = int(size) logger.debug("tablespace[%s]=%d", tablespace, int(size)) struct[arg] = tablespaced From 180a261261ec3ebbd662c98a03626aa9e4f3865b Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Mon, 23 Mar 2026 12:21:27 -0400 Subject: [PATCH 41/48] Update epmtlib.py --- src/epmt/epmtlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/epmt/epmtlib.py b/src/epmt/epmtlib.py index c24f0db20..dfa975ebd 100644 --- a/src/epmt/epmtlib.py +++ b/src/epmt/epmtlib.py @@ -23,7 +23,7 @@ # third element is the patch or bugfix number # Since we are saving as a tuple you can do a simple # compare of two version tuples and python will do the right thing -_version = (5, 0, 0) +_version = (5, 0, 1) # pre-release / development version __version__ = ".".join([str(i) for i in _version]) From 80c8678f6edbbc68faef21d57756fb41e32356f1 Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Mon, 23 Mar 2026 12:22:25 -0400 Subject: [PATCH 42/48] Update version in meta.yaml to pre5.0.1 --- meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta.yaml b/meta.yaml index 1f2211234..7a27526e4 100644 --- a/meta.yaml +++ b/meta.yaml @@ -7,7 +7,7 @@ package: name: epmt # Version is read from the git tag at build time. # When building locally without a tag, the version will be "develop". - version: '{{ environ.get("GIT_DESCRIBE_TAG", "pre5.0.0") }}' + version: '{{ environ.get("GIT_DESCRIBE_TAG", "pre5.0.1") }}' source: path: . From c5f94c724461051d0d7531b3eb478925d8a29529 Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Mon, 23 Mar 2026 12:25:03 -0400 Subject: [PATCH 43/48] Enable Docker build on tag pushes Add support for Docker builds on tag pushes. --- .github/workflows/docker_build_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker_build_test.yml b/.github/workflows/docker_build_test.yml index 395967c10..bdced41b4 100644 --- a/.github/workflows/docker_build_test.yml +++ b/.github/workflows/docker_build_test.yml @@ -5,6 +5,8 @@ on: push: branches: - main + tags: + - '*' # cancel running jobs if there's a newer push concurrency: From b7752d78a94b6ab7edc112a82998b16460a18702 Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Mon, 23 Mar 2026 12:25:28 -0400 Subject: [PATCH 44/48] Add tag trigger to create_test_conda_env workflow --- .github/workflows/create_test_conda_env.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index aa4240ea4..bf1c7fe89 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -7,6 +7,8 @@ on: push: branches: - main + tags: + - '*' # cancel running jobs if theres a newer push concurrency: From 946d1cd12cebf74ef445b7dee55b75d2e7350cba Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Mon, 23 Mar 2026 12:34:48 -0400 Subject: [PATCH 45/48] Update epmtlib.py --- src/epmt/epmtlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/epmt/epmtlib.py b/src/epmt/epmtlib.py index dfa975ebd..4996f4595 100644 --- a/src/epmt/epmtlib.py +++ b/src/epmt/epmtlib.py @@ -23,7 +23,7 @@ # third element is the patch or bugfix number # Since we are saving as a tuple you can do a simple # compare of two version tuples and python will do the right thing -_version = (5, 0, 1) # pre-release / development version +_version = (5, 0, 1) __version__ = ".".join([str(i) for i in _version]) From 7126bef651185542fa9bb1bb1c82ce26df92f635 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Tue, 24 Mar 2026 17:58:52 -0400 Subject: [PATCH 46/48] Add installation instructions and simplify README (#207) * Initial plan * Add installation instructions to README and move detailed content to DEVELOPER.md Closes #205 - adds conda/pip/source installation instructions Closes #11 - simplifies README to a concise quick-start guide Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com> Agent-Logs-Url: https://github.com/NOAA-GFDL/epmt/sessions/d7c05a3a-fca7-4e0e-a6e6-12ece98dc840 * Rename EPMT to Experiment Process / Metadata Tool Updated README to reflect new project name and features. * complete overhaul of advancedd/developer style information * remove no longer existing sections * last tweaks --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: ilaflott <6273252+ilaflott@users.noreply.github.com> Co-authored-by: Ian Laflotte --- DEVELOPER.md | 595 +++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 586 +++++--------------------------------------------- 2 files changed, 646 insertions(+), 535 deletions(-) create mode 100644 DEVELOPER.md diff --git a/DEVELOPER.md b/DEVELOPER.md new file mode 100644 index 000000000..4ca585bdd --- /dev/null +++ b/DEVELOPER.md @@ -0,0 +1,595 @@ +# `epmt` Developer and Advanced Usage Guide +This document contains detailed information about `epmt` configuration, data collection, +database submission, analysis, metrics, debugging, and CI/CD infrastructure. + +For installation and quick-start instructions, see [README.md](./README.md). + + + + +## Table of Contents + +- [Configuration](#configuration) + - [Environment Variables](#environment-variables) + - [Getting Current Configuration Information](#getting-current-configuration-information) +- [The Three Modes of `epmt`](#the-three-modes-of-epmt) + - [The First Mode, Data Collection](#the-first-mode-data-collection) + - [Collecting Performance Data](#collecting-performance-data) + - [Data Collection with SLURM epilog and prolog](#data-collection-with-slurm-epilog-and-prolog) + - [The Second Mode, Data Submission](#the-second-mode-data-submission) + - [Manual Submission Example](#manual-submission-example) + - [Compressed Directory Submission Exmple](#compressed-directory-submission-exmple) + - [Internal-batch Job Submission Example](#internal-batch-job-submission-example) + - [Data From Current Session Submission Example](#data-from-current-session-submission-example) + - [The Third Mode, Data Analysis and Visualization](#the-third-mode-data-analysis-and-visualization) +- [Debugging](#debugging) +- [Performance Metrics Data Dictionary](#performance-metrics-data-dictionary) + - [Addition of new metrics](#addition-of-new-metrics) +- [CI/CD Workflows and Caching](#cicd-workflows-and-caching) + - [Workflows](#workflows) + - [Caches and Invalidation](#caches-and-invalidation) + - [Cache Invalidation Gap vs. `make`](#cache-invalidation-gap-vs-make) + - [Forcing a Rebuild](#forcing-a-rebuild) + - [Weekly Pre-warming](#weekly-pre-warming) +- [Troubleshooting](#troubleshooting) + - [Virtual Environments:](#virtual-environments) + - [Common Error Messages](#common-error-messages) + - [`version GLIBC_x.xx not found`](#version-glibc_xxx-not-found) + + + + +## Configuration +`epmt` houses default settings (`epmt_default_settings.py`), user settings (`settings.py`), +and a submodule (`epmt_settings.py`) that appends the user settings to the end of the +default settings, overriding them. + +Custom user settings should only be included by developers who understand `epmt`'s requirements. +The default settings should first be evaluated, they look a little bit like the following: +```bash +$ cat src/epmt/epmt_default_settings.py + ... + papiex_options = "COLLATED_TSV" + epmt_output_prefix = "/tmp/epmt/" + input_pattern = "*-papiex*.[ct]sv" + ... + db_params = { + 'url': 'sqlite:///:memory:', + 'echo': False, + } + ... +``` + + + +### Environment Variables +The following variables replace, at run-time, the values in the `db_params` dictionary found in `settings.py`: +``` +EPMT_DB_PROVIDER +EPMT_DB_USER +EPMT_DB_PASSWORD +EPMT_DB_HOST +EPMT_DB_DBNAME +EPMT_DB_FILENAME +``` + + + +### Getting Current Configuration Information +You can examine all current settings by passing the `--help` option: +```bash +$ ./epmt --help +usage: epmt [-n] [-d] [-h] [-a] [--drop] + [epmt_cmd] [epmt_cmd_args [epmt_cmd_args ...]] + +positional arguments: + epmt_cmd start, run, stop, submit, dump + epmt_cmd_args Additional arguments, command dependent + +optional arguments: + -n, --dry-run Don't touch the database + -v, --verbose Increase level of verbosity/debug + -h, --help Show this help message and exit + -a, --auto Do start/stop when running + --drop Drop all tables/data and recreate before importing + +settings.py (overridden by below env. vars): +db_params {'host': 'localhost', 'password': 'example', 'user': 'postgres', 'dbname': 'EPMT', 'provider': 'postgres'} +debug False +input_pattern *-papiex-[0-9]*-[0-9]*.csv +install_prefix ../papiex-oss/papiex-oss-install/ +papiex_options PERF_COUNT_SW_CPU_CLOCK +epmt_output_prefix /tmp/epmt/ + +environment variables (overrides settings.py): +``` + + + + +## The Three Modes of `epmt` +There are three modes to `epmt` usage; data collection, data submission, and data analysis. Each +has different dependencies: + +* **Collection** requires Python 2.6 or higher, and compiled [`papiex`](https://github.com/noaa-gfdl/papiex) +libraries for process tagging +* **Submission** requires Python packages for SQL and database interactions (`sqlalchemy`, `sqlite`, `postgres`) +* **Analysis** requires `jupyter`, `iPython`, and additionally [`epmt-dash`](https://github.com/noaa-gfdl/epmt-dash) +and `plotly` for dashboard-style analytics + + + +### The First Mode, Data Collection + + + +#### Collecting Performance Data +Assuming you have `epmt` installed and in your path, let's modify a job file: +```bash +$ cat my_job.sh +#!/bin/bash +# Example job script for Torque or SLURM +./compute_the_world --debug +``` + +This becomes: +```bash +$ cat my_job_epmt.sh +#!/bin/bash +# Example job script for Torque or SLURM +epmt start +epmt run ./compute_the_world --debug +epmt stop +``` + +Or more succintlty by automating the start/stop cycle with the `--auto` or `-a` flag: +```bash +$ cat my_job_epmt2.sh +#!/bin/bash +# Example job script for Torque or SLURM +epmt -a run ./compute_the_world --debug +``` + +But usually we want to run more than one executable. We could have any number of run statements: +```bash +$ cat my_job_epmt3.sh +#!/bin/bash +# Example job script for Torque or SLURM +epmt start +epmt run ./initialize_the_world --random +epmt run ./compute_the_world +epmt run ./postprocess_the_world +epmt stop +``` + +Let's skip all the markup and do it with only environment variables. `epmt` +provides the configuration to export to the environment through the `source` +command. This is intended for use in a job file and should be evaluated by the +running shell, be it `bash` or `csh`. `epmt source` prints the required +environment variables in Bash format unless either the `SHELL` or `_` +environment variable ends in `csh`. + +**Note the unset of `LD_PRELOAD` before stop!** This prevents the data +collection routine from running on `epmt stop` itself. +```bash +$ cat my_job_bash.sh +#!/bin/bash +# Example job script for Torque or SLURM + +### Preamble, collect job metadata and monitor all processes/threads +epmt start +eval `epmt source` + +./initialize_the_world --random +./compute_the_world +./postprocess_the_world + +# Postamble, disable monitoring and collect job metadata +unset LD_PRELOAD +epmt stop +``` + +Here's an example for `csh`, when run interactively, +```bash +$ /bin/csh +> epmt -j1 source +setenv PAPIEX_OPTIONS PERF_COUNT_SW_CPU_CLOCK; +setenv PAPIEX_OUTPUT /tmp/epmt/1/; +setenv LD_PRELOAD /Users/phil/Work/GFDL/epmt.git/../papiex-oss/papiex-oss-install/lib/libpapiex.so:/Users/phil/Work/GFDL/epmt.git/../papiex-oss/papiex-oss-install/lib/libpapi.so:/Users/phil/Work/GFDL/epmt.git/../papiex-oss/papiex-oss-install/lib/libpfm.so:/Users/phil/Work/GFDL/epmt.git/../papiex-oss/papiex-oss-install/lib/libmonitor.so +``` + +#### Data Collection with SLURM epilog and prolog +Using configured prolog and epilogs with SLURM tasks allows one to skip job +instrumentation entirely, except for job tags (`EPMT_JOB_TAGS`) and process +tags (`PAPIEX_TAGS`). These are configured in `slurm.conf` for jobs +submitted with `sbatch` but can be tested on the command line when using `srun`. + +The above Csh job is equivalent to the below sequence using a prolog and +epilog, ***with the exception of the trailing submit statement.*** + +```bash +$ SLURM_TASK_SCRIPT_DIR=${EPMT_PREFIX}/epmt-install/slurm +$ srun -n1 \\ + --task-prolog=${SLURM_TASK_SCRIPT_DIR}/slurm_task_prolog_epmt.sh \\ + --task-epilog=${SLURM_TASK_SCRIPT_DIR}/slurm_task_epilog_epmt.sh +$ sleep 1 +``` + +For this job to work using `sbatch`, make the following modifications in +`slurm.conf`, substituting the appropriate path for `$EPMT_PREFIX`: +```bash +$ SLURM_TASK_SCRIPT_DIR=${EPMT_PREFIX}/epmt-install/slurm +$ TaskProlog=${SLURM_TASK_SCRIPT_DIR}/slurm_task_prolog_epmt.sh +$ TaskEpilog=${SLURM_TASK_SCRIPT_DIR}/slurm_task_epilog_epmt.sh +``` + +If this fails, the `papiex` installation is likely either missing or +misconfigured in `settings.py`. The `-a` flag tells `epmt` to treat +this run as an entire job. + + + +### The Second Mode, Data Submission +After collecting the data, jobs (groups of processes) are imported into the +database with the `submit` command. This command takes arguments in the form of +directories or tar files that must contain a `job_metadata` file. + +Normal operation is to submit one or more directories: +```bash +$ epmt submit [...] +``` + +One can also submit a list of compressed tar files: +```bash +$ epmt submit [...] +``` + +There is also a mode where the current environment is used to determine where to find the data. +```bash +$ epmt submit +``` + + +#### Manual Submission Example +We can submit our previous job to the database defined in `settings.py` by +running the `epmt submit` command with the directory returned by `stage` (found +in the location set by `settings.py` `epmt_output_prefix`): +```bash +$ epmt -v submit /tmp/epmt/1/ +INFO:epmt_cmds:submit_to_db(/tmp/epmt/1/,*-papiex-[0-9]*-[0-9]*.csv,False) +INFO:epmt_cmds:Unpickling from /tmp/epmt/1/job_metadata +INFO:epmt_cmds:1 files to submit +INFO:epmt_cmds:1 hosts found: ['linuxkit-025000000001-'] +INFO:epmt_cmds:host linuxkit-025000000001-: 1 files to import +INFO:epmt_job:Binding to DB: {'filename': ':memory:', 'provider': 'sqlite'} +INFO:epmt_job:Generating mapping from schema... +INFO:epmt_job:Processing job id 1 +INFO:epmt_job:Creating user root +INFO:epmt_job:Creating job 1 +INFO:epmt_job:Creating host linuxkit-025000000001- +INFO:epmt_job:Creating metricname usertime +INFO:epmt_job:Creating metricname systemtime +INFO:epmt_job:Creating metricname rssmax +INFO:epmt_job:Creating metricname minflt +INFO:epmt_job:Creating metricname majflt +INFO:epmt_job:Creating metricname inblock +INFO:epmt_job:Creating metricname outblock +INFO:epmt_job:Creating metricname vol_ctxsw +INFO:epmt_job:Creating metricname invol_ctxsw +INFO:epmt_job:Creating metricname num_threads +INFO:epmt_job:Creating metricname starttime +INFO:epmt_job:Creating metricname processor +INFO:epmt_job:Creating metricname delayacct_blkio_time +INFO:epmt_job:Creating metricname guest_time +INFO:epmt_job:Creating metricname rchar +INFO:epmt_job:Creating metricname wchar +INFO:epmt_job:Creating metricname syscr +INFO:epmt_job:Creating metricname syscw +INFO:epmt_job:Creating metricname read_bytes +INFO:epmt_job:Creating metricname write_bytes +INFO:epmt_job:Creating metricname cancelled_write_bytes +INFO:epmt_job:Creating metricname time_oncpu +INFO:epmt_job:Creating metricname time_waiting +INFO:epmt_job:Creating metricname timeslices +INFO:epmt_job:Creating metricname rdtsc_duration +INFO:epmt_job:Creating metricname PERF_COUNT_SW_CPU_CLOCK +INFO:epmt_job:Adding 1 processes to job +INFO:epmt_job:Earliest process start: 2019-03-06 15:36:56.948350 +INFO:epmt_job:Latest process end: 2019-03-06 15:37:06.996065 +INFO:epmt_job:Computed duration of job: 10047715.000000 us, 0.17 m +INFO:epmt_job:Staged import of 1 processes, 1 threads +INFO:epmt_job:Staged import took 0:00:00.189151, 5.286781 processes per second +INFO:epmt_cmds:Committed job 1 to database: Job[u'1'] +``` + + +#### Compressed Directory Submission Exmple +This might happen at the end of the day via a cron job: +```bash +$ epmt submit /*tgz +``` + + +#### Internal-batch Job Submission Example +These commands could be part of every users job, or in the batch systems configurable preambles/postambles. +```bash +$ cat my_job.sh +#!/bin/bash +# Example job script for Torque or SLURM +echo "$PBS_JOBID or $SLURM_JOBID" +epmt start +epmt run ./compute_the_world --debug +epmt stop +epmt submit +``` + +The start/stop cycle can be removed with the `--auto` or `-a` flag, which performs start and stop for you: +``` +$ epmt -a run ./debug_the_world --outliers +$ epmt submit +``` + + +#### Data From Current Session Submission Example +If not inside of a batch environment, `epmt` will *attempt to fake-and-bake a job id*. This is useful +when performing interactive runs. You may not be able to submit these jobs to a shared database due to +constraints on job ID uniqueness, sincet he session ID is not guaranteed to be unique across reboots, +much less other systems. However, this use case is perfectly acceptable when using a private database: +```bash +$ epmt start +WARNING:epmt_cmds:JOB_ID unset: Using session id 6948 as JOB_ID +WARNING:epmt_cmds:JOB_NAME unset: Using job id 6948 as JOB_NAME +WARNING:epmt_cmds:JOB_SCRIPTNAME unset: Using process name 6948 as JOB_SCRIPTNAME +WARNING:epmt_cmds:JOB_USER unset: Using username phil as JOB_USE$ epmt run ./debug_the_world --outliers +$ epmt stop +$ epmt submit +``` + + + +### The Third Mode, Data Analysis and Visualization +`epmt` uses an `IPython` notebook data analytics environment. Starting the +Jupyter notebook is easy from the `epmt notebook` command: +```bash +$ epmt notebook +[I 15:39:24.236 NotebookApp] Serving notebooks from local directory: /home/chris/Documents/playground/MM/build/epmt +[I 15:39:24.236 NotebookApp] The Jupyter Notebook is running at: +[I 15:39:24.236 NotebookApp] http://localhost:8888/?token=9c7529e19e12cb8121d66ff471e96fdd3056f6acc4480274 +[I 15:39:24.236 NotebookApp] or http://127.0.0.1:8888/?token=9c7529e19e12cb8121d66ff471e96fdd3056f6acc4480274 +[I 15:39:24.236 NotebookApp] Use Control-C to stop this server and shut down all kernels (twice to skip confirmation). +[C 15:39:24.263 NotebookApp] + + To access the notebook, open this file in a browser: + file:///home/chris/.local/share/jupyter/runtime/nbserver-18690-open.html + Or copy and paste one of these URLs: + http://localhost:8888/?token=9c7529e19e12cb8121d66ff471e96fdd3056f6acc4480274 + or http://127.0.0.1:8888/?token=9c7529e19e12cb8121d66ff471e96fdd3056f6acc4480274 +``` + +The notebook command supports passing parameters to jupyter, such as host IP +for sharing access with machines on the local network, notebook token, and +notebook password: +```bash +$ epmt notebook -- --ip 0.0.0.0 --NotebookApp.token='thisisatoken' --NotebookApp.password='hereisa$upersecurepassword' +``` + + + + +## Debugging +`epmt` can be passed both `-n` (dry-run) and `-v` (verbosity) to help +with debugging. Add more `-v` flags to increase verbosity level (`-vvv`). +```bash +$ epmt -v start +``` + +Or to attempt a submit without touching the database: +```bash +$ epmt -vv submit -n /dir/to/jobdata +``` + +Also, one can decode and dump the job_metadata file in a dir or compressed dir. +```bash +$ epmt dump ~/Downloads/yrs05-25.20190221/CM4_piControl_C_atmos_00050101.papiex.gfdl.19712961.tgz +exp_component atmos +exp_jobname CM4_piControl_C_atmos_00050101 +exp_name CM4_piControl_C +exp_oname 00050101 +job_el_env_changes {} +job_el_env_changes_len 0 +job_el_from_batch [] +job_el_status 0 +job_el_stop 2019-02-20 22:13:23.131187 +job_pl_env {'LANG': 'en_US', 'PBS_QUEUE': 'batch', 'SHELL': '/bin/csh', 'PBS_ENVIRONMENT': 'PBS_BATCH', 'PAPIEX_TAGS': 'atmos', 'SHLVL': '3', 'PBS_WALLTIME': '216000', 'MOAB_NODELIST': 'pp057.princeton.rdhpcs.noaa.gov', 'PBS_VERSION': 'TORQUE-6.0.2', 'PAPIEX_OUTPUT': '/vftmp/Foo.Bar/pbs20345339/papiex', 'LOADEDMODULES': '', 'LC_TIME': 'C', 'MACHTYPE': 'x86_64', 'PAPIEX_OPTIONS': 'PERF_COUNT_SW_CPU_CLOCK', 'MOAB_GROUP': 'f'} +job_pl_env_len 81 +job_pl_from_batch [] +job_pl_groupnames ['f', 'f'] +job_pl_hostname pp057 +job_pl_id 20345339.moab01.princeton.rdhpcs.noaa.gov +job_pl_jobname CM4_piControl_C_atmos_00050101 +job_pl_scriptname CM4_piControl_C_atmos_00050101 +job_pl_start 2019-02-20 19:58:41.274267 +job_pl_submit 2019-02-20 19:58:41.274463 +job_pl_username Foo.Bar +``` + + + + +## Performance Metrics Data Dictionary +`epmt` collects data both from the job runtime and the applications run in that +environment. See the `src/epmt/models/` directory for fixed data stored related to each +object. Metric data is stored differently; the data collector's data dictionary +can be found in papiex-oss/README.md. At the time of this writing, it looked +like this: +| Key | Scope | Description | +|--------------------------- |--------- |-------------------------------------------------------- | +| 1. tags | Process | User specified tags for this executable | +| 2. hostname | Process | hostname | +| 3. exename | Process | Name of the application, usually argv[0] | +| 4. path | Process | Path to the application | +| 5. args | Process | All arguments to exe excluding argv[0] | +| 6. exitcode | Process | Exit code | +| 7. exitsignal | Process | Exited due to a signal | +| 8. pid | Process | Process id | +| 9. generation | Process | Incremented after every exec() or PID wrap | +| 10. ppid | Process | Parent process id | +| 11. pgid | Process | Process group id | +| 12. sid | Process | Process session id | +| 13. numtids | Process | Number of threads caught by instrumentation | +| 14. numranks | Process | Number of MPI ranks detected | +| 15. tid | Process | Thread id | +| 16. mpirank | Thread | MPI rank | +| 17. start | Process | Microsecond timestamp at start | +| 18. end | Process | Microsecond timestamp at end | +| 19. usertime | Thread | Microsecond user time | +| 20. systemtime | Thread | Microsecond system time | +| 21. rssmax | Thread | Kb max resident set size | +| 22. minflt | Thread | Minor faults (TLB misses/new page frames) | +| 23. majflt | Thread | Major page faults (requiring I/O) | +| 24. inblock | Thread | 512B blocks read from I/O | +| 25. outblock | Thread | 512B blocks written to I/O | +| 26. vol_ctxsw | Thread | Voluntary context switches (yields) | +| 27. invol_ctxsw | Thread | Involuntary context switches (preemptions) | +| 28. cminflt | Process | minflt (20) for all wait()ed children | +| 29. cmajflt | Thread | majflt (21) for all wait()ed children | +| 30. cutime | Process | utime (17) for all wait()ed children | +| 31. cstime | Thread | stime (18) for all wait()ed children | +| 32. num_threads | Process | Threads in process at finish | +| 33. starttime | Thread | Timestamp in jiffies after boot thread was started | +| 34. processor | Thread | CPU this thread last ran on | +| 35. delayacct_blkio_time | Thread | Jiffies process blocked in D state on I/O device | +| 36. guest_time | Thread | Jiffies running a virtual CPU for a guest OS | +| 37. rchar | Thread | Bytes read via syscall (maybe from cache not dev I/O) | +| 38. wchar | Thread | Bytes written via syscall (maybe to cache not dev I/O) | +| 39. syscr | Thread | Read syscalls | +| 40. syscw | Thread | Write syscalls | +| 41. read_bytes | Thread | Bytes read from I/O device | +| 42. write_bytes | Thread | Bytes written to I/O device | +| 43. cancelled_write_bytes | Thread | Bytes discarded by truncation | +| 44. time_oncpu | Thread | Nanoseconds spent running | +| 45. time_waiting | Thread | Nanoseconds runnable but waiting | +| 46. timeslices | Thread | Number of run periods on CPU | +| 47. rdtsc_duration | Thread | If PAPI, real time cycle duration of thread | +| * | Thread | PAPI metrics | + + + +### Addition of new metrics +Additional metrics can be configured either in two ways: +* The `papiex_options` string in `settings.py` if using `epmt run` or `epmt source` +* The value of the `PAPIEX_OPTIONS` environment variable if using `LD_PRELOAD` directly. + +The value of these should be a comma separated string: +```bash +$ export PAPIEX_OPTIONS="PERF_COUNT_SW_CPU_CLOCK,PAPI_CYCLES" +``` + +To list available and functioning metrics, use one of the included command line tools: + * `papi_avail` and `papi_native_avail` (via `papi`) + * `check_events` and `showevtinfo` (via `libpfm`) + * `perf list` (`linux`) + +The `PERF_COUNT_SW_*` events should work on any system that has the proper +`/proc/sys/kernel/perf_event_paranoid` setting. + +One should verify the functionality of the metric using the `papi_command_line` tool: +```bash +$ papi_command_line PERF_COUNT_SW_CPU_CLOCK +$ papi_command_line CYCLES +``` + + + + +## CI/CD Workflows and Caching +`epmt`'s GitHub Actions CI is split into focused workflows that use +`actions/cache` to avoid rebuilding expensive artifacts on every pull request +run. + + + +### Workflows +| Workflow | Trigger | Purpose | +|---|---|---| +| `docker_build_test.yml` | `push` to `main`, `pull_request` | Full build + test pipeline; restores cached artifacts before building | +| `slurm_image_build.yml` | Weekly (Mon 06:00 UTC), `workflow_dispatch` | Builds the `slurm-cluster` Docker image from source and saves it to cache | +| `weekly_tarball_build.yml` | Weekly (Mon 06:00 UTC), `workflow_dispatch` | Compiles `papiex` and downloads `epmt-dash`, then saves both to cache | +| `build_and_test_epmt.yml` | `push` to `main`, `pull_request` | Source-tree unit tests (no Docker) | + + + +### Caches and Invalidation +Each cache step is keyed on its build prerequisites — analogous to how `make` +uses file modification times to decide whether a target must be rebuilt. Changing +a prerequisite produces a new cache key, causing a cache miss and forcing a fresh +build. + +| Cache | Cache key components | Invalidation trigger | Notes | +|---|---|---|---| +| `epmt-build` Docker image | `OS_TARGET` + `PYTHON_VERSION` +
`SQLITE_VERSION` +
`hashFiles(Dockerfile, requirements.txt.py3)` | Edit the Dockerfile or requirements file, or bump any version variable | Fully content-hash based — closest analogy to `make` | +| `papiex` compiled tarball | `PAPIEX_VERSION` +
`OS_TARGET` | Bump `PAPIEX_VERSION` in `docker_build_test.yml`,
`weekly_tarball_build.yml`, and `Makefile` | Version-gated; relies on papiex using
immutable release tags | +| `test-release` Docker image | `OS_TARGET` + `PYTHON_VERSION` +
`SQLITE_VERSION` +
`hashFiles(Dockerfile, requirements.txt.py3)` +
`github.sha` | Always rebuilds (by design) —
`restore-keys` prefix reuses
unchanged early layers via
`--cache-from` | Image content changes
every commit; layer reuse
keeps it fast | +| `slurm-cluster` Docker image | `IMAGE_TAG` +
`SLURM_TAG` +
`SLURM_CLUSTER_TAG` | Bump any of the three version
variables in `docker_build_test.yml`
and `slurm_image_build.yml` | Version-string based; upstream
tag mutations without a version
bump won't invalidate | +| `epmt-dash` UI directory | `EPMT_DASH_SRC_BRANCH` | Change `EPMT_DASH_SRC_BRANCH`
in `docker_build_test.yml`,
`weekly_tarball_build.yml`,
and `Makefile` | Branch-name based; new commits
to same branch don't invalidate —
weekly workflow bounds staleness
to ≤1 week | + + + +### Cache Invalidation Gap vs. `make` +`make` detects prerequisite changes via file modification times regardless of +version numbers. The GitHub Actions caches above use version strings or content +hashes instead, so: +* `epmt-build` is fully `make`-like — changing the Dockerfile or + requirements file immediately produces a different hash and forces a rebuild. +* `papiex` and `slurm-cluster` require a deliberate version bump in the + workflow env block to trigger a rebuild. Remote source changes without a + version bump go undetected until the next weekly build. +* `epmt-dash` requires either a branch rename or waiting for the weekly + build. The weekly `weekly_tarball_build.yml` workflow always fetches fresh + content, bounding maximum staleness to one week. + + + +### Forcing a Rebuild +To force any individual cache to rebuild, bump the relevant version variable +in the `env:` section of both the weekly workflow and `docker_build_test.yml`, +and update the `Makefile` to match: +```yaml +# docker_build_test.yml (and weekly_tarball_build.yml) +env: + PAPIEX_VERSION: "2.3.16" # bump to force papiex rebuild + EPMT_DASH_SRC_BRANCH: "new-branch" # change to force epmt-dash rebuild + IMAGE_TAG: "25.05.4" # bump to force slurm-cluster rebuild +``` + +The `epmt-build` cache is invalidated automatically whenever +`Dockerfiles/Dockerfile.rocky-8-epmt-build` or `requirements.txt.py3` is +modified — no manual version bump is needed. + + + +### Weekly Pre-warming +`slurm_image_build.yml` and `weekly_tarball_build.yml` run every Monday +morning to pre-warm their respective caches before the working week begins. +`docker_build_test.yml` then restores those caches on pull request and push +runs, skipping the expensive Docker compile steps. If the cache is missing (first +run, eviction, or new key), `docker_build_test.yml` falls back to building the +artifact inline so the pipeline never silently skips a required build step. + + + + +## Troubleshooting + + + +### Virtual Environments: +Note that often in virtual environments, hardware counters are not often available in the VM. + + + +### Common Error Messages + +#### `version GLIBC_x.xx not found` +The collector library may not have been built for the current environment or the release +OS version does not match the current environment. \ No newline at end of file diff --git a/README.md b/README.md index bc71131c2..dcb743b17 100644 --- a/README.md +++ b/README.md @@ -1,570 +1,86 @@ -# EPMT +# **Experiment Process / Metadata Tool** (`epmt`) + +`epmt` collects metadata and performance data about shell processes, down to individual threads in individual processes. Currently, +`epmt` is particularly specialized for interfacing with Slurm batch jobs associated with earth modeling workflows, but is generalizable to +other computational workflow contexts. It also offers entrypoints to analyzing your data by interfacing with `jupyter` for easy access to +a notebook-style interface. -[![build_and_test_epmt](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml) -[![docker_build_test](https://github.com/NOAA-GFDL/epmt/actions/workflows/docker_build_test.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/docker_build_test.yml) [![codecov](https://codecov.io/gh/NOAA-GFDL/epmt/branch/main/graph/badge.svg)](https://codecov.io/gh/NOAA-GFDL/epmt) [![pylint](https://img.shields.io/badge/pylint-%E2%89%A58.1-brightgreen)](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml) +[![weekly_cache_builds](https://github.com/NOAA-GFDL/epmt/actions/workflows/weekly_cache_builds.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/weekly_cache_builds.yml) +[![build_conda](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_conda.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_conda.yml) +[![publish_conda](https://github.com/NOAA-GFDL/epmt/actions/workflows/publish_conda.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/publish_conda.yml) -**Experiment Performance Management Tool** aka -**WorkflowDB** aka -**PerfMiner** - -This is a tool to collect metadata and performance data about an entire job down to the individual threads in individual processes. This tool uses **papiex** to perform the process monitoring. This tool is targeted at batch or ephemeral jobs, not daemon processes. - -The software contained in this repository was written by Philip Mucci of Minimal Metrics LLC. - -## Verifying Installation - -It is best to check your installation and configuration using the ```epmt check``` command. - -Here is an example: -``` -$ epmt check -settings.db_params = {'url': 'sqlite:////home/chris/EPMT_DB_2.2.7.sqlite', 'echo': False} Pass -settings.install_prefix = /home/chris/Downloads/epmt-2.2.7/papiex-epmt-install/ Pass -settings.epmt_output_prefix = /tmp/epmt/ Pass -/proc/sys/kernel/perf_event_paranoid = 0 Pass -settings.papiex_options = PERF_COUNT_SW_CPU_CLOCK Pass -epmt stage functionality Pass -WARNING:epmtlib:No job name found, defaulting to unknown -epmt run functionality Pass -``` - -## Modes of EPMT +| Workflow | Python 3.9 | Python 3.10 | Python 3.11 | +|----------|------------|-------------|------------| +| **create_test_conda_env** | [![3.9](https://github.com/NOAA-GFDL/epmt/actions/workflows/create_test_conda_env.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/create_test_conda_env.yml?query=branch%3Amain+python-version%3A3.9) | [![3.10](https://github.com/NOAA-GFDL/epmt/actions/workflows/create_test_conda_env.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/create_test_conda_env.yml?query=branch%3Amain+python-version%3A3.10) | [![3.11](https://github.com/NOAA-GFDL/epmt/actions/workflows/create_test_conda_env.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/create_test_conda_env.yml?query=branch%3Amain+python-version%3A3.11) | -There are three modes to **EPMT** usage, collection, submission and analysis, and have an increasing number of dependencies: +| Workflow | SQLite | PostgreSQL | +|----------|--------|-----------| +| **docker_build_test** | [![sqlite](https://github.com/NOAA-GFDL/epmt/actions/workflows/docker_build_test.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/docker_build_test.yml?query=branch%3Amain+db_backend%3Asqlite) | [![postgres](https://github.com/NOAA-GFDL/epmt/actions/workflows/docker_build_test.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/docker_build_test.yml?query=branch%3Amain+db_backend%3Apostgres) | +| **build_and_test_epmt** | [![sqlite](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml?query=branch%3Amain+db_backend%3Asqlite) | [![postgres](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml?query=branch%3Amain+db_backend%3Apostgres) | -* **Collection** only requires a minimal Python installation of 2.6.x or higher -* **Submission** requires Python packages for data and database interaction -* **Analysis** requires [Jupyter](https://jupyter.org), an iPython notebook environment, as well as additional python data analysis libraries. -#### Configuration - -All three modes reference the **settings.py** file as well as **environment variables**. EPMT uses uses a in-memory, temporary database by default, see **Configuring a Database**. - -```text - $ cat settings.py - db_params = {'provider': 'sqlite', 'filename': ':memory:'} - papiex_options = "PERF_COUNT_SW_CPU_CLOCK" - epmt_output_prefix = "/tmp/epmt/" - debug = False - input_pattern = "*-papiex-[0-9]*-[0-9]*.csv" - install_prefix = "../papiex-oss/papiex-oss-install/" - # DO NOT TOUCH THIS -``` - -### Collection +## Installation +These are not-yet *fully* functional installations, as `epmt` was designed in an era where virtual environments were not as ubiquitous as +they are today. For full-featured build/installation approaches, consult the `Makefile`, `.github/workflows`, and [`DEVELOPER.md`](./DEVELOPER.md) -Automatic **Collection** with SLURM ---- +### With `conda` (recommended) -Using configured prolog and epilogs with SLURM tasks allows one to skip job instrumentation entirely, with the exception of job tags (***EPMT_JOB_TAGS***) and process tags (***PAPIEX_TAGS***). These are configured in `slurm.conf` for jobs submitted with `sbatch` but they can be tested on the command line when using `srun`. +The `conda` installation is currently favored as a quick-start for new users. -The above Csh job is equivalent to the below sequence using a prolog and epilog, ***with the exception of the trailing submit statement.*** - -``` -srun -n1 \\ ---task-prolog=$EPMT_PREFIX/epmt-install/slurm/slurm_task_prolog_epmt.sh \\ ---task-epilog=$EPMT_PREFIX/epmt-install/slurm/slurm_task_epilog_epmt.sh \\ -sleep 1 +```bash +conda install noaa-gfdl::epmt ``` -For this job to work using `sbatch` the following modifications in the `slurm.conf` would be made, substituting the appropriate path for EPMT_PREFIX: +### From repo checkout +The following creates a whole `epmt` conda environment with `epmt` accessible via an editable `pip` installation. +```bash +git clone https://github.com/NOAA-GFDL/epmt.git +cd epmt +conda env create -f environment.yaml +conda activate epmt ``` -TaskProlog=EPMT_PREFIX/epmt-install/slurm/slurm_task_prolog_epmt.sh -TaskEpilog=EPMT_PREFIX/epmt-install/slurm/slurm_task_epilog_epmt.sh -``` - -If this fails, then it's likely the papiex installation is either missing or misconfigured in **settings.py**. The **-a** flag tells **EPMT** to treat this run as an entire **job**. See **README.md** for further details. - -### Submission ---- -We can submit our previous job to the database defined in **settings.py** just run the epmt submit command with the directory returned by stage (found in location set by settings.py epmt_output_prefix): - -```text -$ epmt -v submit /tmp/epmt/1/ -INFO:epmt_cmds:submit_to_db(/tmp/epmt/1/,*-papiex-[0-9]*-[0-9]*.csv,False) -INFO:epmt_cmds:Unpickling from /tmp/epmt/1/job_metadata -INFO:epmt_cmds:1 files to submit -INFO:epmt_cmds:1 hosts found: ['linuxkit-025000000001-'] -INFO:epmt_cmds:host linuxkit-025000000001-: 1 files to import -INFO:epmt_job:Binding to DB: {'filename': ':memory:', 'provider': 'sqlite'} -INFO:epmt_job:Generating mapping from schema... -INFO:epmt_job:Processing job id 1 -INFO:epmt_job:Creating user root -INFO:epmt_job:Creating job 1 -INFO:epmt_job:Creating host linuxkit-025000000001- -INFO:epmt_job:Creating metricname usertime -INFO:epmt_job:Creating metricname systemtime -INFO:epmt_job:Creating metricname rssmax -INFO:epmt_job:Creating metricname minflt -INFO:epmt_job:Creating metricname majflt -INFO:epmt_job:Creating metricname inblock -INFO:epmt_job:Creating metricname outblock -INFO:epmt_job:Creating metricname vol_ctxsw -INFO:epmt_job:Creating metricname invol_ctxsw -INFO:epmt_job:Creating metricname num_threads -INFO:epmt_job:Creating metricname starttime -INFO:epmt_job:Creating metricname processor -INFO:epmt_job:Creating metricname delayacct_blkio_time -INFO:epmt_job:Creating metricname guest_time -INFO:epmt_job:Creating metricname rchar -INFO:epmt_job:Creating metricname wchar -INFO:epmt_job:Creating metricname syscr -INFO:epmt_job:Creating metricname syscw -INFO:epmt_job:Creating metricname read_bytes -INFO:epmt_job:Creating metricname write_bytes -INFO:epmt_job:Creating metricname cancelled_write_bytes -INFO:epmt_job:Creating metricname time_oncpu -INFO:epmt_job:Creating metricname time_waiting -INFO:epmt_job:Creating metricname timeslices -INFO:epmt_job:Creating metricname rdtsc_duration -INFO:epmt_job:Creating metricname PERF_COUNT_SW_CPU_CLOCK -INFO:epmt_job:Adding 1 processes to job -INFO:epmt_job:Earliest process start: 2019-03-06 15:36:56.948350 -INFO:epmt_job:Latest process end: 2019-03-06 15:37:06.996065 -INFO:epmt_job:Computed duration of job: 10047715.000000 us, 0.17 m -INFO:epmt_job:Staged import of 1 processes, 1 threads -INFO:epmt_job:Staged import took 0:00:00.189151, 5.286781 processes per second -INFO:epmt_cmds:Committed job 1 to database: Job[u'1'] +If you already have an environment created that you wish to install `epmt`, and it's already activated: +```bash +git clone https://github.com/NOAA-GFDL/epmt.git +cd epmt +pip install ``` -## Analysis and Visualization ---- - -EPMT Uses a **ipython notebook** data analytics environment. Starting the jupyter notebook is easy from the **epmt notebook** command. +### Verifying an Installation -```text -$ epmt notebook -[I 15:39:24.236 NotebookApp] Serving notebooks from local directory: /home/chris/Documents/playground/MM/build/epmt -[I 15:39:24.236 NotebookApp] The Jupyter Notebook is running at: -[I 15:39:24.236 NotebookApp] http://localhost:8888/?token=9c7529e19e12cb8121d66ff471e96fdd3056f6acc4480274 -[I 15:39:24.236 NotebookApp] or http://127.0.0.1:8888/?token=9c7529e19e12cb8121d66ff471e96fdd3056f6acc4480274 -[I 15:39:24.236 NotebookApp] Use Control-C to stop this server and shut down all kernels (twice to skip confirmation). -[C 15:39:24.263 NotebookApp] - - To access the notebook, open this file in a browser: - file:///home/chris/.local/share/jupyter/runtime/nbserver-18690-open.html - Or copy and paste one of these URLs: - http://localhost:8888/?token=9c7529e19e12cb8121d66ff471e96fdd3056f6acc4480274 - or http://127.0.0.1:8888/?token=9c7529e19e12cb8121d66ff471e96fdd3056f6acc4480274 +The `check` command is a first-stop sanity-check of your `epmt` installation. Call it with +```bash +$ epmt check ``` -The notebook command offers passing paramaters to jupyter such as host IP for sharing access to the notebook with machines on the local network, notebook token and notebook password -``` -$ epmt notebook -- --ip 0.0.0.0 --NotebookApp.token='thisisatoken' --NotebookApp.password='hereisa$upersecurepassword' +Verify the version: +```bash +$ epmt -V ``` +## Quickstart: Watch `epmt` work -## Collecting Performance Data - -Assuming you have EPMT installed and in your path, let's modify a job file: - -``` -$ cat my_job.sh -#!/bin/bash -# Example job script for Torque or SLURM -./compute_the_world --debug -``` - -This becomes: +Try wrapping your commands with `epmt start` / `epmt stop`: -``` -$ cat my_job_epmt.sh -#!/bin/bash -# Example job script for Torque or SLURM +```bash epmt start -epmt run ./compute_the_world --debug +epmt run ./compute_the_world --debug epmt stop +epmt submit ``` -Or more succintlty by automating the start/stop cycle with the **--auto** or **-a** flag: +Or use the **--auto** (`-a`) flag to automate the start/stop cycle: -``` -$ cat my_job_epmt2.sh -#!/bin/bash -# Example job script for Torque or SLURM +```bash epmt -a run ./compute_the_world --debug -``` - -But usually we want to run more than one executable. We could have any number of run statements: - -``` -$ cat my_job_epmt3.sh -#!/bin/bash -# Example job script for Torque or SLURM -# -epmt start -epmt run ./initialize_the_world --random -epmt run ./compute_the_world -epmt run ./postprocess_the_world -epmt stop -``` - -Let's skip all the markup, as we can do it with only environment variables. **EPMT** provides the configuration to export to the environment through the **source** command. The use for this is in a job file and is meant to be evaluated by the running shell, be that some form of Bash or Csh. **epmt source** just prints the required environment variables in Bash format **unless either the "SHELL" or "_" environment variable ends in csh**. - -**Note the unset of LD_PRELOAD before stop!** This line is to prevent the data collection routine from running on **epmt stop** itself. - -``` -$ cat my_job_bash.sh -#!/bin/bash -# Example job script for Torque or SLURM - -### Preamble, collect job metadata and monitor all processes/threads -epmt start -eval `epmt source` -# -./initialize_the_world --random -./compute_the_world -./postprocess_the_world -# -# Postamble, disable monitoring and collect job metadata -unset LD_PRELOAD -epmt stop -``` - -Here's an example for Csh, when run interactively, - - -``` -$ /bin/csh -> epmt -j1 source -setenv PAPIEX_OPTIONS PERF_COUNT_SW_CPU_CLOCK; -setenv PAPIEX_OUTPUT /tmp/epmt/1/; -setenv LD_PRELOAD /Users/phil/Work/GFDL/epmt.git/../papiex-oss/papiex-oss-install/lib/libpapiex.so:/Users/phil/Work/GFDL/epmt.git/../papiex-oss/papiex-oss-install/lib/libpapi.so:/Users/phil/Work/GFDL/epmt.git/../papiex-oss/papiex-oss-install/lib/libpfm.so:/Users/phil/Work/GFDL/epmt.git/../papiex-oss/papiex-oss-install/lib/libmonitor.so -``` - -## Importing Data Into the Database - -After collecting the data, jobs (groups of processes) are imported into the database with the **submit** command. This command takes arguments in the form of directories or tar files that must contain a *job_metadata* file. - -Normal operation is to submit one or more directories: - -```epmt submit [...]``` - -One can also submit a list of compressed tar files: - -```epmt submit [...]``` - -There is also a mode where the current environment is used to determine where to find the data. - -```epmt submit``` - - -### Examples - -#### Submitting a directory of compressed job data - -This might happen at the end of the day via a cron job: - -``` -$ epmt submit /*tgz -``` - -### Submitting data directly from within a job - -These commands could be part of every users job, or in the batch systems configurable preambles/postambles. - -``` -$ cat my_job.sh -#!/bin/bash -# Example job script for Torque or SLURM -echo "$PBS_JOBID or $SLURM_JOBID" -epmt start -epmt run ./compute_the_world --debug -epmt stop epmt submit - -``` - - -The start/stop cycle can be removed with the **--auto** or **-a** flag, which performs start and stop for you. - -``` -$ epmt -a run ./debug_the_world --outliers -$ epmt submit -``` - -### Submitting data from the current session - -If not inside of a batch environment, **epmt** will *attempt to fake-and-bake a job id*. This is quite useful when just performing interactive runs. **Note you may not be able to submit these jobs to a shared database. The session ID is not guaranteed to be unique across reboots, much less other systems** However, this use case is perfectly acceptable when using a private database. - -``` -$ epmt start -WARNING:epmt_cmds:JOB_ID unset: Using session id 6948 as JOB_ID -WARNING:epmt_cmds:JOB_NAME unset: Using job id 6948 as JOB_NAME -WARNING:epmt_cmds:JOB_SCRIPTNAME unset: Using process name 6948 as JOB_SCRIPTNAME -WARNING:epmt_cmds:JOB_USER unset: Using username phil as JOB_USE$ epmt run ./debug_the_world --outliers -$ epmt stop -$ epmt submit -``` - -## Usage and Configuration - -**EPMT** gets all of it's configuration from two places, environment variables and the **settings.py** file. One can examine all the current settings by passing the **--help** option. - -``` -$ ./epmt help -usage: epmt [-n] [-d] [-h] [-a] [--drop] - [epmt_cmd] [epmt_cmd_args [epmt_cmd_args ...]] - -positional arguments: - epmt_cmd start, run, stop, submit, dump - epmt_cmd_args Additional arguments, command dependent - -optional arguments: - -n, --dry-run Don't touch the database - -v, --verbose Increase level of verbosity/debug - -h, --help Show this help message and exit - -a, --auto Do start/stop when running - --drop Drop all tables/data and recreate before importing - -settings.py (overridden by below env. vars): -db_params {'host': 'localhost', 'password': 'example', 'user': 'postgres', 'dbname': 'EPMT', 'provider': 'postgres'} -debug False -input_pattern *-papiex-[0-9]*-[0-9]*.csv -install_prefix ../papiex-oss/papiex-oss-install/ -papiex_options PERF_COUNT_SW_CPU_CLOCK -epmt_output_prefix /tmp/epmt/ - -environment variables (overrides settings.py): -``` - -## Environment Variables - -The following variables replace, at run-time, the values in the **db_params** dictionary found in **settings.py**. - -``` -EPMT_DB_PROVIDER -EPMT_DB_USER -EPMT_DB_PASSWORD -EPMT_DB_HOST -EPMT_DB_DBNAME -EPMT_DB_FILENAME ``` -### settings.py - -There are a number of example files provided. See **INSTALL.md** for more details. - -``` -$ ls settings -settings_pg_container.py settings_sqlite_inmem.py -settings_pg_localhost.py settings_sqlite_localfile.py -$ -$ # In memory only, disappears after run -$ cp /path/to/install/settings/settings_sqlite_inmem.py /path/to/install/settings.py -$ -$ # Persistent and on disk -$ cp /path/to/install/settings/settings_sqlite_localfile.py /path/to/install/settings.py -$ epmt -v -v submit /dir/to/jobdata -``` - -## Debugging - -**EPMT** can be passed both **-n** (dry-run) and **-v** (verbosity) to help with debugging. Add more **-v** flags to increase the level of information printed **-vvv**. - -``` -$ epmt -v start -``` - -Or to attempt a submit without touching the database: - -``` -$ epmt -vv submit -n /dir/to/jobdata -``` - -Also, one can decode and dump the job_metadata file in a dir or compressed dir. - -``` -$ epmt dump ~/Downloads/yrs05-25.20190221/CM4_piControl_C_atmos_00050101.papiex.gfdl.19712961.tgz -exp_component atmos -exp_jobname CM4_piControl_C_atmos_00050101 -exp_name CM4_piControl_C -exp_oname 00050101 -job_el_env_changes {} -job_el_env_changes_len 0 -job_el_from_batch [] -job_el_status 0 -job_el_stop 2019-02-20 22:13:23.131187 -job_pl_env {'LANG': 'en_US', 'PBS_QUEUE': 'batch', 'SHELL': '/bin/csh', 'PBS_ENVIRONMENT': 'PBS_BATCH', 'PAPIEX_TAGS': 'atmos', 'SHLVL': '3', 'PBS_WALLTIME': '216000', 'MOAB_NODELIST': 'pp057.princeton.rdhpcs.noaa.gov', 'PBS_VERSION': 'TORQUE-6.0.2', 'PAPIEX_OUTPUT': '/vftmp/Foo.Bar/pbs20345339/papiex', 'LOADEDMODULES': '', 'LC_TIME': 'C', 'MACHTYPE': 'x86_64', 'PAPIEX_OPTIONS': 'PERF_COUNT_SW_CPU_CLOCK', 'MOAB_GROUP': 'f'} -job_pl_env_len 81 -job_pl_from_batch [] -job_pl_groupnames ['f', 'f'] -job_pl_hostname pp057 -job_pl_id 20345339.moab01.princeton.rdhpcs.noaa.gov -job_pl_jobname CM4_piControl_C_atmos_00050101 -job_pl_scriptname CM4_piControl_C_atmos_00050101 -job_pl_start 2019-02-20 19:58:41.274267 -job_pl_submit 2019-02-20 19:58:41.274463 -job_pl_username Foo.Bar - -``` - - -## Performance Metrics Data Dictionary - -EPMT collects data both from the job runtime and the applications run in that environment. See the **models/** directory for what fixed data is stored related to each object. Metric data is stored differently and the data collector's data directionary can be found in papiex-oss/README.md. At the time of this writing it looked like this: - - -| Key | Scope | Description | -|--------------------------- |--------- |-------------------------------------------------------- | -| 1. tags | Process | User specified tags for this executable | -| 2. hostname | Process | hostname | -| 3. exename | Process | Name of the application, usually argv[0] | -| 4. path | Process | Path to the application | -| 5. args | Process | All arguments to exe excluding argv[0] | -| 6. exitcode | Process | Exit code | -| 7. exitsignal | Process | Exited due to a signal | -| 8. pid | Process | Process id | -| 9. generation | Process | Incremented after every exec() or PID wrap | -| 10. ppid | Process | Parent process id | -| 11. pgid | Process | Process group id | -| 12. sid | Process | Process session id | -| 13. numtids | Process | Number of threads caught by instrumentation | -| 14. numranks | Process | Number of MPI ranks detected | -| 15. tid | Process | Thread id | -| 16. mpirank | Thread | MPI rank | -| 17. start | Process | Microsecond timestamp at start | -| 18. end | Process | Microsecond timestamp at end | -| 19. usertime | Thread | Microsecond user time | -| 20. systemtime | Thread | Microsecond system time | -| 21. rssmax | Thread | Kb max resident set size | -| 22. minflt | Thread | Minor faults (TLB misses/new page frames) | -| 23. majflt | Thread | Major page faults (requiring I/O) | -| 24. inblock | Thread | 512B blocks read from I/O | -| 25. outblock | Thread | 512B blocks written to I/O | -| 26. vol_ctxsw | Thread | Voluntary context switches (yields) | -| 27. invol_ctxsw | Thread | Involuntary context switches (preemptions) | -| 28. cminflt | Process | minflt (20) for all wait()ed children | -| 29. cmajflt | Thread | majflt (21) for all wait()ed children | -| 30. cutime | Process | utime (17) for all wait()ed children | -| 31. cstime | Thread | stime (18) for all wait()ed children | -| 32. num_threads | Process | Threads in process at finish | -| 33. starttime | Thread | Timestamp in jiffies after boot thread was started | -| 34. processor | Thread | CPU this thread last ran on | -| 35. delayacct_blkio_time | Thread | Jiffies process blocked in D state on I/O device | -| 36. guest_time | Thread | Jiffies running a virtual CPU for a guest OS | -| 37. rchar | Thread | Bytes read via syscall (maybe from cache not dev I/O) | -| 38. wchar | Thread | Bytes written via syscall (maybe to cache not dev I/O) | -| 39. syscr | Thread | Read syscalls | -| 40. syscw | Thread | Write syscalls | -| 41. read_bytes | Thread | Bytes read from I/O device | -| 42. write_bytes | Thread | Bytes written to I/O device | -| 43. cancelled_write_bytes | Thread | Bytes discarded by truncation | -| 44. time_oncpu | Thread | Nanoseconds spent running | -| 45. time_waiting | Thread | Nanoseconds runnable but waiting | -| 46. timeslices | Thread | Number of run periods on CPU | -| 47. rdtsc_duration | Thread | If PAPI, real time cycle duration of thread | -| * | Thread | PAPI metrics | - -### Addition of new metrics - -Additional metrics can be configured either in two ways: -* The papiex_options string In **settings.py** if using ```epmt run``` or ```epmt source``` -* The value of the **PAPIEX_OPTIONS** environment variable if using ```LD_PRELOAD``` directly. - -The value of these should be a comma separated string: -``` -$ export PAPIEX_OPTIONS="PERF_COUNT_SW_CPU_CLOCK,PAPI_CYCLES" -``` - -To list available and functioning metrics, use one of the included command line tools: - * papi_avail - * papi_native_avail - * check_events (libpfm) - * showevtinfo (libpfm) - * perf list (linux) - -**The PERF_COUNT_SW_* events should work on any system that has the proper /proc/sys/kernel/perf_event_paranoid setting**. - -One should verify the functionality of the metric using the ```papi_command_line``` tool: - -``` -$ papi_command_line PERF_COUNT_SW_CPU_CLOCK - -$ papi_command_line CYCLES - -``` - - -## Troubleshooting - -### Error: `version GLIBC_x.xx not found` - -The collector library may not have been built for the current environment or the release -OS version does not match the current environment. - -### Virtual Environments: -Note that often in virtual environments, hardware counters are not often available in the VM. - ---- - -## CI/CD Workflows and Caching - -EPMT's GitHub Actions CI is split into focused workflows that use `actions/cache` to avoid -rebuilding expensive artifacts on every pull request run. - -### Workflows - -| Workflow | Trigger | Purpose | -|---|---|---| -| `docker_build_test.yml` | `push` to `main`, `pull_request` | Full build + test pipeline; restores cached artifacts before building | -| `slurm_image_build.yml` | Weekly (Mon 06:00 UTC), `workflow_dispatch` | Builds the `slurm-cluster` Docker image from source and saves it to cache | -| `weekly_tarball_build.yml` | Weekly (Mon 06:00 UTC), `workflow_dispatch` | Compiles `papiex` and downloads `epmt-dash`, then saves both to cache | -| `build_and_test_epmt.yml` | `push` to `main`, `pull_request` | Source-tree unit tests (no Docker) | - -### Caches and Invalidation - -Each cache step is keyed on its build prerequisites — analogous to how `make` uses file -modification times to decide whether a target must be rebuilt. Changing a prerequisite -produces a new cache key, causing a cache miss and forcing a fresh build. - -| Cache | Cache key components | Invalidation trigger | Notes | -|---|---|---|---| -| `epmt-build` Docker image | `OS_TARGET` + `PYTHON_VERSION` + `SQLITE_VERSION` + `hashFiles(Dockerfile, requirements.txt.py3)` | Edit the Dockerfile or requirements file, or bump any version variable | Fully content-hash based — closest analogy to `make` | -| `papiex` compiled tarball | `PAPIEX_VERSION` + `OS_TARGET` | Bump `PAPIEX_VERSION` in `docker_build_test.yml`, `weekly_tarball_build.yml`, and the `Makefile` | Version-gated; relies on papiex using immutable release tags | -| `test-release` Docker image | `OS_TARGET` + `PYTHON_VERSION` + `SQLITE_VERSION` + `hashFiles(Dockerfile, requirements.txt.py3)` + `github.sha` | Always rebuilds (by design) — `restore-keys` prefix reuses unchanged early layers via `--cache-from` | Image content changes every commit; layer reuse keeps it fast | -| `slurm-cluster` Docker image | `IMAGE_TAG` + `SLURM_TAG` + `SLURM_CLUSTER_TAG` | Bump any of the three version variables in `docker_build_test.yml` and `slurm_image_build.yml` | Version-string based; upstream tag mutations without a version bump won't invalidate | -| `epmt-dash` UI directory | `EPMT_DASH_SRC_BRANCH` | Change `EPMT_DASH_SRC_BRANCH` in `docker_build_test.yml`, `weekly_tarball_build.yml`, and the `Makefile` | Branch-name based; new commits to the same branch don't invalidate — the weekly workflow bounds staleness to ≤1 week | - -### Cache Invalidation Gap vs. `make` - -`make` detects prerequisite changes via file modification times regardless of version numbers. -The GitHub Actions caches above use version strings or content hashes instead, so: - -* **`epmt-build`** is fully `make`-like — changing the Dockerfile or requirements file immediately - produces a different hash and forces a rebuild. -* **`papiex`** and **`slurm-cluster`** require a deliberate version bump in the workflow env block - to trigger a rebuild. Remote source changes without a version bump go undetected until the next - weekly build. -* **`epmt-dash`** requires either a branch rename or waiting for the weekly build. The weekly - `weekly_tarball_build.yml` workflow always fetches fresh content, bounding maximum staleness to - one week. - -### Forcing a Rebuild - -To force any individual cache to rebuild, bump the relevant version variable in the `env:` section -of **both** the weekly workflow and `docker_build_test.yml`, and update the `Makefile` to match: - -``` -# docker_build_test.yml (and weekly_tarball_build.yml) -env: - PAPIEX_VERSION: "2.3.16" # bump to force papiex rebuild - EPMT_DASH_SRC_BRANCH: "new-branch" # change to force epmt-dash rebuild - IMAGE_TAG: "25.05.4" # bump to force slurm-cluster rebuild -``` - -The `epmt-build` cache is invalidated automatically whenever `Dockerfiles/Dockerfile.rocky-8-epmt-build` -or `requirements.txt.py3` is modified — no manual version bump is needed. - -### Weekly Pre-warming +## Further Documentation -`slurm_image_build.yml` and `weekly_tarball_build.yml` run every Monday morning to pre-warm their -respective caches before the working week begins. `docker_build_test.yml` then restores those -caches on pull request and push runs, skipping the expensive Docker compile steps. If the cache -is missing (first run, eviction, or new key), `docker_build_test.yml` falls back to building the -artifact inline so the pipeline never silently skips a required build step. +For detailed information on configuration, data collection, SLURM integration, database submission, analysis, performance metrics, debugging, and CI/CD, see [DEVELOPER.md](DEVELOPER.md). From e20b2ab494fab371d2ceec3e00f3506d52d1b784 Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Tue, 24 Mar 2026 18:01:54 -0400 Subject: [PATCH 47/48] tweaks (#204) * Update pylint thresholds in runtests script * Update pylint threshold to 8.2 * Update pylint thresholds for epmt module and ui * Refactor settings import to remove error handling Simplified the import of user-specific settings by removing the try-except block. * Remove find_diffs_in_envs function Removed the find_diffs_in_envs function from epmt_cmds.py. it's not covered, and it's not called, and everything still works! so it must not be needed. --- .github/workflows/build_and_test_epmt.yml | 4 ++-- Makefile | 2 +- runtests | 8 ++------ src/epmt/epmt_cmds.py | 19 ------------------- src/epmt/epmt_settings.py | 6 +----- 5 files changed, 6 insertions(+), 33 deletions(-) diff --git a/.github/workflows/build_and_test_epmt.yml b/.github/workflows/build_and_test_epmt.yml index bddaf6cb4..bd2addef6 100644 --- a/.github/workflows/build_and_test_epmt.yml +++ b/.github/workflows/build_and_test_epmt.yml @@ -520,11 +520,11 @@ jobs: - name: run pylint on epmt module, ignoring ui (epmt-dash) run: | - pylint --rcfile pylintrc --fail-under 8.1 --ignore-paths src/epmt/ui src/epmt + pylint --rcfile pylintrc --fail-under 8.2 --ignore-paths src/epmt/ui src/epmt - name: run pylint on ui submodule (epmt-dash) run: | - pylint --rcfile pylintrc --fail-under 5.5 src/epmt/ui + pylint --rcfile pylintrc --fail-under 6.0 src/epmt/ui - name: upload pip installable uses: actions/upload-artifact@v4 diff --git a/Makefile b/Makefile index f4ab207cb..de40be7dc 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,7 @@ epmt-build compile build: lint: @echo "(lint) whoami: $(shell whoami)" cd src - python3 -m pylint --fail-under 5.5 --max-line-length 120 --max-args 6 -ry src/epmt + python3 -m pylint --fail-under 8.2 --max-line-length 120 --max-args 6 -ry src/epmt # virtual environment options install-py3-conda: diff --git a/runtests b/runtests index fd5db94e7..7be6b8ee2 100755 --- a/runtests +++ b/runtests @@ -97,15 +97,11 @@ integration_kernel_compile() { _epmt_run_integration "${EPMT_INTEGRATION_DIR}/te run_lint() { echo "" echo "── pylint: epmt module (excluding ui) ──" - pylint --rcfile pylintrc --fail-under 7.0 --ignore-paths src/epmt/ui src/epmt + pylint --rcfile pylintrc --fail-under 8.2 --ignore-paths src/epmt/ui src/epmt echo "" echo "── pylint: ui submodule (epmt-dash) ──" - pylint --rcfile pylintrc --fail-under 5.5 src/epmt/ui - - echo "" - echo "── pylint: epmt entry-point script ──" - pylint --rcfile pylintrc --fail-under 10 src/scripts/epmt + pylint --rcfile pylintrc --fail-under 6.0 src/epmt/ui } # ── coverage report ──────────────────────────────────────────────── diff --git a/src/epmt/epmt_cmds.py b/src/epmt/epmt_cmds.py index 7c3ae9ac0..c413ebcc0 100644 --- a/src/epmt/epmt_cmds.py +++ b/src/epmt/epmt_cmds.py @@ -42,25 +42,6 @@ class bcolors: -def find_diffs_in_envs(start_env, stop_env): - env = {} - for e in start_env.keys(): - if e in stop_env.keys(): - if start_env[e] == stop_env[e]: - logger.debug("Found %s", e) - else: - logger.debug("Different %s", e) - env[e] = stop_env[e] - else: - logger.debug("Deleted %s", e) - env[e] = start_env[e] - for e in stop_env.keys(): - if e not in start_env.keys(): - logger.debug("Added %s", e) - env[e] = stop_env[e] - return env - - def dump_config(outf, sep=":"): print("\nsettings.py:", file=outf) for key, value in sorted(settings.__dict__.items()): diff --git a/src/epmt/epmt_settings.py b/src/epmt/epmt_settings.py index da2ef8a47..44c467a89 100644 --- a/src/epmt/epmt_settings.py +++ b/src/epmt/epmt_settings.py @@ -6,8 +6,4 @@ # now load the user-specific settings.py so they override the defaults # if you want your own configuraions, put it in settings.py, not here -try: - from epmt.settings import * -except Exception as e: - raise ModuleNotFoundError('alternate epmt.settings import approach did not' + - ' work and neither did the first attempt!') from e +from epmt.settings import * From 9c80daf7f24a5442613abefe0889ab01851edf35 Mon Sep 17 00:00:00 2001 From: "Ian L." <6273252+ilaflott@users.noreply.github.com> Date: Wed, 25 Mar 2026 14:49:14 -0400 Subject: [PATCH 48/48] adjust dbcare command output (#208) --- src/epmt/epmt_cmd_dbcare.py | 9 ++-- src/epmt/epmt_cmd_help.py | 1 - src/epmt/epmt_cmd_list.py | 1 - src/epmt/epmt_cmd_show.py | 4 +- src/epmt/epmt_concat.py | 34 ------------ src/epmt/epmt_convert_csv.py | 5 -- src/epmt/epmt_daemon.py | 2 +- src/epmt/epmt_rootcause.py | 102 +++++++++++++++++------------------ src/epmt/epmtlib.py | 4 -- 9 files changed, 60 insertions(+), 102 deletions(-) diff --git a/src/epmt/epmt_cmd_dbcare.py b/src/epmt/epmt_cmd_dbcare.py index ba4075ab2..f747bf349 100644 --- a/src/epmt/epmt_cmd_dbcare.py +++ b/src/epmt/epmt_cmd_dbcare.py @@ -24,8 +24,9 @@ def _get_dead_row_stats(): "ORDER BY relname" ) result = orm_raw_sql(sql) - rows = result.fetchall() - return rows + logger.debug('result = %s', result) + #logger.debug('type(result) = %s', type(result)) + return result def _vacuum_tables(): @@ -47,7 +48,9 @@ def _vacuum_tables(): 'last_vacuum=%s last_autovacuum=%s', row[0], row[1], row[2], row[3], row[4]) except Exception as e: - logger.warning('could not query dead row stats: %s', e) + logger.error('could not query dead row stats: %s', e) + raise Exception from e + # VACUUM requires autocommit — use a raw DBAPI connection raw_conn = engine.raw_connection() diff --git a/src/epmt/epmt_cmd_help.py b/src/epmt/epmt_cmd_help.py index 93168aba4..5b5f98cf6 100644 --- a/src/epmt/epmt_cmd_help.py +++ b/src/epmt/epmt_cmd_help.py @@ -1,7 +1,6 @@ """ EPMT help command module - provides help functionality. """ -# from __future__ import print_function from inspect import signature from sys import stderr diff --git a/src/epmt/epmt_cmd_list.py b/src/epmt/epmt_cmd_list.py index d8a2bc168..0865b09d2 100644 --- a/src/epmt/epmt_cmd_list.py +++ b/src/epmt/epmt_cmd_list.py @@ -2,7 +2,6 @@ EPMT list command module - handles job listing functionality. """ -# from __future__ import print_function from sys import stderr from logging import getLogger diff --git a/src/epmt/epmt_cmd_show.py b/src/epmt/epmt_cmd_show.py index 2b95c45bc..a4ca38001 100644 --- a/src/epmt/epmt_cmd_show.py +++ b/src/epmt/epmt_cmd_show.py @@ -2,12 +2,12 @@ EPMT show command module - handles job display functionality. """ -# from __future__ import print_function from logging import getLogger -logger = getLogger(__name__) import epmt.epmt_query as eq +logger = getLogger(__name__) + def epmt_show_job(jobid, key=None): if isinstance(jobid, list): jobid = jobid[0] diff --git a/src/epmt/epmt_concat.py b/src/epmt/epmt_concat.py index 958c627f8..a3ca7b828 100755 --- a/src/epmt/epmt_concat.py +++ b/src/epmt/epmt_concat.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 ''' Data Flow: Always Check for and Store comments @@ -18,7 +17,6 @@ Jobid: `goodSample_123` ''' -from __future__ import unicode_literals from epmt.epmtlib import epmt_logging_init, logfn from sys import exit as sysexit @@ -375,35 +373,3 @@ def csvjoiner(indir, return False, None, badfiles_renamed -if __name__ == '__main__': - # inl: remove this entry point? - from argparse import ArgumentParser - parser = ArgumentParser(description="Concatenate CSV files. It returns 0 on success and < 0 on error") - parser.add_argument( - 'files', nargs='+', metavar='FILE', - help='Two or more CSV files to concatenate OR a directory containing CSV files') - parser.add_argument( - '-v', '--verbose', - action="count", - default=0, - help="increase verbosity") - parser.add_argument( - '-o', '--output-file', - help="Name of the output file, determined from input if not specified", - default='') - parser.add_argument( - '-e', '--error', - action='store_true', - help="Exit at the first sign of trouble", - default=False) - parser.add_argument( - '-E', '--error-dir', - help="Name of the directory to save files with errors, disabled if not specified", - default='') - args = parser.parse_args() - retval, of, bf = csvjoiner(debug=args.verbose, - indir=(args.files[0] if len(args.files) == 1 else args.files), - outfile=args.output_file, - keep_going=not args.error, - errdir=args.error_dir) - sysexit(0 if retval else -1) diff --git a/src/epmt/epmt_convert_csv.py b/src/epmt/epmt_convert_csv.py index 1966a9552..5ed425264 100755 --- a/src/epmt/epmt_convert_csv.py +++ b/src/epmt/epmt_convert_csv.py @@ -384,8 +384,3 @@ def extract_jobid_from_collated_csv(collated_csv): ''' return collated_csv.split('papiex')[-1].split('-')[1] - -if __name__ == "__main__": - import sys - epmt_logging_init(intlvl=2) - convert_csv_in_tar(sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else '') diff --git a/src/epmt/epmt_daemon.py b/src/epmt/epmt_daemon.py index 6374185e8..995dec358 100644 --- a/src/epmt/epmt_daemon.py +++ b/src/epmt/epmt_daemon.py @@ -1,7 +1,7 @@ """ EPMT daemon module - handles background daemon functionality. """ -# from __future__ import print_function + from getpass import getuser from os import path, kill, unlink, getppid from sys import exit as sysexit diff --git a/src/epmt/epmt_rootcause.py b/src/epmt/epmt_rootcause.py index 5d011788a..ff488e037 100644 --- a/src/epmt/epmt_rootcause.py +++ b/src/epmt/epmt_rootcause.py @@ -88,57 +88,57 @@ def rootcause(ref, input, features, methods=[rootcause_zscore]): return False, None, None -if __name__ == "__main__": - # Synthesize 10 feature names - n_features = 6 - features = [f'{x:c}' for x in range(97, 97 + n_features)] - print("Features:\n", features) +#if __name__ == "__main__": +# # Synthesize 10 feature names +# n_features = 6 +# features = [f'{x:c}' for x in range(97, 97 + n_features)] +# print("Features:\n", features) - def multiple(): - ''' - Check with multiple outliers - Narrow range of reference values - ''' - np.random.seed(104) - random_reference_df = pd.DataFrame(np.random.randint(50, 100, size=(100, n_features)), columns=features) - print("Reference:\n", random_reference_df.head()) - # Wider range input values for test set - random_input_df = pd.DataFrame(np.random.randint(25, 125, size=(1, n_features)), columns=features) - print("Input:\n", random_input_df.head()) - retval, df, dct = rootcause(random_reference_df, random_input_df, features) - print("Retval:\n", retval) - print("Result:\n", df) - print("Result:\n", dct) +def ex_multiple(features, n_features): + ''' + Check with multiple outliers + Narrow range of reference values + ''' + np.random.seed(104) + random_reference_df = pd.DataFrame(np.random.randint(50, 100, size=(100, n_features)), columns=features) + print("Reference:\n", random_reference_df.head()) + # Wider range input values for test set + random_input_df = pd.DataFrame(np.random.randint(25, 125, size=(1, n_features)), columns=features) + print("Input:\n", random_input_df.head()) + retval, df, dct = rootcause(random_reference_df, random_input_df, features) + print("Retval:\n", retval) + print("Result:\n", df) + print("Result:\n", dct) - def none(): - ''' - Check with no outliers - Narrow range of reference values - ''' - np.random.seed(103) - random_reference_df = pd.DataFrame(np.random.randint(50, 100, size=(100, n_features)), columns=features) - print("Reference:\n", random_reference_df.head()) - # Wider range input values for test set - random_input_df = pd.DataFrame(np.random.randint(25, 125, size=(1, n_features)), columns=features) - print("Input:\n", random_input_df.head()) - retval, df, dct = rootcause(random_reference_df, random_input_df, features) - print("Retval:\n", retval) - print("Result:\n", df) - print("Result:\n", dct) +def ex_none(features, n_features): + ''' + Check with no outliers + Narrow range of reference values + ''' + np.random.seed(103) + random_reference_df = pd.DataFrame(np.random.randint(50, 100, size=(100, n_features)), columns=features) + print("Reference:\n", random_reference_df.head()) + # Wider range input values for test set + random_input_df = pd.DataFrame(np.random.randint(25, 125, size=(1, n_features)), columns=features) + print("Input:\n", random_input_df.head()) + retval, df, dct = rootcause(random_reference_df, random_input_df, features) + print("Retval:\n", retval) + print("Result:\n", df) + print("Result:\n", dct) + +def ex_one(features, n_features): + ''' + Check with one outlier + Narrow range of reference values + ''' + np.random.seed(102) + random_reference_df = pd.DataFrame(np.random.randint(50, 100, size=(100, n_features)), columns=features) + print("Reference:\n", random_reference_df.head()) + # Wider range input values for test set + random_input_df = pd.DataFrame(np.random.randint(25, 125, size=(1, n_features)), columns=features) + print("Input:\n", random_input_df.head()) + retval, df, dct = rca(random_reference_df, random_input_df, features) + print("Retval:\n", retval) + print("Result:\n", df) + print("Result:\n", dct) - def one(): - ''' - Check with one outlier - Narrow range of reference values - ''' - np.random.seed(102) - random_reference_df = pd.DataFrame(np.random.randint(50, 100, size=(100, n_features)), columns=features) - print("Reference:\n", random_reference_df.head()) - # Wider range input values for test set - random_input_df = pd.DataFrame(np.random.randint(25, 125, size=(1, n_features)), columns=features) - print("Input:\n", random_input_df.head()) - retval, df, dct = rca(random_reference_df, random_input_df, features) - print("Retval:\n", retval) - print("Result:\n", df) - print("Result:\n", dct) - one() diff --git a/src/epmt/epmtlib.py b/src/epmt/epmtlib.py index 4996f4595..9e39288e8 100644 --- a/src/epmt/epmtlib.py +++ b/src/epmt/epmtlib.py @@ -1224,7 +1224,3 @@ def set_signal_handlers(signals=[], handler=None): logger.debug('Finished restoring signal handlers to defaults') else: logger.debug('Finished setting up signal handlers') - - -if __name__ == "__main__": - print(version_str(True))