Integrate papiex C library compilation into pip install (setup.py + compile_papiex.sh)

.github/workflows/build_and_test_epmt.yml

@@ -143,38 +143,36 @@ jobs:
         run: |
           make epmt-dash
 
-      # ── papiex tarball (OUTSIDE_DOCKER): cached across runs ────────────────
-      # Cache key: version + OS target + "outside-docker" suffix to distinguish
-      # from the Docker-built variant cached by docker_build_test.
-      # The weekly_tarball_build workflow pre-warms this cache every Monday;
-      # the build below is a fallback for cache misses.
-      - name: Check Cache for papiex tarball (OUTSIDE_DOCKER)
-        id: cache-papiex
-        uses: actions/cache@v4
-        with:
-          path: papiex-epmt-${{ env.PAPIEX_VERSION }}-${{ env.OS_TARGET }}.tgz
-          key: papiex-outside-docker-${{ env.PAPIEX_VERSION }}-${{ env.OS_TARGET }}
-
-      - name: make papiex tarball
-        if: steps.cache-papiex.outputs.cache-hit != 'true'
-        run: |
-          make OUTSIDE_DOCKER='YUP' papiex-dist
-
-      - name: make epmt pip-packaging
-        run: |
-          make dist python-dist dist-test
-
-      - name: pip install epmt pip-package
-        run: |
-          ls src/dist/epmt*gz
-          python3 -m pip install src/dist/epmt*gz
+      # ── papiex compilation is handled by compile_papiex.sh at pip install time ──
+      # For bare-metal (non-Docker) installs, setup.py invokes compile_papiex.sh
+      # which downloads and compiles papiex from source. The Docker release path
+      # still uses the Makefile's papiex-dist target (via docker-dist).
+
+      # ── Test pip install ./src with papiex compilation via compile_papiex.sh ──
+      # This exercises the new setup.py hook: compile_papiex.sh downloads
+      # papiex source from GitHub, compiles it, and installs .so files into
+      # src/epmt/lib/ so they are bundled into the pip package.
+      # gcc, make, and curl are already available from the earlier yum install step.
+      # compile_papiex.sh always exits 0 on graceful skips (missing tools, network
+      # errors), and setup.py catches non-zero exits and warns without failing the
+      # install — so pip install ./src should always succeed.
+      - name: Test pip install ./src (papiex compiled via compile_papiex.sh)
+        run: |
+          # Remove any pre-copied .so files to exercise the compilation path
+          rm -rf src/epmt/lib/
+          # Install directly from source; setup.py invokes compile_papiex.sh
+          python3 -m pip install -v -v ./src
+          SITE_PACKAGES=$(python3 -c "import site; print(site.getsitepackages()[0])")
+          if ls ${SITE_PACKAGES}/epmt/lib/*.so* 2>/dev/null | head -1 | grep -q .; then
+            echo "papiex shared libraries found — compile_papiex.sh succeeded"
+          else
+            echo "WARNING: papiex libraries not found after pip install ./src"
+            echo "         (compile_papiex.sh may have skipped or failed gracefully)"
+          fi
+          epmt -v check || true
 
-      - name: create links to papiex executables, adjust path
+      - name: adjust path
         run: |
-          SITE_PACKAGES=$(python3 -c "import site; print(site.getsitepackages()[0])")
-          mkdir -p ${SITE_PACKAGES}/papiex-epmt-install
-          ln -s  ${SITE_PACKAGES}/epmt/lib ${SITE_PACKAGES}/papiex-epmt-install/
-          ln -s  ${SITE_PACKAGES}/epmt/bin ${SITE_PACKAGES}/papiex-epmt-install/
           export PATH='/usr/bin:${PATH}:/usr/local/libexec:/usr/local/bin'
 
       - name: Configure epmt to use PostgreSQL

.github/workflows/weekly_cache_builds.yml

@@ -187,24 +187,9 @@ jobs:
           path: src/epmt/ui
           key: epmt-dash-${{ env.EPMT_DASH_SRC_BRANCH }}
 
-      # ── papiex (OUTSIDE_DOCKER) ────────────────────────────────────
-      - name: Build papiex-dist (OUTSIDE_DOCKER)
-        run: make OUTSIDE_DOCKER='YUP' papiex-dist
-
-      - name: Verify papiex tarball exists
-        run: |
-          TARBALL="papiex-epmt-${PAPIEX_VERSION}-${OS_TARGET}.tgz"
-          if [ ! -f "${TARBALL}" ]; then
-            echo "::error::Papiex OUTSIDE_DOCKER tarball '${TARBALL}' not found."
-            exit 1
-          fi
-          echo "Papiex OUTSIDE_DOCKER tarball '${TARBALL}' exists."
-
-      - name: Cache papiex tarball (OUTSIDE_DOCKER)
-        uses: actions/cache/save@v4
-        with:
-          path: papiex-epmt-${{ env.PAPIEX_VERSION }}-${{ env.OS_TARGET }}.tgz
-          key: papiex-outside-docker-${{ env.PAPIEX_VERSION }}-${{ env.OS_TARGET }}
+      # ── papiex (OUTSIDE_DOCKER) removed ─────────────────────────
+      # Bare-metal papiex compilation is now handled by compile_papiex.sh
+      # at pip install time. Only the Docker-based papiex cache remains.
 
       # ── Python + SQLite from source (container-path cache) ──────────
       # Pre-warm the Python/SQLite build cache so build_and_test_epmt

.gitignore

@@ -63,7 +63,23 @@ src/epmt.egg-info/*
 src/papiex-epmt-install/*
 src/epmt/settings.py
 
+# papiex compile_papiex.sh / pip install artifacts #
+#####################################################
+# vendored papiex C source (populated by 'make vendor-papiex')
+src/vendor/
+# compiled papiex shared libraries (populated by compile_papiex.sh or make python-dist)
+src/epmt/lib/
+# pip build log from setup.py
+install_papiex.log
+
 # virtualenv
 venv*
 
+# setuptools build artifacts
+src/build/
+# papiex compilation byproducts (headers and helper scripts)
+src/epmt/bin/
+src/epmt/include/
 
+# virtualenv
+venv*

.gitlab-ci.yml

@@ -36,9 +36,6 @@ build:python3:
     - make dist python-dist dist-test
 
     - pip3 install src/dist/epmt*gz
-    - mkdir -p /usr/lib/python3.10/site-packages/papiex-epmt-install
-    - ln -s  /usr/lib/python3.10/site-packages/epmt/lib /usr/lib/python3.10/site-packages/papiex-epmt-install/
-    - ln -s  /usr/lib/python3.10/site-packages/epmt/bin /usr/lib/python3.10/site-packages/papiex-epmt-install/
 
     - export PATH='/usr/bin:${PATH}:/usr/local/bin:/usr/local/libexec'
 

Dockerfiles/Dockerfile.centos-7-epmt-test-release

@@ -69,11 +69,6 @@ WORKDIR ${install_path}_archive
 COPY ${epmt_full_release} .
 COPY src/dist/${epmt_python_full_release} .
 RUN python3 -m pip install ${epmt_python_full_release}
-
-RUN mkdir /usr/lib/python3.10/site-packages/papiex-oss \
-    && tar zxf ${epmt_python_full_release}  \
-    && mv epmt-${epmt_version}/papiex-epmt-install /usr/lib/python3.10/site-packages/papiex-oss 
-#    && rm -rf *.tgz
 
 COPY utils/epmt-installer .
 WORKDIR ${install_path}

Dockerfiles/Dockerfile.rocky-8-epmt-test-release

@@ -82,9 +82,6 @@ COPY src/dist/${epmt_python_full_release} .
 
 RUN python3 -m pip install ${epmt_python_full_release}
 
-RUN tar zxf ${epmt_python_full_release}  \
-    && mv epmt-${epmt_version}/papiex-epmt-install /usr/lib/python3.10/site-packages/
-#    && rm -rf *.tgz
 COPY utils/epmt-installer .
 
 WORKDIR ${install_path}

Makefile

@@ -66,7 +66,7 @@ PWD=$(shell pwd)
 	install-py3-conda install-py3-pyenv install-deps \\
 	dist python-dist dist-test docker-dist docker-dist-test \\
 	epmt-dash \\
-	papiex-dist \\
+	papiex-dist vendor-papiex pip-install \\
 	epmt-full-release check-release \\
 	release \\
 	clean-extra clean-all clean distclean dashclean dockerclean papiexclean \\
@@ -161,10 +161,14 @@ $(EPMT_RELEASE) dist:
 	@echo "**********************************************************"
 	tar -czf $(EPMT_RELEASE) epmt-install
 
-# runs setuptools
-# note that this step requires EPMT_RELEASE and PAPIEX_RELEASE, but we don't explicitly state it here, b.c.
-# when we go in the docker container, the time-zone changes and therefore the timestamp comparison triggers re-making
-# targets that do not need to be remade
+# runs setuptools to build the sdist.
+# Used in the Docker release flow: make papiex-dist (Docker path) builds the papiex
+# tarball, then python-dist extracts the pre-compiled .so files into epmt/lib/ before
+# building the sdist. For bare-metal installs, papiex is compiled at pip install time
+# via compile_papiex.sh instead.
+# NOTE: we don't explicitly list EPMT_RELEASE and PAPIEX_RELEASE as dependencies, b.c.
+# when we go in the docker container, the time-zone changes and therefore the timestamp
+# comparison triggers re-making targets that do not need to be remade
 python-dist:
 	@echo "(python-dist) whoami: $(shell whoami)"
 	@echo "**********************************************************"
@@ -173,6 +177,7 @@ python-dist:
 	cd src && echo "GOOD: cd src" || echo "I FAILED: cd src" ; \
 	tar zxf ../$(PAPIEX_RELEASE) && echo "GOOD: tar -zxf ../PAPIEX_RELEASE" || echo "I FAILED: tar zxf ../PAPIEX_RELEASE" ; \
 	mkdir -p epmt/lib && cp -a papiex-epmt-install/lib/*.so* epmt/lib/ && echo "GOOD: cp papiex libs to epmt/lib" || echo "I FAILED: cp papiex libs to epmt/lib" ; \
+	rm -rf papiex-epmt-install && echo "GOOD: rm papiex-epmt-install" || echo "I FAILED: rm papiex-epmt-install" ; \
 	pip3 install --quiet build && echo "GOOD: pip3 install build" || echo "I FAILED: pip3 install build" ; \
 	python3 -m build --sdist && echo "GOOD: python3 -m build --sdist" || echo "I FAILED: python3 -m build --sdist" ; \
 	chmod a+r dist/* && echo "GOOD: chmod a+r dist/*" || echo "I FAILED: chmod a+r dist/*"
@@ -282,15 +287,7 @@ $(PAPIEX_SRC)/$(PAPIEX_RELEASE): $(PAPIEX_SRC)
 	@echo
 	@echo
 	@echo "################### BEGIN MAKE PAPIEX TARBALL : papiex-dist ########################################"
-	if [ -n "${OUTSIDE_DOCKER}" ]; then \
-	echo "within docker. make and make check within PAPIEX_SRC/PAPIEX_RELEASE target" ; \
-	make -C $(PAPIEX_SRC) CONFIG_PAPIEX_PAPI=$(CONFIG_PAPIEX_PAPI) CONFIG_PAPIEX_DEBUG=$(CONFIG_PAPIEX_DEBUG) OS_TARGET=$(OS_TARGET) distclean install dist ; \
-	make -C $(PAPIEX_SRC) CONFIG_PAPIEX_PAPI=$(CONFIG_PAPIEX_PAPI) CONFIG_PAPIEX_DEBUG=$(CONFIG_PAPIEX_DEBUG) OS_TARGET=$(OS_TARGET) dist-test ; \
-	make -C $(PAPIEX_SRC) CONFIG_PAPIEX_PAPI=$(CONFIG_PAPIEX_PAPI) CONFIG_PAPIEX_DEBUG=$(CONFIG_PAPIEX_DEBUG) OS_TARGET=$(OS_TARGET) check ; \
-	else \
-	echo "outside docker. making docker-dist within PAPIEX_SRC/PAPIEX_RELEASE target" ; \
-	make -C $(PAPIEX_SRC) CONFIG_PAPIEX_PAPI=$(CONFIG_PAPIEX_PAPI) CONFIG_PAPIEX_DEBUG=$(CONFIG_PAPIEX_DEBUG) OS_TARGET=$(OS_TARGET) DOCKER_RUN_OPTS="$(DOCKER_RUN_OPTS)" docker-dist ; \
-	fi
+	make -C $(PAPIEX_SRC) CONFIG_PAPIEX_PAPI=$(CONFIG_PAPIEX_PAPI) CONFIG_PAPIEX_DEBUG=$(CONFIG_PAPIEX_DEBUG) OS_TARGET=$(OS_TARGET) DOCKER_RUN_OPTS="$(DOCKER_RUN_OPTS)" docker-dist
 
 $(PAPIEX_SRC): $(PAPIEX_SRC_TARBALL)
 	@echo "(PAPIEX_SRC) whoami: $(shell whoami)"
@@ -305,6 +302,33 @@ $(PAPIEX_SRC_TARBALL):
 	@echo "(PAPIEX_SRC_TARBALL) whoami: $(shell whoami)"
 	curl -L --fail --retry 3 --retry-delay 5 -O $(PAPIEX_SRC_URL) ; \
 	ls $(PAPIEX_SRC_TARBALL)
+
+# Download papiex source into src/vendor/papiex/ so compile_papiex.sh
+# can find it without needing network access at pip-install time.
+vendor-papiex:
+	@echo "(vendor-papiex) whoami: $(shell whoami)"
+	@echo " ------ VENDOR PAPIEX SOURCE INTO src/vendor/papiex ------- "
+	@echo "PAPIEX_SRC_URL = ${PAPIEX_SRC_URL}"
+	if [ -d "src/vendor/papiex" ] && [ -f "src/vendor/papiex/Makefile" ]; then \
+	echo "src/vendor/papiex already exists, skipping download." ; \
+	else \
+	mkdir -p src/vendor ; \
+	curl -L --fail --retry 3 --retry-delay 5 \
+	-o /tmp/papiex-vendor-$(PAPIEX_SRC_BRANCH).tar.gz $(PAPIEX_SRC_URL) ; \
+	tar zxf /tmp/papiex-vendor-$(PAPIEX_SRC_BRANCH).tar.gz -C /tmp ; \
+	TOP_DIR=$$(tar ztf /tmp/papiex-vendor-$(PAPIEX_SRC_BRANCH).tar.gz | head -1 | cut -d/ -f1) ; \
+	mv /tmp/$${TOP_DIR} src/vendor/papiex ; \
+	rm -f /tmp/papiex-vendor-$(PAPIEX_SRC_BRANCH).tar.gz ; \
+	echo "papiex source vendored at src/vendor/papiex" ; \
+	ls src/vendor/papiex ; \
+	fi
+
+# Install epmt directly from src/ using pip, triggering compile_papiex.sh.
+# Run 'make vendor-papiex' first to avoid a network download at install time.
+pip-install: vendor-papiex
+	@echo "(pip-install) whoami: $(shell whoami)"
+	@echo " ------ PIP INSTALL epmt with PAPIEX COMPILATION ------- "
+	pip3 install ./src
 # ----------- \end PAPIEX THINGS ---------- #
 
 
@@ -453,6 +477,8 @@ papiexclean:
 	@echo "(papiexclean) whoami: $(shell whoami)"
 	- rm -fr $(PAPIEX_SRC) 
 	- rm -f $(PAPIEX_SRC_TARBALL) $(PAPIEX_RELEASE)
+	- rm -rf src/vendor/papiex
+	- rm -rf src/epmt/lib
 # ----------- \end CLEANING ---------- #
 
 

environment.yaml

@@ -78,4 +78,4 @@ dependencies:
 #   - conda-forge::pyinstaller-hooks-contrib==2023.11
 
   - pip:
-    - -e src/              # editable install of epmt into this environment
+    - src/              # editable install of epmt into this environment

epmtdocs/docs/INSTALL.md

@@ -8,6 +8,67 @@ batch or ephemeral jobs, not daemon processes.
 
 The software contained in this repository was written by Philip Mucci of Minimal Metrics LLC.
 
+## Installation via pip (recommended)
+
+EPMT can be installed directly with pip.  When build tools (`gcc`, `make`,
+`curl`) are present on the system, `pip install epmt` will automatically
+compile the **papiex** C library and bundle the resulting shared libraries
+into the installed package.  If the build tools are missing, EPMT installs
+cleanly as a pure-Python package and hardware counter collection is simply
+unavailable at runtime (detectable with `epmt check`).
+
+### Quick install
+
+```bash
+pip install epmt
+```
+
+### Install from source (with papiex compilation)
+
+```bash
+# Clone the repository
+git clone https://github.com/NOAA-GFDL/epmt.git
+cd epmt
+
+# (Optional) Pre-download the papiex C source to avoid a network fetch
+# at install time — useful in air-gapped environments.
+make vendor-papiex
+
+# Install; setup.py will compile papiex via compile_papiex.sh
+pip install ./src
+```
+
+### Build environment variables
+
+The following environment variables control how papiex is compiled during
+`pip install`:
+
+| Variable | Default | Description |
+|---|---|---|
+| `PAPIEX_SRC_BRANCH` | `main` | Branch or tag of the papiex repository to download |
+| `CONFIG_PAPIEX_PAPI` | `n` | Set to `y` to enable PAPI hardware counter support |
+| `CONFIG_PAPIEX_DEBUG` | `n` | Set to `y` to enable a debug-mode build |
+
+Example:
+
+```bash
+CONFIG_PAPIEX_PAPI=y pip install ./src
+```
+
+### Verify installation
+
+Run `epmt check` after installation to verify that papiex libraries are
+present and that hardware counter collection is working:
+
+```bash
+epmt check
+```
+
+If papiex was not compiled (e.g. because `gcc` was unavailable), `epmt check`
+will report this and EPMT will still function for metadata collection.
+
+---
+
 ## Installation With Release File
 
 The release file includes EPMT, Data Collection Libraries, Notebook and EPMT Workflow GUI.

src/MANIFEST.in

@@ -1,2 +1,3 @@
-recursive-include papiex-epmt-install/lib *.so*
 recursive-include epmt/lib *.so*
+recursive-include epmt/bin *
+include compile_papiex.sh