diff --git a/.cspell.json b/.cspell.json index 5bd726026..4ad06d442 100644 --- a/.cspell.json +++ b/.cspell.json @@ -7,7 +7,7 @@ { "name": "ok-unknown-words", "path": "./.cspell/ok-unknown-words.txt", - "addWords": false, + "addWords": false } ], "dictionaries": [ @@ -19,10 +19,13 @@ "ignorePaths": [ "cspell.json", "*.cdl", + // local packaging and buid byproducts "build", "*.egg-info", "**/*.egg-info/**", - "notebooks", + "epmtdocs/site/**", + // notebooks/ contains lots of html + "notebooks", // sample/ contains test workload scripts and staged job data "sample/**", // test/data/ contains binary fixtures, CSVs, and outlier datasets diff --git a/.cspell/ok-unknown-words.txt b/.cspell/ok-unknown-words.txt index c35f6928b..3591bc345 100644 --- a/.cspell/ok-unknown-words.txt +++ b/.cspell/ok-unknown-words.txt @@ -1,40 +1,535 @@ -CMIP -cmip -docstrings -EPMT epmt -EPMTDB -epmtdocs -epmtlib -jobid -jobids -noaa -NOAA -noarch -noqa -numpy -omip -PAPI -papi -PAPIEX +EPMT papiex -PPAN -ppan -pylint -pylintrc -pytest -pyyaml -RDHPCS -rdhpcs -scikit -scipy -setuptools +jobid +procs slurm +epmtlib +PAPIEX +ndays SLURM -slurmctld -slurmd -SLURMD +refmodels +rdtsc +numtids +ctxsw +tcsh +refmodel +usertime +systemtime +jobname +fltr +exename +majflt +rssmax +minflt +hsmget sqlalchemy +pyod +dbcare +timeslices +syscr +PAPI +mkdocs +jobids +invol +inblock +delayacct +blkio +syscw +rchar +pytest +outblock +oncpu +ppid +pgid +unanalyzed +dframe +pylint +mvod +papi +JOBID +pyinstaller +exitsignal +uninstrument +outl +logfn +fdet +timavg +dbsize +slurmctld +setuptools +plotly +funcs +wrapit srun sbatch -SBATCH +findwhat +Durachta +untar +rootcause +nprocs +ncrcat +ncatted +LOCALID +fregrid +DSQLITE +destdir +vftmp +tushar +SQLA +slurmd +ocsvm +modulefiles +metricname +hbos +epmtdocs +CPUS +byhost +psycopg +pids +ondelete +ncrc +ncpus +ncks +mpirank +llnl +infile +indir +filedict +csvjoiner +bycpu +gethostname +exitcode +dmput +datas +autouse +splitvars +SHLVL +pyproject +nans +MVOD +madz +libsqlite +kwargify +jobdatafile +dlist +badfiles +uvod +usejson +sysexit +slurmdbd +scipy +scikit +randn +pytz +numpy +NODELIST +mpinumranks +maxiters +libpfm +ipykernel +intlvl +initdb +fini +errdir +dlst +dbcopy +cmip +backref +anysh +ABOD +wchar +tuser +tepmt +syscalls +syscall +sqla +scriptname +mirrorlist +MACHTYPE +LOGNAME +localfile +LOADEDMODULES +linuxkit +libpapiex +labapp +jupyterlab +ipywidgets +ipython +dotdict +docstrings +Dockerfiles +cpus +warnopts +usebytes +unprocd +undercsh +TAVG +Smirnov +SLURMD +Slurmd +Slurmctld +Slurm +sleeptest +sklearn +sinfo +SIGUSR +RUSAGE +rmean +rmax +RLIMIT +retlist +qtconsole +pyenv +pkgs +pidf +outf +onupdate +omip +OLDPWD +oldproctag +nrecs +NODEID +noaa +nestlevel +MPLCONFIGDIR +modulecmd +mjob +MADZ +livereload +libmonitor +lesspipe +ldlib +kstest +Kolmogorov +joblist +inputfile +inputf +inmem +GTIDS +getrusage +filetostage +devtest +devhome +deets +csvfile +covsuffix +cntmax +CMIP +classfiers +bcolors +argslist +Wilk +unproc +unittests +retstr +repr +rdhpcs +rcfile +PYTHONSTARTUP +pystartup +printsettings +PLATFORMTHEME +pidfile +papiexclean +outrows +outrow +OSTYPE +nthreads +NTASKS +NPROCS +NODENAME +noconfirm +NNODES +MODULESHOME +MODULEPATH +mempeak +MANPATH +lzma +LSCOLORS +libpapi +LESSOPEN +LESSCLOSE +ldconfig +lastid +jobscriptname +jobdir +inbetween +HOSTTYPE +HISTSIZE +groupnames +GOPATH +GOBIN +exenames +eventlist +dockerclean +distpath +didsomething +dictlist +dbname +dashclean +cutime +csvt +cstime +cpio +coveragerc +cminflt +cmajflt +CLICOLOR +Bxegedabagaced +appmenu +utime +tsums +tablespaced +sucess +stime +schedstat +runtests +qualname +PYTEST +primaryjoin +outpath +numranks +njobs +MATPLOTLIBDATA +libexec +kvdict +Jobid +inlinevar +indexd +giovtorres +getpid +fdist +epmt's +echoerr +dumperr +drilldown +docsapp +dmget +D'Agostino +contextmanager +autovacuum +apage +alist +Yoyodyne +workflowdb +Werkzeug +WALLTIME +virtualenv +VARCHAR +Unpickling +thismodule +thisisatoken +tablename +Srun +signum +signo +sdist +showevtinfo +sessionmaker +rockylinux +pylintrc +pydatetime +pycache +psql +PPAN +postambles +Postamble +OKBLUE +nots +normaltest +noqa +noheader +nbserver +minmax +metaclass +matplotlib +mapperlib +levelname +joblib +jobdata +jmsg +gsmput +hostnames +goodpath +getuid +gettimeofday +EPMTDB +distutils +decomp +csvjoin +conftest +contextlib +classmethod +bitrot +astext +argparse +arctan +arrayfuncs +arraywise +asdict +autoflush +automap +cmor +getenv +getpgid +getppid +getsid +getsitepackages +hiddenimports +hookspath +isort +jobacct +jobtags +Kolkata +lambda +lasttime +linuxproc +localtime +snum +salloc +testpip +textposition +typevar +typealias +unsetenv +untarring +venv +Waittime +abstractproperty +hsmput +elif +endswith +cython +csvdir +modz +Mucci +multimodel +multifactor +metadatafile +minimalmetrics +mmperftools +MEMLOCK +LASTVAL +FSIZE +globus +cmdname +jobcomp +filetxt +Fairshare +sched +cleandb +secondaryjoin +raiseerr +NTCF +KUBECONFIG +NOFILE +perfevent +gettid +tids +NOAAGFDL +slurmctl +encfeat +encdf +xspf +svgz +rmvb +PRIO +tmpvikl +pdsbspm +proctrack +Proctrack +readlink +schemaname +PGPASSWORD +multimodal +unimodal +trimodal +perc +clfs +esac +uncollated +postprocruns +overgeneral +booleaness +mypackage +mymodule +nosec +mypy +optparse +ixin +NOTSET +datefmt +postproc +dbitem +OKGREEN +metadatafile +dateutil +rgxs +pygtk +msvs +nbhome +noarch +parso +noarchive +datapub +dockerhub +readcsv +rownum +poolclass +Eeuo +outform +decdf +decfeat +arcname +zxvf +filert +spcname +rsplit +dtype +ints +datname +blas +pathex +ESRCH +EINVAL +idempotently +kneedle +lmdd +LMDD +cblof +CBLOF +iforest +npts +scoreatpercentile +ddof +EOSQL +gettz +cvzf +Frob +codehilite +tmpn +hereisasupersecurepassword +aggr +perlbrew +ENDC +EEXIST +reerr +rdstc +dotdicts +asus +outfd +outcsv +outtar +tweakable +devel +openmp +flac +nrows +skiprows +isin +ndarray +bitmask +Uncategorized +xaxis +parseable diff --git a/.github/workflows/md_lint.yml b/.github/workflows/md_lint.yml index 1c0924159..5de579960 100644 --- a/.github/workflows/md_lint.yml +++ b/.github/workflows/md_lint.yml @@ -24,7 +24,6 @@ jobs: uses: actions/checkout@v4 - name: Lint markdown files - continue-on-error: true uses: DavidAnson/markdownlint-cli2-action@v19 with: globs: | diff --git a/DEVELOPER.md b/DEVELOPER.md index 796b4b600..04dfa29b0 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -1,4 +1,5 @@ # `epmt` Developer and Advanced Usage Guide + This document contains detailed information about `epmt` configuration, data collection, database submission, analysis, metrics, debugging, and CI/CD infrastructure. @@ -15,7 +16,7 @@ For installation and quick-start instructions, see [README.md](./README.md). - [Data Collection with SLURM epilog and prolog](#data-collection-with-slurm-epilog-and-prolog) - [The Second Mode, Data Submission](#the-second-mode-data-submission) - [Manual Submission Example](#manual-submission-example) - - [Compressed Directory Submission Exmple](#compressed-directory-submission-exmple) + - [Compressed Directory Submission Example](#compressed-directory-submission-example) - [Internal-batch Job Submission Example](#internal-batch-job-submission-example) - [Data From Current Session Submission Example](#data-from-current-session-submission-example) - [The Third Mode, Data Analysis and Visualization](#the-third-mode-data-analysis-and-visualization) @@ -34,12 +35,14 @@ For installation and quick-start instructions, see [README.md](./README.md). - [`version GLIBC_x.xx not found`](#version-glibc_xxx-not-found) ## Configuration -`epmt` houses default settings (`epmt_default_settings.py`), user settings (`settings.py`), -and a submodule (`epmt_settings.py`) that appends the user settings to the end of the + +`epmt` houses default settings (`epmt_default_settings.py`), user settings (`settings.py`), +and a submodule (`epmt_settings.py`) that appends the user settings to the end of the default settings, overriding them. Custom user settings should only be included by developers who understand `epmt`'s requirements. The default settings should first be evaluated, they look a little bit like the following: + ```bash $ cat src/epmt/epmt_default_settings.py ... @@ -55,8 +58,10 @@ $ cat src/epmt/epmt_default_settings.py ``` ### Environment Variables + The following variables replace, at run-time, the values in the `db_params` dictionary found in `settings.py`: -``` + +```text EPMT_DB_PROVIDER EPMT_DB_USER EPMT_DB_PASSWORD @@ -66,7 +71,9 @@ EPMT_DB_FILENAME ``` ### Getting Current Configuration Information + You can examine all current settings by passing the `--help` option: + ```bash $ ./epmt --help usage: epmt [-n] [-d] [-h] [-a] [--drop] @@ -95,19 +102,22 @@ environment variables (overrides settings.py): ``` ## The Three Modes of `epmt` + There are three modes to `epmt` usage; data collection, data submission, and data analysis. Each has different dependencies: -* **Collection** requires Python 2.6 or higher, and compiled [`papiex`](https://github.com/noaa-gfdl/papiex) +- **Collection** requires Python 2.6 or higher, and compiled [`papiex`](https://github.com/noaa-gfdl/papiex) libraries for process tagging -* **Submission** requires Python packages for SQL and database interactions (`sqlalchemy`, `sqlite`, `postgres`) -* **Analysis** requires `jupyter`, `iPython`, and additionally [`epmt-dash`](https://github.com/noaa-gfdl/epmt-dash) +- **Submission** requires Python packages for SQL and database interactions (`sqlalchemy`, `sqlite`, `postgres`) +- **Analysis** requires `jupyter`, `iPython`, and additionally [`epmt-dash`](https://github.com/noaa-gfdl/epmt-dash) and `plotly` for dashboard-style analytics ### The First Mode, Data Collection #### Collecting Performance Data + Assuming you have `epmt` installed and in your path, let's modify a job file: + ```bash $ cat my_job.sh #!/bin/bash @@ -116,6 +126,7 @@ $ cat my_job.sh ``` This becomes: + ```bash $ cat my_job_epmt.sh #!/bin/bash @@ -125,7 +136,8 @@ epmt run ./compute_the_world --debug epmt stop ``` -Or more succintlty by automating the start/stop cycle with the `--auto` or `-a` flag: +Or more succinctly by automating the start/stop cycle with the `--auto` or `-a` flag: + ```bash $ cat my_job_epmt2.sh #!/bin/bash @@ -134,6 +146,7 @@ epmt -a run ./compute_the_world --debug ``` But usually we want to run more than one executable. We could have any number of run statements: + ```bash $ cat my_job_epmt3.sh #!/bin/bash @@ -150,10 +163,11 @@ provides the configuration to export to the environment through the `source` command. This is intended for use in a job file and should be evaluated by the running shell, be it `bash` or `csh`. `epmt source` prints the required environment variables in Bash format unless either the `SHELL` or `_` -environment variable ends in `csh`. +environment variable ends in `csh`. **Note the unset of `LD_PRELOAD` before stop!** This prevents the data collection routine from running on `epmt stop` itself. + ```bash $ cat my_job_bash.sh #!/bin/bash @@ -172,7 +186,8 @@ unset LD_PRELOAD epmt stop ``` -Here's an example for `csh`, when run interactively, +Here's an example for `csh`, when run interactively, + ```bash $ /bin/csh > epmt -j1 source @@ -182,10 +197,11 @@ setenv LD_PRELOAD /Users/phil/Work/GFDL/epmt.git/../papiex-oss/papiex-oss-instal ``` #### Data Collection with SLURM epilog and prolog + Using configured prolog and epilogs with SLURM tasks allows one to skip job instrumentation entirely, except for job tags (`EPMT_JOB_TAGS`) and process tags (`PAPIEX_TAGS`). These are configured in `slurm.conf` for jobs -submitted with `sbatch` but can be tested on the command line when using `srun`. +submitted with `sbatch` but can be tested on the command line when using `srun`. The above Csh job is equivalent to the below sequence using a prolog and epilog, ***with the exception of the trailing submit statement.*** @@ -200,10 +216,11 @@ $ sleep 1 For this job to work using `sbatch`, make the following modifications in `slurm.conf`, substituting the appropriate path for `$EPMT_PREFIX`: + ```bash -$ SLURM_TASK_SCRIPT_DIR=${EPMT_PREFIX}/epmt-install/slurm -$ TaskProlog=${SLURM_TASK_SCRIPT_DIR}/slurm_task_prolog_epmt.sh -$ TaskEpilog=${SLURM_TASK_SCRIPT_DIR}/slurm_task_epilog_epmt.sh +SLURM_TASK_SCRIPT_DIR=${EPMT_PREFIX}/epmt-install/slurm +TaskProlog=${SLURM_TASK_SCRIPT_DIR}/slurm_task_prolog_epmt.sh +TaskEpilog=${SLURM_TASK_SCRIPT_DIR}/slurm_task_epilog_epmt.sh ``` If this fails, the `papiex` installation is likely either missing or @@ -211,29 +228,35 @@ misconfigured in `settings.py`. The `-a` flag tells `epmt` to treat this run as an entire job. ### The Second Mode, Data Submission + After collecting the data, jobs (groups of processes) are imported into the database with the `submit` command. This command takes arguments in the form of -directories or tar files that must contain a `job_metadata` file. +directories or tar files that must contain a `job_metadata` file. Normal operation is to submit one or more directories: + ```bash -$ epmt submit [...] +epmt submit [...] ``` One can also submit a list of compressed tar files: + ```bash -$ epmt submit [...] +epmt submit [...] ``` There is also a mode where the current environment is used to determine where to find the data. + ```bash -$ epmt submit +epmt submit ``` #### Manual Submission Example + We can submit our previous job to the database defined in `settings.py` by running the `epmt submit` command with the directory returned by `stage` (found in the location set by `settings.py` `epmt_output_prefix`): + ```bash $ epmt -v submit /tmp/epmt/1/ INFO:epmt_cmds:submit_to_db(/tmp/epmt/1/,*-papiex-[0-9]*-[0-9]*.csv,False) @@ -282,14 +305,18 @@ INFO:epmt_job:Staged import took 0:00:00.189151, 5.286781 processes per second INFO:epmt_cmds:Committed job 1 to database: Job[u'1'] ``` -#### Compressed Directory Submission Exmple +#### Compressed Directory Submission Example + This might happen at the end of the day via a cron job: + ```bash -$ epmt submit /*tgz +epmt submit /*tgz ``` #### Internal-batch Job Submission Example + These commands could be part of every users job, or in the batch systems configurable preambles/postambles. + ```bash $ cat my_job.sh #!/bin/bash @@ -302,16 +329,19 @@ epmt submit ``` The start/stop cycle can be removed with the `--auto` or `-a` flag, which performs start and stop for you: -``` -$ epmt -a run ./debug_the_world --outliers -$ epmt submit + +```bash +epmt -a run ./debug_the_world --outliers +epmt submit ``` #### Data From Current Session Submission Example -If not inside of a batch environment, `epmt` will *attempt to fake-and-bake a job id*. This is useful -when performing interactive runs. You may not be able to submit these jobs to a shared database due to -constraints on job ID uniqueness, sincet he session ID is not guaranteed to be unique across reboots, + +If not inside of a batch environment, `epmt` will *attempt to fake-and-bake a job id*. This is useful +when performing interactive runs. You may not be able to submit these jobs to a shared database due to +constraints on job ID uniqueness, since the session ID is not guaranteed to be unique across reboots, much less other systems. However, this use case is perfectly acceptable when using a private database: + ```bash $ epmt start WARNING:epmt_cmds:JOB_ID unset: Using session id 6948 as JOB_ID @@ -323,8 +353,10 @@ $ epmt submit ``` ### The Third Mode, Data Analysis and Visualization + `epmt` uses an `IPython` notebook data analytics environment. Starting the Jupyter notebook is easy from the `epmt notebook` command: + ```bash $ epmt notebook [I 15:39:24.236 NotebookApp] Serving notebooks from local directory: /home/chris/Documents/playground/MM/build/epmt @@ -344,23 +376,29 @@ $ epmt notebook The notebook command supports passing parameters to jupyter, such as host IP for sharing access with machines on the local network, notebook token, and notebook password: + ```bash -$ epmt notebook -- --ip 0.0.0.0 --NotebookApp.token='thisisatoken' --NotebookApp.password='hereisa$upersecurepassword' +epmt notebook -- --ip 0.0.0.0 --NotebookApp.token='thisisatoken' --NotebookApp.password='hereisasupersecurepassword' ``` ## Debugging + `epmt` can be passed both `-n` (dry-run) and `-v` (verbosity) to help with debugging. Add more `-v` flags to increase verbosity level (`-vvv`). + ```bash -$ epmt -v start +epmt -v start ``` Or to attempt a submit without touching the database: + ```bash -$ epmt -vv submit -n /dir/to/jobdata +epmt -vv submit -n /dir/to/jobdata ``` Also, one can decode and dump the job_metadata file in a dir or compressed dir. + + ```bash $ epmt dump ~/Downloads/yrs05-25.20190221/CM4_piControl_C_atmos_00050101.papiex.gfdl.19712961.tgz exp_component atmos @@ -384,13 +422,16 @@ job_pl_start 2019-02-20 19:58:41.274267 job_pl_submit 2019-02-20 19:58:41.274463 job_pl_username Foo.Bar ``` + ## Performance Metrics Data Dictionary + `epmt` collects data both from the job runtime and the applications run in that environment. See the `src/epmt/models/` directory for fixed data stored related to each object. Metric data is stored differently; the data collector's data dictionary can be found in papiex-oss/README.md. At the time of this writing, it looked like this: + | Key | Scope | Description | |--------------------------- |--------- |-------------------------------------------------------- | | 1. tags | Process | User specified tags for this executable | @@ -443,48 +484,59 @@ like this: | * | Thread | PAPI metrics | ### Addition of new metrics + Additional metrics can be configured either in two ways: -* The `papiex_options` string in `settings.py` if using `epmt run` or `epmt source` -* The value of the `PAPIEX_OPTIONS` environment variable if using `LD_PRELOAD` directly. + +- The `papiex_options` string in `settings.py` if using `epmt run` or `epmt source` +- The value of the `PAPIEX_OPTIONS` environment variable if using `LD_PRELOAD` directly. The value of these should be a comma separated string: + ```bash -$ export PAPIEX_OPTIONS="PERF_COUNT_SW_CPU_CLOCK,PAPI_CYCLES" +export PAPIEX_OPTIONS="PERF_COUNT_SW_CPU_CLOCK,PAPI_CYCLES" ``` To list available and functioning metrics, use one of the included command line tools: - * `papi_avail` and `papi_native_avail` (via `papi`) - * `check_events` and `showevtinfo` (via `libpfm`) - * `perf list` (`linux`) + +- `papi_avail` and `papi_native_avail` (via `papi`) +- `check_events` and `showevtinfo` (via `libpfm`) +- `perf list` (`linux`) The `PERF_COUNT_SW_*` events should work on any system that has the proper `/proc/sys/kernel/perf_event_paranoid` setting. One should verify the functionality of the metric using the `papi_command_line` tool: + ```bash -$ papi_command_line PERF_COUNT_SW_CPU_CLOCK -$ papi_command_line CYCLES +papi_command_line PERF_COUNT_SW_CPU_CLOCK +papi_command_line CYCLES ``` ## CI/CD Workflows and Caching + `epmt`'s GitHub Actions CI is split into focused workflows that use `actions/cache` to avoid rebuilding expensive artifacts on every pull request run. ### Workflows + + | Workflow | Trigger | Purpose | |---|---|---| | `docker_build_test.yml` | `push` to `main`, `pull_request` | Full build + test pipeline; restores cached artifacts before building | | `slurm_image_build.yml` | Weekly (Mon 06:00 UTC), `workflow_dispatch` | Builds the `slurm-cluster` Docker image from source and saves it to cache | | `weekly_tarball_build.yml` | Weekly (Mon 06:00 UTC), `workflow_dispatch` | Compiles `papiex` and downloads `epmt-dash`, then saves both to cache | | `build_and_test_epmt.yml` | `push` to `main`, `pull_request` | Source-tree unit tests (no Docker) | + ### Caches and Invalidation + Each cache step is keyed on its build prerequisites — analogous to how `make` uses file modification times to decide whether a target must be rebuilt. Changing a prerequisite produces a new cache key, causing a cache miss and forcing a fresh build. + | Cache | Cache key components | Invalidation trigger | Notes | |---|---|---|---| | `epmt-build` Docker image | `OS_TARGET` + `PYTHON_VERSION` +
`SQLITE_VERSION` +
`hashFiles(Dockerfile, requirements.txt.py3)` | Edit the Dockerfile or requirements file, or bump any version variable | Fully content-hash based — closest analogy to `make` | @@ -492,24 +544,29 @@ build. | `test-release` Docker image | `OS_TARGET` + `PYTHON_VERSION` +
`SQLITE_VERSION` +
`hashFiles(Dockerfile, requirements.txt.py3)` +
`github.sha` | Always rebuilds (by design) —
`restore-keys` prefix reuses
unchanged early layers via
`--cache-from` | Image content changes
every commit; layer reuse
keeps it fast | | `slurm-cluster` Docker image | `IMAGE_TAG` +
`SLURM_TAG` +
`SLURM_CLUSTER_TAG` | Bump any of the three version
variables in `docker_build_test.yml`
and `slurm_image_build.yml` | Version-string based; upstream
tag mutations without a version
bump won't invalidate | | `epmt-dash` UI directory | `EPMT_DASH_SRC_BRANCH` | Change `EPMT_DASH_SRC_BRANCH`
in `docker_build_test.yml`,
`weekly_tarball_build.yml`,
and `Makefile` | Branch-name based; new commits
to same branch don't invalidate —
weekly workflow bounds staleness
to ≤1 week | + ### Cache Invalidation Gap vs. `make` + `make` detects prerequisite changes via file modification times regardless of version numbers. The GitHub Actions caches above use version strings or content hashes instead, so: -* `epmt-build` is fully `make`-like — changing the Dockerfile or + +- `epmt-build` is fully `make`-like — changing the Dockerfile or requirements file immediately produces a different hash and forces a rebuild. -* `papiex` and `slurm-cluster` require a deliberate version bump in the +- `papiex` and `slurm-cluster` require a deliberate version bump in the workflow env block to trigger a rebuild. Remote source changes without a version bump go undetected until the next weekly build. -* `epmt-dash` requires either a branch rename or waiting for the weekly +- `epmt-dash` requires either a branch rename or waiting for the weekly build. The weekly `weekly_tarball_build.yml` workflow always fetches fresh content, bounding maximum staleness to one week. ### Forcing a Rebuild + To force any individual cache to rebuild, bump the relevant version variable in the `env:` section of both the weekly workflow and `docker_build_test.yml`, and update the `Makefile` to match: + ```yaml # docker_build_test.yml (and weekly_tarball_build.yml) env: @@ -523,6 +580,7 @@ The `epmt-build` cache is invalidated automatically whenever modified — no manual version bump is needed. ### Weekly Pre-warming + `slurm_image_build.yml` and `weekly_tarball_build.yml` run every Monday morning to pre-warm their respective caches before the working week begins. `docker_build_test.yml` then restores those caches on pull request and push @@ -533,10 +591,12 @@ artifact inline so the pipeline never silently skips a required build step. ## Troubleshooting ### Virtual Environments + Note that often in virtual environments, hardware counters are not often available in the VM. ### Common Error Messages #### `version GLIBC_x.xx not found` + The collector library may not have been built for the current environment or the release OS version does not match the current environment. diff --git a/Dockerfiles/Dockerfile.centos-7-epmt-test-release b/Dockerfiles/Dockerfile.centos-7-epmt-test-release index e578fc824..0d4bd6e47 100644 --- a/Dockerfiles/Dockerfile.centos-7-epmt-test-release +++ b/Dockerfiles/Dockerfile.centos-7-epmt-test-release @@ -86,9 +86,6 @@ RUN rm -f *.tgz epmt-installer -# -# ian attempting smth here -# ##won't work, read-only file system, even sudo won't do it #RUN sudo -n echo 2 > /proc/sys/kernel/perf_event_paranoid diff --git a/Makefile b/Makefile index 681d109e1..815d50bb3 100644 --- a/Makefile +++ b/Makefile @@ -163,7 +163,7 @@ $(EPMT_RELEASE) dist: # runs setuptools # note that this step requires EPMT_RELEASE and PAPIEX_RELEASE, but we don't explicitly state it here, b.c. -# when we go in the docker container, the time-zone changes and therefore thet timestamp comparison triggers re-making +# when we go in the docker container, the time-zone changes and therefore the timestamp comparison triggers re-making # targets that do not need to be remade python-dist: @echo "(python-dist) whoami: $(shell whoami)" @@ -192,7 +192,7 @@ test-$(EPMT_RELEASE) dist-test: @echo "WARNING removing directory to clean up: rm -rf epmt-install-tests" rm -rf epmt-install-tests -# this target 1) builds an image with an environment inwhich we'd like to build our applicaiton +# this target 1) builds an image with an environment in which we'd like to build our application # 2) builds that application within a running container of that image # NOTE: bind mounts to current working directory, usually the repository directory docker-dist: @@ -204,7 +204,7 @@ docker-dist: @echo @echo @echo " - docker build Dockerfiles/Dockerfile.$(OS_TARGET)-epmt-build" - @echo " we are creating a container environment inwhich to build the python distribution" + @echo " we are creating a container environment in which to build the python distribution" $(DOCKER_BUILD) Dockerfiles/Dockerfile.$(OS_TARGET)-epmt-build -t $(OS_TARGET)-epmt-build:$(EPMT_VERSION) \ --build-arg sqlite_version=$(SQLITE_VERSION) \ --build-arg sqlite_year=$(SQLITE_YEAR) \ @@ -212,7 +212,7 @@ docker-dist: @echo @echo @echo " - docker run " - @echo " within a running contianer of the image we just built, now build the python application." + @echo " within a running container of the image we just built, now build the python application." @echo " i.e. running make dist python-dist dist-test inside $(OS_TARGET)-epmt-build" $(DOCKER_RUN) $(DOCKER_RUN_OPTS) --privileged \ --volume=$(PWD):$(PWD) \ diff --git a/README.md b/README.md index 0c8916a20..e7318aeb0 100644 --- a/README.md +++ b/README.md @@ -1,34 +1,39 @@ # **Experiment Process / Metadata Tool** (`epmt`) -`epmt` collects metadata and performance data about shell processes, down to individual threads in individual processes. Currently, -`epmt` is particularly specialized for interfacing with Slurm batch jobs associated with earth modeling workflows, but is generalizable to -other computational workflow contexts. It also offers entrypoints to analyzing your data by interfacing with `jupyter` for easy access to +`epmt` collects metadata and performance data about shell processes, down to individual threads in individual +processes. Currently, `epmt` is particularly specialized for interfacing with Slurm batch jobs associated with +earth modeling workflows, but is generalizable to other computational workflow contexts. It also offers +entrypoints to analyzing your data by interfacing with `jupyter` for easy access to a notebook-style interface. [![readthedocs](https://app.readthedocs.org/projects/epmt/badge/?version=latest&style=flat)](https://epmt.readthedocs.io/en/latest/) [![codecov](https://codecov.io/gh/NOAA-GFDL/epmt/branch/main/graph/badge.svg)](https://codecov.io/gh/NOAA-GFDL/epmt) -[![pylint](https://img.shields.io/badge/pylint-%E2%89%A58.6-brightgreen)](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml) +[![pylint](https://img.shields.io/badge/pylint-%E2%89%A58.7-brightgreen)](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml) [![weekly_cache_builds](https://github.com/NOAA-GFDL/epmt/actions/workflows/weekly_cache_builds.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/weekly_cache_builds.yml) [![build_conda](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_conda.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_conda.yml?query=branch%3Amain) +[![markdown lint](https://github.com/NOAA-GFDL/epmt/actions/workflows/md_lint.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/md_lint.yml) +[![spell check](https://github.com/NOAA-GFDL/epmt/actions/workflows/spell_check.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/spell_check.yml) -| Workflow | Python 3.10 | Python 3.11 | -|----------|-------------|------------| + +| Workflow | Python 3.10 | Python 3.11 | +| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | **create_test_conda_env** | [![3.10](https://github.com/NOAA-GFDL/epmt/actions/workflows/create_test_conda_env.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/create_test_conda_env.yml?query=branch%3Amain+python-version%3A3.10) | [![3.11](https://github.com/NOAA-GFDL/epmt/actions/workflows/create_test_conda_env.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/create_test_conda_env.yml?query=branch%3Amain+python-version%3A3.11) | -| Workflow | SQLite | PostgreSQL | -|----------|--------|-----------| -| **docker_build_test** | [![sqlite](https://github.com/NOAA-GFDL/epmt/actions/workflows/docker_build_test.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/docker_build_test.yml?query=branch%3Amain+db_backend%3Asqlite) | [![postgres](https://github.com/NOAA-GFDL/epmt/actions/workflows/docker_build_test.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/docker_build_test.yml?query=branch%3Amain+db_backend%3Apostgres) | +| Workflow | SQLite | PostgreSQL | +| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **docker_build_test** | [![sqlite](https://github.com/NOAA-GFDL/epmt/actions/workflows/docker_build_test.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/docker_build_test.yml?query=branch%3Amain+db_backend%3Asqlite) | [![postgres](https://github.com/NOAA-GFDL/epmt/actions/workflows/docker_build_test.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/docker_build_test.yml?query=branch%3Amain+db_backend%3Apostgres) | | **build_and_test_epmt** | [![sqlite](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml?query=branch%3Amain+db_backend%3Asqlite) | [![postgres](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml/badge.svg)](https://github.com/NOAA-GFDL/epmt/actions/workflows/build_and_test_epmt.yml?query=branch%3Amain+db_backend%3Apostgres) | - + ## Installation -These are not-yet *fully* functional installations, as `epmt` was designed in an era where virtual environments were not as ubiquitous as -they are today. For full-featured build/installation approaches, consult the `Makefile`, `.github/workflows`, and [`DEVELOPER.md`](./DEVELOPER.md) +These are not-yet *fully* functional installations, as `epmt` was designed in an era where virtual environments +were not as ubiquitous as they are today. For full-featured build/installation approaches, consult the +`Makefile`, `.github/workflows`, and [`DEVELOPER.md`](./DEVELOPER.md) ### With `conda` (recommended) -The `conda` installation is currently favored as a quick-start for new users. +The `conda` installation is currently favored as a quick-start for new users. ```bash conda install noaa-gfdl::epmt @@ -37,6 +42,7 @@ conda install noaa-gfdl::epmt ### From repo checkout The following creates a whole `epmt` conda environment with `epmt` accessible via an editable `pip` installation. + ```bash git clone https://github.com/NOAA-GFDL/epmt.git cd epmt @@ -45,6 +51,7 @@ conda activate epmt ``` If you already have an environment created that you wish to install `epmt`, and it's already activated: + ```bash git clone https://github.com/NOAA-GFDL/epmt.git cd epmt @@ -54,16 +61,18 @@ pip install src/ ### Verifying an Installation The `check` command is a first-stop sanity-check of your `epmt` installation. Call it with + ```bash -$ epmt check +epmt check ``` Verify the version: + ```bash -$ epmt -V +epmt -V ``` -## Quickstart: Watch `epmt` work +## Quickstart: Watch `epmt` work Try wrapping your commands with `epmt start` / `epmt stop`: @@ -83,4 +92,5 @@ epmt submit ## Further Documentation -For detailed information on configuration, data collection, SLURM integration, database submission, analysis, performance metrics, debugging, and CI/CD, see [DEVELOPER.md](DEVELOPER.md). +For detailed information on configuration, data collection, SLURM integration, database submission, analysis, +performance metrics, debugging, and CI/CD, see [DEVELOPER.md](DEVELOPER.md). diff --git a/epmtdocs/docs/DEVELOPER.md b/epmtdocs/docs/DEVELOPER.md index 796b4b600..04dfa29b0 100644 --- a/epmtdocs/docs/DEVELOPER.md +++ b/epmtdocs/docs/DEVELOPER.md @@ -1,4 +1,5 @@ # `epmt` Developer and Advanced Usage Guide + This document contains detailed information about `epmt` configuration, data collection, database submission, analysis, metrics, debugging, and CI/CD infrastructure. @@ -15,7 +16,7 @@ For installation and quick-start instructions, see [README.md](./README.md). - [Data Collection with SLURM epilog and prolog](#data-collection-with-slurm-epilog-and-prolog) - [The Second Mode, Data Submission](#the-second-mode-data-submission) - [Manual Submission Example](#manual-submission-example) - - [Compressed Directory Submission Exmple](#compressed-directory-submission-exmple) + - [Compressed Directory Submission Example](#compressed-directory-submission-example) - [Internal-batch Job Submission Example](#internal-batch-job-submission-example) - [Data From Current Session Submission Example](#data-from-current-session-submission-example) - [The Third Mode, Data Analysis and Visualization](#the-third-mode-data-analysis-and-visualization) @@ -34,12 +35,14 @@ For installation and quick-start instructions, see [README.md](./README.md). - [`version GLIBC_x.xx not found`](#version-glibc_xxx-not-found) ## Configuration -`epmt` houses default settings (`epmt_default_settings.py`), user settings (`settings.py`), -and a submodule (`epmt_settings.py`) that appends the user settings to the end of the + +`epmt` houses default settings (`epmt_default_settings.py`), user settings (`settings.py`), +and a submodule (`epmt_settings.py`) that appends the user settings to the end of the default settings, overriding them. Custom user settings should only be included by developers who understand `epmt`'s requirements. The default settings should first be evaluated, they look a little bit like the following: + ```bash $ cat src/epmt/epmt_default_settings.py ... @@ -55,8 +58,10 @@ $ cat src/epmt/epmt_default_settings.py ``` ### Environment Variables + The following variables replace, at run-time, the values in the `db_params` dictionary found in `settings.py`: -``` + +```text EPMT_DB_PROVIDER EPMT_DB_USER EPMT_DB_PASSWORD @@ -66,7 +71,9 @@ EPMT_DB_FILENAME ``` ### Getting Current Configuration Information + You can examine all current settings by passing the `--help` option: + ```bash $ ./epmt --help usage: epmt [-n] [-d] [-h] [-a] [--drop] @@ -95,19 +102,22 @@ environment variables (overrides settings.py): ``` ## The Three Modes of `epmt` + There are three modes to `epmt` usage; data collection, data submission, and data analysis. Each has different dependencies: -* **Collection** requires Python 2.6 or higher, and compiled [`papiex`](https://github.com/noaa-gfdl/papiex) +- **Collection** requires Python 2.6 or higher, and compiled [`papiex`](https://github.com/noaa-gfdl/papiex) libraries for process tagging -* **Submission** requires Python packages for SQL and database interactions (`sqlalchemy`, `sqlite`, `postgres`) -* **Analysis** requires `jupyter`, `iPython`, and additionally [`epmt-dash`](https://github.com/noaa-gfdl/epmt-dash) +- **Submission** requires Python packages for SQL and database interactions (`sqlalchemy`, `sqlite`, `postgres`) +- **Analysis** requires `jupyter`, `iPython`, and additionally [`epmt-dash`](https://github.com/noaa-gfdl/epmt-dash) and `plotly` for dashboard-style analytics ### The First Mode, Data Collection #### Collecting Performance Data + Assuming you have `epmt` installed and in your path, let's modify a job file: + ```bash $ cat my_job.sh #!/bin/bash @@ -116,6 +126,7 @@ $ cat my_job.sh ``` This becomes: + ```bash $ cat my_job_epmt.sh #!/bin/bash @@ -125,7 +136,8 @@ epmt run ./compute_the_world --debug epmt stop ``` -Or more succintlty by automating the start/stop cycle with the `--auto` or `-a` flag: +Or more succinctly by automating the start/stop cycle with the `--auto` or `-a` flag: + ```bash $ cat my_job_epmt2.sh #!/bin/bash @@ -134,6 +146,7 @@ epmt -a run ./compute_the_world --debug ``` But usually we want to run more than one executable. We could have any number of run statements: + ```bash $ cat my_job_epmt3.sh #!/bin/bash @@ -150,10 +163,11 @@ provides the configuration to export to the environment through the `source` command. This is intended for use in a job file and should be evaluated by the running shell, be it `bash` or `csh`. `epmt source` prints the required environment variables in Bash format unless either the `SHELL` or `_` -environment variable ends in `csh`. +environment variable ends in `csh`. **Note the unset of `LD_PRELOAD` before stop!** This prevents the data collection routine from running on `epmt stop` itself. + ```bash $ cat my_job_bash.sh #!/bin/bash @@ -172,7 +186,8 @@ unset LD_PRELOAD epmt stop ``` -Here's an example for `csh`, when run interactively, +Here's an example for `csh`, when run interactively, + ```bash $ /bin/csh > epmt -j1 source @@ -182,10 +197,11 @@ setenv LD_PRELOAD /Users/phil/Work/GFDL/epmt.git/../papiex-oss/papiex-oss-instal ``` #### Data Collection with SLURM epilog and prolog + Using configured prolog and epilogs with SLURM tasks allows one to skip job instrumentation entirely, except for job tags (`EPMT_JOB_TAGS`) and process tags (`PAPIEX_TAGS`). These are configured in `slurm.conf` for jobs -submitted with `sbatch` but can be tested on the command line when using `srun`. +submitted with `sbatch` but can be tested on the command line when using `srun`. The above Csh job is equivalent to the below sequence using a prolog and epilog, ***with the exception of the trailing submit statement.*** @@ -200,10 +216,11 @@ $ sleep 1 For this job to work using `sbatch`, make the following modifications in `slurm.conf`, substituting the appropriate path for `$EPMT_PREFIX`: + ```bash -$ SLURM_TASK_SCRIPT_DIR=${EPMT_PREFIX}/epmt-install/slurm -$ TaskProlog=${SLURM_TASK_SCRIPT_DIR}/slurm_task_prolog_epmt.sh -$ TaskEpilog=${SLURM_TASK_SCRIPT_DIR}/slurm_task_epilog_epmt.sh +SLURM_TASK_SCRIPT_DIR=${EPMT_PREFIX}/epmt-install/slurm +TaskProlog=${SLURM_TASK_SCRIPT_DIR}/slurm_task_prolog_epmt.sh +TaskEpilog=${SLURM_TASK_SCRIPT_DIR}/slurm_task_epilog_epmt.sh ``` If this fails, the `papiex` installation is likely either missing or @@ -211,29 +228,35 @@ misconfigured in `settings.py`. The `-a` flag tells `epmt` to treat this run as an entire job. ### The Second Mode, Data Submission + After collecting the data, jobs (groups of processes) are imported into the database with the `submit` command. This command takes arguments in the form of -directories or tar files that must contain a `job_metadata` file. +directories or tar files that must contain a `job_metadata` file. Normal operation is to submit one or more directories: + ```bash -$ epmt submit [...] +epmt submit [...] ``` One can also submit a list of compressed tar files: + ```bash -$ epmt submit [...] +epmt submit [...] ``` There is also a mode where the current environment is used to determine where to find the data. + ```bash -$ epmt submit +epmt submit ``` #### Manual Submission Example + We can submit our previous job to the database defined in `settings.py` by running the `epmt submit` command with the directory returned by `stage` (found in the location set by `settings.py` `epmt_output_prefix`): + ```bash $ epmt -v submit /tmp/epmt/1/ INFO:epmt_cmds:submit_to_db(/tmp/epmt/1/,*-papiex-[0-9]*-[0-9]*.csv,False) @@ -282,14 +305,18 @@ INFO:epmt_job:Staged import took 0:00:00.189151, 5.286781 processes per second INFO:epmt_cmds:Committed job 1 to database: Job[u'1'] ``` -#### Compressed Directory Submission Exmple +#### Compressed Directory Submission Example + This might happen at the end of the day via a cron job: + ```bash -$ epmt submit /*tgz +epmt submit /*tgz ``` #### Internal-batch Job Submission Example + These commands could be part of every users job, or in the batch systems configurable preambles/postambles. + ```bash $ cat my_job.sh #!/bin/bash @@ -302,16 +329,19 @@ epmt submit ``` The start/stop cycle can be removed with the `--auto` or `-a` flag, which performs start and stop for you: -``` -$ epmt -a run ./debug_the_world --outliers -$ epmt submit + +```bash +epmt -a run ./debug_the_world --outliers +epmt submit ``` #### Data From Current Session Submission Example -If not inside of a batch environment, `epmt` will *attempt to fake-and-bake a job id*. This is useful -when performing interactive runs. You may not be able to submit these jobs to a shared database due to -constraints on job ID uniqueness, sincet he session ID is not guaranteed to be unique across reboots, + +If not inside of a batch environment, `epmt` will *attempt to fake-and-bake a job id*. This is useful +when performing interactive runs. You may not be able to submit these jobs to a shared database due to +constraints on job ID uniqueness, since the session ID is not guaranteed to be unique across reboots, much less other systems. However, this use case is perfectly acceptable when using a private database: + ```bash $ epmt start WARNING:epmt_cmds:JOB_ID unset: Using session id 6948 as JOB_ID @@ -323,8 +353,10 @@ $ epmt submit ``` ### The Third Mode, Data Analysis and Visualization + `epmt` uses an `IPython` notebook data analytics environment. Starting the Jupyter notebook is easy from the `epmt notebook` command: + ```bash $ epmt notebook [I 15:39:24.236 NotebookApp] Serving notebooks from local directory: /home/chris/Documents/playground/MM/build/epmt @@ -344,23 +376,29 @@ $ epmt notebook The notebook command supports passing parameters to jupyter, such as host IP for sharing access with machines on the local network, notebook token, and notebook password: + ```bash -$ epmt notebook -- --ip 0.0.0.0 --NotebookApp.token='thisisatoken' --NotebookApp.password='hereisa$upersecurepassword' +epmt notebook -- --ip 0.0.0.0 --NotebookApp.token='thisisatoken' --NotebookApp.password='hereisasupersecurepassword' ``` ## Debugging + `epmt` can be passed both `-n` (dry-run) and `-v` (verbosity) to help with debugging. Add more `-v` flags to increase verbosity level (`-vvv`). + ```bash -$ epmt -v start +epmt -v start ``` Or to attempt a submit without touching the database: + ```bash -$ epmt -vv submit -n /dir/to/jobdata +epmt -vv submit -n /dir/to/jobdata ``` Also, one can decode and dump the job_metadata file in a dir or compressed dir. + + ```bash $ epmt dump ~/Downloads/yrs05-25.20190221/CM4_piControl_C_atmos_00050101.papiex.gfdl.19712961.tgz exp_component atmos @@ -384,13 +422,16 @@ job_pl_start 2019-02-20 19:58:41.274267 job_pl_submit 2019-02-20 19:58:41.274463 job_pl_username Foo.Bar ``` + ## Performance Metrics Data Dictionary + `epmt` collects data both from the job runtime and the applications run in that environment. See the `src/epmt/models/` directory for fixed data stored related to each object. Metric data is stored differently; the data collector's data dictionary can be found in papiex-oss/README.md. At the time of this writing, it looked like this: + | Key | Scope | Description | |--------------------------- |--------- |-------------------------------------------------------- | | 1. tags | Process | User specified tags for this executable | @@ -443,48 +484,59 @@ like this: | * | Thread | PAPI metrics | ### Addition of new metrics + Additional metrics can be configured either in two ways: -* The `papiex_options` string in `settings.py` if using `epmt run` or `epmt source` -* The value of the `PAPIEX_OPTIONS` environment variable if using `LD_PRELOAD` directly. + +- The `papiex_options` string in `settings.py` if using `epmt run` or `epmt source` +- The value of the `PAPIEX_OPTIONS` environment variable if using `LD_PRELOAD` directly. The value of these should be a comma separated string: + ```bash -$ export PAPIEX_OPTIONS="PERF_COUNT_SW_CPU_CLOCK,PAPI_CYCLES" +export PAPIEX_OPTIONS="PERF_COUNT_SW_CPU_CLOCK,PAPI_CYCLES" ``` To list available and functioning metrics, use one of the included command line tools: - * `papi_avail` and `papi_native_avail` (via `papi`) - * `check_events` and `showevtinfo` (via `libpfm`) - * `perf list` (`linux`) + +- `papi_avail` and `papi_native_avail` (via `papi`) +- `check_events` and `showevtinfo` (via `libpfm`) +- `perf list` (`linux`) The `PERF_COUNT_SW_*` events should work on any system that has the proper `/proc/sys/kernel/perf_event_paranoid` setting. One should verify the functionality of the metric using the `papi_command_line` tool: + ```bash -$ papi_command_line PERF_COUNT_SW_CPU_CLOCK -$ papi_command_line CYCLES +papi_command_line PERF_COUNT_SW_CPU_CLOCK +papi_command_line CYCLES ``` ## CI/CD Workflows and Caching + `epmt`'s GitHub Actions CI is split into focused workflows that use `actions/cache` to avoid rebuilding expensive artifacts on every pull request run. ### Workflows + + | Workflow | Trigger | Purpose | |---|---|---| | `docker_build_test.yml` | `push` to `main`, `pull_request` | Full build + test pipeline; restores cached artifacts before building | | `slurm_image_build.yml` | Weekly (Mon 06:00 UTC), `workflow_dispatch` | Builds the `slurm-cluster` Docker image from source and saves it to cache | | `weekly_tarball_build.yml` | Weekly (Mon 06:00 UTC), `workflow_dispatch` | Compiles `papiex` and downloads `epmt-dash`, then saves both to cache | | `build_and_test_epmt.yml` | `push` to `main`, `pull_request` | Source-tree unit tests (no Docker) | + ### Caches and Invalidation + Each cache step is keyed on its build prerequisites — analogous to how `make` uses file modification times to decide whether a target must be rebuilt. Changing a prerequisite produces a new cache key, causing a cache miss and forcing a fresh build. + | Cache | Cache key components | Invalidation trigger | Notes | |---|---|---|---| | `epmt-build` Docker image | `OS_TARGET` + `PYTHON_VERSION` +
`SQLITE_VERSION` +
`hashFiles(Dockerfile, requirements.txt.py3)` | Edit the Dockerfile or requirements file, or bump any version variable | Fully content-hash based — closest analogy to `make` | @@ -492,24 +544,29 @@ build. | `test-release` Docker image | `OS_TARGET` + `PYTHON_VERSION` +
`SQLITE_VERSION` +
`hashFiles(Dockerfile, requirements.txt.py3)` +
`github.sha` | Always rebuilds (by design) —
`restore-keys` prefix reuses
unchanged early layers via
`--cache-from` | Image content changes
every commit; layer reuse
keeps it fast | | `slurm-cluster` Docker image | `IMAGE_TAG` +
`SLURM_TAG` +
`SLURM_CLUSTER_TAG` | Bump any of the three version
variables in `docker_build_test.yml`
and `slurm_image_build.yml` | Version-string based; upstream
tag mutations without a version
bump won't invalidate | | `epmt-dash` UI directory | `EPMT_DASH_SRC_BRANCH` | Change `EPMT_DASH_SRC_BRANCH`
in `docker_build_test.yml`,
`weekly_tarball_build.yml`,
and `Makefile` | Branch-name based; new commits
to same branch don't invalidate —
weekly workflow bounds staleness
to ≤1 week | + ### Cache Invalidation Gap vs. `make` + `make` detects prerequisite changes via file modification times regardless of version numbers. The GitHub Actions caches above use version strings or content hashes instead, so: -* `epmt-build` is fully `make`-like — changing the Dockerfile or + +- `epmt-build` is fully `make`-like — changing the Dockerfile or requirements file immediately produces a different hash and forces a rebuild. -* `papiex` and `slurm-cluster` require a deliberate version bump in the +- `papiex` and `slurm-cluster` require a deliberate version bump in the workflow env block to trigger a rebuild. Remote source changes without a version bump go undetected until the next weekly build. -* `epmt-dash` requires either a branch rename or waiting for the weekly +- `epmt-dash` requires either a branch rename or waiting for the weekly build. The weekly `weekly_tarball_build.yml` workflow always fetches fresh content, bounding maximum staleness to one week. ### Forcing a Rebuild + To force any individual cache to rebuild, bump the relevant version variable in the `env:` section of both the weekly workflow and `docker_build_test.yml`, and update the `Makefile` to match: + ```yaml # docker_build_test.yml (and weekly_tarball_build.yml) env: @@ -523,6 +580,7 @@ The `epmt-build` cache is invalidated automatically whenever modified — no manual version bump is needed. ### Weekly Pre-warming + `slurm_image_build.yml` and `weekly_tarball_build.yml` run every Monday morning to pre-warm their respective caches before the working week begins. `docker_build_test.yml` then restores those caches on pull request and push @@ -533,10 +591,12 @@ artifact inline so the pipeline never silently skips a required build step. ## Troubleshooting ### Virtual Environments + Note that often in virtual environments, hardware counters are not often available in the VM. ### Common Error Messages #### `version GLIBC_x.xx not found` + The collector library may not have been built for the current environment or the release OS version does not match the current environment. diff --git a/epmtdocs/docs/INSTALL.md b/epmtdocs/docs/INSTALL.md index fdeb89677..6d75e7289 100644 --- a/epmtdocs/docs/INSTALL.md +++ b/epmtdocs/docs/INSTALL.md @@ -1,12 +1,16 @@ +# EPMT Installation Guide + Experiment Performance Management Tool a.k.a Workflow DB -This is a tool to collect metadata and performance data about an entire job down to the individual threads in individual processes. This tool uses **papiex** to perform the process monitoring. This tool is targeted at batch or ephemeral jobs, not daemon processes. +This is a tool to collect metadata and performance data about an entire job down to the individual threads in +individual processes. This tool uses **papiex** to perform the process monitoring. This tool is targeted at +batch or ephemeral jobs, not daemon processes. The software contained in this repository was written by Philip Mucci of Minimal Metrics LLC. ## Installation With Release File -The release file includes EPMT, Data Collection Libraries, Notebook and EPMT Workflow GUI. +The release file includes EPMT, Data Collection Libraries, Notebook and EPMT Workflow GUI. For installing with a release file you'll need: @@ -16,9 +20,9 @@ For installing with a release file you'll need: ### Run Install Script -Use the provided epmt-installer script +Use the provided epmt-installer script -``` +```bash $ ./epmt-installer EPMT-release-3.8.20-centos-7.tgz Using release: /tmp/ep-inst/EPMT-release-3.8.20-centos-7.tgz @@ -45,7 +49,7 @@ $ ./epmt-installer EPMT-release-3.8.20-centos-7.tgz If you prefer using modules, you can instead do: module load /tmp/ep-inst/epmt-3.8.20/modulefiles/epmt *********************************************************************** -``` +```bash ### Add EPMT to path @@ -55,7 +59,7 @@ $ export PATH="/tmp/ep-inst/epmt-3.8.20/epmt-install/epmt:$PATH" $ cd /tmp/ $ epmt --version EPMT 3.8.20 -``` +```bash ### Verify installation @@ -71,15 +75,14 @@ settings.papiex_options = PERF_COUNT_SW_CPU_CLOCK Pass epmt stage functionality Pass WARNING: epmtlib: No job name found, defaulting to unknown epmt run functionality Pass -``` - +```bash --- - ### Perf Event System Setting -For detailed hardware and software performance metrics to collected by non-privileged users, the following setting must be verified/modified: +For detailed hardware and software performance metrics to collected by non-privileged users, +the following setting must be verified/modified: ```text # A value of 3 means the system is totally disabled @@ -90,15 +93,20 @@ For detailed hardware and software performance metrics to collected by non-privi $ cat /proc/sys/kernel/perf_event_paranoid 1 -``` +```bash -This isn't necessary unless one would like to collect metrics exposed by [PAPI](http://icl.utk.edu/papi/), [libpfm](http://perfmon2.sourceforge.net/) and the [perfevent](http://web.eece.maine.edu/~vweaver/projects/perf_events/) subsystems. Collecting subsystem data is the premise of EPMT. See [Stack Overflow](https://stackoverflow.com/questions/51911368/what-restriction-is-perf-event-paranoid-1-actually-putting-on-x86-perf) for a discussion of the setting. A setting of 1 is perfectly safe for production systems. +This isn't necessary unless one would like to collect metrics exposed by +[PAPI](http://icl.utk.edu/papi/), [libpfm](http://perfmon2.sourceforge.net/) and the +[perfevent](http://web.eece.maine.edu/~vweaver/projects/perf_events/) subsystems. +Collecting subsystem data is the premise of EPMT. +See [Stack Overflow](https://stackoverflow.com/questions/51911368/what-restriction-is-perf-event-paranoid-1-actually-putting-on-x86-perf) +for a discussion of the setting. A setting of 1 is perfectly safe for production systems. ## Generation (compilation) of release This is done using Docker images. -``` +```bash # You'll want to remove all the old images, bitrot! # Extreme case: docker rmi $(docker images -a) docker system prune -a @@ -117,4 +125,4 @@ make release-all ls release-`date "+%d%m%Y"` # (it will be todays date stamp) EPMT-release-4.9.1-centos-7.tgz papiex-epmt-2.3.14-centos-7.tgz epmt-4.9.1-centos-7.tgz test-epmt-4.9.1-centos-7.tgz -``` +```bash diff --git a/epmtdocs/docs/LICENSE.md b/epmtdocs/docs/LICENSE.md index 3a67f7b30..9a0a65e34 100644 --- a/epmtdocs/docs/LICENSE.md +++ b/epmtdocs/docs/LICENSE.md @@ -1,3 +1,7 @@ +# GNU Lesser General Public License v2.1 + + + GNU LESSER GENERAL PUBLIC LICENSE Version 2.1, February 1999 @@ -498,7 +502,7 @@ necessary. Here is a sample; alter the names: library `Frob' (a library for tweaking knobs) written by James Random Hacker. - , 1 April 1990 + \, 1 April 1990 Ty Coon, President of Vice That's all there is to it! diff --git a/epmtdocs/docs/Quickstart.md b/epmtdocs/docs/Quickstart.md index c863510ed..d71d3fdf6 100644 --- a/epmtdocs/docs/Quickstart.md +++ b/epmtdocs/docs/Quickstart.md @@ -1,18 +1,18 @@ # EPMT Release Quickstart ## Download -You should have been provided link to the release tar EPMT-2.1.0.tgz + +You should have been provided link to the release tar EPMT-2.1.0.tgz as well as an installer script (epmt-installer). Download both of them and place them in a folder. - ## Install ### Automatic install using epmt-installer Just launch the installer and point it to the release tar. -``` +```bash $ ./epmt-installer ./EPMT-2.1.0.tgz Using release: /home/tushar/src/EPMT/EPMT-2.1.0.tgz @@ -34,14 +34,12 @@ export PATH="/home/tushar/src/EPMT/epmt-2.1.0/epmt-install/epmt:$PATH" Or, for C shell/tcsh: setenv PATH "/home/tushar/src/EPMT/epmt-2.1.0/epmt-install/epmt:$PATH" -``` +```bash Once the installer finishes successfully, you should follow the printed instructions and update your shell startup file. Then logout and login to update your PATH. - - ### Manual Install It is recommended that you do the automatic install using @@ -50,21 +48,21 @@ manual install instructions: Untar the release tar (EPMT-2.1.0.tgz), you will get three files - - papiex-epmt-x.y.z.tgz - - epmt-x.y.z.tgz - - test-epmt-x.y.z.tgz - +- papiex-epmt-x.y.z.tgz +- epmt-x.y.z.tgz +- test-epmt-x.y.z.tgz + Then set the environment variables below: -``` +```bash EPMT_VERSION=2.1.0 EPMT_PREFIX=/path/to/install EPMT_DOWNLOAD=/path/to/download -``` +```bash Now perform the following steps: -``` +```bash mkdir -p $EPMT_PREFIX cd $EPMT_PREFIX tar xf $EPMT_DOWNLOAD/epmt-$EPMT_VERSION.tgz @@ -72,46 +70,52 @@ tar xf $EPMT_DOWNLOAD/papiex-epmt-$EPMT_VERSION.tgz tar xf $EPMT_DOWNLOAD/test-epmt-$EPMT_VERSION.tgz cp epmt-install/preset_settings/settings_sqlite_localfile_sqlalchemy.py epmt-install/epmt/settings.py $EPMT_PREFIX/epmt-install/epmt/epmt -V -``` +```bash ***IMPORTANT*** -***The below instructions uses a SQLite database with an on-disk data store, which is scalable only to a few thousand jobs. You may use `settings_pg_localhost_sqlalchemy.py` to set up a connection to a PostGres database instance, which is the recommended configuration. `settings_sqlite_inmem_sqlalchemy.py` is also available for ephemeral testing.*** +***The below instructions uses a SQLite database with an on-disk data store, which is scalable only to a few +thousand jobs. You may use `settings_pg_localhost_sqlalchemy.py` to set up a connection to a PostGres +database instance, which is the recommended configuration. `settings_sqlite_inmem_sqlalchemy.py` is also +available for ephemeral testing.*** Modify install, output and stage paths, and possibly the database connection string, in your copied `settings.py` file. `vi $EPMT_PREFIX/epmt-install/epmt/settings.py` -``` +```bash install_prefix = /papiex-epmt-install/ epmt_output_prefix = "/tmp/epmt/" stage_command = "mv" stage_command_dest = "./" -``` +```bash Now check everything works as expected. -``` +```bash $EPMT_PREFIX/epmt-install/epmt/epmt -V $EPMT_PREFIX/epmt-install/epmt/epmt check -``` +```bash If everything looks good, add `epmt` to your path. For ***Bash***: -``` + +```bash export PATH=$EPMT_PREFIX/epmt-install/epmt:$PATH -``` +```bash + or for ***C shell***: -``` + +```bash setenv PATH $EPMT_PREFIX/epmt-install/epmt:$PATH -``` +```bash + ## Usage ### Manual Usage See examples in the `$EPMT_PREFIX/epmt-install/examples/` directory. - -``` +```bash $ cat epmt-example.csh #!/bin/tcsh @@ -125,7 +129,7 @@ set f=`epmt stage` # Move to medium term storage ($PWD) epmt submit $f # Submit to DB, should do elsewhere $ sbatch epmt-example.csh -``` +```bash ***IMPORTANT*** @@ -133,23 +137,28 @@ $ sbatch epmt-example.csh ### Automatic instrumentation using SLURM -Using configured prolog and epilogs with SLURM tasks allows one to skip job instrumentation entirely, with the exception of job tags (***EPMT_JOB_TAGS***) and process tags (***PAPIEX_TAGS***). These are configured in `slurm.conf` for jobs submitted with `sbatch` but they can be tested on the command line when using `srun`. +Using configured prolog and epilogs with SLURM tasks allows one to skip job instrumentation entirely, +with the exception of job tags (***EPMT_JOB_TAGS***) and process tags (***PAPIEX_TAGS***). These are +configured in `slurm.conf` for jobs submitted with `sbatch` but they can be tested on the command line +when using `srun`. -The above Csh job is equivalent to the below sequence using a prolog and epilog, ***with the exception of the trailing submit statement.*** +The above Csh job is equivalent to the below sequence using a prolog and epilog, +***with the exception of the trailing submit statement.*** -``` +```bash srun -n1 \\ --task-prolog=$EPMT_PREFIX/epmt-install/slurm/slurm_task_prolog_epmt.sh \\ --task-epilog=$EPMT_PREFIX/epmt-install/slurm/slurm_task_epilog_epmt.sh \\ sleep 1 -``` +```bash -For this job to work using `sbatch` the following modifications in the `slurm.conf` would be made, substituting the appropriate path for EPMT_PREFIX: +For this job to work using `sbatch` the following modifications in the `slurm.conf` would be made, +substituting the appropriate path for EPMT_PREFIX: -``` +```bash TaskProlog=EPMT_PREFIX/epmt-install/slurm/slurm_task_prolog_epmt.sh TaskEpilog=EPMT_PREFIX/epmt-install/slurm/slurm_task_epilog_epmt.sh -``` +```bash ## Testing (Optional) @@ -162,26 +171,26 @@ template before running the tests below.*** Saving your existing `settings.py` and using an in-memory sqlite template for the tests: -``` -$ mv $EPMT_PREFIX/epmt-install/epmt/settings.py $EPMT_PREFIX/epmt-install/epmt/settings.py.backup -$ cp $EPMT_PREFIX/epmt-install/preset_settings/settings_sqlite_inmem_sqlalchemy.py $EPMT_PREFIX/epmt-install/epmt/settings.py -``` +```bash +mv $EPMT_PREFIX/epmt-install/epmt/settings.py $EPMT_PREFIX/epmt-install/epmt/settings.py.backup +cp $EPMT_PREFIX/epmt-install/preset_settings/settings_sqlite_inmem_sqlalchemy.py $EPMT_PREFIX/epmt-install/epmt/settings.py +```bash ### Run integration tests -``` -$ cd $EPMT_PREFIX/epmt-install/epmt -$ pytest -x -vv test/integration/test_integration_*.py -``` +```bash +cd $EPMT_PREFIX/epmt-install/epmt +pytest -x -vv test/integration/test_integration_*.py +```bash ### Run unit tests -``` -$ pytest src/epmt/test/ -``` +```bash +pytest src/epmt/test/ +```bash Tip: Don't forget to restore your `settings.py` file after the tests! -``` -$ mv $EPMT_PREFIX/epmt-install/epmt/settings.py.backup $EPMT_PREFIX/epmt-install/epmt/settings.py -``` +```bash +mv $EPMT_PREFIX/epmt-install/epmt/settings.py.backup $EPMT_PREFIX/epmt-install/epmt/settings.py +```bash diff --git a/epmtdocs/docs/RELEASE_NOTES.md b/epmtdocs/docs/RELEASE_NOTES.md index 841838d58..ea1388491 100644 --- a/epmtdocs/docs/RELEASE_NOTES.md +++ b/epmtdocs/docs/RELEASE_NOTES.md @@ -1,97 +1,93 @@ -Version 4.9.1 -============= +# Release Notes + +## Version 4.9.1 Release date: 09/28/2002 The list includes features and fixes since version 4.7.3. - - Fixed logging of SQLA exceptions - - Fixed INDEX field size overruns when inserting into analyses table (get_comparable_jobs may be huge) - - Fixed numerous versioning issues in requirements - - Added --no-analysis argument to daemon - - PAPIEX 2.3.13 - - Removed bitrot throughout the dependencies +- Fixed logging of SQLA exceptions +- Fixed INDEX field size overruns when inserting into analyses table (get_comparable_jobs may be huge) +- Fixed numerous versioning issues in requirements +- Added --no-analysis argument to daemon +- PAPIEX 2.3.13 +- Removed bitrot throughout the dependencies -Version 4.7.3 -============= +## Version 4.7.3 Release date: 07/29/2020 The list includes features and fixes since version 4.5.2. - - DB schema migration to use bigint for primary keys instead of 4-byte int - - epmt migrate CLI support - - EPMT_DB_URL in the environment supersedes settings.py - - get_jobs supports a flag to avoid triggering post-processing - - Query API enhancements - - verify_jobs allows validating data in jobs in the database - - procs_histogram now supports arbitrary metrics' aggregation - - annotate jobs with papiex errors during submission - - bugs related to post-processing fixed - - improvements to unit and integration tests +- DB schema migration to use bigint for primary keys instead of 4-byte int +- epmt migrate CLI support +- EPMT_DB_URL in the environment supersedes settings.py +- get_jobs supports a flag to avoid triggering post-processing +- Query API enhancements + - verify_jobs allows validating data in jobs in the database + - procs_histogram now supports arbitrary metrics' aggregation +- annotate jobs with papiex errors during submission +- bugs related to post-processing fixed +- improvements to unit and integration tests -Version 4.5.2 -============= +## Version 4.5.2 Release date: 06/19/2020 The list includes features and fixes since version 3.7.22. - - Significant speed up in job submission rates for SQLAlchemy +- Significant speed up in job submission rates for SQLAlchemy under PostgreSQL (20x and higher depending on hw/sw) - - Direct-copy ingestion of CSV using PostgreSQL COPY - - Staging of process data into a separate table for faster ingestion - - Seamless post-processing of staged data on first-use - - Speed-up in job collation by using `O_APPEND` in papiex - - EPMT supports unit and integration tests from the CLI - - EPMT CLI supports conversion of old CSV to faster TSV format - - Multi-method scoring for outlier detection is now the default - - Fixes and enhancements to the Query, Outlier detection and Statistics API - - Outlier detection for processes and threads - - -Version 3.7.22 -============== + - Direct-copy ingestion of CSV using PostgreSQL COPY + - Staging of process data into a separate table for faster ingestion + - Seamless post-processing of staged data on first-use + - Speed-up in job collation by using `O_APPEND` in papiex +- EPMT supports unit and integration tests from the CLI +- EPMT CLI supports conversion of old CSV to faster TSV format +- Multi-method scoring for outlier detection is now the default +- Fixes and enhancements to the Query, Outlier detection and Statistics API +- Outlier detection for processes and threads + +## Version 3.7.22 Release date: 04/22/2020 The list includes features added since version 3.3.20. - - Support for automatic database migration under SQLAlchemy added - - `epmt help api` and `epmt help api ` provide +- Support for automatic database migration under SQLAlchemy added +- `epmt help api` and `epmt help api ` provide concise list of API index and function docstrings - - Improved API docstrings - - API support to find jobs based on experiment name, components, +- Improved API docstrings +- API support to find jobs based on experiment name, components, times and exit status - - API support to find missing time-segments in an experiment - - Revamp of the outliers notebook with the latest data - - Daemon mode now supports ingestion and retire functions - - `epmt submit` now supports `--remove` to delete on successful submits - - bug fixes - - papiex memory/cache consistency issues resolved - - fixes to support for `PAPIEX_TAGS` - - resolved race in submit which could cause jobs to remain unprocessed - - epmt annotate supports special handling for `EPMT_JOB_TAGS` - - Additional univariate classifiers added - - API improvements for PCA-based feature ranking - - Improved handling of staging and concatenation errors - - Improvements to the GUI - -Version 3.3.20 -============== +- API support to find missing time-segments in an experiment +- Revamp of the outliers notebook with the latest data +- Daemon mode now supports ingestion and retire functions +- `epmt submit` now supports `--remove` to delete on successful submits +- bug fixes + - papiex memory/cache consistency issues resolved + - fixes to support for `PAPIEX_TAGS` + - resolved race in submit which could cause jobs to remain unprocessed +- epmt annotate supports special handling for `EPMT_JOB_TAGS` +- Additional univariate classifiers added +- API improvements for PCA-based feature ranking +- Improved handling of staging and concatenation errors +- Improvements to the GUI + +## Version 3.3.20 Release date: 02/28/2020 The list below includes features added since version 2.2.7. - - 7x to 10x speedup in ingestion performance for SQLAlchemy +- 7x to 10x speedup in ingestion performance for SQLAlchemy (tested against SQLite and PostgreSQL backends) - - Principal Component Analysis (PCA) support added for outlier detection - - Multivariate Outlier Detection (MVOD) support added (pyod classifiers +- Principal Component Analysis (PCA) support added for outlier detection +- Multivariate Outlier Detection (MVOD) support added (pyod classifiers are supported at present) - - 100x improvement in delete performance with PostgreSQL - - `epmt explore` command-line support to enable GFDL-specific +- 100x improvement in delete performance with PostgreSQL +- `epmt explore` command-line support to enable GFDL-specific explorations into experiments - - `epmt retire` supports period job deletion based on policies - - `epmt annotate` supports appending metrics to a job archive or in the DB - - `epmt dump` now shows job archives and details of jobs in the database +- `epmt retire` supports period job deletion based on policies +- `epmt annotate` supports appending metrics to a job archive or in the DB +- `epmt dump` now shows job archives and details of jobs in the database diff --git a/epmtdocs/docs/db-multiple-users.md b/epmtdocs/docs/db-multiple-users.md index 8dbb3f136..313b23f6a 100644 --- a/epmtdocs/docs/db-multiple-users.md +++ b/epmtdocs/docs/db-multiple-users.md @@ -1,13 +1,15 @@ +# EPMT with Multiple Database Users + EPMT can work with multiple DB accounts with varying privilege levels. -Single-account permissions --------------------------- +## Single-account permissions + Here a single DB account is used and all operations use the same singular DB account. In such a case, you can set the `db_params:url` parameter -in `settings.py` to the singular account and password. +in `settings.py` to the singular account and password. + +## Multiple accounts -Multiple accounts ------------------ EPMT supports an environment variable `EPMT_DB_URL`, which can be set to a valid postgres URI of the form: @@ -17,19 +19,18 @@ When EPMT finds `EPMT_DB_URL` set in the environment, it will use that URI to connect to the database, and ignore the `db_params:url` parameter in `settings.py`. - Please view `migrations/docker-entrypoint-initdb.d/init-user-db.sh` It is configured with 3 different accounts: -1. postgres database admin account (DB ADMIN): -This account is of the owner of the EPMT database. This account can do -all operations including creating and deleting tables. When EPMT is +1. postgres database admin account (DB ADMIN): +This account is of the owner of the EPMT database. This account can do +all operations including creating and deleting tables. When EPMT is first-installed or after a new drop, it is recommended you run the following. -``` -$ EPMT_DB_URL=postgresql://:@postgres-host:5432/EPMT epmt -v migrate -``` +```bash +EPMT_DB_URL=postgresql://:@postgres-host:5432/EPMT epmt -v migrate +```bash This will create the necessary tables and apply migrations. It is necessary to apply migrations using the admin account as shown above. @@ -40,23 +41,22 @@ and only needed if you want to not use the main database admin account, and have a separate admin account for EPMT. If you decide to uncomment the `epmt-admin` section, then you should use those credentials in the command above. - -2. A read-write user account: +1. A read-write user account: This account can do everything the admin account can do, except create/delete tables. It cannot apply database migrations either. You can use this account -for all other EPMT tasks, such as ingestion, querying and deleting jobs, -post-processing, etc. This account is named as `epmt-rw` in the +for all other EPMT tasks, such as ingestion, querying and deleting jobs, +post-processing, etc. This account is named as `epmt-rw` in the `migrations/docker-entrypoint-initdb.d/init-user-db.sh` script. -3. A read-only user account: -This account provides read-only access to the EPMT database, and as such +2. A read-only user account: +This account provides read-only access to the EPMT database, and as such can be used for querying and outlier detection. It cannot modify the database in anyway. It cannot perform post-processing either, since that requires database writes. When performing queries on unprocessed jobs, R/O user accounts will see certain fields such as `proc_sums` (in the job model) empty as post-processing is required to populate such fields. -This account is named `epmt-ro` in the +This account is named `epmt-ro` in the `migrations/docker-entrypoint-initdb.d/init-user-db.sh` script. It is recommended that you edit `settings.py` and set `db_params:url` to @@ -64,18 +64,16 @@ the DB URI for the R/O account. All other account credentials for R/W and admin user can be configured by setting `EPMT_DB_URL` from the environment when such privilege is needed. -Configuring a docker postgres image with multiple user accounts -=============================================================== +## Configuring a docker postgres image with multiple user accounts 1. Edit `migrations/docker-entrypoint-initdb.d/init-user-db.sh` to suit your needs. You should only need to modify the passwords for `epmt-rw` -and `epmt-ro`. +and `epmt-ro`. 2. Now edit `settings.py` and set the `db_params:url` parameter to the postgres URI for the read-only user. This will ensure the default permissions give read-only DB access. - 3. Now run postgres container. The script `init-user-db.sh` needs to be bind-mounted under `/docker-entrypoint-initdb.d/init-user-db.sh`. And, postgres will *only execute the script if the database is empty* and it's @@ -84,21 +82,28 @@ using a `psql` client. A sample docker invocation is: -``` -$ docker run --rm --name postgres -v $PWD/migrations/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d -v /path/to/data/dir:/var/lib/postgresql/data -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=example -e POSTGRES_DB=EPMT -p 5432:5432 postgres:latest -``` +```bash +docker run --rm --name postgres \ + -v $PWD/migrations/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d \ + -v /path/to/data/dir:/var/lib/postgresql/data \ + -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=example -e POSTGRES_DB=EPMT \ + -p 5432:5432 \ + postgres:latest +```bash Here the admin user is `postgres` and has a password `example`. To first apply migrations as the admin user, you would do: -``` + +```bash EPMT_DB_URL=postgresql://postgres:example@localhost:5432/EPMT epmt -v migrate -``` +```bash Then you can submit a job using the R/W user as follows: -``` + +```bash EPMT_DB_URL=postgresql://epmt-rw:@localhost:5432/EPMT epmt -v submit xyz.tgz -``` +```bash Finally, an epmt command executed without `EPMT_DB_URL` will use the credentials in `settings.py` and default to the read-only user. diff --git a/epmtdocs/docs/epmt-explore-cli.md b/epmtdocs/docs/epmt-explore-cli.md index 5058f79a2..d1fa67e90 100644 --- a/epmtdocs/docs/epmt-explore-cli.md +++ b/epmtdocs/docs/epmt-explore-cli.md @@ -1,13 +1,15 @@ +# epmt explore CLI + To reproduce this example, you will need the following test data: -``` -$ epmt submit test/data/outliers_nb/*.tgz -``` +```text +epmt submit test/data/outliers_nb/*.tgz +```text You run the CLI by typing `epmt explore` and passing it an experiment name: -``` +```text $ epmt explore ESM4_hist-piAer_D1 top 10 components by sum(duration): @@ -110,11 +112,11 @@ duration by time segment: 19340101 44092005094 19390101 23934831859 19440101 38736281301 -``` +```text The output is self-explanatory. The outliers are marked with asterisks, -the more the asterisks the greater the outlier. We use multimode score -using a number of univariate classifiers. +the more the asterisks the greater the outlier. We use multimodal scoring +by combining a number of univariate classifiers. You will notice that the first four time-segments take a bulk of the time. That's an artifact of the fact that we loaded all the jobs of the first @@ -125,7 +127,7 @@ In the example above we use the default metric -- `duration`. It shows quite cle that the time-segment `18590101` is affected and took far longer. The output with `cpu_time` metric is also instructive. Have a look: -``` +```text $ epmt explore --metric cpu_time ESM4_hist-piAer_D1 top 10 components by sum(cpu_time): @@ -228,8 +230,8 @@ cpu_time by time segment: 19340101 16479797891 19390101 15447854905 19440101 15803303653 -``` - -The `cpu_time` data suggests that the `18590101` took less cpu cycles. Yet it took longer to finish. One hypothesis that could explain the seeming contradiction is if the node(s) where the `18590101` were time-sharing with other concurrently running jobs. - +```text +The `cpu_time` data suggests that the `18590101` took less cpu cycles. Yet it took longer +to finish. One hypothesis that could explain the seeming contradiction is if the node(s) +where the `18590101` were time-sharing with other concurrently running jobs. diff --git a/epmtdocs/docs/epmt_cmds.md b/epmtdocs/docs/epmt_cmds.md index 2223fcee9..0ba926ed0 100644 --- a/epmtdocs/docs/epmt_cmds.md +++ b/epmtdocs/docs/epmt_cmds.md @@ -1,3 +1,5 @@ +# epmt Commands + ## epmt check Check will verify basic epmt configuration and functionality. @@ -8,23 +10,27 @@ Source provides commands to begin automatic performance instrumentation of all subsequent shell commands. Standard use of this is via the shell's eval method inside job scripts or batch system wrappers. For example: - eval `epmt source` in Bash or Csh - eval `epmt source --slurm` for a SLURM prolog. +```text +eval `epmt source` in Bash or Csh +eval `epmt source --slurm` for a SLURM prolog. +```text Two shell functions/aliases are created to pause/restart instrumentation: - epmt_uninstrument - to pause automatic instrumentation - epmt_instrument - to renable automatic instruction. + +```text +epmt_uninstrument - to pause automatic instrumentation +epmt_instrument - to re-enable automatic instruction. +```text **SLURM USERS NOTE** Use in SLURM's prolog, requires a special syntax enabled here with the -s or --slurm option. For more info, see: -https://slurm.schedmd.com/prolog_epilog.html + ## epmt start Start will create a metadata log file with the current environment variables. - ## epmt run Run will execute a command in the shell, typically used with the auto -a flag @@ -64,11 +70,13 @@ displays the commands leading up to submission. Information about a job -The EPMT Dump command will return all metadata about a job, job username, job tags job exit code all can be found +The EPMT Dump command will return all metadata about a job, job username, job tags job exit code all can be found here. This command can be run on job archives, a job in the database or directly a job_metadata file. + ### Dump Job metadata from Archive -``` + +```text epmt$ epmt dump sample/ppr-batch-sow3/1909/2587750.tgz checked True job_el_env {'TERM': 'linux', 'HOME': '/home/Jeffrey.Durachta', 'SHELL': '/bin/tcsh', 'USER': 'Jeffrey.Durachta', 'LOGNAME': 'Jeffrey.Durachta', 'PATH': '/home/gfdl/bin2:/usr/local/bin:/bin:/usr/bin:.', 'HOSTTYPE': 'x86_64-linux', 'VENDOR': 'unknown', 'OSTYPE': 'linux', 'MACHTYPE': 'x86_64', 'SHLVL': '2', 'PWD': '/vftmp/Jeffrey.Durachta/job2587750', 'GROUP': 'f', 'HOST': 'pp063', 'LANG': 'en_US', 'LC_TIME': 'C', 'MANPATH': '/home/gfdl/man:/usr/local/man:/usr/share/man', 'OMP_NUM_THREADS': '1', 'ARCHIVE': '/archive/Jeffrey.Durachta', 'MODULE_VERSION': '3.2.10', 'MODULE_VERSION_STACK': '3.2.10', 'MODULESHOME': '/usr/local/Modules/3.2.10', 'MODULEPATH': '/usr/local/Modules/modulefiles:/home/fms/local/modulefiles', 'LOADEDMODULES': '', 'SLURM_JOB_NAME': 'ESM4_hist-piAer_D1_ocean_cobalt_sfc_19090101', 'SLURM_PRIO_PROCESS': '0', 'SLURM_SUBMIT_DIR': '/home/Jeffrey.Durachta/CMIP6/ESM4/AerChemMIP/ESM4_hist-piAer_D1/gfdl.ncrc4-intel16-prod-openmp/scripts/postProcess', 'SLURM_SUBMIT_HOST': 'an107', 'SLURM_GET_USER_ENV': '1', 'SLURM_NPROCS': '1', 'SLURM_NTASKS': '1', 'SLURM_CLUSTER_NAME': 'gfdl', 'SLURM_JOB_ID': '2587750', 'SLURM_JOB_NUM_NODES': '1', 'SLURM_JOB_NODELIST': 'pp063', 'SLURM_JOB_PARTITION': 'batch', 'SLURM_NODE_ALIASES': '(null)', 'SLURM_JOB_CPUS_PER_NODE': '1', 'ENVIRONMENT': 'BATCH', 'HOSTNAME': 'pp063', 'SLURM_JOBID': '2587750', 'SLURM_NNODES': '1', 'SLURM_NODELIST': 'pp063', 'SLURM_TASKS_PER_NODE': '1', 'SLURM_JOB_ACCOUNT': 'gfdl_f', 'SLURM_JOB_QOS': 'Added as default', 'SLURM_TOPOLOGY_ADDR': 'pp063', 'SLURM_TOPOLOGY_ADDR_PATTERN': 'node', 'SLURM_CPUS_ON_NODE': '1', 'SLURM_TASK_PID': '2834', 'SLURM_NODEID': '0', 'SLURM_PROCID': '0', 'SLURM_LOCALID': '0', 'SLURM_GTIDS': '0', 'SLURM_CHECKPOINT_IMAGE_DIR': '/var/slurm/checkpoint', 'SLURM_JOB_UID': '4067', 'SLURM_JOB_USER': 'Jeffrey.Durachta', 'SLURM_WORKING_CLUSTER': 'gfdl:slurm01:6817:8448', 'SLURM_JOB_GID': '70', 'SLURMD_NODENAME': 'pp063', 'TMPDIR': '/vftmp/Jeffrey.Durachta/job2587750', 'TMP': '/vftmp/Jeffrey.Durachta/job2587750', 'JOB_ID': '2587750', 'EPMT_JOB_TAGS': 'exp_name:ESM4_hist-piAer_D1;exp_component:ocean_cobalt_sfc;exp_time:19090101;atm_res:c96l49;ocn_res:0.5l75;script_name:ESM4_hist-piAer_D1_ocean_cobalt_sfc_19090101', 'jobname': 'ESM4_hist-piAer_D1_ocean_cobalt_sfc_19090101', 'EPMT_PREFIX': '/home/Jeffrey.Durachta/workflowDB/EPMT/epmt-2.1.2-centos-6', 'EPMT_PATH': '/home/Jeffrey.Durachta/workflowDB/EPMT/epmt-2.1.2-centos-6/epmt-install/epmt', 'EPMT': '/home/Jeffrey.Durachta/workflowDB/EPMT/epmt-2.1.2-centos-6/epmt-install/epmt/epmt', 'pp_script': '/home/Jeffrey.Durachta/CMIP6/ESM4/AerChemMIP/ESM4_hist-piAer_D1/gfdl.ncrc4-intel16-prod-openmp/scripts/postProcess/ESM4_hist-piAer_D1_ocean_cobalt_sfc_19090101.tags', 'LD_LIBRARY_PATH': '/home/Jeffrey.Durachta/workflowDB/EPMT/epmt-2.1.2-centos-6/epmt-install/epmt', 'MPLCONFIGDIR': '/vftmp/Jeffrey.Durachta/job2587750/tmpvikl5v1b', 'MATPLOTLIBDATA': '/home/Jeffrey.Durachta/workflowDB/EPMT/epmt-2.1.2-centos-6/epmt-install/epmt/mpl-data'} @@ -83,10 +91,11 @@ job_pl_start_ts 2019-12-31 07:54:06.222683-05:00 job_pl_submit_ts 2019-12-31 07:54:06.222683-05:00 job_pl_username Jeffrey.Durachta job_tags {'exp_name': 'ESM4_hist-piAer_D1', 'exp_component': 'ocean_cobalt_sfc', 'exp_time': '19090101', 'atm_res': 'c96l49', 'ocn_res': '0.5l75', 'script_name': 'ESM4_hist-piAer_D1_ocean_cobalt_sfc_19090101'} -``` +```text ### Dump Job metadata in database -``` + +```text ./epmt dump 4899590 PERF_COUNT_SW_CPU_CLOCK 294801284824 all_proc_tags [{'op': 'cp', 'op_instance': '3'}, {'op': 'dmput', 'op_instance': '2'}, {'op': 'fregrid', 'op_instance': '2'}, {'op': 'hsmget', 'op_instance': '1'}, {'op': 'hsmget', 'op_instance': '3'}, {'op': 'hsmget', 'op_instance': '4'}, {'op': 'hsmget', 'op_instance': '6'}, {'op': 'hsmget', 'op_instance': '7'}, {'op': 'mv', 'op_instance': '1'}, {'op': 'mv', 'op_instance': '3'}, {'op': 'ncatted', 'op_instance': '4'}, {'op': 'ncrcat', 'op_instance': '1'}, {'op': 'rm', 'op_instance': '1'}, {'op': 'rm', 'op_instance': '2'}, {'op': 'splitvars', 'op_instance': '2'}, {'op': 'untar', 'op_instance': '2'}] @@ -136,10 +145,11 @@ usertime 276883051 vol_ctxsw 121339 wchar 29175525734 write_bytes 27895468032 -``` +```text ### Dump job metadata file -``` + +```text $ ./epmt dump sample/kernel/run_output/job_metadata {'job_pl_id': 'kernel-build-20190606-150222', 'job_pl_submit_ts': datetime.datetime(2019, 6, 6, 15, 2, 22, 541326), 'job_pl_start_ts': datetime.datetime(2019, 6, 6, 15, 2, 22, 541326), 'job_el_reason': 'none', 'job_el_exitcode': 0, 'job_el_stop_ts': datetime.datetime(2019, 6, 6, 20, 39, 17, 792259), 'job_el_env': {'GOPATH': '/home/tushar/devhome/go', 'LS_COLORS': 'rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.Z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.jpg=01;35:*.jpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:', 'rvm_version': '1.29.7 (latest)', 'GOBIN': '/home/tushar/devhome/platform/Linux-x86_64/bin', 'rvm_path': '/home/tushar/.rvm', 'LESSOPEN': '| /usr/bin/lesspipe %s', 'SSH_CLIENT': '192.168.254.108 32832 22', 'LOGNAME': 'tushar', 'USER': 'tushar', 'HOME': '/home/tushar', 'PATH': '/home/tushar/.rvm/bin:/usr/local/bin:.:./bin:/home/tushar/bin:/home/tushar/devhome/platform/Linux-x86_64/bin:/home/tushar/.rvm/bin:/usr/local/bin:.:./bin:/home/tushar/bin:/home/tushar/devhome/platform/Linux-x86_64/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/home/tushar/.rvm/bin:/home/tushar/.rvm/bin:/home/tushar/devhome/bin:/home/tushar/.local/bin:/home/tushar/.rvm/bin:/home/tushar/.rvm/bin:/home/tushar/devhome/bin:/home/tushar/.local/bin', 'HISTSIZE': '10000000', 'LANG': 'en_IN', 'TERM': 'screen', 'SHELL': '/bin/bash', 'K8_KVM_SECONDARY_DISK': '25G', 'K8_KVM_NODE_MEMORY': '2048', 'LANGUAGE': 'en_IN:en', 'SHLVL': '2', 'K8_KVM_NUM_SLAVES': '2', 'QT_QPA_PLATFORMTHEME': 'appmenu-qt5', 'KUBECONFIG': '/home/tushar/src/k8s-kvm/kube.config', 'K8_KVM_MASTER_MEMORY': '2048', 'LIBVIRT_DEFAULT_URI': 'qemu:///system', 'rvm_bin_path': '/home/tushar/.rvm/bin', 'XDG_RUNTIME_DIR': '/run/user/1000', 'rvm_prefix': '/home/tushar', 'TMUX': '/tmp/tmux-1000/default,3744,0', 'EDITOR': 'vim', 'XDG_DATA_DIRS': '/usr/local/share:/usr/share:/var/lib/snapd/desktop', 'XDG_SESSION_ID': '107075', 'TMPDIR': '/scratch', 'SUBNET': '192.168.40', 'LSCOLORS': 'GxFxCxDxBxegedabagaced', 'LESSCLOSE': '/usr/bin/lesspipe %s %s', 'EPMT_JOB_TAGS': 'model:linux-kernel;compiler:gcc', 'PYTHONSTARTUP': '/home/tushar/devhome/rc/pystartup', 'SSH_TTY': '/dev/pts/0', 'OLDPWD': '/home/tushar', 'CLICOLOR': '1', 'NUM_SLAVES': '2', 'PWD': '/home/tushar/mm/epmt/build/epmt', 'MAIL': '/var/mail/tushar', 'SSH_CONNECTION': '192.168.254.108 42414 192.168.254.121 22', 'TMUX_PANE': '%13'}, 'job_pl_env': {'GOPATH': '/home/tushar/devhome/go', 'rvm_version': '1.29.7 (latest)', 'GOBIN': '/home/tushar/devhome/platform/Linux-x86_64/bin', 'rvm_path': '/home/tushar/.rvm', 'LESSOPEN': '| /usr/bin/lesspipe %s', 'SSH_CLIENT': '192.168.254.108 32832 22', 'LOGNAME': 'tushar', 'USER': 'tushar', 'HOME': '/home/tushar', 'PATH': '/home/tushar/.rvm/bin:/usr/local/bin:.:./bin:/home/tushar/bin:/home/tushar/devhome/platform/Linux-x86_64/bin:/home/tushar/.rvm/bin:/usr/local/bin:.:./bin:/home/tushar/bin:/home/tushar/devhome/platform/Linux-x86_64/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/home/tushar/.rvm/bin:/home/tushar/.rvm/bin:/home/tushar/devhome/bin:/home/tushar/.local/bin:/home/tushar/.rvm/bin:/home/tushar/.rvm/bin:/home/tushar/devhome/bin:/home/tushar/.local/bin', 'KUBECONFIG': '/home/tushar/src/k8s-kvm/kube.config', 'SSH_CONNECTION': '192.168.254.108 42414 192.168.254.121 22', 'LANG': 'en_IN', 'TERM': 'screen', 'SHELL': '/bin/bash', 'K8_KVM_SECONDARY_DISK': '25G', 'K8_KVM_NODE_MEMORY': '2048', 'LANGUAGE': 'en_IN:en', 'NUM_SLAVES': '2', 'SHLVL': '2', 'K8_KVM_NUM_SLAVES': '2', 'QT_QPA_PLATFORMTHEME': 'appmenu-qt5', 'HISTSIZE': '10000000', 'K8_KVM_MASTER_MEMORY': '2048', 'LIBVIRT_DEFAULT_URI': 'qemu:///system', 'rvm_bin_path': '/home/tushar/.rvm/bin', 'XDG_RUNTIME_DIR': '/run/user/1000', 'rvm_prefix': '/home/tushar', 'TMUX': '/tmp/tmux-1000/default,3744,0', 'EDITOR': 'vim', 'XDG_SESSION_ID': '107075', 'TMPDIR': '/scratch', 'SUBNET': '192.168.40', 'LSCOLORS': 'GxFxCxDxBxegedabagaced', 'LESSCLOSE': '/usr/bin/lesspipe %s %s', 'EPMT_JOB_TAGS': 'model:linux-kernel;compiler:gcc', 'PYTHONSTARTUP': '/home/tushar/devhome/rc/pystartup', 'SSH_TTY': '/dev/pts/0', 'OLDPWD': '/home/tushar', 'CLICOLOR': '1', 'XDG_DATA_DIRS': '/usr/local/share:/usr/share:/var/lib/snapd/desktop', 'PWD': '/home/tushar/mm/epmt/build/epmt', 'MAIL': '/var/mail/tushar', 'LS_COLORS': 'rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.Z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.jpg=01;35:*.jpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:', 'TMUX_PANE': '%13'}} job_el_env {'GOPATH': '/home/tushar/devhome/go', 'LS_COLORS': 'rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.Z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.jpg=01;35:*.jpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:', 'rvm_version': '1.29.7 (latest)', 'GOBIN': '/home/tushar/devhome/platform/Linux-x86_64/bin', 'rvm_path': '/home/tushar/.rvm', 'LESSOPEN': '| /usr/bin/lesspipe %s', 'SSH_CLIENT': '192.168.254.108 32832 22', 'LOGNAME': 'tushar', 'USER': 'tushar', 'HOME': '/home/tushar', 'PATH': '/home/tushar/.rvm/bin:/usr/local/bin:.:./bin:/home/tushar/bin:/home/tushar/devhome/platform/Linux-x86_64/bin:/home/tushar/.rvm/bin:/usr/local/bin:.:./bin:/home/tushar/bin:/home/tushar/devhome/platform/Linux-x86_64/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/home/tushar/.rvm/bin:/home/tushar/.rvm/bin:/home/tushar/devhome/bin:/home/tushar/.local/bin:/home/tushar/.rvm/bin:/home/tushar/.rvm/bin:/home/tushar/devhome/bin:/home/tushar/.local/bin', 'HISTSIZE': '10000000', 'LANG': 'en_IN', 'TERM': 'screen', 'SHELL': '/bin/bash', 'K8_KVM_SECONDARY_DISK': '25G', 'K8_KVM_NODE_MEMORY': '2048', 'LANGUAGE': 'en_IN:en', 'SHLVL': '2', 'K8_KVM_NUM_SLAVES': '2', 'QT_QPA_PLATFORMTHEME': 'appmenu-qt5', 'KUBECONFIG': '/home/tushar/src/k8s-kvm/kube.config', 'K8_KVM_MASTER_MEMORY': '2048', 'LIBVIRT_DEFAULT_URI': 'qemu:///system', 'rvm_bin_path': '/home/tushar/.rvm/bin', 'XDG_RUNTIME_DIR': '/run/user/1000', 'rvm_prefix': '/home/tushar', 'TMUX': '/tmp/tmux-1000/default,3744,0', 'EDITOR': 'vim', 'XDG_DATA_DIRS': '/usr/local/share:/usr/share:/var/lib/snapd/desktop', 'XDG_SESSION_ID': '107075', 'TMPDIR': '/scratch', 'SUBNET': '192.168.40', 'LSCOLORS': 'GxFxCxDxBxegedabagaced', 'LESSCLOSE': '/usr/bin/lesspipe %s %s', 'EPMT_JOB_TAGS': 'model:linux-kernel;compiler:gcc', 'PYTHONSTARTUP': '/home/tushar/devhome/rc/pystartup', 'SSH_TTY': '/dev/pts/0', 'OLDPWD': '/home/tushar', 'CLICOLOR': '1', 'NUM_SLAVES': '2', 'PWD': '/home/tushar/mm/epmt/build/epmt', 'MAIL': '/var/mail/tushar', 'SSH_CONNECTION': '192.168.254.108 42414 192.168.254.121 22', 'TMUX_PANE': '%13'} @@ -150,11 +160,12 @@ job_pl_env {'GOPATH': '/home/tushar/devhome/go', 'rvm_version': '1. job_pl_id kernel-build-20190606-150222 job_pl_start_ts 2019-06-06 15:02:22.541326 job_pl_submit_ts 2019-06-06 15:02:22.541326 -``` +```text You can pass the ***-k*** key switch and a requested key parameter also. ```text ./epmt dump -k job_tags sample/ppr-batch-sow3/1909/2587750.tgz {'exp_name': 'ESM4_hist-piAer_D1', 'exp_component': 'ocean_cobalt_sfc', 'exp_time': '19090101', 'atm_res': 'c96l49', 'ocn_res': '0.5l75', 'script_name': 'ESM4_hist-piAer_D1_ocean_cobalt_sfc_19090101'} -``` +```text + diff --git a/epmtdocs/docs/metrics.md b/epmtdocs/docs/metrics.md index 997254e00..7315faa0b 100644 --- a/epmtdocs/docs/metrics.md +++ b/epmtdocs/docs/metrics.md @@ -1,55 +1,56 @@ -Data Dictionary ---------------- +# Data Dictionary papiex collects the following data for each thread: -| Key | Source | Datatype | Scope | Description | -|--------------------------- |----------------------------------------|---------------- |--------- |-------------------------------------------------------- | -| 1. tags | getenv("PAPIEX_TAGS") | Escaped string | Process | User specified tags for this executable | -| 2. hostname | gethostname() | String | Process | hostname | -| 3. exename | PAPI | String | Process | Name of the application, usually argv[0] | -| 4. path | PAPI | String | Process | Path to the application | -| 5. args | monitor | Escaped string | Process | All arguments to exe excluding argv[0] | -| 6. exitcode | exit() | Integer | Process | Exit code | -| 7. exitsignal | monitor | Integer | Process | Exited due to a signal | -| 8. pid | getpid() | Integer | Process | Process id | -| 9. generation | monitor | Integer | Process | Incremented after every exec() or PID wrap | -| 10. ppid | getppid() | Integer | Process | Parent process id | -| 11. pgid | getpgid() | Integer | Process | Process group id | -| 12. sid | getsid() | Integer | Process | Process session id | -| 13. numtids | monitor | Integer | Process | Number of threads caught by instrumentation | -| 14. numranks | monitor(MPI) | Integer | Process | Number of MPI ranks detected | -| 15. tid | gettid() | Integer | Process | Thread id | -| 16. mpirank | monitor(MPI) | Integer | Thread | MPI rank | -| 17. start | gettimeofday() | Integer | Process | Microsecond timestamp at start | -| 18. end | gettimeofday() | Integer | Process | Microsecond timestamp at end | -| 19. usertime | getrusage(RUSAGE_THREAD) | Integer | Thread | Microsecond user time | -| 20. systemtime | getrusage(RUSAGE_THREAD) | Integer | Thread | Microsecond system time | -| 21. rssmax | getrusage(RUSAGE_THREAD) | Integer | Thread | Kb max resident set size | -| 22. minflt | getrusage(RUSAGE_THREAD) | Integer | Thread | Minor faults (TLB misses/new page frames) | -| 23. majflt | getrusage(RUSAGE_THREAD) | Integer | Thread | Major page faults (requiring I/O) | -| 24. inblock | getrusage(RUSAGE_THREAD) | Integer | Thread | 512B blocks read from I/O | -| 25. outblock | getrusage(RUSAGE_THREAD) | Integer | Thread | 512B blocks written to I/O | -| 26. vol_ctxsw | getrusage(RUSAGE_THREAD) | Integer | Thread | Voluntary context switches (yields) | -| 27. invol_ctxsw | getrusage(RUSAGE_THREAD) | Integer | Thread | Involuntary context switches (preemptions) | -| 28. cminflt | /proc//task//stat field 11 | Integer | Process | minflt (20) for all wait()ed children | -| 29. cmajflt | /proc//task//stat field 22 | Integer | Thread | majflt (21) for all wait()ed children | -| 30. cutime | /proc//task//stat field 20 | Integer | Process | utime (17) for all wait()ed children | -| 31. cstime | /proc//task//stat field 22 | Integer | Thread | stime (18) for all wait()ed children | -| 32. num_threads | /proc//task//stat field 20 | Integer | Process | Threads in process at finish | -| 33. starttime | /proc//task//stat field 22 | Integer | Thread | Timestamp in jiffies after boot thread was started | -| 34. processor | /proc//task//stat field 39 | Integer | Thread | CPU this thread last ran on | -| 35. delayacct_blkio_time | /proc//task//stat field 42 | Integer | Thread | Jiffies process blocked in D state on I/O device | -| 36. guest_time | /proc//task//stat field 43 | Integer | Thread | Jiffies running a virtual CPU for a guest OS | -| 37. rchar | /proc//task//io line 1 | Integer | Thread | Bytes read via syscall (maybe from cache not dev I/O) | -| 38. wchar | /proc//task//io line 2 | Integer | Thread | Bytes written via syscall (maybe to cache not dev I/O) | -| 39. syscr | /proc//task//io line 3 | Integer | Thread | Read syscalls | -| 40. syscw | /proc//task//io line 4 | Integer | Thread | Write syscalls | -| 41. read_bytes | /proc//task//io line 5 | Integer | Thread | Bytes read from I/O device | -| 42. write_bytes | /proc//task//io line 6 | Integer | Thread | Bytes written to I/O device | -| 43. cancelled_write_bytes | /proc//task//io line 7 | Integer | Thread | Bytes discarded by truncation | -| 44. time_oncpu | /proc//task//schedstat | Integer | Thread | Nanoseconds spent running | -| 45. time_waiting | /proc//task//schedstat | Integer | Thread | Nanoseconds runnable but waiting | -| 46. timeslices | /proc//task//schedstat | Integer | Thread | Number of run periods on CPU | -| 47. rdtsc_duration | PAPI | Integer | Thread | If PAPI, real time cycle duration of thread | -| * | PAPI | Integer | Thread | PAPI metrics | + +| Key | Source | Datatype | Scope | Description | +| -------------------------- |------------------------------------------- | --------------- | -------- | ------------------------------------------------------- | +| 1. tags | getenv("PAPIEX_TAGS") | Escaped string | Process | User specified tags for this executable | +| 2. hostname | gethostname() | String | Process | hostname | +| 3. exename | PAPI | String | Process | Name of the application, usually argv[0] | +| 4. path | PAPI | String | Process | Path to the application | +| 5. args | monitor | Escaped string | Process | All arguments to exe excluding argv[0] | +| 6. exitcode | exit() | Integer | Process | Exit code | +| 7. exitsignal | monitor | Integer | Process | Exited due to a signal | +| 8. pid | getpid() | Integer | Process | Process id | +| 9. generation | monitor | Integer | Process | Incremented after every exec() or PID wrap | +| 10. ppid | getppid() | Integer | Process | Parent process id | +| 11. pgid | getpgid() | Integer | Process | Process group id | +| 12. sid | getsid() | Integer | Process | Process session id | +| 13. numtids | monitor | Integer | Process | Number of threads caught by instrumentation | +| 14. numranks | monitor(MPI) | Integer | Process | Number of MPI ranks detected | +| 15. tid | gettid() | Integer | Process | Thread id | +| 16. mpirank | monitor(MPI) | Integer | Thread | MPI rank | +| 17. start | gettimeofday() | Integer | Process | Microsecond timestamp at start | +| 18. end | gettimeofday() | Integer | Process | Microsecond timestamp at end | +| 19. usertime | getrusage(RUSAGE_THREAD) | Integer | Thread | Microsecond user time | +| 20. systemtime | getrusage(RUSAGE_THREAD) | Integer | Thread | Microsecond system time | +| 21. rssmax | getrusage(RUSAGE_THREAD) | Integer | Thread | Kb max resident set size | +| 22. minflt | getrusage(RUSAGE_THREAD) | Integer | Thread | Minor faults (TLB misses/new page frames) | +| 23. majflt | getrusage(RUSAGE_THREAD) | Integer | Thread | Major page faults (requiring I/O) | +| 24. inblock | getrusage(RUSAGE_THREAD) | Integer | Thread | 512B blocks read from I/O | +| 25. outblock | getrusage(RUSAGE_THREAD) | Integer | Thread | 512B blocks written to I/O | +| 26. vol_ctxsw | getrusage(RUSAGE_THREAD) | Integer | Thread | Voluntary context switches (yields) | +| 27. invol_ctxsw | getrusage(RUSAGE_THREAD) | Integer | Thread | Involuntary context switches (preemptions) | +| 28. cminflt | /proc/\/task/\/stat field 11 | Integer | Process | minflt (20) for all wait()ed children | +| 29. cmajflt | /proc/\/task/\/stat field 22 | Integer | Thread | majflt (21) for all wait()ed children | +| 30. cutime | /proc/\/task/\/stat field 20 | Integer | Process | utime (17) for all wait()ed children | +| 31. cstime | /proc/\/task/\/stat field 22 | Integer | Thread | stime (18) for all wait()ed children | +| 32. num_threads | /proc/\/task/\/stat field 20 | Integer | Process | Threads in process at finish | +| 33. starttime | /proc/\/task/\/stat field 22 | Integer | Thread | Timestamp in jiffies after boot thread was started | +| 34. processor | /proc/\/task/\/stat field 39 | Integer | Thread | CPU this thread last ran on | +| 35. delayacct_blkio_time | /proc/\/task/\/stat field 42 | Integer | Thread | Jiffies process blocked in D state on I/O device | +| 36. guest_time | /proc/\/task/\/stat field 43 | Integer | Thread | Jiffies running a virtual CPU for a guest OS | +| 37. rchar | /proc/\/task/\/io line 1 | Integer | Thread | Bytes read via syscall (maybe from cache not dev I/O) | +| 38. wchar | /proc/\/task/\/io line 2 | Integer | Thread | Bytes written via syscall (maybe to cache not dev I/O) | +| 39. syscr | /proc/\/task/\/io line 3 | Integer | Thread | Read syscalls | +| 40. syscw | /proc/\/task/\/io line 4 | Integer | Thread | Write syscalls | +| 41. read_bytes | /proc/\/task/\/io line 5 | Integer | Thread | Bytes read from I/O device | +| 42. write_bytes | /proc/\/task/\/io line 6 | Integer | Thread | Bytes written to I/O device | +| 43. cancelled_write_bytes | /proc/\/task/\/io line 7 | Integer | Thread | Bytes discarded by truncation | +| 44. time_oncpu | /proc/\/task/\/schedstat | Integer | Thread | Nanoseconds spent running | +| 45. time_waiting | /proc/\/task/\/schedstat | Integer | Thread | Nanoseconds runnable but waiting | +| 46. timeslices | /proc/\/task/\/schedstat | Integer | Thread | Number of run periods on CPU | +| 47. rdtsc_duration | PAPI | Integer | Thread | If PAPI, real time cycle duration of thread | +| * | PAPI | Integer | Thread | PAPI metrics | +