diff --git a/concourse-ci/.gitignore b/concourse-ci/.gitignore new file mode 100644 index 0000000000..ad3b616bb6 --- /dev/null +++ b/concourse-ci/.gitignore @@ -0,0 +1,5 @@ +._* +.mypy_cache +credentials +env +env_ok diff --git a/concourse-ci/.isort.cfg b/concourse-ci/.isort.cfg new file mode 100644 index 0000000000..dbaab0a044 --- /dev/null +++ b/concourse-ci/.isort.cfg @@ -0,0 +1,7 @@ + +[settings] +multi_line_output=3 +include_trailing_comma=True +force_grid_wrap=0 +use_parentheses=True +line_length=88 diff --git a/concourse-ci/Makefile b/concourse-ci/Makefile new file mode 100644 index 0000000000..7aaa613574 --- /dev/null +++ b/concourse-ci/Makefile @@ -0,0 +1,75 @@ +py_dirs = assemble_jobs +py_files = $(wildcard assemble_jobs/*.py) + + +.PHONY: fmt +fmt: env_ok + env/bin/isort -sp .isort.cfg $(py_files) + env/bin/black $(py_files) + + +.PHONY: test +test: check + env/bin/python -m unittest discover $(py_dirs) -p "*.py" -v + + +.PHONY: check +check: env_ok + env/bin/python -m mypy \ + --check-untyped-defs \ + --ignore-missing-imports \ + assemble_jobs + env/bin/python -m flake8 --select F $(py_dirs) + env/bin/isort -sp .isort.cfg --check $(py_files) + env/bin/black --check $(py_files) + + +env_ok: requirements.txt + rm -rf env env_ok + python3 -m venv env + env/bin/pip install -r requirements.txt + touch env_ok + + +.PHONY: clean +clean: + rm -rf env env_ok + + +.PHONY: fly_remote_login +fly_remote_login: + fly -t remote login \ + -c https://gpu1next.ific.uv.es + +.PHONY: fly_execute +fly_execute: + fly -t local execute \ + --include-ignored \ + --input IC=../ \ + --config run-tests.yml + +.PHONY: set_local_pr_pipeline +set_local_pr_pipeline: credentials/github_access_token credentials/key_concourse + fly -t local set-pipeline \ + --pipeline IC-pull-request \ + --config pr-pipeline.yml \ + -v "SSH_PRIVATE_KEY=$$(cat credentials/key_concourse)" \ + -v github_access_token=$$(cat credentials/github_access_token) + +.PHONY: set_pr_pipeline +set_pr_pipeline: credentials/github_access_token credentials/key_concourse + fly -t remote set-pipeline \ + --pipeline IC-pull-request \ + --config pr-pipeline.yml \ + -v "SSH_PRIVATE_KEY=$$(cat credentials/key_concourse)" \ + -v github_access_token=$$(cat credentials/github_access_token) + + +.PHONY: launch_prod_concourse +launch_prod_concourse: credentials/CONCOURSE_TEST_PASSWORD + docker-compose -f prod-docker-compose.yml down -v --remove-orphans + docker-compose -f prod-docker-compose.yml run make_dummy_certs + CONCOURSE_TEST_PASSWORD=$$(credentials/CONCOURSE_TEST_PASSWORD) docker-compose -f prod-docker-compose.yml up -d nginx + docker-compose -f prod-docker-compose.yml run erase_certs + docker-compose -f prod-docker-compose.yml run create_certs + docker-compose -f prod-docker-compose.yml exec nginx nginx -s reload diff --git a/concourse-ci/README.md b/concourse-ci/README.md new file mode 100644 index 0000000000..f87c6c65b7 --- /dev/null +++ b/concourse-ci/README.md @@ -0,0 +1,128 @@ +* [Overview](#overview) +* [Usage](#usage) + * [Concourse server setup](#concourse-server-setup) + * [Online demo version](#online-demo-version) + * [Walkthrough for executing concourse locally](#walkthrough-for-executing-concourse-locally) + * [Running on a server](#running-on-a-server) + * [Developing cluster tests](#developing-cluster-tests) + * [running the test script via the command line](#running-the-test-script-via-the-command-line) + * [setting the concourse pipeline](#setting-the-concourse-pipeline) + * [running locally](#running-locally) + * [running in production](#running-in-production) + * [updating the concourse pipeline](#updating-the-concourse-pipeline) + +# Overview + +**currently moving to [https://gpu1next.ific.uv.es](https://gpu1next.ific.uv.es), still need to sync docker compose file that @jocarbur is working on into this repo** + +Proof of concept is up at [https://gpu1next.ific.uv.es](https://gpu1next.ific.uv.es). It's password protected, ask @mmkekic for access if you want to poke around, authorization for the real version would be mediated via oauth by membership in the [nextic github organization](https://github.com/nextic). + +This sets up an example CI pipeline for the invisible cities project. The idea is that, whenever a pull request to a [protected branch](https://help.github.com/en/articles/about-protected-branches) happens: + +1. the CI runs the unit tests (currently what your travis CI runs) and errors out if they fail +2. if unit tests pass, the CI submits a job to the majorana cluster that performs sanity checks; a report is generated and put somewhere (an ific http server) +3. If the sanity checks also pass, the CI marks the PR as approved in github and the merge can now be performed. + +I set this up using [concourse](https://concourse-ci.org/) because: +* as a side effect of dragging my current company's devs into the 19th century, I can now set up continuous integration environments based on concourse in my sleep +* concourse is awesome; it does have a learning curve (as do all CI tools) but it's very flexible and forces you to decouple stuff in a way that makes deploying in cloud environments easy. + +# Usage + +## Concourse server setup + +### Online demo version + +The concourse interface is up at [https://gpu1next.ific.uv.es/](https://gpu1next.ific.uv.es/). + +The easiest way to understand what's going on is just to open a test PR to the master branch in the [https://github.com/miguelsimon/IC](https://github.com/miguelsimon/IC) repo, you should see: +* Merges are disallowed until the build passes +* At least 1 review is required. + +### Walkthrough for executing concourse locally + +#### Prerequisites + +* [docker-compose](https://www.digitalocean.com/community/tutorials/how-to-install-docker-compose-on-ubuntu-18-04) +* make ie if you're on some unix flavor you're fine +* required credentials are text files that belong in a `credentials` directory which is .gitignored to avoid committing credentials; for a local deploy, these are: + * `credentials/github_access_token` a github access token to write PR status back to github + * `credentials/key_concourse` is an ssh private key which lets the `icdev` user access the majorana cluster + +These steps must be executed from within this directory: + +1. launch the local concourse instance: + `docker-compose up -d` +2. navigate to [http://localhost:8080](http://localhost:8080) and login with username: test password: test +3. download the [fly cli](https://concourse-ci.org/fly.html) from your local concourse installation +4. log in to concourse via the command line: + `fly -t local login -c http://localhost:8080` +5. push the pr pipeline (you'll need appropriate credentials) + `make set_local_pr_pipeline` + +Voilà, you can now unpause the pipeline. + +### Running on a server + +I've set it up at [https://gpu1next.ific.uv.es/](https://gpu1next.ific.uv.es/). + +Doing that involves quite a bit of onerous and annoying details, dealing with dns, certificates, hosting etc. These are just annoying if you know what you're doing but require loads of time to learn all that boring trivia if you don't. I can talk to you guys and help you set it up in your context. + +The set up is mostly sane but the top priority was getting it working in < 3 hours so I'm mainly using stuff I'm comfortable with. + +Birds-eye overview of the current setup: +* The deploy is specified in docker compose; I'm running postgres, concourse, an nginx frontend and [certbot](https://certbot.eff.org/) to set up free certificates via letsencrypt. +* I'm running it on a trial google compute engine VM (google cloud is a very sane cloud provider when compared to others *cough* amazon *cough*) +* access control is via username - password now, we'd delegate access control to github via oauth to avoid operational hassles + +## Developing cluster tests + +**Code in the [assemble_jobs](assemble_jobs) directory is meant for a later stage and should't be used for now** + +An ssh script called [simple-cluster-tests.sh](simple-cluster-tests.sh) is responsible for: +1. copying the required context to the majorana cluster + * the PR code for the IC repo + * the master code for the IC repo + * the scripts that submit the jobs, found in the [jobs](jobs) directory +2. submitting the jobs on majorana and waiting for them to complete +3. fetching the job outputs from majorana via rsync +4. **Not yet implemented** Running the comparison functions on the outputs and generating a report. + +This script is meant to be called from a concourse pipeline. + +### running the test script via the command line + +To test the `simple-cluster-tests.sh` script it's convenient to launch it using local content; you can do this via fly execute (if you've got the proper credentials); the following example uses an IC_master checkout to populate both the IC and IC_master inputs to the script, and leaves results in the `outputs` directory: + +``` +mkdir outputs + +SSH_PRIVATE_KEY=$(cat credentials/key_concourse) \ + fly -t remote execute \ + --include-ignored \ + --input IC=~/IC_master \ + --input IC_master=~/IC_master \ + --input IC_operations=../ \ + --output outputs=./outputs \ + --config simple-cluster-tests.yml +``` + +### setting the concourse pipeline + +The [pr-pipeline.yml](pr-pipeline.yml) script contains the pipeline definition needed to tell the concourse server how and when to invoke the cluster tests script. + +The [Makefile](Makefile) contains the commands and required files needed to configure the pipelines on the concourse server. + +#### running locally + +`make set_local_pr_pipeline` will upload the pipeline to the concourse server running on localhost. + +#### running in production + +`make set_pr_pipeline` will upload the pipeline to the production server. + +### updating the concourse pipeline + +If you make changes to the pipeline itself (eg. modifying the pr-pipeline.yml or cluster-tests.yml files) `make set_pr_pipeline` will apply those changes. + +If you make changes to the code, committing and pushing your changes is enough; the concourse pipeline will detect the changes in the code and automatically rebuild the test image. diff --git a/concourse-ci/assemble_jobs/__init__.py b/concourse-ci/assemble_jobs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/concourse-ci/assemble_jobs/city_job.py b/concourse-ci/assemble_jobs/city_job.py new file mode 100644 index 0000000000..3304776be4 --- /dev/null +++ b/concourse-ci/assemble_jobs/city_job.py @@ -0,0 +1,32 @@ +from typing import NamedTuple + +CITY_JOB_TEMPLATE = """#!/bin/bash +source {bashrc} +source {conda_sh} +conda activate {conda_activate} +export LD_LIBRARY_PATH="{conda_lib}:$LD_LIBRARY_PATH" +export ICTDIR={ictdir} +export ICDIR="$ICTDIR/invisible_cities" +export PATH="$ICTDIR/bin:$PATH" +export PYTHONPATH="$ICTDIR:$PYTHONPATH" + +city {city} \\ + -i {input_path} \\ + -o {output_path} \\ + {conf_path} +""" + + +class CityJob(NamedTuple): + city: str + input_path: str + output_path: str + conf_path: str + ictdir: str + bashrc: str + conda_sh: str + conda_activate: str + conda_lib: str + + def to_sh(self) -> str: + return CITY_JOB_TEMPLATE.format(**self._asdict()) diff --git a/concourse-ci/assemble_jobs/compare_kdst.py b/concourse-ci/assemble_jobs/compare_kdst.py new file mode 100644 index 0000000000..ae43359f57 --- /dev/null +++ b/concourse-ci/assemble_jobs/compare_kdst.py @@ -0,0 +1,35 @@ +from typing import NamedTuple + +COMPARE_KDST_JOB_TEMPLATE = """#!/bin/bash +source {bashrc} +source {conda_sh} +conda activate {conda_activate} +export LD_LIBRARY_PATH="{conda_lib}:$LD_LIBRARY_PATH" +export ICTDIR={ictdir} +export ICDIR="$ICTDIR/invisible_cities" +export PATH="$ICTDIR/bin:$PATH" +export PYTHONPATH="$ICTDIR:$PYTHONPATH" + +mkdir {output_path} + +h5diff -c {master_path} {pr_path} > {output_path}/h5diff.txt +if [ $? -eq 0 ]; then + echo "ok" > {output_path}/status +else + echo "DIFFERENCES" > {output_path}/status +fi +""" + + +class CompareKdstJob(NamedTuple): + master_path: str + pr_path: str + output_path: str + ictdir: str + bashrc: str + conda_sh: str + conda_activate: str + conda_lib: str + + def to_sh(self) -> str: + return COMPARE_KDST_JOB_TEMPLATE.format(**self._asdict()) diff --git a/concourse-ci/assemble_jobs/compare_pmap.py b/concourse-ci/assemble_jobs/compare_pmap.py new file mode 100644 index 0000000000..4e2d538e56 --- /dev/null +++ b/concourse-ci/assemble_jobs/compare_pmap.py @@ -0,0 +1,35 @@ +from typing import NamedTuple + +COMPARE_PMAP_JOB_TEMPLATE = """#!/bin/bash +source {bashrc} +source {conda_sh} +conda activate {conda_activate} +export LD_LIBRARY_PATH="{conda_lib}:$LD_LIBRARY_PATH" +export ICTDIR={ictdir} +export ICDIR="$ICTDIR/invisible_cities" +export PATH="$ICTDIR/bin:$PATH" +export PYTHONPATH="$ICTDIR:$PYTHONPATH" + +mkdir {output_path} + +h5diff -c {master_path} {pr_path} > {output_path}/h5diff.txt +if [ $? -eq 0 ]; then + echo "ok" > {output_path}/status +else + echo "DIFFERENCES" > {output_path}/status +fi +""" + + +class ComparePmapJob(NamedTuple): + master_path: str + pr_path: str + output_path: str + ictdir: str + bashrc: str + conda_sh: str + conda_activate: str + conda_lib: str + + def to_sh(self) -> str: + return COMPARE_PMAP_JOB_TEMPLATE.format(**self._asdict()) diff --git a/concourse-ci/assemble_jobs/ic_jobs.py b/concourse-ci/assemble_jobs/ic_jobs.py new file mode 100644 index 0000000000..00b97e4b83 --- /dev/null +++ b/concourse-ci/assemble_jobs/ic_jobs.py @@ -0,0 +1,159 @@ +import os +import shutil +from typing import NamedTuple, Sequence, Union + +from assemble_jobs.city_job import CityJob +from assemble_jobs.compare_pmap import ComparePmapJob + +CHAIN_TEMPLATE = """#!/bin/bash + +rm -f {all_ok_file} + +function echo_info () {{ + echo + echo "`qstat`" + echo +}} + +(cd master && git lfs pull && source manage.sh work_in_python_version_no_tests 3.7) +(cd pr && git lfs pull && source manage.sh work_in_python_version_no_tests 3.7) + +{job_chain} + +all_ok=$(qsub -W depend=afterok:${last_job} jobs/all_ok.sh) + +echo_info +status=`qstat | grep $all_ok` +while [ -n "$status" ] # while $status is not empty + do + sleep 10 + echo_info + status=`qstat | grep $all_ok` + done + + +if [[ -f {all_ok_file} ]]; then + echo "all ok" +else + echo "not all ok" + exit 1 +fi + +""" + +ALL_OK_TEMPLATE = """#!/bin/bash +touch {all_ok_file} +""" + + +class Config(NamedTuple): + bashrc: str + conda_sh: str + conda_activate: str + conda_lib: str + remote_dir: str + + def get_conf_path(self, city: str) -> str: + return os.path.join(self.remote_dir, "conf", "{0}.conf".format(city)) + + def get_ictdir(self, version: str) -> str: + assert version in ["pr", "master"] + return os.path.join(self.remote_dir, version) + + +class CitySpec(NamedTuple): + city: str + input_path: str + output_path: str + ic_version: str + + def get_job(self, config: Config) -> CityJob: + return CityJob( + city=self.city, + input_path=self.input_path, + output_path=self.output_path, + conf_path=config.get_conf_path(self.city), + ictdir=config.get_ictdir(self.ic_version), + bashrc=config.bashrc, + conda_sh=config.conda_sh, + conda_activate=config.conda_activate, + conda_lib=config.conda_lib, + ) + + +class ComparePmapSpec(NamedTuple): + master_path: str + pr_path: str + output_path: str + ic_version: str + + def get_job(self, config: Config) -> ComparePmapJob: + return ComparePmapJob( + master_path=self.master_path, + pr_path=self.pr_path, + output_path=self.output_path, + ictdir=config.get_ictdir(self.ic_version), + bashrc=config.bashrc, + conda_sh=config.conda_sh, + conda_activate=config.conda_activate, + conda_lib=config.conda_lib, + ) + + +Spec = Union[CitySpec, ComparePmapSpec] + +Job = Union[CityJob, ComparePmapJob] + + +class LocalAssembly(NamedTuple): + master_path: str + pr_path: str + city_conf_dir: str + + def assemble_skeleton(self, config: Config, target_dir: str) -> None: + os.mkdir(os.path.join(target_dir, "jobs")) + os.mkdir(os.path.join(target_dir, "outputs")) + os.mkdir(os.path.join(target_dir, "outputs", "master")) + os.mkdir(os.path.join(target_dir, "outputs", "pr")) + os.mkdir(os.path.join(target_dir, "comparison_outputs")) + + shutil.copytree(self.city_conf_dir, os.path.join(target_dir, "conf")) + shutil.copytree(self.master_path, os.path.join(target_dir, "master")) + shutil.copytree(self.pr_path, os.path.join(target_dir, "pr")) + + def assemble_jobs(self, jobs: Sequence[Job], target_dir: str) -> None: + + # TODO: factor out this hardcode + all_ok_file = "/data_extra2/icdev/miguel_scratch/all_ok" + + job_lines = [] + + for i, job in enumerate(jobs): + path = os.path.join(target_dir, "jobs", "job_{0}.sh".format(i)) + with open(path, "w") as f: + f.write(job.to_sh()) + + if i == 0: + line = "job_0=$(qsub jobs/job_0.sh)" + else: + line = "job_{0}=$(qsub -W depend=afterok:$job_{1} jobs/job_{0}.sh)".format( + i, i - 1 + ) + job_lines.append(line) + + job_chain = "\n".join(job_lines) + + chain_sh = CHAIN_TEMPLATE.format( + **dict( + job_chain=job_chain, + last_job="job_{0}".format(i), + all_ok_file=all_ok_file, + ) + ) + + with open(os.path.join(target_dir, "chain.sh"), "w") as f: + f.write(chain_sh) + + with open(os.path.join(target_dir, "jobs", "all_ok.sh"), "w") as f: + all_ok_sh = ALL_OK_TEMPLATE.format(**dict(all_ok_file=all_ok_file)) + f.write(all_ok_sh) diff --git a/concourse-ci/assemble_jobs/miguel_jobs.py b/concourse-ci/assemble_jobs/miguel_jobs.py new file mode 100644 index 0000000000..75bbbf6bd5 --- /dev/null +++ b/concourse-ci/assemble_jobs/miguel_jobs.py @@ -0,0 +1,83 @@ +import argparse +from typing import List + +from assemble_jobs.ic_jobs import CitySpec, ComparePmapSpec, Config, LocalAssembly, Spec + +config = Config( + bashrc="/home/icdev/.bashrc", + conda_sh="/home/icdev/miniconda/etc/profile.d/conda.sh", + conda_activate="IC-3.7-2018-11-14", + conda_lib="/home/icdev/miniconda/lib", + remote_dir="/data_extra2/icdev/miguel_scratch", +) + +specs: List[Spec] = [ + CitySpec( + city="irene", + input_path="/data_extra2/mmkekic/example_inputs/run_6971_0009_trigger1_waveforms.h5", + output_path="/data_extra2/icdev/miguel_scratch/outputs/master_run_6971_0009_trigger1_pmaps.h5", + ic_version="master", + ), + CitySpec( + city="dorothea", + input_path="/data_extra2/icdev/miguel_scratch/outputs/master_run_6971_0009_trigger1_pmaps.h5", + output_path="/data_extra2/icdev/miguel_scratch/outputs/master_run_6971_0009_trigger1_kdst.h5", + ic_version="master", + ), + CitySpec( + city="irene", + input_path="/data_extra2/mmkekic/example_inputs/run_6971_0009_trigger1_waveforms.h5", + output_path="/data_extra2/icdev/miguel_scratch/outputs/pr_run_6971_0009_trigger1_pmaps.h5", + ic_version="pr", + ), + CitySpec( + city="dorothea", + input_path="/data_extra2/icdev/miguel_scratch/outputs/pr_run_6971_0009_trigger1_pmaps.h5", + output_path="/data_extra2/icdev/miguel_scratch/outputs/pr_run_6971_0009_trigger1_kdst.h5", + ic_version="pr", + ), + ComparePmapSpec( + master_path="/data_extra2/icdev/miguel_scratch/outputs/master_run_6971_0009_trigger1_pmaps.h5", + pr_path="/data_extra2/icdev/miguel_scratch/outputs/pr_run_6971_0009_trigger1_pmaps.h5", + output_path="/data_extra2/icdev/miguel_scratch/comparison_outputs/run_6971_0009_trigger1_pmaps", + ic_version="master", + ), +] + + +jobs = [spec.get_job(config) for spec in specs] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--target_dir", type=str, required=True) + + parser.add_argument( + "--master_dir", + type=str, + required=True, + help="source directory to use as master", + ) + + parser.add_argument( + "--pr_dir", type=str, required=True, help="source directory to use as pr" + ) + + parser.add_argument( + "--city_conf_dir", + type=str, + required=True, + help="source directory where city configurations are found", + ) + + args = parser.parse_args() + + local_assembly = LocalAssembly( + master_path=args.master_dir, + pr_path=args.pr_dir, + city_conf_dir=args.city_conf_dir, + ) + + local_assembly.assemble_skeleton(config, args.target_dir) + local_assembly.assemble_jobs(jobs, args.target_dir) diff --git a/concourse-ci/assemble_jobs/production_jobs.py b/concourse-ci/assemble_jobs/production_jobs.py new file mode 100644 index 0000000000..50cd110481 --- /dev/null +++ b/concourse-ci/assemble_jobs/production_jobs.py @@ -0,0 +1,94 @@ +import argparse +import os +import re +from typing import List + +from assemble_jobs.ic_jobs import CitySpec, ComparePmapSpec, Config, LocalAssembly, Spec + +config = Config( + bashrc="/home/icdev/.bashrc", + conda_sh="/home/icdev/miniconda/etc/profile.d/conda.sh", + conda_activate="IC-3.7-2018-11-14", + conda_lib="/home/icdev/miniconda/lib", + remote_dir="/data_extra2/icdev/miguel_scratch", +) + +inputs = """run_6971_0324_trigger1_waveforms.h5 run_6971_0658_trigger1_waveforms.h5 run_6971_0992_trigger1_waveforms.h5 +run_6971_0325_trigger1_waveforms.h5 run_6971_0659_trigger1_waveforms.h5 run_6971_0993_trigger1_waveforms.h5 +run_6971_0326_trigger1_waveforms.h5 run_6971_0660_trigger1_waveforms.h5 run_6971_0994_trigger1_waveforms.h5 +run_6971_0327_trigger1_waveforms.h5 run_6971_0661_trigger1_waveforms.h5 run_6971_0995_trigger1_waveforms.h5 +run_6971_0328_trigger1_waveforms.h5 run_6971_0662_trigger1_waveforms.h5 run_6971_0996_trigger1_waveforms.h5 +run_6971_0329_trigger1_waveforms.h5 run_6971_0663_trigger1_waveforms.h5 run_6971_0997_trigger1_waveforms.h5 +run_6971_0330_trigger1_waveforms.h5 run_6971_0664_trigger1_waveforms.h5 run_6971_0998_trigger1_waveforms.h5 +run_6971_0331_trigger1_waveforms.h5 run_6971_0665_trigger1_waveforms.h5 run_6971_0999_trigger1_waveforms.h5""".split() + +specs: List[Spec] = [] + +for filename in inputs: + pmap_filename = re.sub("waveforms", "pmaps", filename) + input_path = os.path.join("/analysis/6971/hdf5/data", filename) + + for branch in ["master", "pr"]: + output_path = os.path.join( + "/data_extra2/icdev/miguel_scratch/outputs/", branch, pmap_filename + ) + specs.append( + CitySpec( + city="irene", + input_path=input_path, + output_path=output_path, + ic_version=branch, + ) + ) + + specs.append( + ComparePmapSpec( + master_path=os.path.join( + "/data_extra2/icdev/miguel_scratch/outputs/", "master", pmap_filename + ), + pr_path=os.path.join( + "/data_extra2/icdev/miguel_scratch/outputs/", "pr", pmap_filename + ), + output_path=os.path.join( + "/data_extra2/icdev/miguel_scratch/comparison_outputs", pmap_filename + ), + ic_version="master", + ) + ) + +jobs = [spec.get_job(config) for spec in specs] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--target_dir", type=str, required=True) + + parser.add_argument( + "--master_dir", + type=str, + required=True, + help="source directory to use as master", + ) + + parser.add_argument( + "--pr_dir", type=str, required=True, help="source directory to use as pr" + ) + + parser.add_argument( + "--city_conf_dir", + type=str, + required=True, + help="source directory where city configurations are found", + ) + + args = parser.parse_args() + + local_assembly = LocalAssembly( + master_path=args.master_dir, + pr_path=args.pr_dir, + city_conf_dir=args.city_conf_dir, + ) + + local_assembly.assemble_skeleton(config, args.target_dir) + local_assembly.assemble_jobs(jobs, args.target_dir) diff --git a/concourse-ci/assemble_jobs/reports.py b/concourse-ci/assemble_jobs/reports.py new file mode 100644 index 0000000000..8535b256c0 --- /dev/null +++ b/concourse-ci/assemble_jobs/reports.py @@ -0,0 +1,148 @@ +import argparse +import os +import tempfile +import unittest +from typing import List, NamedTuple + +import yattag + +TOGGLE_JS = """ + +""" + + +class Result(NamedTuple): + name: str + status: str + output: str + + def to_txt(self): + return "{0}\t{1}".format(self.name, self.status.strip()) + + def to_html(self, doc, tag, text): + status = self.status.strip() + if status == "ok": + style = "color:green" + else: + style = "color:red" + with tag("h3", style=style): + text("{0} {1}".format(self.name, status)) + button = """""".format( + self.name + ) + doc.asis(button) + with tag("div", id=self.name, style="display:none;"): + with tag("pre"): + text(self.output) + + +class Report(NamedTuple): + results: List[Result] + + def to_txt(self): + return "\n".join([res.to_txt() for res in self.results]) + + def to_html(self): + doc, tag, text = yattag.Doc().tagtext() + doc.asis(TOGGLE_JS) + for res in self.results: + res.to_html(doc, tag, text) + return doc.getvalue() + + +class CmpSpec(NamedTuple): + name: str + path: str + + def get_result(self) -> Result: + status_path = os.path.join(self.path, "status") + output_path = os.path.join(self.path, "h5diff.txt") + + if not os.path.exists(status_path) or not os.path.exists(output_path): + return Result(name=self.name, status="bad_output", output="") + else: + with open(status_path, "r") as status_f, open(output_path, "r") as output_f: + return Result( + name=self.name, status=status_f.read(), output=output_f.read() + ) + + +class ReportSpec(NamedTuple): + specs: List[CmpSpec] + + def get_report(self): + results = [cmp_spec.get_result() for cmp_spec in self.specs] + results.sort(key=lambda res: (res.status, res.name)) + return Report(results) + + +def dir_to_ReportSpec(target_dir: str) -> ReportSpec: + specs: List[CmpSpec] = [] + for output in sorted(os.listdir(target_dir)): + path = os.path.join(target_dir, output) + specs.append(CmpSpec(output, path)) + return ReportSpec(specs) + + +def make_dummy_output(target_path, status, output): + os.mkdir(target_path) + status_path = os.path.join(target_path, "status") + output_path = os.path.join(target_path, "h5diff.txt") + + with open(status_path, "w") as f: + f.write(status) + + with open(output_path, "w") as f: + f.write(output) + + +class Test(unittest.TestCase): + def test(self): + with tempfile.TemporaryDirectory() as target_dir: + a_path = os.path.join(target_dir, "a") + b_path = os.path.join(target_dir, "b") + + make_dummy_output(a_path, "ok", "no output") + + report_spec = ReportSpec( + [CmpSpec(name="a", path=a_path), CmpSpec(name="b", path=b_path)] + ) + + report = report_spec.get_report() + + self.assertEqual(report.results[1].status, "ok") + self.assertEqual(report.results[0].status, "bad_output") + + print(report.to_txt()) + print(report.to_html()) + + def test_example_comparison_output(self): + report_spec = dir_to_ReportSpec("example_comparison_outputs") + report = report_spec.get_report() + print(report.to_txt()) + print(report.to_html()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output_dir", type=str, required=True) + parser.add_argument("--format", type=str, choices=["txt", "html"], required=True) + args = parser.parse_args() + + report_spec = dir_to_ReportSpec(args.output_dir) + report = report_spec.get_report() + if args.format == "txt": + print(report.to_txt()) + elif args.format == "html": + print(report.to_html()) + else: + raise Exception("unknown format {0}".format(args.format)) diff --git a/concourse-ci/assemble_jobs/test_ic_jobs.py b/concourse-ci/assemble_jobs/test_ic_jobs.py new file mode 100644 index 0000000000..885d2326bc --- /dev/null +++ b/concourse-ci/assemble_jobs/test_ic_jobs.py @@ -0,0 +1,70 @@ +import os +import tempfile +import unittest + +from assemble_jobs.ic_jobs import CitySpec, ComparePmapSpec, Config, LocalAssembly + + +class Test(unittest.TestCase): + def get_config(self): + return Config( + bashrc="/home/icdev/.bashrc", + conda_sh="/home/icdev/miniconda/etc/profile.d/conda.sh", + conda_activate="IC-3.7-2018-11-14", + conda_lib="home/icdev/miniconda/lib", + remote_dir="home/data_extra2/icdev/miguel_scratch", + ) + + def test_Config(self): + config = self.get_config() + + print(config.get_conf_path("irene")) + print(config.get_ictdir("master")) + + def test_CitySpec_CityJob(self): + config = self.get_config() + spec = CitySpec( + city="irene", + input_path="/data_extra2/mmkekic/example_inputs/run_6971_0009_trigger1_waveforms.h5", + output_path="/data_extra2/icdev/miguel_scratch/outputs/run_6971_0009_trigger1_pmaps.h5", + ic_version="master", + ) + + job = spec.get_job(config) + print(job) + print(job.to_sh()) + + def test_ComparePmapSpec_ComparePmapJob(self): + config = self.get_config() + spec = ComparePmapSpec( + master_path="/data_extra2/icdev/miguel_scratch/outputs/master_run_6971_0009_trigger1_pmaps.h5", + pr_path="/data_extra2/icdev/miguel_scratch/outputs/pr_run_6971_0009_trigger1_pmaps.h5", + output_path="/data_extra2/icdev/miguel_scratch/outputs/compare_run_6971_0009_trigger1_pmaps.txt", + ic_version="master", + ) + + job = spec.get_job(config) + print(job) + print(job.to_sh()) + + def test_LocalAssembly(self): + config = self.get_config() + spec = CitySpec( + city="irene", + input_path="/data_extra2/mmkekic/example_inputs/run_6971_0009_trigger1_waveforms.h5", + output_path="/data_extra2/icdev/miguel_scratch/outputs/run_6971_0009_trigger1_pmaps.h5", + ic_version="master", + ) + + jobs = [spec.get_job(config)] + with tempfile.TemporaryDirectory() as dummy_ic_dir: + local_assembly = LocalAssembly( + master_path=dummy_ic_dir, + pr_path=dummy_ic_dir, + city_conf_dir=dummy_ic_dir, + ) + + with tempfile.TemporaryDirectory() as target_dir: + local_assembly.assemble_skeleton(config, target_dir) + print(os.listdir(target_dir)) + local_assembly.assemble_jobs(jobs, target_dir) diff --git a/concourse-ci/conf/dorothea.conf b/concourse-ci/conf/dorothea.conf new file mode 100644 index 0000000000..5d287cc751 --- /dev/null +++ b/concourse-ci/conf/dorothea.conf @@ -0,0 +1,51 @@ +# Dorothea computes a KDST after selecting PMAPS according to an S12 selector. + +include('$ICDIR/config/s12_selector.conf') + +# override the default input/output files: + +files_in = '$ICDIR/database/test_data/Kr83_nexus_v5_03_00_ACTIVE_7bar_10evts_PMP.h5' +file_out = 'KrDST.h5' +compression = 'ZLIB4' + +# run number 0 is for MC +run_number = 6971 +detector_db = 'new' + +# How frequently to print events +print_mod = 1 + +# Dorothea uses s12 selector parameters +# and can re-write some of them +# for example s2 parameters are re-written here + +event_range = all + +drift_v = 1 * mm / mus +s1_nmin = 1 +s1_nmax = 5 +s1_emin = 0 * pes +s1_emax = 1e6 * pes +s1_wmin = 100 * ns +s1_wmax = 1e6 * ns +s1_hmin = 0 * pes +s1_hmax = 1e6 * pes +s1_ethr = 0.5 * pes + +s2_nmin = 1 +s2_nmax = 100 +s2_emin = 0 * pes +s2_emax = 1e8 * pes +s2_wmin = 1 * mus +s2_wmax = 10 * ms +s2_hmin = 0 * pes +s2_hmax = 1e6 * pes +s2_ethr = 0 * pes +s2_nsipmmin = 1 +s2_nsipmmax = 2000 + + +global_reco_params = dict( + Qthr = 0 * pes, + lm_radius = -1 * mm +) \ No newline at end of file diff --git a/concourse-ci/conf/irene.conf b/concourse-ci/conf/irene.conf new file mode 100644 index 0000000000..869ffc8738 --- /dev/null +++ b/concourse-ci/conf/irene.conf @@ -0,0 +1,35 @@ +files_in = 'lalalalla' +file_out = 'lululululu' +compression = 'ZLIB4' +run_number = 6971 +detector_db = 'new' +print_mod = 1 +event_range = 20 +n_baseline = 62400 # for a window of 800 mus (DEBATABLE WHETHER OPTIMAL) +# Set MAU for calibrated sum +n_mau = 100 +thr_mau = 3 * adc +thr_sipm = 1.0 * pes +thr_sipm_type = "common" +# Set thresholds for calibrated sum +thr_csum_s1 = 0.5 * pes +thr_csum_s2 = 2.0 * pes ## REDUCED TO 1 TO AVOID BIAS, ESP. MUONS +# Set parameters to search for S1 +s1_tmin = 0 * mus # look from start up to trigger +s1_tmax = 790 * mus # change tmin and tmax if S1 not at 100 mus +s1_stride = 4 # minimum number of 25 ns bins in S1 searches +s1_lmin = 5 # 8 x 25 = 200 ns +s1_lmax = 30 # 20 x 25 = 500 ns +s1_rebin_stride = 1 # Do not rebin S1 by default +# Set parameters to search for S2 +s2_tmin = 795 * mus # CHANGED TO LOOK IN WHOLE WINDOW +s2_tmax = 1600 * mus # end of the window +s2_stride = 40 # CHANGED TO ONLY ACCEPT CONTINUOUS +s2_lmin = 80 # 40 x 25 = 1 mus +s2_lmax = 200000 # maximum value of S2 width +s2_rebin_stride = 40 # Rebin by default +# Set S2Si parameters +thr_sipm_s2 = 5 * pes # CHANGED TO BE UNUSED = 0.1 +# SiPM selection +#thr_sipm = 0.99 +#thr_sipm_type = "Individual" diff --git a/concourse-ci/conf/penthesilea.conf b/concourse-ci/conf/penthesilea.conf new file mode 100644 index 0000000000..42833e240b --- /dev/null +++ b/concourse-ci/conf/penthesilea.conf @@ -0,0 +1,55 @@ +files_in = '/analysis/6561/hdf5/prod/v0.9.9/20181011/pmaps/trigger2/*.h5' +file_out = '/home/ausonandres/test_penthesilea/hdst/trigger2/hdst_6561_trigger2.h5' + +# run number 0 is for MC +run_number = 6791 + +# How frequently to print events +print_mod = 1 +# max number of events to run +event_range = all + +# compression library +compression = 'ZLIB4' + +# s12 selector +drift_v = 1 * mm / mus # Expected drift velocity + +s1_nmin = 1 +s1_nmax = 1 +s1_emin = 40 * pes # Min S1 energy integral +s1_emax = 1e+6 * pes # Max S1 energy integral +s1_wmin = 175 * ns # min width for S1 +s1_wmax = 1.e6 * ns # Max width +s1_hmin = 0 * pes # Min S1 height +s1_hmax = 1e+6 * pes # Max S1 height +s1_ethr = 0.5 * pes # Energy threshold for S1 + +s2_nmin = 1 +s2_nmax = 100 # Max number of S2 signals +s2_emin = 0 * pes # Min S2 energy integral +s2_emax = 1e+8 * pes # Max S2 energy integral in pes +s2_wmin = 2.5 * mus # Min width +s2_wmax = 10 * ms # Max width +s2_hmin = 0 * pes # Min S2 height +s2_hmax = 1e+6 * pes # Max S2 height +s2_nsipmmin = 1 # Min number of SiPMs touched +s2_nsipmmax = 2000 # Max number of SiPMs touched +s2_ethr = 0. * pes # Energy threshold for S2 + +rebin = 2 + + +slice_reco_params=dict( + Qthr = 3 * pes, # charge threshold, ignore all SiPMs with less than Qthr pes + Qlm = 35 * pes, # every Cluster must contain at least one SiPM with charge >= Qlm + lm_radius = 0 *mm, + new_lm_radius = 15*mm, + msipm = 6. ) + +global_reco_params=dict( + Qthr = 1. * pes, # charge threshold, ignore all SiPMs with less than Qthr pes + Qlm = 0 * pes, # every Cluster must contain at least one SiPM with charge >= Qlm + lm_radius = -1 *mm, + new_lm_radius = -1*mm, + msipm = -1. ) \ No newline at end of file diff --git a/concourse-ci/docker-compose.yml b/concourse-ci/docker-compose.yml new file mode 100644 index 0000000000..7b9b8cadb4 --- /dev/null +++ b/concourse-ci/docker-compose.yml @@ -0,0 +1,27 @@ +version: '3' + +services: + concourse-db: + image: postgres + environment: + POSTGRES_DB: concourse + POSTGRES_PASSWORD: concourse_pass + POSTGRES_USER: concourse_user + PGDATA: /database + + concourse: + image: concourse/concourse + command: quickstart + privileged: true + depends_on: [concourse-db] + ports: ["8080:8080"] + environment: + CONCOURSE_POSTGRES_HOST: concourse-db + CONCOURSE_POSTGRES_USER: concourse_user + CONCOURSE_POSTGRES_PASSWORD: concourse_pass + CONCOURSE_POSTGRES_DATABASE: concourse + CONCOURSE_EXTERNAL_URL: http://localhost:8080 + CONCOURSE_ADD_LOCAL_USER: test:test + CONCOURSE_MAIN_TEAM_LOCAL_USER: test + + diff --git a/concourse-ci/example_comparison_outputs/run_6971_0324_trigger1_pmaps.h5/h5diff.txt b/concourse-ci/example_comparison_outputs/run_6971_0324_trigger1_pmaps.h5/h5diff.txt new file mode 100644 index 0000000000..f34d659f78 --- /dev/null +++ b/concourse-ci/example_comparison_outputs/run_6971_0324_trigger1_pmaps.h5/h5diff.txt @@ -0,0 +1,40 @@ +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset diff --git a/concourse-ci/example_comparison_outputs/run_6971_0324_trigger1_pmaps.h5/status b/concourse-ci/example_comparison_outputs/run_6971_0324_trigger1_pmaps.h5/status new file mode 100644 index 0000000000..9766475a41 --- /dev/null +++ b/concourse-ci/example_comparison_outputs/run_6971_0324_trigger1_pmaps.h5/status @@ -0,0 +1 @@ +ok diff --git a/concourse-ci/example_comparison_outputs/run_6971_0330_trigger1_pmaps.h5/h5diff.txt b/concourse-ci/example_comparison_outputs/run_6971_0330_trigger1_pmaps.h5/h5diff.txt new file mode 100644 index 0000000000..f34d659f78 --- /dev/null +++ b/concourse-ci/example_comparison_outputs/run_6971_0330_trigger1_pmaps.h5/h5diff.txt @@ -0,0 +1,40 @@ +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset diff --git a/concourse-ci/example_comparison_outputs/run_6971_0330_trigger1_pmaps.h5/status b/concourse-ci/example_comparison_outputs/run_6971_0330_trigger1_pmaps.h5/status new file mode 100644 index 0000000000..207aed1cf7 --- /dev/null +++ b/concourse-ci/example_comparison_outputs/run_6971_0330_trigger1_pmaps.h5/status @@ -0,0 +1 @@ +DIFFERENCES diff --git a/concourse-ci/example_comparison_outputs/run_6971_0331_trigger1_pmaps.h5/h5diff.txt b/concourse-ci/example_comparison_outputs/run_6971_0331_trigger1_pmaps.h5/h5diff.txt new file mode 100644 index 0000000000..f34d659f78 --- /dev/null +++ b/concourse-ci/example_comparison_outputs/run_6971_0331_trigger1_pmaps.h5/h5diff.txt @@ -0,0 +1,40 @@ +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset +Not comparable: or is an empty dataset diff --git a/concourse-ci/example_comparison_outputs/run_6971_0331_trigger1_pmaps.h5/status b/concourse-ci/example_comparison_outputs/run_6971_0331_trigger1_pmaps.h5/status new file mode 100644 index 0000000000..9766475a41 --- /dev/null +++ b/concourse-ci/example_comparison_outputs/run_6971_0331_trigger1_pmaps.h5/status @@ -0,0 +1 @@ +ok diff --git a/concourse-ci/jobs/all_ok.sh b/concourse-ci/jobs/all_ok.sh new file mode 100644 index 0000000000..e439c096d4 --- /dev/null +++ b/concourse-ci/jobs/all_ok.sh @@ -0,0 +1,2 @@ +#!/bin/bash +touch /data_extra2/icdev/miguel_scratch/all_ok diff --git a/concourse-ci/jobs/irene.conf b/concourse-ci/jobs/irene.conf new file mode 100644 index 0000000000..869ffc8738 --- /dev/null +++ b/concourse-ci/jobs/irene.conf @@ -0,0 +1,35 @@ +files_in = 'lalalalla' +file_out = 'lululululu' +compression = 'ZLIB4' +run_number = 6971 +detector_db = 'new' +print_mod = 1 +event_range = 20 +n_baseline = 62400 # for a window of 800 mus (DEBATABLE WHETHER OPTIMAL) +# Set MAU for calibrated sum +n_mau = 100 +thr_mau = 3 * adc +thr_sipm = 1.0 * pes +thr_sipm_type = "common" +# Set thresholds for calibrated sum +thr_csum_s1 = 0.5 * pes +thr_csum_s2 = 2.0 * pes ## REDUCED TO 1 TO AVOID BIAS, ESP. MUONS +# Set parameters to search for S1 +s1_tmin = 0 * mus # look from start up to trigger +s1_tmax = 790 * mus # change tmin and tmax if S1 not at 100 mus +s1_stride = 4 # minimum number of 25 ns bins in S1 searches +s1_lmin = 5 # 8 x 25 = 200 ns +s1_lmax = 30 # 20 x 25 = 500 ns +s1_rebin_stride = 1 # Do not rebin S1 by default +# Set parameters to search for S2 +s2_tmin = 795 * mus # CHANGED TO LOOK IN WHOLE WINDOW +s2_tmax = 1600 * mus # end of the window +s2_stride = 40 # CHANGED TO ONLY ACCEPT CONTINUOUS +s2_lmin = 80 # 40 x 25 = 1 mus +s2_lmax = 200000 # maximum value of S2 width +s2_rebin_stride = 40 # Rebin by default +# Set S2Si parameters +thr_sipm_s2 = 5 * pes # CHANGED TO BE UNUSED = 0.1 +# SiPM selection +#thr_sipm = 0.99 +#thr_sipm_type = "Individual" diff --git a/concourse-ci/jobs/master_job.sh b/concourse-ci/jobs/master_job.sh new file mode 100644 index 0000000000..97a6f76133 --- /dev/null +++ b/concourse-ci/jobs/master_job.sh @@ -0,0 +1,14 @@ +#!/bin/bash +source /home/icdev/.bashrc +source /home/icdev/miniconda/etc/profile.d/conda.sh +conda activate IC-3.7-2018-11-14 +export LD_LIBRARY_PATH="/home/icdev/miniconda/lib:$LD_LIBRARY_PATH" +export ICTDIR=/data_extra2/icdev/miguel_scratch/IC_master +export ICDIR="$ICTDIR/invisible_cities" +export PATH="$ICTDIR/bin:$PATH" +export PYTHONPATH="$ICTDIR:$PYTHONPATH" + +city irene \ + -i /data_extra2/mmkekic/example_inputs/run_6971_0009_trigger1_waveforms.h5 \ + -o /data_extra2/icdev/miguel_scratch/outputs/master_run_6971_0009_trigger1_pmaps.h5 \ + /data_extra2/icdev/miguel_scratch/jobs/irene.conf diff --git a/concourse-ci/jobs/pr_job.sh b/concourse-ci/jobs/pr_job.sh new file mode 100644 index 0000000000..d647774628 --- /dev/null +++ b/concourse-ci/jobs/pr_job.sh @@ -0,0 +1,14 @@ +#!/bin/bash +source /home/icdev/.bashrc +source /home/icdev/miniconda/etc/profile.d/conda.sh +conda activate IC-3.7-2018-11-14 +export LD_LIBRARY_PATH="/home/icdev/miniconda/lib:$LD_LIBRARY_PATH" +export ICTDIR=/data_extra2/icdev/miguel_scratch/IC +export ICDIR="$ICTDIR/invisible_cities" +export PATH="$ICTDIR/bin:$PATH" +export PYTHONPATH="$ICTDIR:$PYTHONPATH" + +city irene \ + -i /data_extra2/mmkekic/example_inputs/run_6971_0009_trigger1_waveforms.h5 \ + -o /data_extra2/icdev/miguel_scratch/outputs/pr_run_6971_0009_trigger1_pmaps.h5 \ + /data_extra2/icdev/miguel_scratch/jobs/irene.conf diff --git a/concourse-ci/jobs/submit.sh b/concourse-ci/jobs/submit.sh new file mode 100644 index 0000000000..cfc89d05b8 --- /dev/null +++ b/concourse-ci/jobs/submit.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +function echo_info () { + echo + echo "`qstat`" + echo +} + +(cd IC_master && git lfs pull && source manage.sh work_in_python_version_no_tests 3.7) +(cd IC && git lfs pull && source manage.sh work_in_python_version_no_tests 3.7) + +master_job=$(qsub jobs/master_job.sh) +pr_job=$(qsub jobs/pr_job.sh) + +all_ok=$(qsub -W depend=afterok:$master_job:$pr_job jobs/all_ok.sh) + +echo_info +status=`qstat | grep $all_ok` +while [ -n "$status" ] # while $status is not empty + do + sleep 10 + echo_info + status=`qstat | grep $all_ok` + done + + +if [[ -f /data_extra2/icdev/miguel_scratch/all_ok ]]; then + echo "all ok" +else + echo "not all ok" + exit 1 +fi diff --git a/concourse-ci/nginx/make_dummy_certs.sh b/concourse-ci/nginx/make_dummy_certs.sh new file mode 100644 index 0000000000..0facdba7dd --- /dev/null +++ b/concourse-ci/nginx/make_dummy_certs.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +set -e + +CERTS=/etc/letsencrypt/live/ci.ific-invisible-cities.com + +mkdir -p $CERTS + +openssl req -x509 -nodes -newkey rsa:1024 -days 1 \ + -keyout $CERTS/privkey.pem \ + -out $CERTS/fullchain.pem \ + -subj '/CN=localhost' diff --git a/concourse-ci/nginx/nginx.conf b/concourse-ci/nginx/nginx.conf new file mode 100644 index 0000000000..38ef9848a5 --- /dev/null +++ b/concourse-ci/nginx/nginx.conf @@ -0,0 +1,27 @@ +events { +} + +http { + server { + listen 80; + + location / { + root /html; + } + } +} + +stream { + upstream concourse_server { + server concourse:8080; + } + + server { + listen 443 ssl; + + ssl_certificate /etc/letsencrypt/live/ci.ific-invisible-cities.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/ci.ific-invisible-cities.com/privkey.pem; + + proxy_pass concourse_server; + } +} diff --git a/concourse-ci/pr-pipeline.yml b/concourse-ci/pr-pipeline.yml new file mode 100644 index 0000000000..f164a5e3da --- /dev/null +++ b/concourse-ci/pr-pipeline.yml @@ -0,0 +1,89 @@ +--- +resource_types: +- name: pull-request + type: docker-image + source: + repository: teliaoss/github-pr-resource +resources: + +- name: IC + type: pull-request + check_every: 1m + source: + repository: miguelsimon/IC + access_token: ((github_access_token)) + +- name: IC_master + type: git + source: + uri: https://github.com/nextic/IC.git + branch: master + +- name: IC_operations + type: git + source: + uri: https://github.com/miguelsimon/IC.git + branch: concourse-ci + +jobs: +- name: unit-tests + serial: true + plan: + - get: IC + trigger: true + version: every + - put: IC + params: + path: IC + status: pending + context: unit-tests + - task: unit-tests + file: IC/concourse-ci/run-tests.yml + on_success: + do: + - put: IC + params: + path: IC + status: success + context: unit-tests + on_failure: + do: + - put: IC + params: + path: IC + status: failure + context: unit-tests + + +- name: cluster-tests + serial: true + plan: + - get: IC + version: every + passed: + - unit-tests + - get: IC_master + - get: IC_operations + - put: IC + params: + path: IC + status: pending + context: cluster-tests + - task: cluster-tests + file: IC/concourse-ci/simple-cluster-tests.yml + params: + SSH_PRIVATE_KEY: ((SSH_PRIVATE_KEY)) + on_success: + do: + - put: IC + params: + path: IC + status: success + context: cluster-tests + on_failure: + do: + - put: IC + params: + path: IC + status: failure + context: cluster-tests diff --git a/concourse-ci/prod-docker-compose.yml b/concourse-ci/prod-docker-compose.yml new file mode 100644 index 0000000000..5bddbb2f39 --- /dev/null +++ b/concourse-ci/prod-docker-compose.yml @@ -0,0 +1,68 @@ +version: '3' + +volumes: + html-volume: + letsencrypt-volume: + +services: + concourse-db: + image: postgres + environment: + - POSTGRES_DB=concourse + - POSTGRES_PASSWORD=concourse_pass + - POSTGRES_USER=concourse_user + - PGDATA=/database + + concourse: + image: concourse/concourse + command: quickstart + privileged: true + depends_on: [concourse-db] + ports: ["8080:8080"] + environment: + - CONCOURSE_POSTGRES_HOST=concourse-db + - CONCOURSE_POSTGRES_USER=concourse_user + - CONCOURSE_POSTGRES_PASSWORD=concourse_pass + - CONCOURSE_POSTGRES_DATABASE=concourse + - CONCOURSE_EXTERNAL_URL=https://ci.ific-invisible-cities.com + - CONCOURSE_WORKER_GARDEN_NETWORK + - CONCOURSE_ADD_LOCAL_USER=test:${CONCOURSE_TEST_PASSWORD} + - CONCOURSE_MAIN_TEAM_LOCAL_USER=test + + nginx: + image: nginx:stable + depends_on: [concourse] + ports: ["80:80", "443:443"] + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf + - html-volume:/html + - letsencrypt-volume:/etc/letsencrypt + + make_dummy_certs: + image: certbot/certbot + volumes: + - letsencrypt-volume:/etc/letsencrypt + - ./nginx/make_dummy_certs.sh:/make_dummy_certs.sh + entrypoint: /bin/sh + command: /make_dummy_certs.sh + + erase_certs: + image: alpine + volumes: + - letsencrypt-volume:/etc/letsencrypt + command: rm -rf /etc/letsencrypt/live/ci.ific-invisible-cities.com + + create_certs: + image: certbot/certbot + depends_on: [nginx] + volumes: + - html-volume:/html + - letsencrypt-volume:/etc/letsencrypt + command: certonly --webroot -w /html -d ci.ific-invisible-cities.com --agree-tos --force-renewal --register-unsafely-without-email + + inspect: + image: certbot/certbot + volumes: + - letsencrypt-volume:/etc/letsencrypt + - html-volume:/html + entrypoint: /bin/sh diff --git a/concourse-ci/requirements.txt b/concourse-ci/requirements.txt new file mode 100644 index 0000000000..eb898465ec --- /dev/null +++ b/concourse-ci/requirements.txt @@ -0,0 +1,5 @@ +yattag +mypy +flake8 +black +isort diff --git a/concourse-ci/run-tests.sh b/concourse-ci/run-tests.sh new file mode 100644 index 0000000000..f79dab16ed --- /dev/null +++ b/concourse-ci/run-tests.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +apt-get update && apt install -y curl build-essential + +curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash +apt-get install -y git-lfs + +git lfs install +git lfs pull + +source manage.sh work_in_python_version_no_tests ${IC_PYTHON_VERSION} + + +HYPOTHESIS_PROFILE=hard bash manage.sh run_tests_par 2 diff --git a/concourse-ci/run-tests.yml b/concourse-ci/run-tests.yml new file mode 100644 index 0000000000..ef7695b401 --- /dev/null +++ b/concourse-ci/run-tests.yml @@ -0,0 +1,13 @@ + +platform: linux +image_resource: + type: docker-image + source: {repository: ubuntu, tag: 18.04} +inputs: + - name: IC +run: + path: bash + args: [ "concourse-ci/run-tests.sh" ] + dir: IC +params: + IC_PYTHON_VERSION: 3.7 diff --git a/concourse-ci/simple-cluster-tests.sh b/concourse-ci/simple-cluster-tests.sh new file mode 100644 index 0000000000..39fb45b95b --- /dev/null +++ b/concourse-ci/simple-cluster-tests.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +set -ex + +# error out if the credentials haven't been provided + +: "${SSH_PRIVATE_KEY:?}" + +echo "$SSH_PRIVATE_KEY" > ssh_key +chmod 600 ssh_key + +apt-get update && apt install -y curl rsync ssh + +# IC holds the PR +# IC_master holds the reference version +# IC_operations holds this code + +ssh \ + -i ssh_key \ + -o StrictHostKeyChecking=no \ + icdev@majorana1.ific.uv.es \ + "cd /data_extra2/icdev && rm -rf miguel_scratch && mkdir -p miguel_scratch/outputs" + +rsync \ + -e "ssh -i ssh_key -o StrictHostKeyChecking=no" \ + -vzr \ + --delete \ + IC_master \ + icdev@majorana1.ific.uv.es:/data_extra2/icdev/miguel_scratch/ + +rsync \ + -e "ssh -i ssh_key -o StrictHostKeyChecking=no" \ + -vzr \ + --delete \ + IC \ + icdev@majorana1.ific.uv.es:/data_extra2/icdev/miguel_scratch/ + +rsync \ + -e "ssh -i ssh_key -o StrictHostKeyChecking=no" \ + -vzr \ + --delete \ + IC_operations/concourse-ci/jobs \ + icdev@majorana1.ific.uv.es:/data_extra2/icdev/miguel_scratch/ + +ssh \ + -i ssh_key \ + -o StrictHostKeyChecking=no \ + icdev@majorana1.ific.uv.es \ + "cd /data_extra2/icdev/miguel_scratch && bash jobs/submit.sh" + +rsync \ + -e "ssh -i ssh_key -o StrictHostKeyChecking=no" \ + -vzr \ + icdev@majorana1.ific.uv.es:/data_extra2/icdev/miguel_scratch/outputs \ + . + +ls outputs + +rsync \ + -e "ssh -i ssh_key -o StrictHostKeyChecking=no" \ + -vzr \ + outputs \ + icdev@html_repo:/downloads/ diff --git a/concourse-ci/simple-cluster-tests.yml b/concourse-ci/simple-cluster-tests.yml new file mode 100644 index 0000000000..1c7a20cec9 --- /dev/null +++ b/concourse-ci/simple-cluster-tests.yml @@ -0,0 +1,16 @@ + +platform: linux +image_resource: + type: docker-image + source: {repository: ubuntu, tag: 18.04} +inputs: + - name: IC + - name: IC_master + - name: IC_operations +outputs: + - name: outputs +run: + path: bash + args: [ "IC_operations/concourse-ci/simple-cluster-tests.sh" ] +params: + SSH_PRIVATE_KEY: ((SSH_PRIVATE_KEY))