diff --git a/.circleci/config.yml b/.circleci/config.yml index 6d842d2389..302a7d3112 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,524 +1,14 @@ -# reusable anchors -_machine_defaults: &machine_defaults - environment: - TZ: "/usr/share/zoneinfo/America/Los_Angeles" - SCRATCH: "/scratch" - machine: - image: ubuntu-2204:current - docker_layer_caching: true - working_directory: /tmp/src/sdcflows - resource_class: large - -_python_defaults: &python_defaults - docker: - - image: cimg/python:3.10.9 - working_directory: /tmp/src/sdcflows - -_docker_auth: &docker_auth - name: Docker authentication - command: | - if [[ -n $DOCKER_PAT ]]; then - echo "$DOCKER_PAT" | docker login -u $DOCKER_USER --password-stdin - fi - -_setup_docker_registry: &setup_docker_registry - name: Set up Docker registry - command: | - if [[ -f /tmp/images/registry.tar.gz ]]; then - echo "Loading saved registry image" - docker load < /tmp/images/registry.tar.gz - else - echo "Pulling registry image from DockerHub" - docker pull registry:2 - fi - docker run -d -p 5000:5000 --restart=always --name=registry \ - -v /tmp/docker:/var/lib/registry registry:2 - -_pull_from_registry: &pull_from_registry - name: Pull and tag image from local registry - command: | - docker pull localhost:5000/sdcflows - docker tag localhost:5000/sdcflows nipreps/sdcflows:latest - version: 2.1 -orbs: - docker: circleci/docker@2.2.0 - codecov: codecov/codecov@3.2.4 jobs: - cache_test_data: - docker: # executor type - - image: nipreps/miniconda:py39_2209.01 - auth: - username: $DOCKER_USER - password: $DOCKER_PAT - - working_directory: /tmp/data - environment: - - TEMPLATEFLOW_HOME: /tmp/templateflow - steps: - - checkout: - path: /tmp/src/sdcflows - - - run: - name: Configure git (pacify datalad) - command: | - git config --global user.name "First Last" - git config --global user.email "email@domain.com" - - - restore_cache: - keys: - - data-v6-{{ .Branch }}-{{ .Revision }} - - data-v6--{{ .Revision }} - - data-v6-{{ .Branch }}- - - data-v6-main- - - data-v6- - - - run: - name: Ensure some templates are cached - command: | - python -c "from templateflow import api as tfapi; \ - tfapi.get('MNI152NLin2009cAsym', resolution=2, desc='brain', suffix='mask'); \ - tfapi.get('MNI152NLin2009cAsym', resolution=2, desc='fMRIPrep', suffix='boldref');" - - - run: - name: Install ds001600 - command: | - datalad install -r https://github.com/nipreps-data/ds001600.git - datalad update -r --merge -d ds001600/ - datalad get -r -d ds001600/ ds001600/sub-1/ - - - run: - name: Install HCP/sub-101006 - command: | - datalad install -r https://github.com/nipreps-data/HCP101006.git - datalad update -r --merge -d HCP101006/ - datalad get -r -d HCP101006 - - - run: - name: Install ds001771 - command: | - datalad install -r https://github.com/nipreps-data/ds001771.git - datalad update -r --merge -d ds001771/ - datalad get -r -d ds001771/ ds001771/sub-36/* - datalad get -r -d ds001771/derivatives ds001771/derivatives/openneuro/sub-36/* - - - run: - name: Install ds000206 - command: | - datalad install -r https://github.com/nipreps-data/ds000206.git - datalad update -r --merge -d ds000206/ - datalad get -r -d ds000206/ ds000206/sub-05/* - - - run: - name: Install ds000054 - command: | - datalad install -r https://github.com/nipreps-data/ds000054.git - datalad update -r --merge -d ds000054/ - datalad get -r -J 2 -d ds000054/ ds000054/* ds000054/derivatives/* - - - run: - name: Install Brain extraction tests - command: | - datalad install -r https://gin.g-node.org/nipreps-data/brain-extraction-tests - datalad update --merge -d brain-extraction-tests/ - datalad get -r -J 2 -d brain-extraction-tests - - - run: - name: Install HCPh fieldmaps - command: | - datalad install -r https://github.com/nipreps-data/hcph-pilot_fieldmaps.git - datalad update -r --merge -d hcph-pilot_fieldmaps/ - datalad get -r -J 2 -d hcph-pilot_fieldmaps/ hcph-pilot_fieldmaps/* - - - save_cache: - key: data-v6-{{ .Branch }}-{{ .Revision }} - paths: - - /tmp/data - - /tmp/templateflow - - - restore_cache: - keys: - - freesurfer-v0-{{ .BuildNum }} - - freesurfer-v0- - - run: - name: Pull FreeSurfer down - command: | - if [[ ! -d /tmp/freesurfer ]]; then - curl -sSL https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.1/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.1.tar.gz | tar zxv --no-same-owner -C /tmp \ - --exclude='freesurfer/diffusion' \ - --exclude='freesurfer/docs' \ - --exclude='freesurfer/fsfast' \ - --exclude='freesurfer/lib/cuda' \ - --exclude='freesurfer/lib/qt' \ - --exclude='freesurfer/matlab' \ - --exclude='freesurfer/mni/share/man' \ - --exclude='freesurfer/subjects/fsaverage_sym' \ - --exclude='freesurfer/subjects/fsaverage3' \ - --exclude='freesurfer/subjects/fsaverage4' \ - --exclude='freesurfer/subjects/cvs_avg35' \ - --exclude='freesurfer/subjects/cvs_avg35_inMNI152' \ - --exclude='freesurfer/subjects/bert' \ - --exclude='freesurfer/subjects/lh.EC_average' \ - --exclude='freesurfer/subjects/rh.EC_average' \ - --exclude='freesurfer/subjects/sample-*.mgz' \ - --exclude='freesurfer/subjects/V1_average' \ - --exclude='freesurfer/trctrain' - echo "b2VzdGViYW5Ac3RhbmZvcmQuZWR1CjMwNzU2CiAqQ1MzYkJ5VXMxdTVNCiBGU2kvUGJsejJxR1V3Cg==" | base64 -d > /tmp/freesurfer/license.txt - else - echo "FreeSurfer was cached." - circleci step halt - fi - - save_cache: - key: freesurfer-v0-{{ .BuildNum }} - paths: - - /tmp/freesurfer - - build_n_pytest: - <<: *machine_defaults - working_directory: /tmp/tests + empty: + machine: + image: ubuntu-2204:current steps: - - restore_cache: - keys: - - build-v2-{{ .Branch }}-{{ epoch }} - - build-v2-{{ .Branch }}- - - build-v2-master- - - build-v2- - paths: - - /tmp/docker - - docker/install-docker-credential-helper - - run: *docker_auth - - run: *setup_docker_registry - - run: - name: Pull Ubuntu/jammy image - command: | - set +e - docker pull localhost:5000/ubuntu - success=$? - set -e - if [[ "$success" = "0" ]]; then - echo "Pulling from local registry" - docker tag localhost:5000/ubuntu ubuntu:jammy - else - echo "Pulling from Docker Hub" - docker pull ubuntu:jammy - docker tag ubuntu:jammy localhost:5000/ubuntu - docker push localhost:5000/ubuntu - fi - - run: - name: Pull SDCFlows Docker image - command: | - set +e - docker pull localhost:5000/sdcflows - success=$? - set -e - if [[ "$success" = "0" ]]; then - echo "Pulling from local registry" - docker tag localhost:5000/sdcflows nipreps/sdcflows:latest - docker tag localhost:5000/sdcflows nipreps/sdcflows - else - echo "Pulling from Docker Hub" - docker pull nipreps/sdcflows:latest - fi - - checkout: - path: /tmp/src/sdcflows - - run: - name: Build Docker image - working_directory: /tmp/src/sdcflows - no_output_timeout: 60m - command: | - export PY3=$( pyenv versions | awk '/^\* 3/ { print $2 }' ) - pyenv local $PY3 - python3 -m pip install -U build hatch hatchling pip twine docutils - - # Get version, update files. - THISVERSION=$( python3 -m hatch version | tail -n1 | xargs ) - if [[ ${THISVERSION:0:1} == "0" ]] ; then - echo "WARNING: latest git tag could not be found" - echo "Please, make sure you fetch all tags from upstream with" - echo "the command ``git fetch --tags --verbose`` and push" - echo "them to your fork with ``git push origin --tags``" - fi - # Build docker image - docker build --rm \ - --cache-from=nipreps/sdcflows \ - -t nipreps/sdcflows:latest \ - --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \ - --build-arg VCS_REF=`git rev-parse --short HEAD` \ - --build-arg VERSION="${CIRCLE_TAG:-$THISVERSION}" . \ - | tee build-output.log - echo "${CIRCLE_TAG:-$THISVERSION}" >> /tmp/.local-version.txt - - run: - name: Check Docker image - working_directory: /tmp/src/sdcflows - command: | - export PY3=$( pyenv versions | awk '/^\* 3/ { print $2 }' ) - pyenv local $PY3 - # Get version, update files. - THISVERSION=$( python3 -m hatch version | tail -n1 | xargs ) - BUILT_VERSION=$( docker run --rm --entrypoint=python nipreps/sdcflows:latest -c "import sdcflows; print(sdcflows.__version__)" ) - BUILT_VERSION=${BUILT_VERSION%$'\r'} - echo "VERSION: \"$THISVERSION\"" - echo "BUILT: \"$BUILT_VERSION\"" - set -e - test "$BUILT_VERSION" = "$THISVERSION" - - run: - name: Docker push to local registry - no_output_timeout: 40m - command: | - docker tag nipreps/sdcflows:latest localhost:5000/sdcflows - docker push localhost:5000/sdcflows - - run: - name: Docker registry garbage collection - command: | - docker exec -it registry /bin/registry garbage-collect --delete-untagged \ - /etc/docker/registry/config.yml - - save_cache: - key: build-v2-{{ .Branch }}-{{ epoch }} - paths: - - /tmp/docker - - - restore_cache: - keys: - - freesurfer-v0-{{ .BuildNum }} - - freesurfer-v0- - - restore_cache: - keys: - - data-v6-{{ .Branch }}-{{ .Revision }} - - data-v6--{{ .Revision }} - - data-v6-{{ .Branch }}- - - data-v6-main- - - data-v6- - - - restore_cache: - keys: - - workdir-v3-{{ .Branch }}- - - workdir-v3-master- - - workdir-v3- - - run: - name: Refreshing cached intermediate results - working_directory: /tmp/src/sdcflows - command: | - COMMIT_MSG=$( git log --format=oneline -n 1 $CIRCLE_SHA1 ) - set +e - do_refresh="$( echo "${COMMIT_MSG}" | grep -i -E '\[refresh[ _]?cache\]' )" - set -e - if [[ "x${do_refresh}" = "x" ]]; then - echo "Did not refresh the workdir." - else - wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \ - -O /tmp/data/workdir.tar.gz "https://files.osf.io/v1/resources/9sy2a/providers/osfstorage/5dcabd60a1cd9e000c751b3c" - rm -rf /tmp/work - mkdir -p /tmp/work - pushd /tmp/work - tar xzfv /tmp/data/workdir.tar.gz --strip 1 - popd - fi - - wipe_dir=$( echo "${COMMIT_MSG}" | sed -n 's/.*\[wipe \([a-zA-Z0-9_\*]*\)\].*/\1/p' ) - if [[ "x${wipe_dir}" != "x" ]]; then - path=/tmp/work/${wipe_dir} - echo "Found tag [wipe ${wipe_dir}] - clearing up $path ..." - rm -rf ${path} - fi - - run: - name: Run tests - no_output_timeout: 2h - command: | - mkdir -p /tmp/work - docker run -it --rm -w /src/sdcflows \ - -e TEST_WORK_DIR=/work \ - -e TEST_DATA_HOME=/data \ - -e TEST_OUTPUT_DIR=/out \ - -e COVERAGE_FILE=/out/.coverage \ - -e FS_LICENSE=/opt/freesurfer/license.txt \ - -v /tmp/data:/data:ro \ - -v /tmp/src:/src \ - -v /tmp/tests:/out \ - -v /tmp/work:/work \ - -v /tmp/freesurfer:/opt/freesurfer:ro \ - -v /tmp/templateflow:/home/sdcflows/.cache/templateflow \ - nipreps/sdcflows:latest \ - pytest -v --junit-xml=/out/pytest.xml \ - --cov sdcflows --cov-report xml:/out/unittests.xml \ - -n auto sdcflows/ - - save_cache: - key: workdir-v3-{{ .Branch }}-{{ .BuildNum }} - paths: - - /tmp/work - - store_artifacts: - path: /tmp/tests - - store_test_results: - path: /tmp/tests - - - codecov/upload: - file: /tmp/tests/unittests.xml - flags: unittests - - build_docs: - <<: *python_defaults - working_directory: /tmp/gh-pages - environment: - - FSLOUTPUTTYPE: NIFTI - - SUBJECTS_DIR: /tmp/subjects - steps: - - checkout - - run: - name: Create subjects folder - command: mkdir -p $SUBJECTS_DIR - - run: - name: Install Graphviz & pandoc - command: | - sudo apt-get update -y - sudo apt-get install -y --no-install-recommends graphviz pandoc texlive - - run: - name: Install deps - command: | - python -m venv /tmp/venv - source /tmp/venv/bin/activate - python -m pip install -U build hatch hatchling pip twine docutils - python -m pip install .[docs] - - run: - name: Build only this commit - command: | - source /tmp/venv/bin/activate - python -m hatch version | tail -n1 | xargs - BRANCH=$( echo $CIRCLE_BRANCH | sed 's+/+_+g' ) - python -c "from templateflow.api import get; get('MNI152NLin2009cAsym', desc='brain', resolution=1, suffix='T1w')" - make -C docs SPHINXOPTS="-W -v" BUILDDIR="$HOME/docs" OUTDIR=${CIRCLE_TAG:-$BRANCH} html - - store_artifacts: - path: ~/docs/ - - deploy_docker: - <<: *machine_defaults - steps: - - restore_cache: - keys: - - build-v2-{{ .Branch }}-{{ epoch }} - - build-v2-{{ .Branch }}- - - build-v2-master- - - build-v2- - paths: - - /tmp/docker - - docker/install-docker-credential-helper - - run: *docker_auth - - run: *setup_docker_registry - - run: *pull_from_registry - - run: - name: Deploy to Docker Hub - no_output_timeout: 40m - command: | - if [[ -n "$DOCKER_PAT" ]]; then - docker push nipreps/sdcflows:latest - docker tag nipreps/sdcflows nipreps/sdcflows:$CIRCLE_TAG - docker push nipreps/sdcflows:$CIRCLE_TAG - fi - - test_package: - <<: *python_defaults - steps: - - checkout - - run: - name: Prepare environment & build - command: | - python -m venv /tmp/buildenv - source /tmp/buildenv/bin/activate - python3 -m pip install -U build hatch hatchling pip twine docutils - python3 -m build - twine check dist/sdcflows* - - store_artifacts: - path: /tmp/src/sdcflows/dist - - persist_to_workspace: - root: /tmp/src/sdcflows - paths: dist - - - run: - name: Validate version - command: | - source /tmp/buildenv/bin/activate - THISVERSION=$( python -m hatch version | tail -n1 | xargs ) - python -m pip install dist/*.whl - mkdir empty - cd empty - INSTALLED=$( python -c 'import sdcflows; print(sdcflows.__version__)' ) - test "${CIRCLE_TAG:-$THISVERSION}" == "$INSTALLED" - - deploy_pypi: - <<: *python_defaults - steps: - - attach_workspace: - at: /tmp/src/sdcflows - - run: - name: Upload to Pypi - command: | - python -m pip install twine - python -m twine check dist/* - python -m twine upload dist/* --non-interactive + - run: echo Not doing nothing. workflows: version: 2 build_deploy: jobs: - - cache_test_data: - context: - - nipreps-common - - fs-license - filters: - branches: - ignore: - - /docs?\/.*/ - tags: - only: /.*/ - - - build_n_pytest: - requires: - - cache_test_data - filters: - branches: - ignore: - - /docs?\/.*/ - tags: - only: /.*/ - - - test_package: - context: - - nipreps-common - filters: - branches: - ignore: - - /docs?\/.*/ - - /tests?\/.*/ - tags: - only: /.*/ - - - deploy_pypi: - context: - - nipreps-common - requires: - - build_docs - - test_package - - build_n_pytest - filters: - branches: - ignore: /.*/ - tags: - only: /.*/ - - - deploy_docker: - context: - - nipreps-common - requires: - - deploy_pypi - filters: - branches: - ignore: /.*/ - tags: - only: /.*/ - - - build_docs: - filters: - branches: - ignore: - - /tests?\/.*/ - tags: - only: /.*/ + - empty