collect-datasets.yml

name: collect-datasets

on:
  schedule:
  # Run at 0 minute past every 12th hour
    - cron:  '0 */12 * * *'
  push:
    branches: [ master ]
  pull_request:
    branches: [ master ]
  workflow_dispatch:

jobs:
  collect-datasets:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v2

    - name: Set up Python 3.8
      uses: actions/setup-python@v1
      with:
        python-version: 3.8
    
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install flake8 pytest
        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
    
    - name: Lint with flake8
      run: |
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
    
    - name: Collect v1 and v2 datasets from Metadata Catalogue
      run: |
        python datasets.py
    
    - name: Perform v1 completeness and schema validation checks
      run: |
        python quality_checks.py
    
    - name: Perform V2 completeness and schema validation checks
      run: |
        python quality_checks_v2.py
    
    - name: Merge V1 and V2 quality scores
      run: |
        python merge_quality_scores.py
    
    - name: Auto-generate README
      run: |
        python update_readme.py

    - uses: stefanzweifel/git-auto-commit-action@v4.1.6
      with:
        commit_message: Commit new datasets
        branch: ${{ github.head_ref }}
        commit_user_name: Susheel Varma
        commit_user_email: susheel.varma@hdruk.ac.uk
        commit_author: Susheel Varma <susheel.varma@hdruk.ac.uk>