Merge pull request #355 from EESSI/develop

merge `develop` into `main` for releasing version v0.10.0

Author: boegel

.github/workflows/markdown-lint.yml

@@ -10,7 +10,7 @@
 #
 
 name: Markdown Lint
-on: [push, pull_request]
+on: [push, pull_request, workflow_dispatch]
 # Declare default permissions as read only.
 permissions: read-all
 
@@ -24,10 +24,11 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.0
         with:
-          node-version: '18'
+          node-version: '20'
 
       - name: Install markdownlint-cli
-        run: npm install -g markdownlint-cli
+        run: npm install -g igorshubovych/markdownlint-cli#192ad822316c3a22fb3d3cc8aa6eafa0b8488360 # v0.45.0
+        # SHA from https://github.com/igorshubovych/markdownlint-cli/commit/192ad822316c3a22fb3d3cc8aa6eafa0b8488360
 
       - name: Run markdownlint
-        run: markdownlint "**/*.md" --ignore .git 
+        run: markdownlint "**/*.md" --ignore .git 

.github/workflows/requirements-lock.txt

@@ -0,0 +1,52 @@
+# NOTE, this file is used with installing packages in CI. Do not use it for running
+#   bot components.
+# try to pin required versions using hashes
+# for some Python versions we may have to use different versions
+pip==25.3; python_version >= "3.9" \
+    --hash=sha256:8d0538dbbd7babbd207f261ed969c65de439f6bc9e5dbd3b3b9a77f25d95f343
+
+exceptiongroup==1.3.0 \
+    --hash=sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88
+
+iniconfig==2.1.0; python_version <= "3.9" \
+    --hash=sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7
+iniconfig==2.3.0; python_version >= "3.10" \
+    --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730
+
+packaging==25.0 \
+    --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f
+
+typing-extensions==4.15.0 \
+    --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466
+
+pluggy==1.6.0 \
+    --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3
+
+pygments==2.19.2 \
+    --hash=sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887
+
+tomli==2.3.0 \
+    --hash=sha256:64be704a875d2a59753d80ee8a533c3fe183e3f06807ff7dc2232938ccb01549
+
+pytest==8.4.2 \
+    --hash=sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01
+
+coverage==7.10.7; python_version <= "3.9" \
+    --hash=sha256:f4ab143ab113be368a3e9b795f9cd7906c5ef407d6173fe9675a902e1fffc239
+coverage==7.11.0; python_version >= "3.10" \
+    --hash=sha256:167bd504ac1ca2af7ff3b81d245dfea0292c5032ebef9d66cc08a7d28c1b8050
+
+pytest-cov==7.0.0 \
+    --hash=sha256:33c97eda2e049a0c5298e91f519302a1334c26ac65c1a483d6206fd458361af1
+
+mccabe==0.7.0 \
+    --hash=sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325
+
+pycodestyle==2.14.0 \
+    --hash=sha256:c4b5b517d278089ff9d0abdec919cd97262a3367449ea1c8b49b91529167b783
+
+pyflakes==3.4.0 \
+    --hash=sha256:b24f96fafb7d2ab0ec5075b7350b3d2d2218eab42003821c06344973d3ea2f58
+
+flake8==7.3.0 \
+    --hash=sha256:fe044858146b9fc69b551a4b490d69cf960fcb78ad1edcb84e7fbb1b4a8e3872

.github/workflows/scorecards.yml

@@ -44,7 +44,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # v2.0.6
+        uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
         with:
           results_file: results.sarif
           results_format: sarif

.github/workflows/tests.yaml

@@ -20,33 +20,29 @@ jobs:
     runs-on: ubuntu-24.04
     strategy:
       matrix:
-        # for now, only test with Python 3.9+ (since we're testing in Ubuntu 24.04)
-        #python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
-        python: ['3.9', '3.10', '3.11']
+        # Waitress 3.0.1 does not support Python 3.8 or older
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
       fail-fast: false
     steps:
       - name: checkout
         uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
-      - name: set up Python
+      - name: set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
         with:
-          python-version: ${{matrix.python}}
+          python-version: ${{ matrix.python-version }}
 
       - name: Install required Python packages + pytest + flake8
         run: |
-          python -m pip install --upgrade pip
           python -m pip install -r requirements.txt
-          python -m pip install pytest
-          python -m pip install --upgrade flake8
+          python -m pip install -r .github/workflows/requirements-lock.txt --require-hashes
 
       - name: Run test suite (without coverage)
         run: |
           ./test.sh --verbose
 
       - name: Run test suite (with coverage)
         run: |
-          python -m pip install pytest-cov
           ./test.sh -q --cov=$PWD
 
       - name: Run flake8 to verify PEP8-compliance of Python code

RELEASE_NOTES

@@ -1,6 +1,22 @@
 This file contains a description of the major changes to the EESSI
 build-and-deploy bot. For more detailed information, please see the git log.
 
+v0.10.0 (13 November 2025)
+--------------------------
+
+This is a minor release of the EESSI build-and-deploy bot.
+
+Bug fixes:
+* show builds for different repositories for `bot: status last_build` command (#345)
+* fix out of range error for `bot: status last_build` command (#346)
+* ensure tarballs with other suffixes than `.tar.gz` are correctly handled (#352, #353)
+
+Improvements:
+* improve security aspects due to OpenSSF Scorecard report (#347, #348, #349, #350)
+
+No changes to 'app.cfg' settings.
+
+
 v0.9.0 (22 August 2025)
 --------------------------
 

containers/Dockerfile.smee-client

@@ -1,7 +1,12 @@
-ARG smee_client_version=4.2.1
+ARG smee_client_version=4.4.1
+# ARG smee_client_version_commit=b837fa85fd05853731160e21356ffd30c8c3e791 # v4.4.1
 
-FROM node:lts-alpine
+# pinning base image to specific hash (corresponding to lts-alpine)
+FROM node@sha256:f36fed0b2129a8492535e2853c64fbdbd2d29dc1219ee3217023ca48aebd3787
 ARG smee_client_version
+# ARG smee_client_version_commit
+
+# Then install
 RUN npm install --global smee-client@${smee_client_version}
 ENTRYPOINT ["smee"]
 CMD ["--help"]

eessi_bot_event_handler.py

@@ -591,6 +591,7 @@ def handle_bot_command_status(self, event_info, bot_command):
         repo_name = event_info['raw_request_body']['repository']['full_name']
         pr_number = event_info['raw_request_body']['issue']['number']
         status_table = request_bot_build_issue_comments(repo_name, pr_number)
+        self.log(f"Retrieved status table from issue comments: {status_table}")
 
         if 'last_build' in bot_command.general_args:
             # If the bot command is something like 'bot:status =last_build', then only retain the last build for each
@@ -603,10 +604,17 @@ def handle_bot_command_status(self, event_info, bot_command):
             # copied, we ignore it, since - as a result of the sorting - the second entry is always older than the
             # first
             dates = status_table['date']
+            status = status_table['status']
             timestamps = []
-            for date in dates:
-                date_object = datetime.strptime(date, "%b %d %X %Z %Y")
-                timestamps.append(int(date_object.timestamp()))
+            for date, state in zip(dates, status):
+                if state == 'finished':
+                    date_object = datetime.strptime(date, "%b %d %X %Z %Y")
+                    timestamps.append(int(date_object.timestamp()))
+                else:
+                    # Add the oldest date possible, so that ongoing builds only show up if there was no other
+                    # completed build yet
+                    timestamps.append(0)
+
             status_table['timestamp'] = timestamps
 
             # Figure out the sorting indices, so that things are sorted first by the 'for arch', and then by 'date'
@@ -625,7 +633,17 @@ def handle_bot_command_status(self, event_info, bot_command):
                 'on arch': [], 'for arch': [], 'for repo': [], 'date': [], 'status': [], 'url': [], 'result': []
             }
             for x in range(0, len(sorted_table['date'])):
-                if sorted_table['for arch'][x] not in status_table_last['for arch']:
+                # Check if the current 'for arch' AND 'for repo' are already in the status_table_last. If not, add it
+                already_present = False
+                for y in range(0, len(status_table_last['for arch'])):
+                    if (
+                        sorted_table['for arch'][x] == status_table_last['for arch'][y]
+                        and sorted_table['for repo'][x] == status_table_last['for repo'][y]
+                    ):
+                        already_present = True
+                        # One match is enough, we don't append in this case, no need to look further
+                        break
+                if not already_present:
                     self.log(f"arch: {sorted_table['for arch'][x]} not yet in status_table_last")
                     for key in status_table_last:
                         self.log(f"Adding to '{key}' and the value {sorted_table[key][x]}")

requirements.txt

@@ -11,7 +11,7 @@
 # license: GPLv2
 #
 PyGithub
-Waitress
-cryptography
+Waitress>=3.0.1 # required to fix vulnerabilities detected by scorecards
+cryptography>=44.0.1 # required to fix vulnerabilities detected by scorecards
 PyGHee>=0.0.3
 retry

tasks/build.py

@@ -1225,11 +1225,20 @@ def request_bot_build_issue_comments(repo_name, pr_number):
             # it is building for)
             # Then, check that it has at least 4 lines so that we can safely index up to that number
             if instance_repo_match and len(comment_body) >= 4:
+                # Set some defaults
+                repo_id = ""
+                on_arch = ""
+                for_arch = ""
+                date = ""
+                status = ""
+                url = ""
+                result = ""
+
                 log(f"{fn}(): found bot build response in issue, processing...")
 
                 # First, extract the repo_id
                 log(f"{fn}(): found build for repository: {instance_repo_match.group('repo_id')}")
-                status_table['for repo'].append(instance_repo_match.group('repo_id'))
+                repo_id = instance_repo_match.group('repo_id')
 
                 # Then, try to match the architecture we build on.
                 # First try this including accelerator, to see if one was defined
@@ -1241,16 +1250,15 @@ def request_bot_build_issue_comments(repo_name, pr_number):
                     # Pattern with accelerator matched, append to status_table
                     log(f"{fn}(): found build on architecture: {on_arch_match.group('on_arch')}, "
                         f"with accelerator {on_arch_match.group('accelerator')}")
-                    status_table['on arch'].append(f"`{on_arch_match.group('on_arch')}`, "
-                                                   f"`{on_arch_match.group('accelerator')}`")
+                    on_arch = f"`{on_arch_match.group('on_arch')}`, `{on_arch_match.group('accelerator')}`"
                 else:
                     # Pattern with accelerator did not match, retry without accelerator
                     on_arch_re = template_to_regex(on_arch_fmt)
                     on_arch_match = re.match(on_arch_re, comment_body[1])
                     if on_arch_match:
                         # Pattern without accelerator matched, append to status_table
                         log(f"{fn}(): found build on architecture: {on_arch_match.group('on_arch')}")
-                        status_table['on arch'].append(f"`{on_arch_match.group('on_arch')}`")
+                        on_arch = f"`{on_arch_match.group('on_arch')}`"
                     else:
                         # This shouldn't happen: we had an instance_repo_match, but no match for the 'on architecture'
                         msg = "Could not match regular expression for extracting the architecture to build on.\n"
@@ -1271,16 +1279,15 @@ def request_bot_build_issue_comments(repo_name, pr_number):
                     # Pattern with accelerator matched, append to status_table
                     log(f"{fn}(): found build for architecture: {for_arch_match.group('for_arch')}, "
                         f"with accelerator {for_arch_match.group('accelerator')}")
-                    status_table['for arch'].append(f"`{for_arch_match.group('for_arch')}`, "
-                                                    f"`{for_arch_match.group('accelerator')}`")
+                    for_arch = f"`{for_arch_match.group('for_arch')}`, `{for_arch_match.group('accelerator')}`"
                 else:
                     # Pattern with accelerator did not match, retry without accelerator
                     for_arch_re = template_to_regex(for_arch_fmt)
                     for_arch_match = re.match(for_arch_re, comment_body[2])
                     if for_arch_match:
                         # Pattern without accelerator matched, append to status_table
                         log(f"{fn}(): found build for architecture: {for_arch_match.group('for_arch')}")
-                        status_table['for arch'].append(f"`{for_arch_match.group('for_arch')}`")
+                        for_arch = f"`{for_arch_match.group('for_arch')}`"
                     else:
                         # This shouldn't happen: we had an instance_repo_match, but no match for the 'on architecture'
                         msg = "Could not match regular expression for extracting the architecture to build for.\n"
@@ -1315,17 +1322,39 @@ def request_bot_build_issue_comments(repo_name, pr_number):
                 # add date, status, url to  status_table if
                 for row in rows:
                     if row['job status'] == 'finished':
-                        status_table['date'].append(row['date'])
-                        status_table['status'].append(row['job status'])
-                        status_table['url'].append(comment['html_url'])
+                        date = row['date']
+                        status = row['job status']
+                        url = comment['html_url']
                         if 'FAILURE' in row['comment']:
-                            status_table['result'].append(':cry: FAILURE')
+                            result = ':cry: FAILURE'
                         elif 'SUCCESS' in row['comment']:
-                            status_table['result'].append(':grin: SUCCESS')
+                            result = ':grin: SUCCESS'
                         elif 'UNKNOWN' in row['comment']:
-                            status_table['result'].append(':shrug: UNKNOWN')
+                            result = ':shrug: UNKNOWN'
                         else:
-                            status_table['result'].append(row['comment'])
+                            result = row['comment']
+                    elif row['job status'] in ['submitted', 'received', 'running']:
+                        # Make sure that if the job is not finished yet, we also put something useful in these fields
+                        # It is useful to know a job is submitted, running, etc
+                        date = row['date']
+                        status = row['job status']
+                        url = comment['html_url']
+                        result = row['comment']
+                    else:
+                        # Don't do anything for the test line for now - we might add an extra entry to the status
+                        # table later to reflect the test result
+                        continue
+
+                # Add all entries to status_table. We do this at the end of this loop so that the operation is
+                # more or less 'atomic', i.e. all vectors in the status_table dict have the same length
+                status_table['for repo'].append(repo_id)
+                status_table['on arch'].append(on_arch)
+                status_table['for arch'].append(for_arch)
+                status_table['date'].append(date)
+                status_table['status'].append(status)
+                status_table['url'].append(url)
+                status_table['result'].append(result)
+
         if len(comments) != 100:
             break
     return status_table