From 0c47cf2d703951aff2f66af23baacbd2b4b0597d Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Fri, 18 Oct 2024 22:57:43 +0200 Subject: [PATCH 01/12] Change tests to not use setup.py, but use the unittest module directly This is due to setup.py test being is newer releases of python --- ci/run_gui_test.sh | 2 +- ci/run_python_tests.sh | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ci/run_gui_test.sh b/ci/run_gui_test.sh index 3c7d92a6d..8a32a462e 100755 --- a/ci/run_gui_test.sh +++ b/ci/run_gui_test.sh @@ -21,7 +21,7 @@ fi echo "Using Python binary ${py}" test_log="/tmp/cuegui_result.log" -PYTHONPATH=pycue xvfb-run -d "${py}" cuegui/setup.py test | tee ${test_log} +PYTHONPATH=pycue xvfb-run -d "${py}" -m unittest discover -s cuegui/tests -t cuegui -p "*.py"| tee ${test_log} grep -Pz 'Ran \d+ tests in [0-9\.]+s\n\nOK' ${test_log} if [ $? -eq 0 ]; then diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index 5f1bfe294..4e1b0212b 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -22,11 +22,12 @@ python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py 2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py -python pycue/setup.py test -PYTHONPATH=pycue python pyoutline/setup.py test -PYTHONPATH=pycue python cueadmin/setup.py test -PYTHONPATH=pycue:pyoutline python cuesubmit/setup.py test -python rqd/setup.py test +python3 -m unittest discover -s pycue/tests -t pycue -p "*.py" +PYTHONPATH=pycue python3 -m unittest discover -s pyoutline/tests -t pyoutline -p "*.py" +PYTHONPATH=pycue python3 -m unittest discover -s cueadmin/tests -t cueadmin -p "*.py" +PYTHONPATH=pycue:pyoutline python3 -m unittest discover -s cuesubmit/tests -t cuesubmit -p "*.py" +python3 -m unittest discover -s rqd/tests -t rqd -p "*.py" + # Xvfb no longer supports Python 2. if [[ "$python_version" =~ "Python 3" && ${args[0]} != "--no-gui" ]]; then From 9db2eb4488747a7a9f4f834b8bde082b52ab27c5 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Fri, 18 Oct 2024 23:12:47 +0200 Subject: [PATCH 02/12] Add tests for rqd using vanilla python and introduce requirements dir with split up requirements --- .github/workflows/testing-pipeline.yml | 60 ++++++++++++++++++++++++++ ci/run_rqd_python_tests.sh | 27 ++++++++++++ requirements/rqd.txt | 10 +++++ requirements/tests.txt | 3 ++ 4 files changed, 100 insertions(+) create mode 100755 ci/run_rqd_python_tests.sh create mode 100644 requirements/rqd.txt create mode 100644 requirements/tests.txt diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index dd82f4dc5..02270ac37 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -124,3 +124,63 @@ jobs: uses: tj-actions/changed-files@v41 - name: Check for Version Change run: ci/check_version_bump.py ${{ steps.get_changed_files.outputs.all_changed_and_modified_files }} + + test_rqd_python_3_7: + name: Test RQD with python 3.7 + runs-on: ubuntu-latest + container: + image: python:3.7 + steps: + - uses: actions/checkout@v4 + - name: Run Python Tests + run: ci/run_rqd_python_tests.sh + + test_rqd_python_3_8: + name: Test RQD with python 3.8 + runs-on: ubuntu-latest + container: + image: python:3.8 + steps: + - uses: actions/checkout@v4 + - name: Run Python Tests + run: ci/run_rqd_python_tests.sh + + test_rqd_python_3_9: + name: Test RQD with python 3.9 + runs-on: ubuntu-latest + container: + image: python:3.9 + steps: + - uses: actions/checkout@v4 + - name: Run Python Tests + run: ci/run_rqd_python_tests.sh + + test_rqd_python_3_10: + name: Test RQD with python 3.10 + runs-on: ubuntu-latest + container: + image: python:3.10 + steps: + - uses: actions/checkout@v4 + - name: Run Python Tests + run: ci/run_rqd_python_tests.sh + + test_rqd_python_3_11: + name: Test RQD with python 3.11 + runs-on: ubuntu-latest + container: + image: python:3.11 + steps: + - uses: actions/checkout@v4 + - name: Run Python Tests + run: ci/run_rqd_python_tests.sh + + test_rqd_python_3_12: + name: Test RQD with python 3.12 + runs-on: ubuntu-latest + container: + image: python:3.12 + steps: + - uses: actions/checkout@v4 + - name: Run Python Tests + run: ci/run_rqd_python_tests.sh diff --git a/ci/run_rqd_python_tests.sh b/ci/run_rqd_python_tests.sh new file mode 100755 index 000000000..6c1dce894 --- /dev/null +++ b/ci/run_rqd_python_tests.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Script for running OpenCue RQD unit tests with vanilla python +# +# This script is written to be run within the OpenCue GitHub Actions environment. +# See `.github/workflows/testing-pipeline.yml`. + +set -e + +args=("$@") +python_version=$(python -V 2>&1) +echo "Will run tests using ${python_version}" + +pip install --user -r requirements/rqd.txt -r requirements/tests.txt + +# Protos need to have their Python code generated in order for tests to pass. +python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto +python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto + +# Fix imports to work in both Python 2 and 3. See +# for more info. +2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py + +python3 -m unittest discover -s pycue/tests -t pycue -p "*.py" +python3 -m unittest discover -s rqd/tests -t rqd -p "*.py" + diff --git a/requirements/rqd.txt b/requirements/rqd.txt new file mode 100644 index 000000000..34a7adb12 --- /dev/null +++ b/requirements/rqd.txt @@ -0,0 +1,10 @@ +2to3==1.0 +future==1.0.0 +grpcio==1.48.2;python_version<"3.7" +grpcio-tools==1.48.2;python_version<"3.7" +grpcio==1.53.2;python_version>="3.7" and python_version<"3.12" +grpcio-tools==1.53.0;python_version>="3.7" and python_version<"3.12" +grpcio==1.66.2;python_version>="3.12" +grpcio-tools==1.66.2;python_version>="3.12" +PyYAML==5.1 +psutil==5.9.8 \ No newline at end of file diff --git a/requirements/tests.txt b/requirements/tests.txt new file mode 100644 index 000000000..9d2efae96 --- /dev/null +++ b/requirements/tests.txt @@ -0,0 +1,3 @@ +mock==2.0.0 +pyfakefs==3.6;python_version<"3.7" +pyfakefs==5.2.3;python_version>="3.7" From 9b1738a23028aadfd0fb91669e4b302c711806f3 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Wed, 23 Oct 2024 22:32:12 +0200 Subject: [PATCH 03/12] Add pynput to tests.txt --- requirements/tests.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/tests.txt b/requirements/tests.txt index 9d2efae96..30ef6a99f 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,3 +1,4 @@ mock==2.0.0 pyfakefs==3.6;python_version<"3.7" pyfakefs==5.2.3;python_version>="3.7" +pynput==1.7.6 \ No newline at end of file From f899968d66ee1f570c0f9965ab8e081dcf7a1a13 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 24 Oct 2024 23:51:31 +0200 Subject: [PATCH 04/12] Add script for converting imports in grpc python modules and remove 2to3 --- ci/build_sphinx_docs.sh | 2 +- ci/fix_compiled_proto.py | 28 ++++++++++++++++++++++++ ci/python_coverage_report.sh | 4 ++-- ci/run_python_lint.sh | 4 ++-- ci/run_python_tests.sh | 16 +++++++------- connectors/prometheus_metrics/Dockerfile | 2 +- cueadmin/Dockerfile | 2 +- cuegui/Dockerfile | 2 +- cuesubmit/Dockerfile | 2 +- proto/README.md | 10 ++++----- pycue/Dockerfile | 2 +- pyoutline/Dockerfile | 2 +- requirements.txt | 1 - rqd/Dockerfile | 2 +- sandbox/install-client-sources.sh | 2 +- 15 files changed, 53 insertions(+), 28 deletions(-) create mode 100644 ci/fix_compiled_proto.py diff --git a/ci/build_sphinx_docs.sh b/ci/build_sphinx_docs.sh index 9f207d51e..b39c68fc2 100755 --- a/ci/build_sphinx_docs.sh +++ b/ci/build_sphinx_docs.sh @@ -10,7 +10,7 @@ python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto # Fix imports to work in both Python 2 and 3. See # for more info. -2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +python ci/fix_compiled_proto.py pycue/opencue/compiled_proto # Build the docs and treat warnings as errors ~/.local/bin/sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html diff --git a/ci/fix_compiled_proto.py b/ci/fix_compiled_proto.py new file mode 100644 index 000000000..ae735cb63 --- /dev/null +++ b/ci/fix_compiled_proto.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python + +"""Script that makes the imports in the generated compiled_proto python files relative. + +""" +import os +import re +import sys +import glob + +PYTHON_SCRIPT_PATH = sys.argv[1] + +print(PYTHON_SCRIPT_PATH) +if os.path.isdir(PYTHON_SCRIPT_PATH): + pattern = re.compile(r"^import \w+ as \w+_pb2") + for filepath in glob.glob(os.path.join(PYTHON_SCRIPT_PATH, "*_pb2*.py")): + filedata = [] + with open(filepath) as f: + for line in f.readlines(): + match = pattern.match(line) + if match is not None: + line = f"from . {line}" + filedata.append(line.strip("\n")) + # print("\n".join(filedata)) + with open(filepath, "w") as f: + f.write("\n".join(filedata)) +else: + print("Argument is not a directory") diff --git a/ci/python_coverage_report.sh b/ci/python_coverage_report.sh index e0c65328f..4477c8d44 100755 --- a/ci/python_coverage_report.sh +++ b/ci/python_coverage_report.sh @@ -9,8 +9,8 @@ python -m pip install coverage pytest-xvfb # Protos need to have their Python code generated in order for tests to pass. python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto -2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py -2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py +python ci/fix_compiled_proto.py pycue/opencue/compiled_proto +python ci/fix_compiled_proto.py rqd/rqd/compiled_proto # Run coverage for each component individually, but append it all into the same report. python -m coverage run --source=pycue/opencue/,pycue/FileSequence/ --omit=pycue/opencue/compiled_proto/* pycue/tests/test_suite.py diff --git a/ci/run_python_lint.sh b/ci/run_python_lint.sh index bd86c9188..ba98d2b6c 100755 --- a/ci/run_python_lint.sh +++ b/ci/run_python_lint.sh @@ -13,8 +13,8 @@ python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc # Fix imports to work in both Python 2 and 3. See # for more info. -2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py -2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py +python ci/fix_compiled_proto.py pycue/opencue/compiled_proto +python ci/fix_compiled_proto.py rqd/rqd/compiled_proto echo "Running lint for pycue/..." cd pycue diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index 4e1b0212b..edb0322cc 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -19,14 +19,14 @@ python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc # Fix imports to work in both Python 2 and 3. See # for more info. -2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py -2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py - -python3 -m unittest discover -s pycue/tests -t pycue -p "*.py" -PYTHONPATH=pycue python3 -m unittest discover -s pyoutline/tests -t pyoutline -p "*.py" -PYTHONPATH=pycue python3 -m unittest discover -s cueadmin/tests -t cueadmin -p "*.py" -PYTHONPATH=pycue:pyoutline python3 -m unittest discover -s cuesubmit/tests -t cuesubmit -p "*.py" -python3 -m unittest discover -s rqd/tests -t rqd -p "*.py" +python ci/fix_compiled_proto.py pycue/opencue/compiled_proto +python ci/fix_compiled_proto.py rqd/rqd/compiled_proto + +python -m unittest discover -s pycue/tests -t pycue -p "*.py" +PYTHONPATH=pycue python -m unittest discover -s pyoutline/tests -t pyoutline -p "*.py" +PYTHONPATH=pycue python -m unittest discover -s cueadmin/tests -t cueadmin -p "*.py" +PYTHONPATH=pycue:pyoutline python -m unittest discover -s cuesubmit/tests -t cuesubmit -p "*.py" +python -m unittest discover -s rqd/tests -t rqd -p "*.py" # Xvfb no longer supports Python 2. diff --git a/connectors/prometheus_metrics/Dockerfile b/connectors/prometheus_metrics/Dockerfile index f710dce0c..357b5a0bf 100644 --- a/connectors/prometheus_metrics/Dockerfile +++ b/connectors/prometheus_metrics/Dockerfile @@ -48,7 +48,7 @@ RUN python -m grpc_tools.protoc \ # Fix imports to work in both Python 2 and 3. See # for more info. -RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +RUN python3 ci/fix_compiled_proto.py pycue/opencue/compiled_proto RUN cd pycue && python setup.py install diff --git a/cueadmin/Dockerfile b/cueadmin/Dockerfile index 20ff77c1e..dd1359edc 100644 --- a/cueadmin/Dockerfile +++ b/cueadmin/Dockerfile @@ -24,7 +24,7 @@ RUN python3 -m grpc_tools.protoc \ # Fix imports to work in both Python 2 and 3. See # for more info. -RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +RUN python3 ci/fix_compiled_proto.py pycue/opencue/compiled_proto COPY cueadmin/README.md ./cueadmin/ COPY cueadmin/setup.py ./cueadmin/ diff --git a/cuegui/Dockerfile b/cuegui/Dockerfile index 3e6630804..6e53c7f04 100644 --- a/cuegui/Dockerfile +++ b/cuegui/Dockerfile @@ -54,7 +54,7 @@ RUN python3.6 -m grpc_tools.protoc \ # Fix imports to work in both Python 2 and 3. See # for more info. -RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +RUN python3 ci/fix_compiled_proto.py pycue/opencue/compiled_proto COPY cuegui/README.md ./cuegui/ COPY cuegui/setup.py ./cuegui/ diff --git a/cuesubmit/Dockerfile b/cuesubmit/Dockerfile index eb2a4902a..47c615f7c 100644 --- a/cuesubmit/Dockerfile +++ b/cuesubmit/Dockerfile @@ -41,7 +41,7 @@ RUN python3.6 -m grpc_tools.protoc \ # Fix imports to work in both Python 2 and 3. See # for more info. -RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +RUN python3 ci/fix_compiled_proto.py pycue/opencue/compiled_proto COPY pyoutline/README.md ./pyoutline/ COPY pyoutline/setup.py ./pyoutline/ diff --git a/proto/README.md b/proto/README.md index 0b039b462..904a112e1 100644 --- a/proto/README.md +++ b/proto/README.md @@ -14,15 +14,14 @@ To generate: ```sh python -m grpc_tools.protoc -I=. --python_out=../rqd/rqd/compiled_proto --grpc_python_out=../rqd/rqd/compiled_proto ./*.proto -2to3 -wn -f import ../rqd/rqd/compiled_proto/*_pb2*.py +python ../ci/fix_compiled_proto.py ../rqd/rqd/compiled_proto ``` For Windows (Powershell): ```powershell python -m grpc_tools.protoc --proto_path=. --python_out=../rqd/rqd/compiled_proto --grpc_python_out=../rqd/rqd/compiled_proto (ls *.proto).Name -cd ..\rqd\rqd\compiled_proto\ -2to3 -wn -f import (ls *_pb2*.py).Name +python ../ci/fix_compiled_proto.py ../rqd/rqd/compiled_proto ``` @@ -32,15 +31,14 @@ To generate: ```sh python -m grpc_tools.protoc -I=. --python_out=../pycue/opencue/compiled_proto --grpc_python_out=../pycue/opencue/compiled_proto ./*.proto -2to3 -wn -f import ../pycue/opencue/compiled_proto/*_pb2*.py +python ../ci/fix_compiled_proto.py ../pycue/opencue/compiled_proto ``` For Windows (Powershell): ```powershell python -m grpc_tools.protoc --proto_path=. --python_out=../pycue/opencue/compiled_proto --grpc_python_out=../pycue/opencue/compiled_proto (ls *.proto).Name -cd ..\pycue\opencue\compiled_proto\ -2to3 -wn -f import (ls *_pb2*.py).Name +python ../ci/fix_compiled_proto.py ../pycue/opencue/compiled_proto ``` diff --git a/pycue/Dockerfile b/pycue/Dockerfile index 9698e94bc..c61d3cf16 100644 --- a/pycue/Dockerfile +++ b/pycue/Dockerfile @@ -25,7 +25,7 @@ RUN python3 -m grpc_tools.protoc \ # Fix imports to work in both Python 2 and 3. See # for more info. -RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +RUN python ci/fix_compiled_proto.py pycue/opencue/compiled_proto COPY VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)" | tee VERSION diff --git a/pyoutline/Dockerfile b/pyoutline/Dockerfile index bc7155daf..6937a5584 100644 --- a/pyoutline/Dockerfile +++ b/pyoutline/Dockerfile @@ -24,7 +24,7 @@ RUN python3 -m grpc_tools.protoc \ # Fix imports to work in both Python 2 and 3. See # for more info. -RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +RUN python ci/fix_compiled_proto.py pycue/opencue/compiled_proto COPY pyoutline/README.md ./pyoutline/ COPY pyoutline/setup.py ./pyoutline/ diff --git a/requirements.txt b/requirements.txt index cceee9237..8af83f243 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -2to3==1.0 enum34==1.1.6 future==1.0.0 grpcio==1.48.2;python_version<"3.7" diff --git a/rqd/Dockerfile b/rqd/Dockerfile index c3a0a0dc1..6847ad10a 100644 --- a/rqd/Dockerfile +++ b/rqd/Dockerfile @@ -35,7 +35,7 @@ RUN python3.9 -m grpc_tools.protoc \ # Fix imports to work in both Python 2 and 3. See # for more info. -RUN 2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py +RUN python ci/fix_compiled_proto.py rqd/rqd/compiled_proto COPY VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)" | tee VERSION diff --git a/sandbox/install-client-sources.sh b/sandbox/install-client-sources.sh index 7e15ed018..80ed1f39e 100755 --- a/sandbox/install-client-sources.sh +++ b/sandbox/install-client-sources.sh @@ -10,7 +10,7 @@ python -m grpc_tools.protoc -I=. \ --python_out=../pycue/opencue/compiled_proto \ --grpc_python_out=../pycue/opencue/compiled_proto ./*.proto cd .. -2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +python ../ci/fix_compiled_proto.py pycue/opencue/compiled_proto # Install all client packages. pip install pycue/ pyoutline/ cueadmin/ cuesubmit/ cuegui/ From cc2fec41e9aef60618ee736b6d386c305c3139c7 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Fri, 25 Oct 2024 00:24:03 +0200 Subject: [PATCH 05/12] Replace 2to3 with convert script and remove 2to3 --- ci/run_rqd_python_tests.sh | 8 ++++---- requirements/rqd.txt | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ci/run_rqd_python_tests.sh b/ci/run_rqd_python_tests.sh index 6c1dce894..5ae974d6d 100755 --- a/ci/run_rqd_python_tests.sh +++ b/ci/run_rqd_python_tests.sh @@ -19,9 +19,9 @@ python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc # Fix imports to work in both Python 2 and 3. See # for more info. -2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py -2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py +python ci/fix_compiled_proto.py pycue/opencue/compiled_proto +python ci/fix_compiled_proto.py rqd/rqd/compiled_proto -python3 -m unittest discover -s pycue/tests -t pycue -p "*.py" -python3 -m unittest discover -s rqd/tests -t rqd -p "*.py" +python -m unittest discover -s pycue/tests -t pycue -p "*.py" +python -m unittest discover -s rqd/tests -t rqd -p "*.py" diff --git a/requirements/rqd.txt b/requirements/rqd.txt index 34a7adb12..32e13ae3b 100644 --- a/requirements/rqd.txt +++ b/requirements/rqd.txt @@ -1,4 +1,3 @@ -2to3==1.0 future==1.0.0 grpcio==1.48.2;python_version<"3.7" grpcio-tools==1.48.2;python_version<"3.7" From 15c80134e27b3f6bfda3c322af40d5ea8d579785 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Fri, 25 Oct 2024 00:24:31 +0200 Subject: [PATCH 06/12] Add python 3.13 rqd tests --- .github/workflows/testing-pipeline.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index 02270ac37..cdb57eb03 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -184,3 +184,13 @@ jobs: - uses: actions/checkout@v4 - name: Run Python Tests run: ci/run_rqd_python_tests.sh + + test_rqd_python_3_13: + name: Test RQD with python 3.13 + runs-on: ubuntu-latest + container: + image: python:3.13 + steps: + - uses: actions/checkout@v4 + - name: Run Python Tests + run: ci/run_rqd_python_tests.sh From ad902f5caf1bc90bd4b220ef50b27435fe5b1448 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Fri, 25 Oct 2024 00:25:26 +0200 Subject: [PATCH 07/12] Remove debug prints --- ci/fix_compiled_proto.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/fix_compiled_proto.py b/ci/fix_compiled_proto.py index ae735cb63..0a5534803 100644 --- a/ci/fix_compiled_proto.py +++ b/ci/fix_compiled_proto.py @@ -10,7 +10,6 @@ PYTHON_SCRIPT_PATH = sys.argv[1] -print(PYTHON_SCRIPT_PATH) if os.path.isdir(PYTHON_SCRIPT_PATH): pattern = re.compile(r"^import \w+ as \w+_pb2") for filepath in glob.glob(os.path.join(PYTHON_SCRIPT_PATH, "*_pb2*.py")): @@ -21,7 +20,6 @@ if match is not None: line = f"from . {line}" filedata.append(line.strip("\n")) - # print("\n".join(filedata)) with open(filepath, "w") as f: f.write("\n".join(filedata)) else: From 7af9ec307ba256257c911a3b2a187eeab079022c Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Sat, 9 Nov 2024 20:33:54 +0100 Subject: [PATCH 08/12] Switch to using pytest for rqd --- ci/run_rqd_python_tests.sh | 3 +-- requirements/tests.txt | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/run_rqd_python_tests.sh b/ci/run_rqd_python_tests.sh index 5ae974d6d..1d5c2d079 100755 --- a/ci/run_rqd_python_tests.sh +++ b/ci/run_rqd_python_tests.sh @@ -23,5 +23,4 @@ python ci/fix_compiled_proto.py pycue/opencue/compiled_proto python ci/fix_compiled_proto.py rqd/rqd/compiled_proto python -m unittest discover -s pycue/tests -t pycue -p "*.py" -python -m unittest discover -s rqd/tests -t rqd -p "*.py" - +python -m pytest rqd/tests diff --git a/requirements/tests.txt b/requirements/tests.txt index 30ef6a99f..2ea7e1f8f 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,4 +1,5 @@ mock==2.0.0 pyfakefs==3.6;python_version<"3.7" pyfakefs==5.2.3;python_version>="3.7" -pynput==1.7.6 \ No newline at end of file +pynput==1.7.6 +pytest==8.3.3 \ No newline at end of file From 75996fffdc647295be0e393eea527fcb388ea1bc Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Sat, 9 Nov 2024 20:47:48 +0100 Subject: [PATCH 09/12] Remove rqd test for python 3.7 --- .github/workflows/testing-pipeline.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index cdb57eb03..169fffb93 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -125,16 +125,6 @@ jobs: - name: Check for Version Change run: ci/check_version_bump.py ${{ steps.get_changed_files.outputs.all_changed_and_modified_files }} - test_rqd_python_3_7: - name: Test RQD with python 3.7 - runs-on: ubuntu-latest - container: - image: python:3.7 - steps: - - uses: actions/checkout@v4 - - name: Run Python Tests - run: ci/run_rqd_python_tests.sh - test_rqd_python_3_8: name: Test RQD with python 3.8 runs-on: ubuntu-latest From e8e240ad7c6806b34c2f6a62595857d6c82375d0 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 14 Nov 2024 23:45:47 +0100 Subject: [PATCH 10/12] Fix test for python 3.12+ which involves removing a real directory --- rqd/tests/rqconstants_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rqd/tests/rqconstants_test.py b/rqd/tests/rqconstants_test.py index 0503994a2..176c2a3ec 100644 --- a/rqd/tests/rqconstants_test.py +++ b/rqd/tests/rqconstants_test.py @@ -29,6 +29,7 @@ import mock import pyfakefs.fake_filesystem_unittest +import unittest import six @@ -91,7 +92,7 @@ class RqConstantTests(pyfakefs.fake_filesystem_unittest.TestCase): tempdir = tempfile.mkdtemp() def setUp(self): - self.setUpPyfakefs() + self.setUpPyfakefs(additional_skip_names=['shutil']) self.fs.add_real_directory(self.tempdir) self.fs.create_file("/proc/cpuinfo", contents=CPUINFO) self.loadavg = self.fs.create_file("/proc/loadavg", contents=LOADAVG_LOW_USAGE) From c81334a02f33992133090c3754e3fe2225a9c999 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Fri, 15 Nov 2024 00:00:31 +0100 Subject: [PATCH 11/12] Update rqconstants_test.py --- rqd/tests/rqconstants_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rqd/tests/rqconstants_test.py b/rqd/tests/rqconstants_test.py index 176c2a3ec..2a1553d81 100644 --- a/rqd/tests/rqconstants_test.py +++ b/rqd/tests/rqconstants_test.py @@ -29,7 +29,6 @@ import mock import pyfakefs.fake_filesystem_unittest -import unittest import six From 4f87aa8d230667d1579b980df12963d5f242f215 Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Thu, 28 Nov 2024 11:06:51 +0100 Subject: [PATCH 12/12] Add docker as a requirement for rqd --- requirements/rqd.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements/rqd.txt b/requirements/rqd.txt index 32e13ae3b..4bf32be84 100644 --- a/requirements/rqd.txt +++ b/requirements/rqd.txt @@ -1,4 +1,5 @@ future==1.0.0 +docker==7.1.0 grpcio==1.48.2;python_version<"3.7" grpcio-tools==1.48.2;python_version<"3.7" grpcio==1.53.2;python_version>="3.7" and python_version<"3.12" @@ -6,4 +7,4 @@ grpcio-tools==1.53.0;python_version>="3.7" and python_version<"3.12" grpcio==1.66.2;python_version>="3.12" grpcio-tools==1.66.2;python_version>="3.12" PyYAML==5.1 -psutil==5.9.8 \ No newline at end of file +psutil==5.9.8