Merge pull request #1 from Totto16/python3.12

Python3.12
Totto16 · Nov 20, 2024 · f0fb08c · f0fb08c
2 parents a5d4991 + f81299e
commit f0fb08c
Show file tree

Hide file tree

Showing 23 changed files with 1,676 additions and 109 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -1,64 +1,67 @@
 name: Python Lints / Checks
 
 on:
+    pull_request:
     push:
         branches:
             - main
 
 jobs:
     mypy_lint:
-        name: Lint with Mypy
-        runs-on: ubuntu-latest
+        name: Lint with Mypy and Ruff
+        runs-on: ubuntu-24.04
         container:
-            image: python:3.11-bookworm
+            image: python:3.12-bookworm
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
               with:
                   fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
             - name: prepare environment
               run: |
+                  apt-get update
+                  apt-get install cmake --no-install-recommends -y
                   pip install pipenv -U --quiet
                   pipenv install --dev
 
             - name: run lint
               run: |
                   pipenv run mypy
-                  pipenv run ruff-ci
+                  pipenv run ruff-ci github
 
     tests:
         name: Test with pytest
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-24.04
         container:
-            image: python:3.11-bookworm
+            image: python:3.12-bookworm
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
               with:
                   fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
             - name: prepare environment
               run: |
+                  apt-get update
+                  apt-get install ffmpeg cmake --no-install-recommends -y
                   pip install pipenv -U --quiet
                   pipenv install --dev
-                  apt-get update
-                  apt-get install ffmpeg -y
 
             - name: run tests
               run: |
                   pipenv run test
 
     sonarqube_checks:
         name: SonarQube Checks
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-24.04
         needs:
             - mypy_lint
             - tests
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
               with:
                   fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
             - name: prepare environment
               run: |
                   sudo apt-get update
-                  sudo apt-get install python3.11 --install-recommends ffmpeg -y
+                  sudo apt-get install python3.12 ffmpeg --no-install-recommends -y
                   pip install pipenv -U --quiet
                   pipenv install --dev
             - name: run coverage report
@@ -75,7 +78,7 @@ jobs:
                   project: "Video Language Detection"
                   path: "."
                   format: HTML
-                  out: "reports" # this is the default, no need to specify unless you wish to override it
+                  out: "reports"
                   args: >
                       --failOnCVSS 7
                       --enableRetired
@@ -85,8 +88,6 @@ jobs:
               env:
                   SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
                   SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
-            # If you wish to fail your job when the Quality Gate is red, uncomment the
-            # following lines. This would typically be used to fail a deployment.
             - uses: sonarsource/sonarqube-quality-gate-action@master
               timeout-minutes: 5
               env:

diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,5 @@
 __pycache__
 .mypy_cache
-Pipfile.lock
 model/
 /src/tmp/
 /data/
@@ -9,3 +8,4 @@ model/
 /.coverage
 /coverage.xml
 /config.ini
+/config.yaml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -7,21 +7,22 @@ stages: # List of stages for jobs, and their order of execution
 python-lint-job:
     stage: python_checks
     allow_failure: false
-    image: python:3.11-bookworm
+    image: python:3.12-bookworm
     script:
         - pip install pipenv -U --quiet
         - pipenv install --dev
-        - pipenv run lint
+        - pipenv run mypy
+        - pipenv run ruff-ci gitlab
 
 python-test-job:
     stage: python_checks
     allow_failure: false
-    image: python:3.11-bookworm
+    image: python:3.12-bookworm
     script:
         - pip install pipenv -U --quiet
         - pipenv install --dev
         - apt-get update
-        - apt-get install ffmpeg -y
+        - apt-get install ffmpeg --no-install-recommends -y
         - pipenv run test
 
 sonarqube-check:
@@ -42,9 +43,8 @@ sonarqube-check:
     script:
         - pip install pipenv -U --quiet
         - pipenv install --dev
-        # TODO:check if this is even debian / ubuntu or alpine and install ffprobe
         - sudo apt-get update
-        - sudo apt-get install ffmpeg -y
+        - sudo apt-get install ffmpeg --no-install-recommends -y
         - pipenv run coverage
         - sonar-scanner
     allow_failure: false

diff --git a/IMDB/imdb-dataset-parser b/IMDB/imdb-dataset-parser
diff --git a/IMDB2sql b/IMDB2sql
diff --git a/Pipfile b/Pipfile
@@ -1,13 +1,16 @@
+
 [[source]]
 url = "https://pypi.org/simple"
 verify_ssl = true
 name = "pypi"
 
+
 [packages]
-python-ffmpeg = "*"
 torchaudio = "*"
-speechbrain = "*"
 torchvision = "*"
+torch = "*"
+speechbrain = "*"
+python-ffmpeg = "*"
 pynvml = "*"
 humanize = "*"
 enlighten = "*"
@@ -32,9 +35,9 @@ chardet = "*"
 [scripts]
 check = "python src/entry.py"
 lint = "bash -c 'pipenv run mypy && pipenv run ruff'"
-mypy = "mypy src/ tests/ ffmpeg/ --strict --ignore-missing-imports --enable-incomplete-feature=Unpack "
+mypy = "mypy src/ tests/ ffmpeg/ --strict --ignore-missing-imports"
 ruff = "ruff check src/ tests/ ffmpeg/"
-ruff-ci = "ruff check src/ tests/ ffmpeg/ --format=github"
+ruff-ci = "ruff check src/ tests/ ffmpeg/ --output-format"
 ruff-fix = "ruff check src/ tests/ ffmpeg/ --fix"
 schema = "python src/entry.py schema"
 test = "pytest tests/"
@@ -43,4 +46,4 @@ extract_translation = "pygettext3 -d video_language_detect -o locales/video_lang
 generate_translation = "bash -c 'for PO_FILE in locales/*/LC_MESSAGES/*.po; do MO_FILE=\"${PO_FILE/.po/.mo}\"; msgfmt -o \"$MO_FILE\" \"$PO_FILE\"; done'"
 
 [requires]
-python_version = "3.11"
+python_version = "3.12"
+2 −1		.gitignore
+40 −0		Pipfile
+2 −2		app.py
+8 −5		config/config.yml
+35 −0		pyproject.toml
+0 −14		requirements.txt
+8 −9		run.py
+2 −2		run_tests.sh
+9 −6		src/dataset_handler.py
+3 −2		src/dataset_loader.py
+69 −47		src/dataset_parser.py
+4 −1		src/schema.py
+17 −0		src/types.py
+33 −10		src/utils.py
+3 −3		tests/test_dataset_loader.py