diff --git a/.github/workflows/eval-job-tests.yaml b/.github/workflows/eval-job-tests.yaml
new file mode 100644
index 0000000..da81467
--- /dev/null
+++ b/.github/workflows/eval-job-tests.yaml
@@ -0,0 +1,50 @@
+# Copyright (c) Microsoft. All rights reserved
+
+name: Evaluation Jobs Tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Branch to run on'
+        required: false
+        default: ''
+
+permissions:
+  contents: read
+
+jobs:
+  compute-test:
+    runs-on: ubuntu-latest-8-cores
+
+    steps:
+      - name: Display system info
+        run: |
+          echo "=== CPU Info ==="
+          nproc
+          lscpu | grep -E '^(CPU\(s\)|Model name|CPU MHz)'
+          echo ""
+          echo "=== Memory Info ==="
+          free -h
+          echo ""
+          echo "=== Disk Info ==="
+          df -h /
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        run: uv python install 3.12
+
+      - name: Run test script
+        run: |
+          uv run --with "eval-recipes @ git+https://github.com/microsoft/eval-recipes@v0.0.29" python -c "
+          import time
+          import eval_recipes
+          print(f'Successfully imported eval_recipes version: {eval_recipes.__version__}')
+          print('Sleeping for 3 seconds...')
+          time.sleep(3)
+          print('Done!')
+          "