[QA] Vizro AI dashboard tests improvements (#935)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
mckinsey · Feb 4, 2025 · 2426ef6 · 2426ef6
1 parent a4b245f
commit 2426ef6
Show file tree

Hide file tree

Showing 8 changed files with 404 additions and 252 deletions.
diff --git a/.github/workflows/test-e2e-dashboard-vizro-ai.yml b/.github/workflows/test-e2e-dashboard-vizro-ai.yml
@@ -5,7 +5,9 @@ defaults:
     working-directory: vizro-ai
 
 on:
-  workflow_dispatch:
+  schedule:
+    - cron: "30 10 * * 1" # run every Monday at 10:30 UTC
+  workflow_dispatch: # helps to run this job manually from GitHub Actions UI
 
 env:
   PYTHONUNBUFFERED: 1
@@ -20,17 +22,8 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - python-version: "3.9"
-            hatch-env: all.py3.9
-          - python-version: "3.10"
-            hatch-env: all.py3.10
-          - python-version: "3.11"
-            hatch-env: all.py3.11
           - python-version: "3.12"
             hatch-env: all.py3.12
-          - python-version: "3.9"
-            hatch-env: lower-bounds
-            label: lower bounds
 
     steps:
       - uses: actions/checkout@v4
@@ -46,17 +39,8 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - python-version: "3.9"
-            hatch-env: all.py3.9
-          - python-version: "3.10"
-            hatch-env: all.py3.10
-          - python-version: "3.11"
-            hatch-env: all.py3.11
           - python-version: "3.12"
             hatch-env: all.py3.12
-          - python-version: "3.9"
-            hatch-env: lower-bounds
-            label: lower bounds
 
     steps:
       - uses: actions/checkout@v4
@@ -77,21 +61,12 @@ jobs:
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          ANTHROPIC_BASE_URL: ${{ secrets.ANTHROPIC_BASE_URL }}
           VIZRO_TYPE: pypi
           BRANCH: ${{ github.head_ref }}
           PYTHON_VERSION: ${{ matrix.config.python-version }}
 
-      - name: Run vizro-ai e2e dashboard tests with local vizro
-        run: |
-          hatch run ${{ matrix.config.hatch-env }}:pip install ../vizro-core
-          hatch run ${{ matrix.config.hatch-env }}:test-e2e-dashboard
-        env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }}
-          VIZRO_TYPE: local
-          BRANCH: ${{ github.head_ref }}
-          PYTHON_VERSION: ${{ matrix.config.python-version }}
-
       - name: Send custom JSON data to Slack
         id: slack
         uses: slackapi/[email protected]
@@ -116,6 +91,7 @@ jobs:
   test-e2e-dashboard-vizro-ai-report:
     needs: test-e2e-dashboard-vizro-ai
     runs-on: ubuntu-latest
+    if: always()
     steps:
       - uses: actions/checkout@v4
 
@@ -131,7 +107,8 @@ jobs:
       - name: Create one csv report
         run: |
           cd /home/runner/work/vizro/vizro/
-          head -n 1 Report-3.11-/report_model_gpt-4o-mini_pypi.csv > report-aggregated-${{ steps.date.outputs.date }}.csv && tail -n+2 -q */*.csv >> report-aggregated-${{ steps.date.outputs.date }}.csv
+          ls */*.csv | head -n1 | xargs head -n1 > report-aggregated-${{ steps.date.outputs.date }}.csv && tail -n+2 -q */*.csv >> report-aggregated-${{ steps.date.outputs.date }}.csv
+          # replace all timestamps in aggregated report to current date
           gawk -F, -i inplace 'FNR>1 {$1="${{ steps.date.outputs.date }}"} {print}' OFS=, report-aggregated-${{ steps.date.outputs.date }}.csv
 
       - name: Report artifacts

diff --git a/vizro-ai/changelog.d/20241224_135602_alexey_snigir_score_tests_improvements.md b/vizro-ai/changelog.d/20241224_135602_alexey_snigir_score_tests_improvements.md
@@ -0,0 +1,48 @@
+<!--
+A new scriv changelog fragment.
+
+Uncomment the section that is right (remove the HTML comment wrapper).
+-->
+
+<!--
+### Highlights ✨
+
+- A bullet item for the Highlights ✨ category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX. ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Removed
+
+- A bullet item for the Removed category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX. ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Added
+
+- A bullet item for the Added category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX. ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Changed
+
+- A bullet item for the Changed category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX. ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Deprecated
+
+- A bullet item for the Deprecated category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX. ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Fixed
+
+- A bullet item for the Fixed category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX. ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Security
+
+- A bullet item for the Security category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX. ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
diff --git a/vizro-ai/hatch.toml b/vizro-ai/hatch.toml
@@ -50,6 +50,7 @@ prep-release = [
 ]
 pypath = "hatch run python -c 'import sys; print(sys.executable)'"
 test = "pytest tests {args}"
+# to run dashboard and plot tests please configure base url and api key for the selected model
 test-e2e-dashboard = "pytest -vs tests/e2e/test_dashboard.py --headless {args}"
 test-e2e-plot = "pytest -vs --reruns 1 tests/e2e/test_plot.py --headless {args}"
 test-unit = "pytest tests/unit {args}"

diff --git a/vizro-ai/pyproject.toml b/vizro-ai/pyproject.toml
@@ -73,7 +73,8 @@ filterwarnings = [
   "ignore:Exception in thread",
   # Ignore deprecation warning as it comes from the plotly default template. In our templates `vizro_dark` and
   # `vizro_light`, we do not use mapbox anymore, see: https://github.com/mckinsey/vizro/pull/974
-  "ignore:.*scattermapbox.*is deprecated.*Use.*scattermap.*instead:DeprecationWarning"
+  "ignore:.*scattermapbox.*is deprecated.*Use.*scattermap.*instead:DeprecationWarning",
+  "ignore:CapturedCallable function is excluded from the schema"
 ]
 pythonpath = ["../tools/tests"]
 

diff --git a/vizro-ai/tests/e2e/data_classes.py b/vizro-ai/tests/e2e/data_classes.py
@@ -0,0 +1,12 @@
+from dataclasses import dataclass
+from typing import Literal
+
+
+@dataclass
+class Component:
+    type: Literal["ag_grid", "card", "graph"]
+
+
+@dataclass
+class Control:
+    type: Literal["filter", "parameter"]
diff --git a/vizro-ai/tests/e2e/prompts.py b/vizro-ai/tests/e2e/prompts.py
@@ -0,0 +1,201 @@
+from data_classes import Component, Control
+
+easy_prompt = {
+    "tier_type": "easy",
+    "prompt_name": "one page + table + card + chart + 2 filters",
+    "prompt_text": """
+    I need a page with 1 table, 1 card and 1 chart.
+    The table shows the tech companies stock data.
+    The card says 'The Gapminder dataset provides historical data on countries' development indicators.'
+    The chart is the scatter plot which uses gapminder dataframe
+    and showing life expectancy vs. GDP per capita by country.
+    Life expectancy on the y axis, GDP per capita on the x axis, and colored by continent.
+
+    The layout uses a grid of 2 columns and 3 rows.
+    The first row contains card
+    The second row contains chart
+    The third row contains table
+
+    Add a filter to filter the scatter plot by continent.
+    Add a second filter to filter the table by companies.
+    """,
+    "expected_config": {
+        "pages": [
+            {
+                "components": [
+                    Component(type="ag_grid"),
+                    Component(type="card"),
+                    Component(type="graph"),
+                ],
+                "controls": [
+                    Control(type="filter"),
+                    Control(type="filter"),
+                ],
+            },
+        ],
+    },
+}
+
+medium_prompt = {
+    "tier_type": "medium",
+    "prompt_name": "4 pages with supported prompt instructions",
+    "prompt_text": """
+    <Page 1>
+    I need a page with 1 table and 1 line chart.
+    The chart shows the stock price trends of GOOG and AAPL.
+    The table shows the stock prices data details.
+
+    <Page 2>
+    I need a second page showing 3 cards and 4 charts.
+    The cards says 'The Gapminder dataset provides historical data on countries' development indicators.'
+    The charts are the scatter plots showing GDP per capita vs. life expectancy.
+    GDP per capita on the x axis, life expectancy on the y axis, and colored by continent.
+    Layout the cards on the left and the chart on the right.
+    Add a filter to filter the scatter plots by continent.
+    Add a second filter to filter the charts by year.
+
+    <Page 3>
+    This page displays the tips dataset. use four different charts to show data
+    distributions. one chart should be a bar chart. the other should be a scatter plot.
+    next chart should be a line chart. last one should be an area plot.
+    first and second charts are on the left and the third and fourth charts are on the right.
+    Add a filter to filter data in every plot by smoker.
+
+    <Page 4>
+    Create 3 cards on this page:
+    1. The first card on top says "This page combines data from various sources
+     including tips, stock prices, and global indicators."
+    2. The second card says "Insights from Gapminder dataset."
+    3. The third card says "Stock price trends over time."
+
+    Layout these 3 cards in this way:
+    create a grid with 3 columns and 2 rows.
+    Row 1: The first row has three columns:
+    - The first column is empty.
+    - The second and third columns span the area for card 1.
+
+    Row 2: The second row also has three columns:
+    - The first column is empty.
+    - The second column is occupied by the area for card 2.
+    - The third column is occupied by the area for card 3.
+    """,
+    "expected_config": {
+        "pages": [
+            {
+                "components": [
+                    Component(type="ag_grid"),
+                    Component(type="graph"),
+                ],
+                "controls": [],
+            },
+            {
+                "components": [
+                    Component(type="card"),
+                    Component(type="graph"),
+                ],
+                "controls": [
+                    Control(type="filter"),
+                    Control(type="filter"),
+                ],
+            },
+            {
+                "components": [
+                    Component(type="graph"),
+                    Component(type="graph"),
+                ],
+                "controls": [
+                    Control(type="filter"),
+                ],
+            },
+            {
+                "components": [
+                    Component(type="card"),
+                    Component(type="card"),
+                    Component(type="card"),
+                ],
+                "controls": [],
+            },
+        ],
+    },
+}
+
+complex_prompt = {
+    "tier_type": "complex",
+    "prompt_name": "4 pages with mix of supported and unsupported prompt instructions",
+    "prompt_text": """
+    <Page 1>
+    Show me 1 table on the first page that shows tips and sorted by day
+    Using export button I want to export data to csv
+    Add filters by bill and by tip amount using slider
+
+    <Page 2>
+    Second page should contain kpi cards with population trends and
+    two popular charts that display population per capita vs. continent.
+    Filter charts by GDP using dropdown.
+    Align kpi cards in one row and charts in different.
+    Both charts should be in tabs.
+
+    <Page 3>
+    Third page should contain 6 charts showing stocks.
+    Each should have separate filter by date.
+    Filter types should include dropdown, datepicker, slider, checklist and radio items.
+    Add parameter for any chart.
+
+    <Page 4>
+    Fourth page contains chart with wind data.
+    Table with population per capita data.
+    Two more charts with stocks and tips representations.
+    Align table beautifully relative to the charts.
+    Every chart should have 2 filters.
+    Table should have 1 filter.
+    """,
+    "expected_config": {
+        "pages": [
+            {
+                "components": [
+                    Component(type="ag_grid"),
+                ],
+                "controls": [Control(type="filter"), Control(type="filter")],
+            },
+            {
+                "components": [Component(type="graph"), Component(type="graph")],
+                "controls": [Control(type="filter")],
+            },
+            {
+                "components": [
+                    Component(type="graph"),
+                    Component(type="graph"),
+                    Component(type="graph"),
+                    Component(type="graph"),
+                    Component(type="graph"),
+                    Component(type="graph"),
+                ],
+                "controls": [
+                    Control(type="filter"),
+                    Control(type="filter"),
+                    Control(type="filter"),
+                    Control(type="filter"),
+                    Control(type="filter"),
+                    Control(type="filter"),
+                ],
+            },
+            {
+                "components": [
+                    Component(type="ag_grid"),
+                    Component(type="graph"),
+                    Component(type="graph"),
+                    Component(type="graph"),
+                ],
+                "controls": [
+                    Control(type="filter"),
+                    Control(type="filter"),
+                    Control(type="filter"),
+                    Control(type="filter"),
+                    Control(type="filter"),
+                    Control(type="filter"),
+                    Control(type="filter"),
+                ],
+            },
+        ],
+    },
+}
diff --git a/vizro-ai/tests/e2e/pytest.ini b/vizro-ai/tests/e2e/pytest.ini