Hotfix/benchmarking redesign

.github/workflows/continuous_integration.yml

@@ -3,11 +3,11 @@ on: [push, pull_request]
 
 jobs:
       lint_and_format:
-            runs-on: ubuntu-latest 
+            runs-on: ubuntu-latest
             steps:
                   - name: Fetch code
                     uses: actions/checkout@v3
-                  # - name: Check linting 
+                  # - name: Check linting
                   #   uses: astral-sh/ruff-action@v1
                   #   with:
                   #       args: "check"
@@ -19,27 +19,27 @@ jobs:
           needs: lint_and_format
           steps:
                 - name: Fetch code
-                  uses: actions/checkout@v3 
+                  uses: actions/checkout@v3
                 - name: "Install dependecies"
                   run: pip install bandit
                 - name: "Check with bandit"
                   run: bandit -r src/
       tests:
-        runs-on: ubuntu-latest 
+        runs-on: ubuntu-latest
         needs: lint_and_format
         strategy:
           matrix:
             python-version: ['3.10', '3.11']
         steps:
           - name: Fetch code
-            uses: actions/checkout@v3 
-          
+            uses: actions/checkout@v3
+
           - name: Set up python ${{ matrix.python-version }}
             uses: actions/setup-python@v4
             with:
               python-version: ${{ matrix.python-version }}
               cache: 'pip'
-          
+
           - name: Cache pip dependencies
             uses: actions/cache@v3
             with:
@@ -48,27 +48,20 @@ jobs:
               restore-keys: |
                 ${{ runner.os }}-pip-${{ matrix.python-version }}-
                 ${{ runner.os }}-pip-
-
-          - name: "Install graphviz"
-            run: |
-              sudo apt-get update
-              sudo apt-get install -y graphviz graphviz-dev
-            shell: bash
-
+
           - name: "Create virtual environment"
             run: |
               python -m venv .venv
             shell: bash
-          
+
           - name: "Install dependencies"
             run: |
               source .venv/bin/activate
               pip install -e ".[dev]"
-              pip install pygraphviz
             shell: bash
-          
+
           - name: Run python tests with coverage
-            env: 
+            env:
                 OPEN_ROUTER_API_KEY: ${{ secrets.OPEN_ROUTER_API_KEY }}
             run: |
               source .venv/bin/activate
@@ -79,13 +72,13 @@ jobs:
                 --cov-report=term:skip-covered \
                 tests/
             shell: bash
-          
+
           - name: "Show coverage summary"
             if: always()
             run: |
                 echo "=== Coverage Summary ==="
                 coverage report || echo "Coverage report not available"
-          
+
           - name: Upload results of test coverage
             uses: actions/upload-artifact@v4
             if: always()

.gitignore

@@ -41,8 +41,11 @@ logs/
 # Temporary files
 *.tmp
 
-agent_run_data/
-im-working.txt
+uv.lock
 
-# examples/**/agent_execution_results/
-*.db
+#Ignore vscode AI rules
+.github/instructions/codacy.instructions.md
+.venv/
+
+
+tests/benchmark/results/

Makefile

@@ -1,84 +1,50 @@
 .PHONY: install format lint docs tests examples-sequential-research examples-sequential-financial examples-supervisor examples-supervisor-sales examples-supervisor-product-research examples-basic examples-parallel-minimal examples-parallel-llm-router examples-mcp-sales-analytics
+
 .DEFAULT_GOAL:= help
+SHELL := /bin/bash
 
 
-# VARIABLES
+# VARIABLES FOR PRETTY PRINT
 RED = \033[31m
 GREEN = \033[32m
 YELLOW = \033[33m
 BLUE = \033[34m
 RESET = \033[0m
 
+# VARIABLES, GENERAL
 VENV_PATH =  ./.venv
 VENV_ACTIVATE = source $(VENV_PATH)/bin/activate
 
-help: 
-	@echo "$(BLUE)Available commands:$(RESET)"
-	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "  $(GREEN)%-15s$(RESET) %s\n", $$1, $$2}' $(MAKEFILE_LIST)
-
-# ============
-# =  SET-UP  =
-# ============
-
-install: #Install dependencies with pip => install graphviz 
-	$(VENV_ACTIVATE) && pip install -e "."
-	$(VENV_ACTIVATE) &&  pre-commit install
-	$(VENV_ACTIVATE) && pip install --config-settings="--global-option=build_ext" \
-		--config-settings="--global-option=-I$$(brew --prefix graphviz)/include/" \
-		--config-settings="--global-option=-L$$(brew --prefix graphviz)/lib/" \
-		pygraphviz
-install-dev: 
-	$(VENV_ACTIVATE) && pip install -e ".[dev]"
-	$(VENV_ACTIVATE) &&  pre-commit install
-	$(VENV_ACTIVATE) && pip install --config-settings="--global-option=build_ext" \
-		--config-settings="--global-option=-I$$(brew --prefix graphviz)/include/" \
-		--config-settings="--global-option=-L$$(brew --prefix graphviz)/lib/" \
-		pygraphviz
-
 
-# ============
-# =   CI/CD  =
-# ============
-format:
-	$(VENV_ACTIVATE) && ruff format .
-
-lint:
-	$(VENV_ACTIVATE) && ruff check --fix .
-docs:	## Renders docs locally
-	$(VENV_ACTIVATE) &&  mkdocs serve
-tests: ## Run tests
-	$(VENV_ACTIVATE) && pytest -vv -s \
-                  --cov=flowgentic \
-                  --cov-report=html:coverage_report \
-                  --cov-report=term-missing \
-                  --cov-report=term:skip-covered \
-                  tests/
-tests-units: 
-	$(VENV_ACTIVATE) && pytest -vv -s \
-				--cov=flowgentic \
-				--cov-report=html:coverage_report \
-				--cov-report=term-missing \
-				--cov-report=term:skip-covered \
-				tests/unit
-
-# ============
-# = EXAMPLES =
-# ============
-
-## LangGraph
-### Design Patterns 
-examples-chatbot-toy: 
-	$(VENV_ACTIVATE) && python3 -m examples.langgraph-integration.design_patterns.chatbot.toy
-examples-sequential-research:
-	$(VENV_ACTIVATE) && python3 -m examples.langgraph-integration.design_patterns.sequential.research_agent.main
-examples-supervisor-toy: 
-	$(VENV_ACTIVATE) && python3 -m examples.langgraph-integration.design_patterns.supervisor.toy.main
-#### Memory
-examples-sequential-research-memory:
-	$(VENV_ACTIVATE) && python3 -m examples.langgraph-integration.design_patterns.sequential.research_agent_memory.main
-### Services
-examples-services-intermittent-task: 
-	$(VENV_ACTIVATE) && python3 -m examples.langgraph-integration.service-task.service-intermittent
-### Miscellaneous
-examples-runtime-graph:
-	$(VENV_ACTIVATE) && python3 -m examples.langgraph-integration.miscellaneous.runtime-graph-creation
+# 1) Set-up
+help: ## Show this help message
+	@printf "$(BLUE) Available commands: $(RESET)\n"
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "  $(GREEN)%-15s$(RESET) %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+install: ## Install core dev dependencies
+	uv venv $(VENV_PATH) --python 3.10
+	uv pip install -e ".[dev]"
+install-benchmark: ## Install dev + benchmark dependencies (langgraph + asyncflow)
+	uv venv $(VENV_PATH) --python 3.10
+	uv pip install -e ".[langgraph,asyncflow,dev]"
+install-all: ## Install all dependencies (frameworks + runtimes + dev)
+	uv venv $(VENV_PATH) --python 3.10
+	uv pip install -e ".[all,dev]"
+
+# 2) Examples
+examples-lg-asyncflow:  ## Langgraph + Asyncflow
+	$(VENV_ACTIVATE) && python3 -m examples.langgraph_asyncflow
+examples-ca-asyncflow:  ## CrewAI + Asyncfow
+	$(VENV_ACTIVATE) && python3 -m examples.crewai_asyncflow
+examples-ag-asyncflow:  ## AutoGen + Asyncfow
+	$(VENV_ACTIVATE) && python3 -m examples.autogen_asyncflow
+examples-lg-parsl:  ## AutoGen + Parsl
+	$(VENV_ACTIVATE) && python3 -m examples.langgraph_parsl
+examples-ag-parsl:  ## AutoGen + Parsl
+	$(VENV_ACTIVATE) && python3 -m examples.autogen_parsl
+
+# 3) Benchmark
+benchmark: ## Run the experiments in the benchmarking
+	$(VENV_ACTIVATE) && python3 -m tests.benchmark.data_generation.run_experiments
+
+benchmark-multiple-workloads: ## Run the full benchmark suite with multiple configs
+	$(VENV_ACTIVATE) && python3 -m tests.benchmark.run_benchmark_suite