-
Notifications
You must be signed in to change notification settings - Fork 4
Expand file tree
/
Copy pathMakefile
More file actions
172 lines (147 loc) · 8.41 KB
/
Copy pathMakefile
File metadata and controls
172 lines (147 loc) · 8.41 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
# =============================================================================
# Enterprise Benchmark — Docker image lifecycle
# =============================================================================
# Targets:
# make download Download benchmark data from HuggingFace
# make build Build the benchmark_environ image from source
# make test Smoke-test the locally built image (file checks + MCP handshakes)
# make validate Validate live MCP connections against running containers
# make validate-output FILES=<path> Validate output JSON files match submission schema
# make setup download → build → test → start → validate (first-time setup)
# make start Start all benchmark containers via docker compose
# make stop Stop and remove all benchmark containers
# make restart Stop and restart all containers
# make logs Tail logs for all running benchmark containers
# make clean Stop containers and remove the local Docker image
# make e2e Run end-to-end benchmark tests (requires HF_TOKEN + OPENAI_API_KEY)
# make e2e-quick Run e2e tests against already-running containers — OpenAI provider
# make e2e-quick-rits Run e2e tests against already-running containers — RITS provider
# make e2e-quick-watsonx Run e2e tests against already-running containers — WatsonX provider
# make e2e-quick-litellm Run e2e tests against already-running containers — LiteLLM provider
# make e2e-quick-anthropic Run e2e tests against already-running containers — Anthropic provider
#
# Override variables (optional):
# DOCKER=podman make build Force use of podman
# PYTHON=python make validate Force use of python instead of python3
# =============================================================================
IMAGE_NAME := benchmark_environ
DOCKERFILE := docker/Dockerfile.unified
# Auto-detect container runtime: prefer docker, fall back to podman
DOCKER ?= $(shell PATH=$$PATH:/usr/bin command -v docker 2>/dev/null || command -v podman 2>/dev/null || echo docker)
# Auto-detect Python interpreter: prefer the active virtualenv, then python3, then python
PYTHON ?= $(shell \
if [ -n "$$VIRTUAL_ENV" ] && [ -x "$$VIRTUAL_ENV/bin/python" ]; then \
echo "$$VIRTUAL_ENV/bin/python"; \
elif [ -x ".venv/bin/python" ]; then \
echo ".venv/bin/python"; \
else \
command -v python3 2>/dev/null | head -1 || command -v python 2>/dev/null | head -1 || echo python3; \
fi)
.PHONY: download build test validate validate-output setup start stop restart logs clean e2e \
e2e-quick e2e-quick-rits e2e-quick-watsonx e2e-quick-litellm e2e-quick-anthropic \
start-capability1 start-capability2 start-capability3 start-capability4
# ---------------------------------------------------------------------------
# Download benchmark data from HuggingFace (prompts for HF token if not set)
# ---------------------------------------------------------------------------
download:
$(PYTHON) benchmark_setup.py --download-data
# ---------------------------------------------------------------------------
# Build
# ---------------------------------------------------------------------------
build:
$(DOCKER) build -t $(IMAGE_NAME) -f $(DOCKERFILE) .
# ---------------------------------------------------------------------------
# Smoke-test (spins up a temp container, no volume mounts needed)
# Checks: required files exist, M3 REST /openapi.json, BPO + M3 MCP handshakes
# ---------------------------------------------------------------------------
test:
bash docker/smoke_test.sh $(IMAGE_NAME)
# ---------------------------------------------------------------------------
# Validate live MCP connections against the running benchmark containers
# Requires: make start (or benchmark_setup.py --start-containers) run first
# ---------------------------------------------------------------------------
validate:
$(PYTHON) benchmark/validate_clients.py
# ---------------------------------------------------------------------------
# Validate output files — check submission JSON files match the required schema
# Usage: make validate-output FILES="results/address.json results/hockey.json"
# make validate-output FILES=results/ (all .json files in a directory)
# ---------------------------------------------------------------------------
validate-output:
$(PYTHON) validate_output.py $(FILES)
# ---------------------------------------------------------------------------
# First-time setup: data + image + containers
# ---------------------------------------------------------------------------
setup: download build test start validate
@echo ""
@echo "Setup complete. Run '$(PYTHON) benchmark_runner.py --capability_id 3 --domain bpo' to start benchmarking."
# ---------------------------------------------------------------------------
# Container lifecycle
# ---------------------------------------------------------------------------
start:
$(DOCKER) compose up -d
stop:
$(DOCKER) compose down --remove-orphans
logs:
@for c in capability_1_bi_apis capability_2_dashboard_apis capability_3_multihop_reasoning capability_4_multiturn; do \
echo ""; \
echo "=== $$c ==="; \
$(DOCKER) logs --tail 20 $$c 2>/dev/null || echo " (not running)"; \
done
# ---------------------------------------------------------------------------
# End-to-end benchmark tests
# Requires: HF_TOKEN and OPENAI_API_KEY env vars set
# ---------------------------------------------------------------------------
e2e:
@if [ -z "$(HF_TOKEN)" ]; then echo "ERROR: HF_TOKEN is not set."; exit 1; fi
@if [ -z "$(OPENAI_API_KEY)" ]; then echo "ERROR: OPENAI_API_KEY is not set."; exit 1; fi
$(PYTHON) -m pytest tests/e2e/test_benchmark_e2e.py -v -s
# ---------------------------------------------------------------------------
# e2e-quick variants: run e2e tests against already-running containers
# Skips data download and container restart. Requires: make start first.
# Each target validates the required credentials for that provider.
# ---------------------------------------------------------------------------
e2e-quick:
@if [ -z "$(OPENAI_API_KEY)" ]; then echo "ERROR: OPENAI_API_KEY is not set."; exit 1; fi
E2E_SKIP_SETUP=1 $(PYTHON) -m pytest tests/e2e/test_benchmark_e2e.py -v -s
e2e-quick-rits:
@if [ -z "$(RITS_API_KEY)" ]; then echo "ERROR: RITS_API_KEY is not set."; exit 1; fi
E2E_SKIP_SETUP=1 $(PYTHON) -m pytest tests/e2e/test_benchmark_e2e.py -v -s
e2e-quick-watsonx:
@if [ -z "$(WATSONX_APIKEY)" ]; then echo "ERROR: WATSONX_APIKEY is not set."; exit 1; fi
@if [ -z "$(WATSONX_PROJECT_ID)" ] && [ -z "$(WATSONX_SPACE_ID)" ]; then \
echo "ERROR: WATSONX_PROJECT_ID or WATSONX_SPACE_ID must be set."; exit 1; fi
E2E_SKIP_SETUP=1 $(PYTHON) -m pytest tests/e2e/test_benchmark_e2e.py -v -s
e2e-quick-litellm:
@if [ -z "$(LITELLM_API_KEY)" ]; then echo "ERROR: LITELLM_API_KEY is not set."; exit 1; fi
@if [ -z "$(LITELLM_BASE_URL)" ]; then echo "ERROR: LITELLM_BASE_URL is not set."; exit 1; fi
E2E_SKIP_SETUP=1 E2E_PROVIDER=litellm $(PYTHON) -m pytest tests/e2e/test_benchmark_e2e.py -v -s
e2e-quick-anthropic:
@if [ -z "$(ANTHROPIC_API_KEY)" ]; then echo "ERROR: ANTHROPIC_API_KEY is not set."; exit 1; fi
E2E_SKIP_SETUP=1 E2E_PROVIDER=anthropic $(PYTHON) -m pytest tests/e2e/test_benchmark_e2e.py -v -s
# ---------------------------------------------------------------------------
# Start a single container (useful when one fails and you don't want to
# restart all four — uses docker compose so the image must already exist)
# ---------------------------------------------------------------------------
restart:
$(DOCKER) compose down --remove-orphans
$(DOCKER) compose up -d
start-capability1:
-$(DOCKER) rm -f capability_1_bi_apis
$(DOCKER) compose up -d capability_1_bi_apis
start-capability2:
-$(DOCKER) rm -f capability_2_dashboard_apis
$(DOCKER) compose up -d capability_2_dashboard_apis
start-capability3:
-$(DOCKER) rm -f capability_3_multihop_reasoning
$(DOCKER) compose up -d capability_3_multihop_reasoning
start-capability4:
-$(DOCKER) rm -f capability_4_multiturn
$(DOCKER) compose up -d capability_4_multiturn
# ---------------------------------------------------------------------------
# Clean — stop & remove containers, then remove the local image
# ---------------------------------------------------------------------------
clean:
$(DOCKER) compose down --remove-orphans
$(DOCKER) rmi -f $(IMAGE_NAME) 2>/dev/null || true
@echo "Removed containers and image '$(IMAGE_NAME)'."