Skip to content

Commit 313342d

Browse files
JSONboredJSONbored
authored
feat(miner): add a Grafana dashboard for coding-agent-driver usage, cost, and attempt outcomes (#5646)
Adds grafana/dashboards/miner-usage.json: a single dashboard (a $provider template variable, not one dashboard per provider) showing per-provider attempt success/fail counts, total cost, and total tokens, reading only the redacted attempt_outcome_summary rows the AMS reporting export exposes via the AMS Attempt Log SQLite datasource -- no Prometheus scraping. Cross-references ai-usage.json (ORB) in both directions via a top-level `links` entry, matching the only existing dashboard-to- dashboard link convention in this repo (resource-hub.json's own links array). ai-usage.json's other panels/variables are untouched.
1 parent db0a34c commit 313342d

3 files changed

Lines changed: 536 additions & 1 deletion

File tree

‎grafana/dashboards/ai-usage.json‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,16 @@
7171
]
7272
},
7373
"annotations": { "list": [] },
74-
"links": [],
74+
"links": [
75+
{
76+
"title": "Miner usage (AMS)",
77+
"type": "link",
78+
"url": "/d/loopover-miner-usage/loopover-miner-usage",
79+
"icon": "external link",
80+
"targetBlank": false,
81+
"tooltip": "AMS's own coding-agent usage dashboard -- a separate local CLI (gittensory-miner) from every provider above, tracked on its own dashboard since it's a structurally different local-SQLite source, not another row in ai_usage_events."
82+
}
83+
],
7584
"panels": [
7685
{
7786
"id": 1,

‎grafana/dashboards/miner-usage.json‎

Lines changed: 266 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,266 @@
1+
{
2+
"uid": "loopover-miner-usage",
3+
"title": "LoopOver — Miner usage (AMS)",
4+
"tags": ["loopover", "miner", "ams"],
5+
"timezone": "browser",
6+
"schemaVersion": 39,
7+
"version": 1,
8+
"refresh": "30s",
9+
"time": {
10+
"from": "now-7d",
11+
"to": "now"
12+
},
13+
"description": "Per-provider coding-agent (claude-cli/codex-cli/agent-sdk) attempt outcomes, cost, and token usage for the Autonomous Miner System (AMS) -- a separate local CLI (gittensory-miner) from ORB, this dashboard's own AI usage. Reads the redacted attempt_outcome_summary rows exported from AMS's local attempt-log ledger (see packages/gittensory-miner/docs/observability.md) via the AMS Attempt Log SQLite datasource -- never the miner's live GITTENSORY_MINER_CONFIG_DIR ledgers, and never scraped from a Prometheus metrics path. All three coding-agent providers share one dashboard (a $provider template variable, not one dashboard per provider) since AMS's local ledger has far lower data volume than ORB's cloud reviewers. tokens_used is honestly 0/NULL until some future coding-agent driver reports real token usage (#5395) -- never fabricated.",
14+
"annotations": {
15+
"list": []
16+
},
17+
"links": [
18+
{
19+
"title": "AI usage (ORB)",
20+
"type": "link",
21+
"url": "/d/loopover-ai-usage/loopover-ai-usage",
22+
"icon": "external link",
23+
"targetBlank": false,
24+
"tooltip": "ORB's own AI-usage dashboard (cloud review providers) -- a dual-role operator running both ORB and AMS on one box wants this one for cloud-review spend, this dashboard for the local miner's coding-agent spend."
25+
}
26+
],
27+
"templating": {
28+
"list": [
29+
{
30+
"name": "provider",
31+
"label": "Provider",
32+
"type": "query",
33+
"datasource": {
34+
"type": "frser-sqlite-datasource",
35+
"uid": "ams-attempt-log"
36+
},
37+
"query": {
38+
"queryType": "table",
39+
"queryText": "SELECT DISTINCT provider FROM attempt_log_events WHERE event_type = 'attempt_outcome_summary' AND provider IS NOT NULL ORDER BY provider",
40+
"rawQueryText": "SELECT DISTINCT provider FROM attempt_log_events WHERE event_type = 'attempt_outcome_summary' AND provider IS NOT NULL ORDER BY provider"
41+
},
42+
"includeAll": true,
43+
"allValue": "$__all",
44+
"multi": false,
45+
"current": {
46+
"text": "All",
47+
"value": "$__all"
48+
},
49+
"refresh": 2,
50+
"sort": 1
51+
}
52+
]
53+
},
54+
"panels": [
55+
{
56+
"id": 1,
57+
"type": "row",
58+
"title": "Summary (AMS attempt outcomes, filtered)",
59+
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }
60+
},
61+
{
62+
"id": 2,
63+
"type": "stat",
64+
"title": "Total attempts",
65+
"description": "Every attempt_outcome_summary row (one per completed attempt) in the selected time range, filtered by Provider.",
66+
"gridPos": { "h": 5, "w": 6, "x": 0, "y": 1 },
67+
"datasource": { "type": "frser-sqlite-datasource", "uid": "ams-attempt-log" },
68+
"fieldConfig": {
69+
"defaults": {
70+
"unit": "short",
71+
"color": { "mode": "fixed", "fixedColor": "blue" }
72+
}
73+
},
74+
"options": {
75+
"reduceOptions": { "calcs": ["lastNotNull"] },
76+
"colorMode": "value",
77+
"graphMode": "none",
78+
"justifyMode": "center"
79+
},
80+
"targets": [
81+
{
82+
"refId": "A",
83+
"queryType": "table",
84+
"queryText": "SELECT count(*) AS attempts FROM attempt_log_events WHERE event_type = 'attempt_outcome_summary' AND (${provider:sqlstring} = '$__all' OR provider = ${provider:sqlstring}) AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds}",
85+
"rawQueryText": "SELECT count(*) AS attempts FROM attempt_log_events WHERE event_type = 'attempt_outcome_summary' AND (${provider:sqlstring} = '$__all' OR provider = ${provider:sqlstring}) AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds}"
86+
}
87+
]
88+
},
89+
{
90+
"id": 3,
91+
"type": "stat",
92+
"title": "Succeeded",
93+
"description": "attempt_outcome_summary rows whose action_class is attempt_submitted -- every other action_class (attempt_abandon/attempt_stale/attempt_blocked/attempt_governed) counts as not-succeeded.",
94+
"gridPos": { "h": 5, "w": 6, "x": 6, "y": 1 },
95+
"datasource": { "type": "frser-sqlite-datasource", "uid": "ams-attempt-log" },
96+
"fieldConfig": {
97+
"defaults": {
98+
"unit": "short",
99+
"color": { "mode": "fixed", "fixedColor": "green" }
100+
}
101+
},
102+
"options": {
103+
"reduceOptions": { "calcs": ["lastNotNull"] },
104+
"colorMode": "value",
105+
"graphMode": "none",
106+
"justifyMode": "center"
107+
},
108+
"targets": [
109+
{
110+
"refId": "A",
111+
"queryType": "table",
112+
"queryText": "SELECT count(*) AS succeeded FROM attempt_log_events WHERE event_type = 'attempt_outcome_summary' AND action_class = 'attempt_submitted' AND (${provider:sqlstring} = '$__all' OR provider = ${provider:sqlstring}) AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds}",
113+
"rawQueryText": "SELECT count(*) AS succeeded FROM attempt_log_events WHERE event_type = 'attempt_outcome_summary' AND action_class = 'attempt_submitted' AND (${provider:sqlstring} = '$__all' OR provider = ${provider:sqlstring}) AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds}"
114+
}
115+
]
116+
},
117+
{
118+
"id": 4,
119+
"type": "stat",
120+
"title": "Total cost",
121+
"description": "Real accumulated cost_usd, summed. 0 for a provider that never reports a cost signal (CLI-subprocess providers today) -- an honest running total, never fabricated.",
122+
"gridPos": { "h": 5, "w": 6, "x": 12, "y": 1 },
123+
"datasource": { "type": "frser-sqlite-datasource", "uid": "ams-attempt-log" },
124+
"fieldConfig": {
125+
"defaults": {
126+
"unit": "currencyUSD",
127+
"decimals": 4,
128+
"color": { "mode": "fixed", "fixedColor": "green" }
129+
}
130+
},
131+
"options": {
132+
"reduceOptions": { "calcs": ["lastNotNull"] },
133+
"colorMode": "value",
134+
"graphMode": "none",
135+
"justifyMode": "center"
136+
},
137+
"targets": [
138+
{
139+
"refId": "A",
140+
"queryType": "table",
141+
"queryText": "SELECT coalesce(sum(cost_usd), 0) AS cost_usd FROM attempt_log_events WHERE event_type = 'attempt_outcome_summary' AND (${provider:sqlstring} = '$__all' OR provider = ${provider:sqlstring}) AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds}",
142+
"rawQueryText": "SELECT coalesce(sum(cost_usd), 0) AS cost_usd FROM attempt_log_events WHERE event_type = 'attempt_outcome_summary' AND (${provider:sqlstring} = '$__all' OR provider = ${provider:sqlstring}) AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds}"
143+
}
144+
]
145+
},
146+
{
147+
"id": 5,
148+
"type": "stat",
149+
"title": "Total tokens",
150+
"description": "Always 0 today: no coding-agent driver reports real token usage yet (#5395) -- an honest gap, not a fabricated number. Will populate automatically once a driver reports real usage; no dashboard change needed then.",
151+
"gridPos": { "h": 5, "w": 6, "x": 18, "y": 1 },
152+
"datasource": { "type": "frser-sqlite-datasource", "uid": "ams-attempt-log" },
153+
"fieldConfig": {
154+
"defaults": {
155+
"unit": "short",
156+
"color": { "mode": "fixed", "fixedColor": "purple" }
157+
}
158+
},
159+
"options": {
160+
"reduceOptions": { "calcs": ["lastNotNull"] },
161+
"colorMode": "value",
162+
"graphMode": "none",
163+
"justifyMode": "center"
164+
},
165+
"targets": [
166+
{
167+
"refId": "A",
168+
"queryType": "table",
169+
"queryText": "SELECT coalesce(sum(tokens_used), 0) AS tokens FROM attempt_log_events WHERE event_type = 'attempt_outcome_summary' AND (${provider:sqlstring} = '$__all' OR provider = ${provider:sqlstring}) AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds}",
170+
"rawQueryText": "SELECT coalesce(sum(tokens_used), 0) AS tokens FROM attempt_log_events WHERE event_type = 'attempt_outcome_summary' AND (${provider:sqlstring} = '$__all' OR provider = ${provider:sqlstring}) AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds}"
171+
}
172+
]
173+
},
174+
{
175+
"id": 6,
176+
"type": "row",
177+
"title": "By provider",
178+
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }
179+
},
180+
{
181+
"id": 7,
182+
"type": "table",
183+
"title": "Usage by provider",
184+
"description": "One row per provider: attempt success/fail counts, total cost, and total tokens -- everything #5185 asks this dashboard to show, in one place.",
185+
"gridPos": { "h": 8, "w": 16, "x": 0, "y": 7 },
186+
"datasource": { "type": "frser-sqlite-datasource", "uid": "ams-attempt-log" },
187+
"options": {
188+
"showHeader": true,
189+
"cellHeight": "sm",
190+
"sortBy": [{ "displayName": "total", "desc": true }]
191+
},
192+
"fieldConfig": {
193+
"defaults": {
194+
"custom": { "filterable": true }
195+
}
196+
},
197+
"targets": [
198+
{
199+
"refId": "A",
200+
"queryType": "table",
201+
"queryText": "SELECT provider, sum(CASE WHEN action_class = 'attempt_submitted' THEN 1 ELSE 0 END) AS succeeded, sum(CASE WHEN action_class != 'attempt_submitted' THEN 1 ELSE 0 END) AS failed, count(*) AS total, coalesce(sum(cost_usd), 0) AS cost_usd, coalesce(sum(tokens_used), 0) AS tokens_used FROM attempt_log_events WHERE event_type = 'attempt_outcome_summary' AND (${provider:sqlstring} = '$__all' OR provider = ${provider:sqlstring}) AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds} GROUP BY provider ORDER BY total DESC",
202+
"rawQueryText": "SELECT provider, sum(CASE WHEN action_class = 'attempt_submitted' THEN 1 ELSE 0 END) AS succeeded, sum(CASE WHEN action_class != 'attempt_submitted' THEN 1 ELSE 0 END) AS failed, count(*) AS total, coalesce(sum(cost_usd), 0) AS cost_usd, coalesce(sum(tokens_used), 0) AS tokens_used FROM attempt_log_events WHERE event_type = 'attempt_outcome_summary' AND (${provider:sqlstring} = '$__all' OR provider = ${provider:sqlstring}) AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds} GROUP BY provider ORDER BY total DESC"
203+
}
204+
]
205+
},
206+
{
207+
"id": 8,
208+
"type": "piechart",
209+
"title": "Attempts by provider",
210+
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 7 },
211+
"datasource": { "type": "frser-sqlite-datasource", "uid": "ams-attempt-log" },
212+
"fieldConfig": {
213+
"defaults": { "unit": "short" }
214+
},
215+
"options": {
216+
"pieType": "donut",
217+
"legend": {
218+
"showLegend": true,
219+
"placement": "right",
220+
"values": ["percent", "value"]
221+
},
222+
"reduceOptions": { "calcs": ["lastNotNull"] },
223+
"displayLabels": ["percent"]
224+
},
225+
"targets": [
226+
{
227+
"refId": "A",
228+
"queryType": "table",
229+
"queryText": "SELECT provider, count(*) AS attempts FROM attempt_log_events WHERE event_type = 'attempt_outcome_summary' AND (${provider:sqlstring} = '$__all' OR provider = ${provider:sqlstring}) AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds} GROUP BY provider ORDER BY attempts DESC",
230+
"rawQueryText": "SELECT provider, count(*) AS attempts FROM attempt_log_events WHERE event_type = 'attempt_outcome_summary' AND (${provider:sqlstring} = '$__all' OR provider = ${provider:sqlstring}) AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds} GROUP BY provider ORDER BY attempts DESC"
231+
}
232+
]
233+
},
234+
{
235+
"id": 9,
236+
"type": "row",
237+
"title": "Recent activity",
238+
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 }
239+
},
240+
{
241+
"id": 10,
242+
"type": "table",
243+
"title": "Recent attempt outcomes",
244+
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 16 },
245+
"datasource": { "type": "frser-sqlite-datasource", "uid": "ams-attempt-log" },
246+
"options": {
247+
"showHeader": true,
248+
"cellHeight": "sm",
249+
"sortBy": [{ "displayName": "created_at", "desc": true }]
250+
},
251+
"fieldConfig": {
252+
"defaults": {
253+
"custom": { "filterable": true }
254+
}
255+
},
256+
"targets": [
257+
{
258+
"refId": "A",
259+
"queryType": "table",
260+
"queryText": "SELECT created_at, attempt_id, provider, action_class, mode, coalesce(cost_usd, 0) AS cost_usd, tokens_used FROM attempt_log_events WHERE event_type = 'attempt_outcome_summary' AND (${provider:sqlstring} = '$__all' OR provider = ${provider:sqlstring}) AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds} ORDER BY created_at DESC LIMIT 100",
261+
"rawQueryText": "SELECT created_at, attempt_id, provider, action_class, mode, coalesce(cost_usd, 0) AS cost_usd, tokens_used FROM attempt_log_events WHERE event_type = 'attempt_outcome_summary' AND (${provider:sqlstring} = '$__all' OR provider = ${provider:sqlstring}) AND unixepoch(created_at) >= ${__from:date:seconds} AND unixepoch(created_at) < ${__to:date:seconds} ORDER BY created_at DESC LIMIT 100"
262+
}
263+
]
264+
}
265+
]
266+
}

0 commit comments

Comments
 (0)