fix(demo): cleanup and fix some features of ShiftIQ demo

Makefile

@@ -3,6 +3,7 @@ BACKEND_PORT ?= 8040
 FRONTEND_PORT ?= 3040
 DOMAIN ?= reddash
 EXTRA_ENV_FILE ?=
+LOAD_DATA_ARGS ?=
 
 .PHONY: help install backend-install frontend-install dev backend frontend \
 	generate-data generate-models load-data setup-surface validate-domain smoke-domain create-domain flush-redis reset \
@@ -42,7 +43,7 @@ generate-data:
 	@uv run python scripts/generate_data.py --domain $(DOMAIN)
 
 load-data:
-	@uv run python scripts/load_data.py --domain $(DOMAIN)
+	@uv run python scripts/load_data.py --domain $(DOMAIN) $(LOAD_DATA_ARGS)
 
 setup-surface:
 	@uv run python scripts/setup_surface.py --domain $(DOMAIN)
@@ -81,7 +82,7 @@ cache-domain-price-csvs:
 	uv run python domains/$(DOMAIN)/fetch_price_csvs.py --years 5
 
 backend:
-	@uv run uvicorn backend.app.main:app --reload --host $(BACKEND_HOST) --port $(BACKEND_PORT)
+	@DEMO_DOMAIN=$(DOMAIN) uv run uvicorn backend.app.main:app --reload --host $(BACKEND_HOST) --port $(BACKEND_PORT)
 
 frontend:
 	@cd frontend && npm run dev -- --host 0.0.0.0 --port $(FRONTEND_PORT)

backend/app/domain_events.py

@@ -5,11 +5,14 @@
 from datetime import datetime, timezone
 from typing import Any
 
+from redis.exceptions import TimeoutError as RedisTimeoutError
+
 from backend.app.core.domain_contract import DomainPack
 from backend.app.redis_connection import create_async_redis_client, create_redis_client
 from backend.app.settings import Settings
 
 DOMAIN_EVENT_STREAM_SUFFIX = "stream:events"
+DOMAIN_EVENT_BLOCK_MS = 5000
 
 
 def domain_event_stream_key(domain: DomainPack) -> str:
@@ -159,7 +162,11 @@ async def stream_domain_events(
                     last_id = newest_history_id
 
         while True:
-            messages = await client.xread({stream_key: last_id}, block=15000, count=50)
+            try:
+                messages = await client.xread({stream_key: last_id}, block=DOMAIN_EVENT_BLOCK_MS, count=50)
+            except RedisTimeoutError:
+                yield ": keepalive\n\n"
+                continue
             if not messages:
                 yield ": keepalive\n\n"
                 continue

domains/finance-researcher/data_generator.py

@@ -269,6 +269,12 @@ def slugify(text: str) -> str:
     return text.strip("-") or "item"
 
 
+def tag_safe_slug(text: str) -> str:
+    text = text.lower()
+    text = re.sub(r"[^a-z0-9]+", "_", text)
+    return text.strip("_") or "item"
+
+
 def safe_filename(name: str) -> str:
     cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", name.strip())
     return cleaned.strip("._") or "file"
@@ -756,7 +762,9 @@ def normalize_event_type(source_type: str) -> str:
 
 
 def make_document_id(*, ticker: str, source_type: str, accession: str, filename: str) -> str:
-    return f"{ticker}_{source_type}_{accession.replace('-', '')}_{slugify(Path(filename).stem)}"
+    accession_slug = tag_safe_slug(accession)
+    filename_slug = tag_safe_slug(Path(filename).stem)
+    return f"{ticker}_{source_type}_{accession_slug}_{filename_slug}"
 
 
 def make_chunk_id(document_id: str, index: int) -> str:

domains/finance-researcher/docs/demo_paths.md

@@ -1,9 +1,22 @@
-# Finance Researcher Domain Spec
+# Finance Researcher Demo Paths
 
 ## Persona
 
 The flagship user is **Morgan Lee**, a buy-side analyst who uses the demo to compare a curated 14-company watchlist across filings, earnings materials, structured metrics, price history, and normalized update events.
 
+## Demo Positioning
+
+ShiftIQ is not a full portfolio, CRM, trading, or research-management system. It is a focused context-engine demo showing how an agent can use Redis-backed context across structured company records, filings, metrics, price time series, and live update events.
+
+Use **Context Surfaces** mode for the primary flow.
+
+The unique value of the agent flow is coordination:
+
+- It starts from the signed-in analyst context and watchlist instead of a generic prompt.
+- It coordinates structured records, document chunks, RedisTimeSeries, and Redis Streams in one conversation.
+- It makes the tool path inspectable, so the audience can see when the answer came from records, time-series data, source documents, or live stream events.
+- It gives a clean bridge to the broader Redis message: context first, orchestration across data planes second, and optimization next through semantic caching and routing.
+
 ## Watchlist
 
 The v1 watchlist is intentionally concentrated in large-cap and AI-adjacent names so the demo can support cross-company comparisons:
@@ -56,45 +69,173 @@ The planned dataset supports these record types:
 
 ## Flagship Demo Paths
 
-### 1. Cross-company narrative comparison
+### 1. Establish analyst and data context
+
+Ask:
+
+```text
+What is my active research context, what companies are in scope, and what data is loaded?
+```
 
-Ask: "Compare the latest NVIDIA, AMD, and Broadcom filings and tell me what changed across the three companies."
+What this shows:
+
+- The agent identifies the current analyst before answering.
+- It inspects dataset coverage instead of relying on prompt memory.
+- It frames the 14-company watchlist as the working context for the rest of the demo.
+
+### 2. Cross-company narrative comparison
+
+Ask:
+
+```text
+Compare the latest NVIDIA, AMD, and Broadcom filings and tell me what changed across the three companies.
+```
 
 What the dataset must support:
 
 - Recent documents for `NVDA`, `AMD`, and `AVGO`
 - Chunks from the most recent filing or earnings materials
 - A way to compare how each company describes demand, guidance, or capital intensity
 
-### 2. Metric-plus-document reasoning
+What this shows:
+
+- Source-grounded research across multiple companies.
+- Coordination between company records, source documents, and document chunks.
+- A research workflow that summarizes what changed before a client or stakeholder conversation.
+
+### 3. Metric-plus-document reasoning
 
-Ask: "Walk me through Oracle's latest quarter using both the filing and the structured metrics."
+Ask:
+
+```text
+Walk me through Oracle's latest quarter using both the filing and the structured metrics. Keep it to three bullets.
+```
 
 What the dataset must support:
 
 - Oracle documents with a clear period label
 - Quarterly metric points for revenue, gross profit, operating income, net income, and cash flow proxies
 - Enough document chunks to explain the narrative behind the numbers
 
-### 3. Peer price and fundamentals trend comparison
+What this shows:
+
+- The agent combines narrative evidence with structured metrics.
+- The response separates "what the numbers say" from "what the filing says."
+- The path is useful as a concise pre-meeting brief.
+
+### 4. Peer price and fundamentals trend comparison
+
+Ask:
 
-Ask: "Compare stock price and fundamentals trends for NVIDIA, AMD, and MongoDB."
+```text
+Compare the one-year close price trend for Apple, Microsoft, and Amazon. Keep it simple for an advisor.
+```
 
 What the dataset must support:
 
-- Daily `PriceBar` rows for `NVDA`, `AMD`, and `MDB`
+- Daily `PriceBar` rows for `AAPL`, `MSFT`, and `AMZN`
 - Comparable metric points across the same companies
 - A consistent benchmark group so the peer set is obvious
 
-### 4. What is new in my watchlist
+What this shows:
+
+- RedisTimeSeries as a first-class agent tool.
+- Chart-ready time-series output from queried data, not invented by the model.
+- Inspectable Redis commands in the trace.
+
+### 5. Live watchlist update context
+
+In another terminal:
 
-Ask: "What's new in my watchlist this week?"
+```bash
+make publish-domain-event DOMAIN=finance-researcher
+```
+
+Then ask:
+
+```text
+A live watchlist update just came in. Read the latest live events and tell me what changed, which ticker it affects, and what an advisor should pay attention to next.
+```
 
 What the dataset must support:
 
 - Normalized `CoverageEvent` rows for new filings, releases, presentations, or notable price moves
 - A way to trace each event back to a `ResearchDocument`
 - Recent timestamps so the agent can answer using the current market or filing window
+- A running backend that exposes `recent_watchlist_events` in `/api/health`
+
+What this shows:
+
+- Redis Streams as live context, not just a UI notification.
+- The application and the agent can read from the same stream-backed event feed.
+- The model can react to something that arrived after page load.
+
+Before using this step, confirm `/api/health` lists `recent_watchlist_events` under `internal_tools`. If it does not, restart the backend so the current finance-researcher domain code is loaded.
+
+## Additional Softball Prompts
+
+Use these when you want a lower-risk path that still exercises useful tools.
+
+### Company record lookup
+
+Ask:
+
+```text
+For Apple, give me the company snapshot from the loaded records and one advisor-facing takeaway.
+```
+
+Shows:
+
+- Direct structured lookup against the company record.
+- Translation from raw company context into a concise advisory takeaway.
+
+### Watchlist segmentation
+
+Ask:
+
+```text
+List the mega-cap technology names in my watchlist and show each ticker with sector and subsector.
+```
+
+Shows:
+
+- The agent starts with the analyst watchlist, then resolves company records.
+- A raw coverage universe becomes a set of client-discussion buckets.
+
+### Fundamentals trend
+
+Ask:
+
+```text
+Compare revenue trends for NVIDIA, AMD, and Broadcom over the max available period. Keep it to three advisor-ready bullets.
+```
+
+Shows:
+
+- RedisTimeSeries on fundamentals, not just prices.
+- A bridge from market context to operating performance.
+
+## Optional RAG Contrast
+
+Switch to **Simple RAG** and ask:
+
+```text
+Compare the one-year close price trend for NVIDIA, AMD, and Microsoft.
+```
+
+Point out:
+
+Simple RAG can retrieve text. It cannot coordinate structured records, time-series data, live events, and durable agent state the same way.
+
+## Prompts To Avoid In Executive Demos
+
+Avoid broad keyword prompts like:
+
+```text
+Search the research corpus for margin pressure commentary across the watchlist.
+```
+
+The current generated corpus is small and can return no matches for generic phrases. Prefer company-specific filing prompts or time-series prompts when you need a reliable demo path.
 
 ## Non-Goals For v1