feat(ENG-5578): add integration tests with recorded market replay

.claude/CODEBASE.md

@@ -28,6 +28,7 @@ Key properties:
 | [`fynd-core`](../fynd-core/CLAUDE.md) | `fynd-core/` | Pure solving logic: algorithms, worker pools, graph, feed, derived data, encoding. No HTTP deps |
 | [`fynd-rpc`](../fynd-rpc/CLAUDE.md) | `fynd-rpc/` | HTTP RPC server builder (Actix Web): API handlers, middleware, `FyndRPCBuilder` |
 | [`fynd-rpc-types`](../fynd-rpc-types/CLAUDE.md) | `fynd-rpc-types/` | Shared DTO types for the RPC API (request/response wire format) |
+| `fynd-test-fixtures` | `test-fixtures/` | Shared types for recorded-market test fixtures: `MarketRecording`, expected outputs, test scenarios. Not published |
 
 ### [Clients](../clients/CLAUDE.md)
 
@@ -44,6 +45,7 @@ Both clients wrap the same OpenAPI spec (`clients/openapi.json`, generated via `
 |---|---|---|
 | `fynd-benchmark` | `tools/benchmark/` | Load testing, solver comparison, trade dataset download |
 | `fynd-swap-cli` | `tools/fynd-swap-cli/` | Quote and execute token swaps (ERC-20 or Permit2) |
+| `record-market` | `tools/record-market/` | Record live Tycho market state and generate expected outputs for the integration tests |
 
 ## Architecture Overview
 
@@ -118,6 +120,7 @@ See `docs/ARCHITECTURE.md` for the full architecture diagram and detailed compon
 ## Testing
 
 - `cargo nextest run --workspace --all-targets --all-features` — full test suite
+- `cargo nextest run -p fynd-core --features test-utils --test integration` — replayed-market integration tests against recorded fixtures (`fynd-core/tests/fixtures/`, Git LFS). See `fynd-core/tests/integration/README.md`; regenerate baselines with the ignored `regenerate_expected_outputs` test, record fresh markets with `tools/record-market`
 - `cargo +nightly clippy --workspace --all-targets --all-features` — lint
 - `cargo +nightly fmt --all --check` — format check
 - `RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --locked --package fynd-core --package fynd-rpc-types --package fynd-rpc --package fynd-client` — doc build (broken links, missing docs)

.gitattributes

@@ -0,0 +1,3 @@
+# Track compressed market recordings with Git LFS to keep the repo lightweight.
+# These fixtures are multi-MB zstd-compressed JSON files used by integration tests.
+fynd-core/tests/fixtures/*.json.zst filter=lfs diff=lfs merge=lfs -text

.github/workflows/ci.yaml

@@ -142,6 +142,32 @@ jobs:
       - run: pnpm --dir clients/typescript --filter @kayibal/fynd-client run lint
       - run: pnpm --dir clients/typescript --filter @kayibal/fynd-client run test
 
+  integration_tests:
+    name: Integration Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    needs: [compile_and_test]
+    env:
+      RUSTFLAGS: "-D warnings"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
+        with:
+          persist-credentials: false
+          lfs: true
+      - name: Setup toolchain
+        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9  # v1
+        with:
+          toolchain: stable
+      - name: Setup Rust Cache
+        uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32  # v2
+        with:
+          cache-on-failure: true
+      - name: Install latest nextest release
+        uses: taiki-e/install-action@2b15ad09ca0f06a76791407e35ab20f0b712d615  # nextest
+      - name: Run integration tests
+        run: cargo nextest run -p fynd-core --features test-utils --test integration
+
   security_audit:
     name: Security Audit
     runs-on: ubuntu-latest

.github/workflows/nightly-canary.yaml

@@ -0,0 +1,70 @@
+name: Nightly Canary
+
+# Records fresh live Tycho data and regenerates expected outputs from that same
+# recording before running the integration tests. The baselines are therefore
+# self-referential: this canary detects Tycho data-format changes, decode
+# failures, and replay-pipeline crashes — NOT solver quality regressions.
+# Quality is guarded by the committed baseline in the PR-triggered
+# integration_tests job in ci.yaml.
+
+on:
+  schedule:
+    - cron: '0 6 * * *'  # Daily at 06:00 UTC
+  workflow_dispatch:  # Allow manual trigger
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: "-D warnings"
+
+jobs:
+  live-canary:
+    name: Live Tycho Canary
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
+        with:
+          persist-credentials: false
+      - name: Setup toolchain
+        uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9  # v1
+        with:
+          toolchain: stable
+      - name: Setup Rust Cache
+        uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32  # v2
+        with:
+          cache-on-failure: true
+      - name: Install latest nextest release
+        uses: taiki-e/install-action@2b15ad09ca0f06a76791407e35ab20f0b712d615  # nextest
+
+      - name: Record fresh market state
+        run: |
+          cargo run -p record-market -- \
+            --tycho-url "$TYCHO_URL" \
+            --rpc-url "$RPC_URL" \
+            --tycho-api-key "$TYCHO_API_KEY" \
+            --duration-secs 120 \
+            --output-dir fynd-core/tests/fixtures
+        env:
+          TYCHO_URL: ${{ secrets.TYCHO_URL }}
+          RPC_URL: ${{ secrets.RPC_URL }}
+          TYCHO_API_KEY: ${{ secrets.TYCHO_API_KEY }}
+
+      - name: Run solver against live market
+        run: |
+          cargo nextest run -p fynd-core --features test-utils --test integration
+
+      - name: Notify on failure
+        if: failure()
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          echo "::error::Nightly canary failed — Tycho data format may have changed or the replay pipeline crashed"
+          if [ -z "$SLACK_WEBHOOK_URL" ]; then
+            echo "::warning::SLACK_WEBHOOK_URL secret not set; skipping Slack notification"
+            exit 0
+          fi
+          curl -sf -X POST -H 'Content-type: application/json' \
+            --data "{\"text\":\":red_circle: Nightly canary failed — Tycho data format may have changed or the replay pipeline crashed. <${RUN_URL}|View run>\"}" \
+            "$SLACK_WEBHOOK_URL"

.gitignore

@@ -8,6 +8,7 @@ scale_results*.json
 .env
 .claude/worktrees/
 .claude/plans/
+.worktrees/
 
 # fynd-gas-audit output artifacts
 tools/fynd-gas-audit/out/

Cargo.toml

@@ -84,6 +84,8 @@ members = [
     "tools/erc20-overrides",
     "tools/fynd-swap-cli",
     "tools/fynd-gas-audit",
+    "tools/record-market",
+    "test-fixtures",
 ]
 resolver = "2"
 
@@ -139,6 +141,9 @@ tycho-execution = ">=0.304"
 tycho-simulation = ">=0.304"
 typetag = "0.2"
 
+# Compression
+zstd = "0.13"
+
 # Graph algorithms
 petgraph = "0.6"
 
@@ -156,13 +161,16 @@ async-trait = "0.1"
 chrono = "0.4"
 itertools = "0.14.0"
 hex = "0.4"
+sha2 = "0.10"
 
 # Internal crates
 fynd-core = { path = "fynd-core", version = "0.78.1" }
 fynd-rpc = { path = "fynd-rpc", version = "0.78.1" }
 fynd-rpc-types = { path = "fynd-rpc-types", version = "0.78.1" }
 fynd-client = { path = "clients/rust", version = "0.78.1" }
 erc20-overrides = { path = "tools/erc20-overrides", version = "0.78.1" }
+fynd-test-fixtures = { path = "test-fixtures" }
 
 # Dev dependencies
 rstest = "0.18"
+tempfile = "3"

Dockerfile

@@ -22,9 +22,12 @@ COPY tools/benchmark/Cargo.toml tools/benchmark/
 COPY tools/fynd-swap-cli/Cargo.toml tools/fynd-swap-cli/
 COPY tools/erc20-overrides/Cargo.toml tools/erc20-overrides/
 COPY tools/fynd-gas-audit/Cargo.toml tools/fynd-gas-audit/
+COPY tools/record-market/Cargo.toml tools/record-market/
+COPY test-fixtures/Cargo.toml test-fixtures/
 RUN mkdir -p src fynd-core/src fynd-rpc/src fynd-rpc-types/src \
         clients/rust/src tools/benchmark/src tools/fynd-swap-cli/src \
-        tools/erc20-overrides/src tools/fynd-gas-audit/src && \
+        tools/erc20-overrides/src tools/fynd-gas-audit/src \
+        tools/record-market/src test-fixtures/src && \
     echo "fn main() {}" > src/main.rs && \
     echo "" > src/lib.rs && \
     echo "" > fynd-core/src/lib.rs && \
@@ -35,10 +38,13 @@ RUN mkdir -p src fynd-core/src fynd-rpc/src fynd-rpc-types/src \
     echo "fn main() {}" > tools/fynd-swap-cli/src/main.rs && \
     echo "" > tools/erc20-overrides/src/lib.rs && \
     echo "fn main() {}" > tools/fynd-gas-audit/src/main.rs && \
+    echo "fn main() {}" > tools/record-market/src/main.rs && \
+    echo "" > test-fixtures/src/lib.rs && \
     cargo build --release --package fynd --package fynd-swap-cli && \
     rm -rf src fynd-core/src fynd-rpc/src fynd-rpc-types/src \
         clients/rust/src tools/benchmark/src tools/fynd-swap-cli/src \
-        tools/erc20-overrides/src tools/fynd-gas-audit/src
+        tools/erc20-overrides/src tools/fynd-gas-audit/src \
+        tools/record-market/src test-fixtures/src
 
 # Copy real source and rebuild
 COPY src/ src/
@@ -48,9 +54,12 @@ COPY fynd-rpc-types/src/ fynd-rpc-types/src/
 COPY clients/rust/src/ clients/rust/src/
 COPY tools/fynd-swap-cli/src/ tools/fynd-swap-cli/src/
 COPY tools/erc20-overrides/src/ tools/erc20-overrides/src/
-RUN mkdir -p tools/benchmark/src tools/fynd-gas-audit/src && \
+RUN mkdir -p tools/benchmark/src tools/fynd-gas-audit/src \
+        tools/record-market/src test-fixtures/src && \
     echo "fn main() {}" > tools/benchmark/src/main.rs && \
     echo "fn main() {}" > tools/fynd-gas-audit/src/main.rs && \
+    echo "fn main() {}" > tools/record-market/src/main.rs && \
+    echo "" > test-fixtures/src/lib.rs && \
     touch src/main.rs src/lib.rs fynd-core/src/lib.rs fynd-rpc/src/lib.rs \
         fynd-rpc-types/src/lib.rs clients/rust/src/lib.rs \
         tools/fynd-swap-cli/src/main.rs tools/erc20-overrides/src/lib.rs && \

fynd-core/CLAUDE.md

@@ -67,6 +67,14 @@ Price guard methods: `price_guard_enabled(bool)`, `register_price_provider(Box<d
 
 See `fynd-core/examples/custom_algorithm.rs` for a walkthrough.
 
+## Integration Tests
+
+`tests/integration/` replays a recorded market (`tests/fixtures/`, Git LFS) through the full
+pipeline and asserts solution availability, quality vs baseline, derived-data metrics, and timing.
+Run with `cargo nextest run -p fynd-core --features test-utils --test integration`. The
+`test-utils` feature gates `Solver::from_recording` and the recording helpers; fixtures are
+recorded with `tools/record-market`. See `tests/integration/README.md`.
+
 ## Data Flow
 
 **Market updates** (every block):

fynd-core/Cargo.toml

@@ -34,6 +34,12 @@ reqwest.workspace = true
 tokio-tungstenite.workspace = true
 alloy = { workspace = true, features = ["sol-types"] }
 
+[features]
+test-utils = []
+
 [dev-dependencies]
 rstest.workspace = true
+tempfile.workspace = true
+toml.workspace = true
+fynd-test-fixtures.workspace = true
 tracing-subscriber.workspace = true

fynd-core/src/feed/protocol_registry.rs

@@ -33,6 +33,20 @@ use tycho_simulation::{
 
 use super::DataFeedError;
 
+/// Register DEX protocol decoders for test tooling (record-market).
+///
+/// Wrapper over [`register_exchanges`] so the recorder builds the same protocol stream as
+/// production without exposing the crate-private `DataFeedError`.
+#[cfg(feature = "test-utils")]
+pub fn register_exchanges_for_recording(
+    builder: ProtocolStreamBuilder,
+    tvl_filter: ComponentFilter,
+    protocols: &[String],
+) -> Result<ProtocolStreamBuilder, String> {
+    register_exchanges(builder, tvl_filter, protocols).map_err(|e| e.to_string())
+}
+
+/// Register DEX protocol decoders on a [`ProtocolStreamBuilder`].
 pub(crate) fn register_exchanges(
     mut builder: ProtocolStreamBuilder,
     tvl_filter: ComponentFilter,

fynd-core/src/feed/tycho_feed.rs

@@ -441,7 +441,7 @@ impl TychoFeed {
 
     /// Handles a message from Tycho stream.
     #[instrument(skip(self, msg))]
-    async fn handle_tycho_message(&self, msg: Update) -> Result<(), DataFeedError> {
+    pub(crate) async fn handle_tycho_message(&self, msg: Update) -> Result<(), DataFeedError> {
         // Collect variables for market shared data update
         let Update {
             new_pairs: added_components,