Productionize pairwise digital-twin evaluation

.env.example

@@ -43,3 +43,15 @@ CORTEX_EMBEDDING_MODEL=text-embedding-3-small
 CORTEX_EMBEDDING_DIMENSIONS=384
 CORTEX_EMBEDDING_STRICT=0
 OPENAI_API_KEY=
+
+# Pairwise evaluation preflight admission. These are operator-owned ceilings;
+# request callers can tighten them but cannot raise them.
+CORTEX_PAIRWISE_MAX_PROVIDER_CALLS=1000
+CORTEX_PAIRWISE_MAX_TOTAL_TOKENS=10000000
+CORTEX_PAIRWISE_MAX_DURATION_SECONDS=86400
+CORTEX_PAIRWISE_MAX_PARALLEL_GENERATIONS=1
+CORTEX_PAIRWISE_MAX_PARALLEL_JUDGMENTS=1
+# Optional: enables short-lived, user-bound admission receipts. Use a distinct
+# random secret of at least 32 bytes; do not reuse an API or provider key.
+CORTEX_PAIRWISE_ADMISSION_SIGNING_KEY=
+CORTEX_PAIRWISE_ADMISSION_RECEIPT_TTL_SECONDS=900

.github/workflows/ci.yml

@@ -17,7 +17,9 @@ jobs:
   python:
     name: Python checks and backend tests
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    # The pairwise suite brings this job close to ten minutes on current
+    # runners. Keep enough headroom for dependency installation and variance.
+    timeout-minutes: 30
 
     steps:
       - name: Check out repository
@@ -135,7 +137,9 @@ jobs:
       - name: Audit backend dependencies for known vulnerabilities
         run: pip-audit --strict -r backend/requirements.txt
 
-      - name: Audit top-level dependencies for known vulnerabilities
+      - name: Audit archived prototype dependencies for known vulnerabilities
+        # rumps is correctly Darwin-marked in the manifest, so this Linux job
+        # resolves and audits the applicable archived dependency graph only.
         run: pip-audit --strict -r requirements.txt
 
   distribution:
@@ -156,7 +160,7 @@ jobs:
         run: python scripts/check_distribution_site.py
 
       - name: Validate site update manifest
-        run: python scripts/validate_update_manifest.py site/downloads/latest.json
+        run: python scripts/validate_update_manifest.py --allow-remote-artifacts site/downloads/latest.json
 
   obsidian-plugin:
     name: Obsidian plugin build

backend/app/config.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import math
 import os
 from dataclasses import dataclass
 from pathlib import Path
@@ -51,6 +52,13 @@ class Settings:
     store_cache_size: int = 512
     rate_limit_per_minute: int = 0
     default_memory_quota: int = 0
+    pairwise_preflight_max_provider_calls: int = 1_000
+    pairwise_preflight_max_total_tokens: int = 10_000_000
+    pairwise_preflight_max_duration_seconds: float = 86_400
+    pairwise_preflight_max_parallel_generations: int = 1
+    pairwise_preflight_max_parallel_judgments: int = 1
+    pairwise_admission_signing_key: str = ""
+    pairwise_admission_receipt_ttl_seconds: int = 15 * 60
     require_scoped_api_tokens: bool = False
     sync_signing_key: str = ""
     hosted_database_url: str = ""
@@ -190,6 +198,22 @@ def _truthy_env(name: str) -> bool:
     return os.environ.get(name, "").strip().lower() in {"1", "true", "yes", "on"}
 
 
+def _positive_int_env(name: str, default: int) -> int:
+    try:
+        value = int(os.environ.get(name, str(default)) or str(default))
+    except (TypeError, ValueError):
+        return default
+    return value if value > 0 else default
+
+
+def _positive_float_env(name: str, default: float) -> float:
+    try:
+        value = float(os.environ.get(name, str(default)) or str(default))
+    except (TypeError, ValueError):
+        return default
+    return value if math.isfinite(value) and value > 0 else default
+
+
 def _load_plan_quotas() -> dict[str, int] | None:
     """Parse CORTEX_PLAN_QUOTAS (JSON object mapping plan name -> int quota).
     Malformed input falls back to the baked-in defaults so a bad env can never
@@ -258,6 +282,37 @@ def load_settings() -> Settings:
         store_cache_size=max(1, int(os.environ.get("CORTEX_STORE_CACHE_SIZE", "512") or "512")),
         rate_limit_per_minute=max(0, int(os.environ.get("CORTEX_RATE_LIMIT_PER_MINUTE", "0") or "0")),
         default_memory_quota=max(0, int(os.environ.get("CORTEX_DEFAULT_MEMORY_QUOTA", "0") or "0")),
+        pairwise_preflight_max_provider_calls=_positive_int_env(
+            "CORTEX_PAIRWISE_MAX_PROVIDER_CALLS",
+            1_000,
+        ),
+        pairwise_preflight_max_total_tokens=_positive_int_env(
+            "CORTEX_PAIRWISE_MAX_TOTAL_TOKENS",
+            10_000_000,
+        ),
+        pairwise_preflight_max_duration_seconds=_positive_float_env(
+            "CORTEX_PAIRWISE_MAX_DURATION_SECONDS",
+            86_400,
+        ),
+        pairwise_preflight_max_parallel_generations=_positive_int_env(
+            "CORTEX_PAIRWISE_MAX_PARALLEL_GENERATIONS",
+            1,
+        ),
+        pairwise_preflight_max_parallel_judgments=_positive_int_env(
+            "CORTEX_PAIRWISE_MAX_PARALLEL_JUDGMENTS",
+            1,
+        ),
+        pairwise_admission_signing_key=os.environ.get(
+            "CORTEX_PAIRWISE_ADMISSION_SIGNING_KEY",
+            "",
+        ),
+        pairwise_admission_receipt_ttl_seconds=min(
+            _positive_int_env(
+                "CORTEX_PAIRWISE_ADMISSION_RECEIPT_TTL_SECONDS",
+                15 * 60,
+            ),
+            60 * 60,
+        ),
         require_scoped_api_tokens=require_scoped_api_tokens,
         sync_signing_key=os.environ.get("CORTEX_SYNC_SIGNING_KEY", ""),
         hosted_database_url=os.environ.get("CORTEX_HOSTED_DATABASE_URL", os.environ.get("DATABASE_URL", "")).strip(),

backend/app/connectors/agent_sessions.py

@@ -254,7 +254,7 @@ def _claude_records(root: Path, path: Path, *, session_cap: int) -> list[AgentSe
         cleaned = _clean_user_text(text)
         if not cleaned:
             continue
-        fingerprint = sha1(cleaned.encode("utf-8")).hexdigest()
+        fingerprint = sha1(cleaned.encode("utf-8"), usedforsecurity=False).hexdigest()
         if fingerprint in seen_texts:
             continue
         seen_texts.add(fingerprint)
@@ -316,7 +316,7 @@ def _codex_records(root: Path, path: Path, *, session_cap: int) -> list[AgentSes
         cleaned = _clean_user_text(str(payload.get("message") or ""))
         if not cleaned:
             continue
-        fingerprint = sha1(cleaned.encode("utf-8")).hexdigest()
+        fingerprint = sha1(cleaned.encode("utf-8"), usedforsecurity=False).hexdigest()
         if fingerprint in seen_texts:
             continue
         seen_texts.add(fingerprint)
@@ -370,7 +370,7 @@ def _cursor_records(root: Path, path: Path, *, mtime: float, session_cap: int) -
         cleaned = _clean_user_text(str(prompt.get("text") or ""))
         if not cleaned:
             continue
-        fingerprint = sha1(cleaned.encode("utf-8")).hexdigest()
+        fingerprint = sha1(cleaned.encode("utf-8"), usedforsecurity=False).hexdigest()
         if fingerprint in seen_texts:
             continue
         seen_texts.add(fingerprint)
@@ -465,7 +465,7 @@ def _stable_external_id(agent: str, session_id: str, anchor: str) -> str:
     raw = f"{agent}:{session_id}:{anchor}"
     if len(raw) <= 240:
         return raw
-    digest = sha1(raw.encode("utf-8")).hexdigest()[:16]
+    digest = sha1(raw.encode("utf-8"), usedforsecurity=False).hexdigest()[:16]
     return f"{raw[:220]}#{digest}"
 
 

backend/app/connectors/notion.py

@@ -3,7 +3,7 @@
 from dataclasses import dataclass
 import json
 import re
-from typing import Any, Callable
+from typing import Any, Callable, Optional
 from urllib.parse import urlencode
 from urllib.request import Request, urlopen
 
@@ -19,7 +19,7 @@
 MAX_BLOCK_TREE_DEPTH = 3
 
 
-RequestJSON = Callable[[str, dict[str, str], dict[str, Any] | None, str], Any]
+RequestJSON = Callable[[str, dict[str, str], Optional[dict[str, Any]], str], Any]
 
 
 @dataclass(frozen=True)