Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,12 @@ Each component is organized into a self-contained subtree:
* Python 3.12+ (Required only for the runtime agent and the benchmarking harness).
* *Responsibility Check:* Ensure you are only scanning code bases you are explicitly authorized to analyze.

You supply your own model access. The runtime agent supports the direct Anthropic API,
AWS Bedrock (OAuth proxy or SigV4), and the [OrcaRouter](https://www.orcarouter.ai)
gateway — `anthropic.auth_mode = "orcarouter"` with either a pasted `sk-orca-…` API key
or an OAuth 2.0 + PKCE login. See
[`vulnhunter-agent/README.md`](vulnhunter-agent/README.md#orcarouter-auth_mode--orcarouter).

### Installation

```bash
Expand Down
19 changes: 19 additions & 0 deletions harness/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,25 @@ has no other runtime dependencies.
Set the scanning/judging model once in `local_harness/config.py` (the `MODEL`
constant) to switch models across every workflow.

### Routing at OrcaRouter

[OrcaRouter](https://www.orcarouter.ai) is an OpenAI-compatible AI gateway that
routes many providers behind one endpoint. To send the harness's `claude`
subprocesses through it, export the same `ORCA_*` variables the runtime agent
honors and the harness injects them into each `claude` invocation:

```bash
export ORCA_API_KEY=sk-orca-... # or ORCA_BASE_URL for self-hosted
export ORCA_API_BASE_URL=https://api.orcarouter.ai/v1
python -m local_harness.batch.run scan
```

There is no `ANTHROPIC_BASE_URL`/`ANTHROPIC_API_KEY` for you to set by hand and
no OAuth flow here: the harness is a credential *consumer*. Obtain the key with
`python -m agent --mode=orcarouter login` (or `--orcarouter-action=store-api-key`)
in `vulnhunter-agent/`, or export `ORCA_API_KEY` directly. With no `ORCA_*`
variable set, the harness inherits the ambient environment exactly as before.

## What's inside

| Path | Purpose |
Expand Down
2 changes: 2 additions & 0 deletions harness/local_harness/benchmark/analyze_misses.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))

from local_harness.config import BENCHMARK_DIR, MODEL, REPO_ROOT, RESULTS_DIR, STATE_FILE, atomic_write_json
from local_harness.orcarouter_env import claude_env

PHASE_TO_PROMPT = {
"phase1": ["skill/phases/phase1_recon.md"],
Expand Down Expand Up @@ -277,6 +278,7 @@ def invoke_diagnostic(finding, phase_key, evidence, results_dir, repo_dir):
try:
result = subprocess.run(
cmd, capture_output=True, text=True, timeout=1200,
env=claude_env(),
)
except subprocess.TimeoutExpired:
elapsed = time.time() - start
Expand Down
2 changes: 2 additions & 0 deletions harness/local_harness/benchmark/judge.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
JUDGE_TIMEOUT,
MODEL,
)
from local_harness.orcarouter_env import claude_env

JUDGE_SYSTEM_PROMPT = """You are a security finding comparator. Your job is to determine whether a vulnerability scanner's output detected specific known vulnerabilities.

Expand Down Expand Up @@ -95,6 +96,7 @@ def judge_findings_batch(results_report, findings, model=None):
"--model", model,
"--system-prompt", JUDGE_SYSTEM_PROMPT],
capture_output=True, text=True, timeout=JUDGE_TIMEOUT,
env=claude_env(),
)
except subprocess.TimeoutExpired:
return [{"finding_id": f["finding_id"], "detected": None,
Expand Down
93 changes: 93 additions & 0 deletions harness/local_harness/orcarouter_env.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
"""OrcaRouter environment bridge for the harness's ``claude`` subprocesses.

The harness drives the ``claude`` CLI directly and relies on the CLI's own
ambient authentication. To route those invocations at OrcaRouter we inject
the inference origin and an API key into the subprocess environment.

The harness is a dev tool with no config file, so it reads the same ``ORCA_*``
environment variables the agent honors. If a key is not supplied via env, it
tries the agent's persisted credential store (an optional dependency here —
imported only when present). Nothing is set unless OrcaRouter is actually
configured, so the harness's default behavior is byte-identical.

There is no OAuth/PKCE here and none should be added: the harness is not a
credential-acquisition surface, it is a consumer of a credential obtained
(either by exporting env or by ``python -m agent --mode=orcarouter login``).
"""

import os

# The CLI appends its version segment to ANTHROPIC_BASE_URL itself, so this
# must be the inference origin WITHOUT a trailing /v1.
_BASE_URL_ENV = "ANTHROPIC_BASE_URL"
_API_KEY_ENV = "ANTHROPIC_API_KEY"

# Env vars the agent also honors, in priority order.
_KEY_ENV_NAMES = ("ORCA_API_KEY", "ORCAROUTER_API_KEY")
_BASE_ENV_NAMES = ("ORCA_API_BASE_URL", "ORCA_BASE_URL")
_DEFAULT_API_BASE = "https://api.orcarouter.ai/v1"


def _configured() -> bool:
return any(os.environ.get(name) for name in (*_KEY_ENV_NAMES, *_BASE_ENV_NAMES))


def _api_base() -> str:
for name in _BASE_ENV_NAMES:
value = os.environ.get(name)
if value and value.strip():
return value.strip()
return _DEFAULT_API_BASE


def _cli_base_url(api_base: str) -> str:
"""Strip a trailing ``/v1`` (the CLI appends its own version segment)."""
base = api_base.rstrip("/")
return base[:-3] if base.endswith("/v1") else base


def _api_key() -> str:
for name in _KEY_ENV_NAMES:
value = os.environ.get(name)
if value and value.strip():
return value.strip()
# Fall back to the agent's persisted credential store (whatever the user
# established via API key or OAuth login). Optional: the agent package is
# a sibling subtree that may not be installed in a harness-only checkout.
try:
from agent.config import load_orcarouter_config
from agent.orcarouter import ConnectCredentialProvider
except ImportError:
return ""
try:
config = load_orcarouter_config()
return ConnectCredentialProvider(config).resolve().api_key
except Exception:
return ""


def orcarouter_env() -> dict:
"""Return the CLI env overrides for OrcaRouter, or an empty dict."""
if not _configured():
return {}
key = _api_key()
if not key:
raise RuntimeError(
"OrcaRouter is configured (ORCA_* env set) but no API key is "
"available. Export ORCA_API_KEY, or run "
"`python -m agent --mode=orcarouter login` first."
)
return {_BASE_URL_ENV: _cli_base_url(_api_base()), _API_KEY_ENV: key}


def claude_env() -> dict | None:
"""Environment for a ``claude`` subprocess, or ``None`` to inherit.

``None`` (the default when OrcaRouter is not configured) tells the
``subprocess`` call to inherit the parent environment unchanged, so the
harness's existing behavior is preserved exactly.
"""
overrides = orcarouter_env()
if not overrides:
return None
return {**os.environ, **overrides}
4 changes: 4 additions & 0 deletions harness/local_harness/scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
SCAN_TIMEOUT,
SKILLS_DIR,
)
from .orcarouter_env import claude_env

# The result of scanning one folder. A namedtuple (rather than a bare 7-tuple)
# so consumers can use attribute access and a field reorder can't silently
Expand Down Expand Up @@ -221,6 +222,9 @@ def scan_folder(folder_path, log_file=None, readonly=False):
stderr=subprocess.STDOUT,
text=True,
cwd=folder_path,
# Route at OrcaRouter when configured; None inherits the parent env
# unchanged, so the default behavior is untouched.
env=claude_env(),
)

event_count = 0
Expand Down
19 changes: 19 additions & 0 deletions harness/tests/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
"""Shared pytest fixtures for the harness test suite.

Keeps OrcaRouter provider configuration out of the test process so a
credential present in the developer/CI environment can never change a
test's outcome (the harness env bridge keys off ``ORCA_*``).
"""

from __future__ import annotations

import os

import pytest


@pytest.fixture(autouse=True)
def _clean_orcarouter_env(monkeypatch: pytest.MonkeyPatch) -> None:
for key in list(os.environ):
if key.startswith(("ORCA_", "ORCAROUTER_")):
monkeypatch.delenv(key, raising=False)
155 changes: 155 additions & 0 deletions harness/tests/test_orcarouter_env.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
"""Tests for the harness OrcaRouter env bridge.

The harness drives the ``claude`` CLI directly. When OrcaRouter is
configured, the bridge injects the inference origin and an API key into the
subprocess environment; otherwise it must be a no-op so existing behavior is
unchanged. The harness acquires no credentials itself.
"""

import importlib

import pytest

import local_harness.orcarouter_env as bridge


ORCA_ENV_KEYS = (
"ORCA_BASE_URL", "ORCA_API_BASE_URL", "ORCA_AUTH_BASE_URL",
"ORCA_API_KEY", "ORCAROUTER_API_KEY",
)


@pytest.fixture(autouse=True)
def _clean_orca_env(monkeypatch):
for key in ORCA_ENV_KEYS:
monkeypatch.delenv(key, raising=False)


def test_noop_when_orcarouter_not_configured():
assert bridge.orcarouter_env() == {}
assert bridge.claude_env() is None


def test_injects_api_origin_and_key(monkeypatch):
monkeypatch.setenv("ORCA_API_KEY", "sk-orca-HARNESS000000000000000")
monkeypatch.setenv("ORCA_API_BASE_URL", "https://api.orcarouter.ai/v1")
env = bridge.orcarouter_env()
assert env["ANTHROPIC_BASE_URL"] == "https://api.orcarouter.ai"
assert env["ANTHROPIC_API_KEY"] == "sk-orca-HARNESS000000000000000"
# Never the /v1 form — the CLI appends the version segment itself.
assert not env["ANTHROPIC_BASE_URL"].endswith("/v1")


def test_shared_base_fallback_used(monkeypatch):
monkeypatch.setenv("ORCA_API_KEY", "sk-orca-x")
monkeypatch.setenv("ORCA_BASE_URL", "https://orca.internal.example/v1")
assert bridge.orcarouter_env()["ANTHROPIC_BASE_URL"] == "https://orca.internal.example"


def test_orcarouter_api_key_env_is_honored(monkeypatch):
monkeypatch.setenv("ORCAROUTER_API_KEY", "sk-orca-fromcampaign")
monkeypatch.setenv("ORCA_BASE_URL", "https://api.orcarouter.ai")
assert bridge.orcarouter_env()["ANTHROPIC_API_KEY"] == "sk-orca-fromcampaign"


def test_missing_key_raises_actionable_error(monkeypatch):
# Configured (base set) but no key available anywhere.
monkeypatch.setenv("ORCA_BASE_URL", "https://api.orcarouter.ai")
monkeypatch.setattr(bridge, "_api_key", lambda: "")
with pytest.raises(RuntimeError, match="ORCA_API_KEY"):
bridge.orcarouter_env()


def test_claude_env_merges_parent_environment(monkeypatch):
monkeypatch.setenv("ORCA_API_KEY", "sk-orca-merge")
monkeypatch.setenv("ORCA_BASE_URL", "https://api.orcarouter.ai")
monkeypatch.setenv("SOME_UNRELATED_VAR", "kept")
env = bridge.claude_env()
assert env["SOME_UNRELATED_VAR"] == "kept"
assert env["ANTHROPIC_API_KEY"] == "sk-orca-merge"


def test_scan_subprocess_receives_env(monkeypatch, tmp_path):
"""The scan Popen call is handed the injected env."""
import json

import local_harness.scan as scan

monkeypatch.setenv("ORCA_API_KEY", "sk-orca-scan")
monkeypatch.setenv("ORCA_BASE_URL", "https://api.orcarouter.ai")

captured = {}

class _FakePopen:
def __init__(self, *a, **k):
captured.update(k)
self.stdout = iter([json.dumps({"type": "result"}) + "\n"])
self.returncode = 0

def wait(self):
return 0

def kill(self):
pass

class _NoTimer:
def __init__(self, *a, **k):
pass

def start(self):
pass

def cancel(self):
pass

folder = tmp_path / "repo"
folder.mkdir()
monkeypatch.setattr(scan, "SKILLS_DIR", str(tmp_path / "skills"))
(tmp_path / "skills").mkdir()
monkeypatch.setattr(scan.subprocess, "Popen", _FakePopen)
monkeypatch.setattr(scan.threading, "Timer", _NoTimer)

scan.scan_folder(str(folder))
assert captured["env"]["ANTHROPIC_BASE_URL"] == "https://api.orcarouter.ai"
assert captured["env"]["ANTHROPIC_API_KEY"] == "sk-orca-scan"


def test_scan_subprocess_inherits_when_unconfigured(monkeypatch, tmp_path):
"""Default behavior: env is None so the subprocess inherits unchanged."""
import json

import local_harness.scan as scan

captured = {}

class _FakePopen:
def __init__(self, *a, **k):
captured.update(k)
self.stdout = iter([json.dumps({"type": "result"}) + "\n"])
self.returncode = 0

def wait(self):
return 0

def kill(self):
pass

class _NoTimer:
def __init__(self, *a, **k):
pass

def start(self):
pass

def cancel(self):
pass

folder = tmp_path / "repo"
folder.mkdir()
monkeypatch.setattr(scan, "SKILLS_DIR", str(tmp_path / "skills"))
(tmp_path / "skills").mkdir()
monkeypatch.setattr(scan.subprocess, "Popen", _FakePopen)
monkeypatch.setattr(scan.threading, "Timer", _NoTimer)

scan.scan_folder(str(folder))
assert captured["env"] is None
Loading