Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,11 @@ jobs:
components: rustfmt, clippy
- if: env.RUN == 'true'
uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2
# The scripted-engine regressions launch the browser suite's Node fixtures.
- if: env.RUN == 'true'
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version: 22
- if: env.RUN == 'true'
run: cargo clippy --workspace --all-targets -- -D warnings
- if: env.RUN == 'true'
Expand Down
158 changes: 158 additions & 0 deletions crates/orbynd/tests/scripted_engines.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
//! The browser suite's Node engines through the production process resolver.
mod support;

use std::{ffi::OsString, path::Path, process::Stdio, time::Duration};

use orbyn_core::host::process::{ManagedChild, engine_command, resolve};
use serde_json::{Value, json};
use support::{FAKE, TempDir};
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};

fn engine(dir: &Path, name: &str, args: &[&str]) -> ManagedChild {
// Spaces and a different cwd catch shims that accidentally rely on either.
let scripts = dir.join("test engines");
std::fs::create_dir_all(&scripts).unwrap();
let source = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../web/e2e/real");
for file in [
format!("{name}.mjs"),
format!("{name}.cmd"),
"scenes.mjs".into(),
] {
std::fs::copy(source.join(&file), scripts.join(file)).unwrap();
}
let extension = if cfg!(windows) { "cmd" } else { "mjs" };
let command = resolve(
scripts
.join(format!("{name}.{extension}"))
.to_str()
.unwrap(),
)
.expect("test engine should resolve");
#[cfg(windows)]
{
assert!(
command
.program
.file_name()
.is_some_and(|name| name.eq_ignore_ascii_case("node.exe")),
"{command:?}"
);
assert_eq!(
command.prefix,
[scripts.join(format!("{name}.mjs")).into_os_string()]
);
}
let args: Vec<OsString> = args.iter().map(OsString::from).collect();
let mut command = engine_command(&command, &args, dir);
command.env_clear().stderr(Stdio::inherit());
for key in ["PATH", "Path", "SystemRoot", "WINDIR", "PATHEXT"] {
if let Some(value) = std::env::var_os(key) {
command.env(key, value);
}
}
for key in [
"HOME",
"USERPROFILE",
"CODEX_HOME",
"CLAUDE_CONFIG_DIR",
"TEMP",
"TMP",
] {
command.env(key, dir);
}
command
.env("ORBYN_E2E_FAKE", FAKE)
.env("FAKE_VERSION", "0.166.0");
ManagedChild::spawn(command).expect("Node test engine should start")
}

#[cfg(windows)]
#[test]
fn a_shebang_script_is_not_a_windows_executable() {
let script =
Path::new(env!("CARGO_MANIFEST_DIR")).join("../../web/e2e/real/scripted-engine.mjs");
let error = std::process::Command::new(script)
.arg("--version")
.spawn()
.expect_err("CreateProcess cannot execute a Node shebang");
assert_eq!(error.raw_os_error(), Some(193));
}

#[tokio::test]
async fn both_script_engines_forward_version_arguments() {
let dir = TempDir::new();
for name in ["scripted-engine", "usage-engine"] {
let output = tokio::time::timeout(
Duration::from_secs(10),
engine(dir.path(), name, &["--version"]).output(),
)
.await
.expect("version probe timed out")
.unwrap();
assert!(output.status.success(), "{name}: {output:?}");
assert_eq!(String::from_utf8(output.stdout).unwrap().trim(), "0.166.0");
}
}

#[tokio::test]
async fn scripted_engine_initializes_and_finishes_the_benchmark_scene() {
let dir = TempDir::new();
let mut child = engine(dir.path(), "scripted-engine", &["app-server"]);
let mut input = child.child_mut().stdin.take().unwrap();
let mut output = BufReader::new(child.child_mut().stdout.take().unwrap()).lines();
let exchange = async {
for request in [
json!({"id":1,"method":"initialize","params":{}}),
json!({"id":2,"method":"thread/start","params":{}}),
] {
input
.write_all(format!("{request}\n").as_bytes())
.await
.unwrap();
let response: Value =
serde_json::from_str(&output.next_line().await.unwrap().unwrap()).unwrap();
assert_eq!(response["id"], request["id"]);
assert!(response.get("result").is_some(), "{response}");
}
let request = json!({"id":3,"method":"turn/start","params":{
"cwd":dir.path(), "input":[{"type":"text","text":"[scene:bench]"}]
}});
input
.write_all(format!("{request}\n").as_bytes())
.await
.unwrap();
let mut text = String::new();
let mut deltas = 0;
loop {
let message: Value = serde_json::from_str(
&output
.next_line()
.await
.unwrap()
.expect("engine exited before turn completed"),
)
.unwrap();
match message["method"].as_str() {
Some("item/agentMessage/delta") => {
text.push_str(message["params"]["delta"].as_str().unwrap());
deltas += 1;
}
Some("turn/completed") => {
assert_eq!(message["params"]["turn"]["status"], "completed");
assert!(!message["params"]["threadId"].as_str().unwrap().is_empty());
break;
}
_ => {}
}
}
assert_eq!(deltas, 200);
assert_eq!(text.len(), 50 * 1024);
assert!(text.ends_with("END."));
};
tokio::time::timeout(Duration::from_secs(15), exchange)
.await
.expect("scripted protocol timed out");
// The process-tree owner also closes the script's protocol-double descendant.
drop(input);
drop(child);
}
2 changes: 1 addition & 1 deletion docs/parity.md
Original file line number Diff line number Diff line change
Expand Up @@ -736,7 +736,7 @@ no row of their own are in I62. A CLI that speaks ACP is found and launched by i
- [x] O24 Hidden helpers: ACP MCP bridge and call, SSH helper, service launcher, Claude history | apps/server/src/cli/acpMcpBridge.ts | c | M | scope: mixed | local: hidden `orbyn acp-mcp-bridge` (stdio JSON-RPC relay to the run's `/mcp/:token`, offered by the ACP driver as a stdio `mcpServers` entry to agents without HTTP MCP, endpoint in the environment only), `orbyn acp-mcp-call <tool> <arguments-json>`, `orbyn service run` as the service launcher, and the credential-free, read-only `orbyn __claude-history getSessionMessages` | excludes: SSH helper; `__claude-history forkSession`, refused because it would write into Claude's own history (`claude --resume --fork-session` does that) | evidence: `crates/orbynd/tests/drivers.rs` runs a fake ACP agent without HTTP MCP through real `orbynd`, which offers the stdio bridge; the agent lists tools through `orbyn acp-mcp-bridge` and calls `skill_search` through `orbyn acp-mcp-call`; `crates/orbynd/tests/hidden_helpers.rs` covers hidden help, usage exits, and real-CLI history reads with Unix and Windows working folders; Core unit tests cover the stdio entry, `.exe` sibling lookup, Windows folder names and id path refusal
- [x] O25 Start in directory: `t3 <path>` starts there, bare names only if the folder exists, existing server reported, `--base-dir` isolates a second server | docs/user/install.md | c | S | server done, as `orbyn <path>`; `--data-dir` isolates a second service
- [x] O26 Try without install: `npx t3@latest`; `t3 help` full reference | docs/user/install.md | c | S | docs/cli.md covers registered commands, options, exit codes, source/bundle invocation and hidden service-run behavior; real CLI triage help tested | status: partial | remaining: no standalone npm launcher package or npx distribution; no try-without-install run is claimed
- [x] O27 Render benchmark: replay a seeded 50 KB reply in fixed deltas into this UI and the reference UI in fresh browsers, record main-thread time, heap and mutations | bench/t3-compare/README.md | c | M | beyond T3 | scope: mixed | local: `web/e2e/real/render-benchmark.spec.ts` uses the seeded `bench` scene (200 deltas of 256 bytes, 10 ms apart, 50 KiB) through real `orbynd` in fresh Chromium; it records task, script, layout and style time, heap and DOM mutations and fails on a lost reply, 30 s main-thread or 512 MiB heap overrun | excludes: running the reference UI side by side; no reference build is available here | evidence: historical macOS record from two heavily loaded runs: about 2.4-2.5 s main-thread task, 1.4 s script, 40-48 MB heap, about 870-890 mutation records; not re-run here. On 2026-11-08, source `42725569`, Windows 11 Pro x64 build 26200, Ryzen 5 5600X (6 cores/12 threads), 32 GiB RAM (9.6 GiB free), CPU load snapshot 2%, all other owned heavy jobs idle: the unchanged spec failed 1/1 because its scripted engine could not start (Win32 error 193); no assistant reply or valid timing/heap/mutation metrics were produced. Reused frozen executables; 949 web inputs, 270 tracked runtime inputs and three binary hashes matched before/after | status: partial | remaining: Windows scripted-engine launch needs separate work before a successful Windows replay; comparative reference numbers and a certified quiet-machine baseline are still missing; owner applications remained open, with no current macOS run
- [x] O27 Render benchmark: replay a seeded 50 KB reply in fixed deltas into this UI and the reference UI in fresh browsers, record main-thread time, heap and mutations | bench/t3-compare/README.md | c | M | beyond T3 | scope: mixed | local: `web/e2e/real/render-benchmark.spec.ts` uses the seeded `bench` scene (200 deltas of 256 bytes, 10 ms apart, 50 KiB) through real `orbynd` in fresh Chromium; it records task, script, layout and style time, heap and DOM mutations and fails on a lost reply, 30 s main-thread or 512 MiB heap overrun | excludes: running the reference UI side by side; no reference build is available here | evidence: historical macOS record from two heavily loaded runs: about 2.4-2.5 s main-thread task, 1.4 s script, 40-48 MB heap, about 870-890 mutation records; not re-run here. On 2026-11-08, source `42725569`, Windows 11 Pro x64 build 26200, Ryzen 5 5600X (6 cores/12 threads), 32 GiB RAM (9.6 GiB free), CPU load snapshot 2%, all other owned heavy jobs idle: the unchanged spec failed 1/1 because its scripted engine could not start (Win32 error 193); no assistant reply or valid timing/heap/mutation metrics were produced. Reused frozen executables; 949 web inputs, 270 tracked runtime inputs and three binary hashes matched before/after. With the scripted-engine launch correction in #336 against base `2e02f9e6`, the unchanged O27 spec passed 1/1 on Windows using Playwright headless shell 1248 and reached the complete assistant reply ending `END.`; this is functional evidence only, with no quiet-machine performance baseline claimed | status: partial | remaining: comparative reference numbers and a certified quiet-machine baseline are still missing; owner applications remained open, with no current macOS run

## P. Desktop shell
- [x] P1 Bundled backend: pool of managed local servers | apps/desktop/src/backend/DesktopBackendPool.ts | d | L | one managed `orbynd`; a pool serves remote hosts, which are out of scope | scope: mixed | local: one bundled local backend | excludes: a pool of remote hosts
Expand Down
6 changes: 6 additions & 0 deletions web/e2e/real/engine-scripts.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
import { resolve } from "node:path";

// Windows follows these npm-style shims to Node; CreateProcess cannot run a shebang.
const extension = process.platform === "win32" ? "cmd" : "mjs";
export const SCRIPTED_ENGINE = resolve(import.meta.dirname, `scripted-engine.${extension}`);
export const USAGE_ENGINE = resolve(import.meta.dirname, `usage-engine.${extension}`);
2 changes: 2 additions & 0 deletions web/e2e/real/scripted-engine.cmd
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
@echo off
node "%~dp0\scripted-engine.mjs" %*
5 changes: 2 additions & 3 deletions web/e2e/real/service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,10 @@ import { tmpdir } from "node:os";
import { delimiter, join, resolve } from "node:path";
import { createInterface } from "node:readline";
import { once } from "node:events";
import { SCRIPTED_ENGINE, USAGE_ENGINE } from "./engine-scripts";

const WEB = resolve(import.meta.dirname, "../..");
const BIN = process.env.ORBYN_BIN ?? resolve(WEB, "../target/debug/orbynd");
const SCRIPTED = resolve(WEB, "e2e/real/scripted-engine.mjs");
const USAGE_ENGINE = resolve(WEB, "e2e/real/usage-engine.mjs");

/** The shell from `npm run test:e2e:install`, so the service's previews never open an installed browser. */
function headlessShell(): string {
Expand Down Expand Up @@ -175,7 +174,7 @@ export async function startService(options: ServiceOptions = {}): Promise<Servic
const dir = mkdtempSync(join(tmpdir(), "orbyn-e2e-"));
const fake = resolve(BIN, "../orbyn-fake-engine");
const engines = options.scenes
? { ORBYN_CLAUDE_BIN: join(dir, "no-claude"), ORBYN_CODEX_BIN: SCRIPTED, ORBYN_E2E_FAKE: fake }
? { ORBYN_CLAUDE_BIN: join(dir, "no-claude"), ORBYN_CODEX_BIN: SCRIPTED_ENGINE, ORBYN_E2E_FAKE: fake }
: options.claudeUsage
? { ORBYN_CLAUDE_BIN: USAGE_ENGINE, ORBYN_CODEX_BIN: fake, ORBYN_E2E_FAKE: fake, ORBYN_E2E_USAGE: JSON.stringify(options.claudeUsage) }
: options.fakeEngines
Expand Down
3 changes: 2 additions & 1 deletion web/e2e/real/thread-lifecycle-parity.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { createServer } from "node:net";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { createInterface } from "node:readline";
import { SCRIPTED_ENGINE } from "./engine-scripts";

const web = resolve(import.meta.dirname, "../..");

Expand Down Expand Up @@ -55,7 +56,7 @@ test("selected routes follow lifecycle sections and limit recovery keeps snooze
const kept = ["PATH", "Path", "SystemRoot", "TEMP", "TMP", "TMPDIR", "LANG"];
const env = { ...Object.fromEntries(kept.filter((key) => process.env[key]).map((key) => [key, process.env[key]])),
HOME: profile, CFFIXED_USER_HOME: profile, USERPROFILE: profile, ORBYN_OFFLINE: "1",
ORBYN_CLAUDE_BIN: join(profile, "no-claude"), ORBYN_CODEX_BIN: resolve(web, "e2e/real/scripted-engine.mjs"), ORBYN_E2E_FAKE: fake,
ORBYN_CLAUDE_BIN: join(profile, "no-claude"), ORBYN_CODEX_BIN: SCRIPTED_ENGINE, ORBYN_E2E_FAKE: fake,
CODEX_HOME: join(profile, "codex"), CLAUDE_CONFIG_DIR: join(profile, "claude"),
GIT_AUTHOR_NAME: "Test", GIT_AUTHOR_EMAIL: "test@example.com", GIT_COMMITTER_NAME: "Test", GIT_COMMITTER_EMAIL: "test@example.com" };
const git = (...args: string[]) => execFileSync("git", args, { cwd: library, env, stdio: "ignore" });
Expand Down
2 changes: 2 additions & 0 deletions web/e2e/real/usage-engine.cmd
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
@echo off
node "%~dp0\usage-engine.mjs" %*
Loading