Skip to content

Commit 8e3053a

Browse files
singaraionaAnton
andcommitted
build: source rayforce from upstream at pinned SHA (#8)
* build: source rayforce from upstream at pinned SHA The vendored rayforce/ copy had drifted from upstream and shipped incomplete (no Makefile). Replace it with an auto-clone flow that sources rayforce from the pinned SHA in `.rayforce-version`. Resolution order in build.rs: 1. RAYFORCE_DIR set -> link external librayforce.a (dev mode, unchanged) 2. vendor/rayforce/ -> source bundled inside the published Makefile exists .crate tarball (or populated by CI before `cargo package`); copy to OUT_DIR, build 3. neither -> shallow clone upstream at the pinned SHA into OUT_DIR, build there All `make lib` work happens in `OUT_DIR/rayforce-build/`. The source tree is never modified — required for `cargo package` to verify clean. CFLAGS override: drop `-march=native` and `-Werror` from the upstream RELEASE_CFLAGS so the produced .a runs on any CPU of the target arch and downstream builds don't break on new compiler warnings. Cargo.toml `include` whitelists vendor/rayforce/{Makefile,LICENSE, src/**,include/**} so the published .crate ships the source needed for offline downstream builds. .gitignore drops vendor/ from tracking; CI populates it explicitly before `cargo package` (publish.yml). Pin format: single-line hex SHA in .rayforce-version. Bump by editing the file and running `cargo clean -p raysense && cargo build`. Verified locally: - cargo test: 123/123 green - cargo fmt --check: clean - cargo package: 167 files, 838 KiB compressed (under crates.io 10 MiB limit). Verification build passes — source dir unmodified. * fmt: collapse read_pin one-liner per rustfmt --------- Co-authored-by: Anton <anton.kundenko@gmail.com>
1 parent 458687d commit 8e3053a

145 files changed

Lines changed: 191 additions & 78756 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/publish.yml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,27 @@ jobs:
4545
- name: Checkout raysense
4646
uses: actions/checkout@v6
4747

48+
- name: Vendor rayforce at pinned SHA
49+
# The published .crate must contain the rayforce source so end users
50+
# can `cargo install raysense` without network access. We clone it
51+
# here, strip .git/, and let `cargo package` pick it up via the
52+
# Cargo.toml `include` whitelist (vendor/ is gitignored otherwise).
53+
shell: bash
54+
run: |
55+
set -euo pipefail
56+
sha="$(cat .rayforce-version | tr -d '[:space:]')"
57+
if [[ -z "$sha" ]]; then
58+
echo "::error::.rayforce-version is empty" >&2
59+
exit 1
60+
fi
61+
mkdir -p vendor
62+
rm -rf vendor/rayforce
63+
git -c advice.detachedHead=false clone --quiet \
64+
https://github.com/RayforceDB/rayforce.git vendor/rayforce
65+
git -C vendor/rayforce checkout --quiet "$sha"
66+
rm -rf vendor/rayforce/.git
67+
test -f vendor/rayforce/Makefile || { echo "::error::Makefile missing" >&2; exit 1; }
68+
4869
- name: Test
4970
run: cargo test
5071

.gitignore

Lines changed: 2 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,7 @@
1-
# Copyright (c) 2025-2026 Anton Kundenko <singaraiona@gmail.com>
2-
# All rights reserved.
3-
#
4-
# Permission is hereby granted, free of charge, to any person obtaining a copy
5-
# of this software and associated documentation files (the "Software"), to deal
6-
# in the Software without restriction, including without limitation the rights
7-
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8-
# copies of the Software, and to permit persons to whom the Software is
9-
# furnished to do so, subject to the following conditions:
10-
#
11-
# The above copyright notice and this permission notice shall be included in all
12-
# copies or substantial portions of the Software.
13-
#
14-
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15-
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16-
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17-
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18-
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19-
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20-
# SOFTWARE.
21-
1+
CLAUDE.md
222
/target/
233
Cargo.lock
244
/local/
255
.codex
266
/deps/rayforce/
7+
/vendor/

.rayforce-version

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
6614c1baebf25f0fd0db02cb0de214a92e22f550

Cargo.toml

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,24 @@ repository = "https://github.com/RayforceDB/raysense"
2828
description = "Architectural X-ray for your codebase. Live, local, agent-ready."
2929
readme = "README.md"
3030
links = "rayforce"
31+
# Whitelist files shipped in the published .crate. `vendor/rayforce/**` is
32+
# included even though it's gitignored — CI populates it from upstream
33+
# rayforce at the pinned SHA before `cargo package` runs.
34+
include = [
35+
"src/**/*.rs",
36+
"build.rs",
37+
"Cargo.toml",
38+
"README.md",
39+
".rayforce-version",
40+
# Bundled rayforce source — narrow to just what `make lib` needs.
41+
# Excludes website/, docs/, test/, examples/, bench/ to keep the
42+
# published .crate well under the crates.io 10 MiB ceiling.
43+
"vendor/rayforce/Makefile",
44+
"vendor/rayforce/LICENSE",
45+
"vendor/rayforce/include/**/*.h",
46+
"vendor/rayforce/src/**/*.c",
47+
"vendor/rayforce/src/**/*.h",
48+
]
3149

3250
[[bin]]
3351
name = "raysense"
@@ -56,5 +74,4 @@ tree-sitter-python = "0.25.0"
5674
tree-sitter-rust = "0.24.2"
5775
tree-sitter-typescript = "0.23.2"
5876

59-
[build-dependencies]
60-
cc = "1"
77+
# build.rs uses only std + git/make subprocesses; no build-deps.

build.rs

Lines changed: 148 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -21,21 +21,56 @@
2121
* SOFTWARE.
2222
*/
2323

24-
//! Compile the vendored C library directly via `cc`. No external checkout
25-
//! required — `cargo build` works from a fresh clone with no extra steps.
26-
//! Set `RAYFORCE_DIR` only if you want to link against an outside build for
27-
//! development.
24+
//! Build raysense by linking the upstream rayforce static library.
25+
//!
26+
//! Three resolution modes, in priority order:
27+
//!
28+
//! 1. `RAYFORCE_DIR` env var — link an externally built `librayforce.a`
29+
//! from a developer-provided rayforce checkout (you build rayforce
30+
//! yourself, point raysense at it).
31+
//!
32+
//! 2. `vendor/rayforce/Makefile` exists in the source tree (bundled
33+
//! inside the published `.crate` tarball, or populated by CI before
34+
//! `cargo package`) — copy that source into `OUT_DIR` and build it
35+
//! there.
36+
//!
37+
//! 3. Otherwise — clone upstream rayforce at the SHA pinned in
38+
//! `.rayforce-version` directly into `OUT_DIR`, then build it there.
39+
//!
40+
//! All `make lib` work happens inside `OUT_DIR/rayforce-build/`. The
41+
//! source tree is never modified — required by `cargo package`'s
42+
//! verification step (build scripts must not write outside `OUT_DIR`).
43+
//!
44+
//! The Makefile's stock `RELEASE_CFLAGS` includes `-march=native`, which
45+
//! bakes the build host's CPU features into the static library and would
46+
//! crash on older CPUs of the same arch. We override `RELEASE_CFLAGS` to
47+
//! a portable baseline so the produced `.a` is shippable across hosts.
2848
2949
use std::env;
50+
use std::fs;
3051
use std::path::{Path, PathBuf};
52+
use std::process::Command;
53+
54+
const RAYFORCE_REPO: &str = "https://github.com/RayforceDB/rayforce.git";
55+
56+
/// Portable release CFLAGS. Differs from upstream `RELEASE_CFLAGS` by
57+
/// dropping `-march=native` (build-host-specific) and `-Werror` (would
58+
/// fail downstream builds on new compiler warnings).
59+
const PORTABLE_CFLAGS: &str = "-fPIC -O3 -fomit-frame-pointer -fno-math-errno \
60+
-funroll-loops -std=c17 -Wall -Wextra -Wstrict-prototypes \
61+
-Wno-unused-parameter";
3162

3263
fn main() {
3364
let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
3465

35-
if let Some(external_dir) = env::var_os("RAYFORCE_DIR") {
36-
link_external(PathBuf::from(external_dir));
66+
if let Some(external) = env::var_os("RAYFORCE_DIR") {
67+
link_external(PathBuf::from(external));
3768
} else {
38-
compile_vendored(&manifest_dir.join("vendor/rayforce"));
69+
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
70+
let build_dir = out_dir.join("rayforce-build");
71+
ensure_build_dir(&manifest_dir, &build_dir);
72+
run_make_lib(&build_dir);
73+
link_static_lib(&build_dir);
3974
}
4075

4176
if env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("linux") {
@@ -46,59 +81,124 @@ fn main() {
4681
}
4782

4883
println!("cargo:rerun-if-env-changed=RAYFORCE_DIR");
84+
println!("cargo:rerun-if-changed=.rayforce-version");
4985
}
5086

51-
/// Default path: build the vendored sources with `cc::Build`. Excludes the
52-
/// REPL binary entry (`src/app/main.c`) since we only need the library.
53-
fn compile_vendored(vendor_dir: &Path) {
54-
let include_dir = vendor_dir.join("include");
55-
let src_dir = vendor_dir.join("src");
56-
let mut build = cc::Build::new();
57-
build
58-
.std("c17")
59-
.include(&include_dir)
60-
.include(&src_dir)
61-
.flag_if_supported("-fPIC")
62-
.flag_if_supported("-Wno-unused-parameter")
63-
.flag_if_supported("-Wno-unused-but-set-variable")
64-
.flag_if_supported("-Wno-unused-variable")
65-
.flag_if_supported("-Wno-unused-function");
66-
67-
if let Ok(profile) = env::var("PROFILE") {
68-
if profile == "release" {
69-
build
70-
.opt_level(3)
71-
.flag_if_supported("-funroll-loops")
72-
.flag_if_supported("-fomit-frame-pointer")
73-
.flag_if_supported("-fno-math-errno");
87+
/// Materialize rayforce source under `build_dir`. Either copy bundled
88+
/// `vendor/rayforce/` from the source tree, or clone upstream at the
89+
/// pinned SHA. Skips work if `build_dir` already holds the right SHA.
90+
fn ensure_build_dir(manifest_dir: &Path, build_dir: &Path) {
91+
let pinned_sha = read_pin(manifest_dir);
92+
let sentinel = build_dir.join(".raysense-built-sha");
93+
94+
if let Ok(prev) = fs::read_to_string(&sentinel) {
95+
if prev.trim() == pinned_sha {
96+
return;
7497
}
7598
}
7699

77-
let mut count = 0usize;
78-
for entry in walk_c_sources(&src_dir) {
79-
if entry.ends_with(Path::new("app/main.c"))
80-
|| entry.ends_with(Path::new("app/repl.c"))
81-
|| entry.ends_with(Path::new("app/term.c"))
82-
{
83-
continue;
84-
}
85-
println!("cargo:rerun-if-changed={}", entry.display());
86-
build.file(&entry);
87-
count += 1;
100+
if build_dir.exists() {
101+
fs::remove_dir_all(build_dir).expect("rm previous rayforce-build/");
102+
}
103+
104+
let bundled = manifest_dir.join("vendor/rayforce");
105+
if bundled.join("Makefile").exists() {
106+
copy_tree(&bundled, build_dir);
107+
} else {
108+
clone_at_pin(build_dir, &pinned_sha);
109+
}
110+
111+
fs::write(&sentinel, &pinned_sha).expect("write sentinel");
112+
}
113+
114+
fn read_pin(manifest_dir: &Path) -> String {
115+
let path = manifest_dir.join(".rayforce-version");
116+
let raw = fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display()));
117+
let sha = raw.trim();
118+
if sha.len() < 7 || !sha.chars().all(|c| c.is_ascii_hexdigit()) {
119+
panic!("`.rayforce-version` does not contain a hex SHA: {sha:?}");
120+
}
121+
sha.to_string()
122+
}
123+
124+
fn copy_tree(src: &Path, dst: &Path) {
125+
fs::create_dir_all(dst).expect("mkdir build_dir");
126+
// `cp -r` is robust on Linux/macOS; the upstream Makefile only
127+
// supports those platforms anyway. Trailing dot copies contents,
128+
// not the src dir itself.
129+
let status = Command::new("cp")
130+
.arg("-R")
131+
.arg(format!("{}/.", src.display()))
132+
.arg(dst)
133+
.status()
134+
.unwrap_or_else(|e| panic!("`cp -R {src:?} -> {dst:?}` failed: {e}"));
135+
if !status.success() {
136+
panic!("`cp -R` exited {status}");
137+
}
138+
}
139+
140+
fn clone_at_pin(build_dir: &Path, sha: &str) {
141+
if let Some(parent) = build_dir.parent() {
142+
fs::create_dir_all(parent).expect("mkdir build_dir parent");
88143
}
89-
if count == 0 {
144+
fs::create_dir_all(build_dir).expect("mkdir build_dir");
145+
146+
run_git(build_dir, &["init", "-q"]);
147+
run_git(build_dir, &["remote", "add", "origin", RAYFORCE_REPO]);
148+
run_git(build_dir, &["fetch", "--depth", "1", "origin", sha]);
149+
run_git(build_dir, &["checkout", "--quiet", "FETCH_HEAD"]);
150+
// Strip .git/ — keeps OUT_DIR small and prevents stale clone state
151+
// from confusing future cache hits.
152+
let dot_git = build_dir.join(".git");
153+
if dot_git.exists() {
154+
let _ = fs::remove_dir_all(&dot_git);
155+
}
156+
}
157+
158+
fn run_git(cwd: &Path, args: &[&str]) {
159+
let status = Command::new("git")
160+
.current_dir(cwd)
161+
.args(args)
162+
.status()
163+
.unwrap_or_else(|e| panic!("failed to run `git {}`: {e}", args.join(" ")));
164+
if !status.success() {
165+
panic!("`git {}` failed with status {}", args.join(" "), status);
166+
}
167+
}
168+
169+
fn run_make_lib(build_dir: &Path) {
170+
let status = Command::new("make")
171+
.current_dir(build_dir)
172+
.arg("lib")
173+
.arg(format!("RELEASE_CFLAGS={PORTABLE_CFLAGS}"))
174+
.status()
175+
.unwrap_or_else(|e| panic!("failed to run `make lib`: {e}"));
176+
if !status.success() {
90177
panic!(
91-
"no C sources found under {} — vendor/ is empty?",
92-
src_dir.display()
178+
"`make lib` in {} exited with status {}",
179+
build_dir.display(),
180+
status
93181
);
94182
}
95-
println!("cargo:rerun-if-changed={}", include_dir.display());
183+
let lib = build_dir.join("librayforce.a");
184+
if !lib.exists() {
185+
panic!(
186+
"expected {} after `make lib`, but it is missing",
187+
lib.display()
188+
);
189+
}
190+
println!("cargo:rerun-if-changed={}", lib.display());
191+
}
192+
193+
fn link_static_lib(build_dir: &Path) {
194+
let include_dir = build_dir.join("include");
96195
println!("cargo:include={}", include_dir.display());
97-
build.compile("rayforce");
196+
println!("cargo:rustc-link-search=native={}", build_dir.display());
197+
println!("cargo:rustc-link-lib=static=rayforce");
98198
}
99199

100-
/// Optional: link against an externally-built `librayforce.a`. Used only for
101-
/// rayforce development; everyone else gets the vendored compile path above.
200+
/// Optional: link against an externally-built `librayforce.a`. Used only
201+
/// for rayforce development; everyone else gets the auto-vendored path.
102202
fn link_external(rayforce_dir: PathBuf) {
103203
let include_dir = rayforce_dir.join("include");
104204
let lib_path = rayforce_dir.join("librayforce.a");
@@ -119,25 +219,3 @@ fn link_external(rayforce_dir: PathBuf) {
119219
include_dir.join("rayforce.h").display()
120220
);
121221
}
122-
123-
/// Walk a directory tree collecting all `*.c` files. Pure-std (no walkdir
124-
/// dep) to keep build-deps minimal.
125-
fn walk_c_sources(root: &Path) -> Vec<PathBuf> {
126-
let mut out = Vec::new();
127-
let mut stack = vec![root.to_path_buf()];
128-
while let Some(dir) = stack.pop() {
129-
let Ok(entries) = std::fs::read_dir(&dir) else {
130-
continue;
131-
};
132-
for entry in entries.flatten() {
133-
let path = entry.path();
134-
if path.is_dir() {
135-
stack.push(path);
136-
} else if path.extension().and_then(|s| s.to_str()) == Some("c") {
137-
out.push(path);
138-
}
139-
}
140-
}
141-
out.sort();
142-
out
143-
}

vendor/rayforce/LICENSE

Lines changed: 0 additions & 21 deletions
This file was deleted.

0 commit comments

Comments
 (0)