Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,7 @@
# unusually well, and shrinking those headers to win a byte count would destroy the most useful thing in
# the files.
test/*.sh linguist-detectable=false

# Frozen evaluation packs are length-prefixed binary artifacts; Git must not rewrite their LF bytes on Windows.
bench/recalleval/*.mdpack -text
bench/recalleval/*.srcpack -text
66 changes: 66 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ concurrency:
# superseded. A queued main run costs runner minutes; a cancelled one costs the answer.
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

permissions:
contents: read

# Never RIPWIRE_NATIVE in CI: -march=native would bake in whatever ISA the CI runner's host happens to
# expose that week, defeating the entire point of testing the portable default.

Expand Down Expand Up @@ -88,6 +91,7 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # churn/co-change gates read real git history — a shallow clone reddens them
persist-credentials: false

- name: Install clang-format / clang-tidy (PINNED major — see the job comment)
run: |
Expand Down Expand Up @@ -209,6 +213,7 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # churn/co-change gates read real git history — a shallow clone reddens them
persist-credentials: false

# clang is installed on BOTH Linux legs, not only the clang one. optremarkscheck's Clang-only
# configure arms pin `clang++` when the default front end is not Clang, so the gcc leg keeps that
Expand Down Expand Up @@ -397,6 +402,7 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
persist-credentials: false

# RHEL 9's default gcc is 11, which does not implement C++23 — CMakeLists sets CXX_STANDARD 23 with
# STANDARD_REQUIRED ON, so the configure would fail outright. gcc-toolset-N is Red Hat's own supported
Expand Down Expand Up @@ -460,6 +466,65 @@ jobs:
- name: G4 — xmllint --noout
run: ./build/ripwire test/fixture --no-cache | xmllint --noout -

# ─── windows: native Windows validation with MSVC ABI + Clang ────────────────────────────────────────
windows:
name: windows (windows-latest, clang)
runs-on: windows-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
Comment thread
coderabbitai[bot] marked this conversation as resolved.
persist-credentials: false

- uses: ilammy/msvc-dev-cmd@v1
with:
arch: x64

- uses: actions/setup-python@v5
with:
python-version: '3.12'

- name: Configure (portable — clang + ninja)
shell: bash
run: cmake -S . -B build -G Ninja -DCMAKE_C_COMPILER=clang-cl -DCMAKE_CXX_COMPILER=clang-cl -DRIPWIRE_LTO=OFF

- name: Build
run: cmake --build build -j

# Build the HEAD comparison binary once, before the suite. The monotonicity gates use it in staged
# mode; building it inside a gate would multiply the Windows compile cost and can race the gate budget.
- name: Stage the HEAD comparison binary (once before the Windows suite)
shell: bash
run: |
export TMPDIR="$( cygpath -u "$RUNNER_TEMP" )"
export RIPWIRE_HEADBIN_BUILD_LOG="$TMPDIR/headbin-build.log"
. test/lib/headbinlib.sh
hb="$( ripwire_head_binary "$PWD" "$TMPDIR" )" || { echo "HEAD binary build failed; last 80 lines of its log:"; tail -n 80 "$RIPWIRE_HEADBIN_BUILD_LOG"; exit 1; }
ripwire_headbin_verify "$hb" "$( git rev-parse HEAD )"
"$hb" --version
echo "RIPWIRE_HEADBIN=$hb" >> "$GITHUB_ENV"

- name: Doctor check
run: .\build\ripwire.exe . --doctor

- name: Self-run on the fixture
run: .\build\ripwire.exe test/fixture --no-cache

- name: det-gate — 2-run byte-identical diff
shell: bash
run: |
./build/ripwire.exe test/fixture --no-cache > run_a.xml
./build/ripwire.exe test/fixture --no-cache > run_b.xml
diff -q run_a.xml run_b.xml

- name: Determinism gate (test/det-gate.sh)
shell: bash
run: bash test/det-gate.sh build/ripwire.exe

- name: Full native Windows gate suite
shell: bash
run: python test/pargates.py . build/ripwire.exe -j 6 --budget-scale 4

asan:
name: asan (${{ matrix.os }})
strategy:
Expand Down Expand Up @@ -490,6 +555,7 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # churn/co-change gates read real git history — a shallow clone reddens them
persist-credentials: false

- name: Install tooling (Linux)
if: runner.os == 'Linux'
Expand Down
118 changes: 111 additions & 7 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,10 @@ option(RIPWIRE_NATIVE "build with -march=native (DEV MACHINES ONLY — bakes in
include(cmake/PortableFlags.cmake)
add_compile_options(${RIPWIRE_ARCH_FLAGS})

if(WIN32)
add_compile_definitions(_CRT_SECURE_NO_WARNINGS _CRT_NONSTDC_NO_DEPRECATE)
endif()

# ---- link-time optimization: -DRIPWIRE_LTO=ON ----
# The one change the optimization-remarks pass actually justified (docs/OPTREMARKS.md, finding F1).
#
Expand Down Expand Up @@ -511,11 +515,16 @@ set(RIPWIRE_SRCS
src/ingest.cpp
src/pagerank.cpp
src/infra/diagnostics.cpp
src/infra/platform_compat.cpp
)

# The global profile uses -ffast-math, but PageRank reductions must not reassociate (the determinism
# contract's no-reassociation rule — docs/ARCHITECTURE.md §3).
set_source_files_properties(src/pagerank.cpp PROPERTIES COMPILE_OPTIONS "-fno-fast-math")
if(CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC")
set_source_files_properties(src/pagerank.cpp PROPERTIES COMPILE_OPTIONS "/fp:precise")
else()
set_source_files_properties(src/pagerank.cpp PROPERTIES COMPILE_OPTIONS "-fno-fast-math")
endif()

# All grammar OBJECT files, gathered once so every target links the same set.
set(RIPWIRE_TS_OBJECTS
Expand Down Expand Up @@ -560,9 +569,19 @@ add_executable(ripwire_probe
${RIPWIRE_SRCS}
${RIPWIRE_TS_OBJECTS})
target_link_libraries(ripwire_probe PRIVATE tree-sitter Threads::Threads)
if(WIN32)
target_link_libraries(ripwire_probe PRIVATE ws2_32)
target_sources(ripwire_probe PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/win32/ripwire.manifest")
if(CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC")
target_compile_options(ripwire_probe PRIVATE /EHsc /FI "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/platform_compat.h")
else()
target_compile_options(ripwire_probe PRIVATE -include "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/platform_compat.h")
endif()
endif()
target_include_directories(ripwire_probe PRIVATE
${tree_sitter_SOURCE_DIR}/lib/include
${_ripwire_generated_dir}
$<$<BOOL:${WIN32}>:${CMAKE_CURRENT_SOURCE_DIR}/src/infra/compat>
src/infra
third_party
src)
Expand All @@ -575,9 +594,19 @@ add_executable(ripwire
${RIPWIRE_SRCS}
${RIPWIRE_TS_OBJECTS})
target_link_libraries(ripwire PRIVATE tree-sitter Threads::Threads)
if(WIN32)
target_link_libraries(ripwire PRIVATE ws2_32)
target_sources(ripwire PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/win32/ripwire.manifest")
if(CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC")
target_compile_options(ripwire PRIVATE /EHsc /FI "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/platform_compat.h")
else()
target_compile_options(ripwire PRIVATE -include "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/platform_compat.h")
endif()
endif()
target_include_directories(ripwire PRIVATE
${tree_sitter_SOURCE_DIR}/lib/include
${_ripwire_generated_dir}
$<$<BOOL:${WIN32}>:${CMAKE_CURRENT_SOURCE_DIR}/src/infra/compat>
src/infra
third_party
src)
Expand Down Expand Up @@ -666,6 +695,18 @@ if(RIPWIRE_TESTS)
target_compile_definitions(ripwire_test_strkern PRIVATE RIPWIRE_TEST_ROOT="${CMAKE_CURRENT_SOURCE_DIR}")
target_link_libraries(ripwire_test_strkern PRIVATE doctest::doctest)
add_test(NAME ripwire.strkern COMMAND ripwire_test_strkern)

if(WIN32)
foreach(_ripwire_test_target ripwire_test_csr ripwire_test_pagerank ripwire_test_radix ripwire_test_strkern)
target_sources(${_ripwire_test_target} PRIVATE src/infra/platform_compat.cpp)
target_link_libraries(${_ripwire_test_target} PRIVATE ws2_32)
if(CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC")
target_compile_options(${_ripwire_test_target} PRIVATE /EHsc /FI "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/platform_compat.h")
else()
target_compile_options(${_ripwire_test_target} PRIVATE -include "${CMAKE_CURRENT_SOURCE_DIR}/src/infra/platform_compat.h")
endif()
endforeach()
endif()
endif()

# ---- self-profiling build (src/infra/profileScope.h): -DRIPWIRE_PROFILE=ON ----
Expand Down Expand Up @@ -857,14 +898,45 @@ file(WRITE "${_ripwire_libstdcxx_ignorelist}"
"src:*/include/c\\+\\+/*/print\n"
"[implicit-unsigned-integer-truncation]\nsrc:*/include/c\\+\\+/*/format\n"
"src:*/include/c\\+\\+/*/print\n")
if(WIN32)
# MSVC's filesystem prefix helper intentionally subtracts the lower bound from an unsigned packed value;
# Clang's integer sanitizer diagnoses that defined range test before any project code runs. Keep the
# exemption limited to the vendor header; project arithmetic remains covered by the complete G1 stack.
file(APPEND "${_ripwire_libstdcxx_ignorelist}"
"[unsigned-integer-overflow]\nsrc:*/include/filesystem\n"
"src:*filesystem\n"
"fun:*_Is_drive_prefix*\n")
endif()

if(RIPWIRE_ASAN)
set(_ripwire_windows_clangcl_asan OFF)
set(_ripwire_no_omit_frame_pointer -fno-omit-frame-pointer)
# CMake's MSVC linker rule invokes CMAKE_LINKER directly. With ClangCL, passing the LLVM
# -fsanitize flags to link.exe instruments the objects but never loads the Clang sanitizer
# runtime, leaving __asan_* unresolved. Use clang-cl as the link driver for this one flavour;
# its driver preserves the MSVC ABI/import libraries and adds the matching runtime.
if(WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC")
set(_ripwire_windows_clangcl_asan ON)
set(_ripwire_no_omit_frame_pointer /Oy-)
string(REPLACE "<CMAKE_LINKER>" "<CMAKE_CXX_COMPILER>" CMAKE_CXX_LINK_EXECUTABLE
"${CMAKE_CXX_LINK_EXECUTABLE}")
# The replacement template does not carry the generator's per-target FLAGS. Keep the
# sanitizer thunk and the instrumented objects on the same dynamic CRT as the compile rule.
string(REPLACE "<CMAKE_CXX_COMPILER> ${CMAKE_CL_NOLOGO} <OBJECTS>"
"<CMAKE_CXX_COMPILER> ${CMAKE_CL_NOLOGO} /MD <OBJECTS> ${RIPWIRE_G1_SANITIZERS}"
CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE}")
string(REPLACE " /out:<TARGET>" " /link /out:<TARGET>"
CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE}")
message(STATUS "RIPWIRE_ASAN: using the ClangCL driver for Windows sanitizer links")
endif()
foreach(_t IN LISTS RIPWIRE_RUNTIME_COMPILE_TARGETS)
target_compile_options(${_t} PRIVATE ${RIPWIRE_G1_SANITIZERS}
-fno-sanitize-recover=all -fno-omit-frame-pointer -O2 -g)
-fno-sanitize-recover=all ${_ripwire_no_omit_frame_pointer} -O2 -g)
endforeach()
foreach(_t IN LISTS RIPWIRE_RUNTIME_LINK_TARGETS)
target_link_options(${_t} PRIVATE ${RIPWIRE_G1_SANITIZERS})
if(NOT _ripwire_windows_clangcl_asan)
target_link_options(${_t} PRIVATE ${RIPWIRE_G1_SANITIZERS})
endif()
endforeach()
# Every exemption below names a check inside the Clang-only `integer` group, or uses
# -fsanitize-ignorelist=, which GCC does not implement. With `integer` absent there is nothing to
Expand All @@ -882,15 +954,47 @@ if(RIPWIRE_ASAN)
endforeach()
endif()

# Xcode's arm64 Darwin runtime rejects both the standalone leak sanitizer and detect_leaks=1 at startup.
# Keep the local ASan/UBSan gate executable and explicit; LeakSanitizer remains a Linux/upstream-runtime
# CI gate using the committed tree-sitter suppressions rather than being falsely claimed on Apple Clang.
if(APPLE)
# Darwin and Windows runtimes reject detect_leaks=1 at startup. Keep the local ASan/UBSan gate
# executable and explicit; LeakSanitizer remains a Linux/upstream-runtime CI gate using the
# committed tree-sitter suppressions rather than being falsely claimed on either platform.
if(APPLE OR WIN32)
set(_ripwire_asan_options "detect_leaks=0:halt_on_error=1:abort_on_error=1")
else()
set(_ripwire_asan_options "detect_leaks=1:halt_on_error=1:abort_on_error=1")
endif()
set(_ripwire_asan_runtime_copy_command)
if(WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
# clang-cl links the ASAN dynamic runtime, but Windows does not search Clang's resource
# directory for DLLs. Discover the matching directory from the active compiler so the fixture
# exercises this build without requiring a system-wide DLL installation. Copying beside the
# executable is deliberate: putting a semicolon-separated Windows PATH in a CMake command is
# parsed as a list of unrelated arguments by the Ninja generator.
execute_process(
COMMAND "${CMAKE_CXX_COMPILER}" -print-resource-dir
OUTPUT_VARIABLE _ripwire_clang_resource_dir
OUTPUT_STRIP_TRAILING_WHITESPACE
ERROR_QUIET)
if(_ripwire_clang_resource_dir)
file(TO_CMAKE_PATH "${_ripwire_clang_resource_dir}/lib/windows" _ripwire_asan_runtime_dir)
list(APPEND _ripwire_asan_runtime_copy_command
COMMAND ${CMAKE_COMMAND} -E copy_if_different
"${_ripwire_asan_runtime_dir}/clang_rt.asan_dynamic-x86_64.dll"
"$<TARGET_FILE_DIR:ripwire>")
message(STATUS "RIPWIRE_ASAN: fixture copies the runtime DLL from ${_ripwire_asan_runtime_dir}")
else()
message(WARNING "RIPWIRE_ASAN: clang resource directory was not discoverable; runtime fixture may miss the ASAN DLL")
endif()
endif()
if(WIN32 AND _ripwire_asan_runtime_dir)
foreach(_t IN LISTS RIPWIRE_RUNTIME_LINK_TARGETS)
add_custom_command(TARGET ${_t} POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_if_different
"${_ripwire_asan_runtime_dir}/clang_rt.asan_dynamic-x86_64.dll"
"$<TARGET_FILE_DIR:${_t}>")
endforeach()
endif()
add_custom_target(ripwire_asan_fixture
${_ripwire_asan_runtime_copy_command}
COMMAND ${CMAKE_COMMAND} -E env
"ASAN_OPTIONS=${_ripwire_asan_options}"
"UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1"
Expand Down
32 changes: 32 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,38 @@ is compiled out and freshness comes from the per-request stat sweep — that is
a degradation, so it is silent and the staleness contract is unchanged. You can build and run that
path on a Mac with `cmake -S . -B build-nokqueue -DCMAKE_CXX_FLAGS=-DRIPWIRE_HAS_KQUEUE=0`.

### Building on Windows

ripwire builds natively on Windows (x64) with Clang and the MSVC ABI, with zero external runtime
dependencies (linking only system `kernel32`, `ws2_32`, `advapi32`, `shell32`).

From an **x64 Native Tools Command Prompt for Visual Studio** (with `clang` and `ninja` on `PATH`):

```cmd
cmake -S . -B build -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++
cmake --build build -j
```

For maximum performance (Release mode with ThinLTO and host-CPU vectorization):

```cmd
cmake -S . -B build-release -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release -DRIPWIRE_NATIVE=ON
cmake --build build-release -j
```

Profile-Guided Optimization (PGO) is also supported on Windows via `scripts/pgobuild.sh` (under Git Bash) or CMake (`-DRIPWIRE_PGO=generate` and `-DRIPWIRE_PGO=use`), providing an additional 2–11% speedup across hot capture, query, and AST linting paths.

Or using the Visual Studio generator with Clang-CL:

```cmd
cmake -S . -B build -T ClangCL
cmake --build build --config Release
```

The resulting binaries (`build/ripwire.exe` and `build/ripwire_probe.exe`) embed an application
manifest opting into `longPathAware` (handling arbitrary deep paths up to NTFS 32k limits) and UTF-8
active code page.

### Determinism gate

Output is a sorted top-K. A sort has no tolerance band, so the contract is byte-identity:
Expand Down
27 changes: 24 additions & 3 deletions bench/agentloop/grade_answers.py
Original file line number Diff line number Diff line change
Expand Up @@ -156,10 +156,25 @@ def globstar_shell():
probed once and a row that NEEDS `**` is REFUSED when no capable shell exists, never guessed at."""
if not _SHELL:
_SHELL.append( None )
for candidate in ( "bash", "/opt/homebrew/bin/bash", "/usr/local/bin/bash", "/bin/bash" ):
if os.name == "nt":
configured = os.environ.get( "RIPWIRE_BASH", "" )
candidates = [ configured, r"C:\\Program Files\\Git\\usr\\bin\\bash.exe",
r"C:\\Program Files (x86)\\Git\\usr\\bin\\bash.exe" ]
candidates = [ c for c in candidates if c and os.path.isfile( c ) ]
else:
candidates = ( "bash", "/opt/homebrew/bin/bash", "/usr/local/bin/bash", "/bin/bash" )
for candidate in candidates:
if os.name == "nt":
normalized = os.path.normcase( os.path.abspath( candidate ) )
if "\\windows\\system32\\" in normalized or "\\windowsapps\\" in normalized:
continue
probe_env = None
if os.name == "nt":
probe_env = os.environ.copy()
probe_env[ "PATH" ] = "/usr/bin:/bin:" + probe_env.get( "PATH", "" )
try:
probe = subprocess.run( [ candidate, "-O", "globstar", "-c", "true" ],
capture_output=True, text=True, timeout=30 )
capture_output=True, text=True, timeout=30, env=probe_env )
except ( OSError, subprocess.SubprocessError ):
continue
if probe.returncode == 0:
Expand All @@ -170,9 +185,15 @@ def globstar_shell():
def run_gt( gt_command, pin_root, timeout_s=300 ):
"""Execute the derivation command at the pin, under bash (never the operator's zsh — §4)."""
shell = globstar_shell()
if os.name == "nt" and shell is None:
return "", "no supported Git Bash executable found", 127
argv = ( [ shell, "-O", "globstar", "-O", "nullglob", "-c", gt_command ] if shell
else [ "bash", "-c", gt_command ] )
proc = subprocess.run( argv, capture_output=True, text=True, cwd=str( pin_root ), timeout=timeout_s )
run_env = None
if os.name == "nt":
run_env = os.environ.copy()
run_env[ "PATH" ] = "/usr/bin:/bin:" + run_env.get( "PATH", "" )
proc = subprocess.run( argv, capture_output=True, text=True, cwd=str( pin_root ), timeout=timeout_s, env=run_env )
return proc.stdout, proc.stderr, proc.returncode

def derive_key( stdout ):
Expand Down
Loading