diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 53ecb358..556fbbf2 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -88,6 +88,18 @@ jobs: env: CBM_SKIP_PERF: ${{ inputs.skip_perf && '1' || '' }} + test-tsan: + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Install deps (Ubuntu) + run: sudo apt-get update && sudo apt-get install -y clang zlib1g-dev + + - name: ThreadSanitizer tests + run: make -f Makefile.cbm test-tsan CC=clang CXX=clang++ + test-windows: needs: setup-matrix strategy: diff --git a/Makefile.cbm b/Makefile.cbm index 299cfcde..d9c0218b 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -67,10 +67,11 @@ CFLAGS_TEST = $(CFLAGS_COMMON) -g -O1 $(SANITIZE) CXXFLAGS_TEST = $(CXXFLAGS_COMMON) -g -O1 $(SANITIZE) # TSan (can't combine with ASan) +TSAN_SANITIZE = -fsanitize=thread -fno-omit-frame-pointer CFLAGS_TSAN = $(CFLAGS_COMMON) -g -O1 \ - -fsanitize=thread -fno-omit-frame-pointer + $(TSAN_SANITIZE) CXXFLAGS_TSAN = $(CXXFLAGS_COMMON) -g -O1 \ - -fsanitize=thread -fno-omit-frame-pointer + $(TSAN_SANITIZE) # Windows needs ws2_32 (Winsock), psapi (GetProcessMemoryInfo), and # --allow-multiple-definition (MinGW CRT symbol clashes). @@ -88,7 +89,7 @@ endif LDFLAGS = -lm -lstdc++ -lpthread -lz $(WIN32_LIBS) $(STATIC_FLAGS) LDFLAGS_TEST = -lm -lstdc++ -lpthread -lz $(SANITIZE) $(WIN32_LIBS) -LDFLAGS_TSAN = -lm -lstdc++ -lpthread -lz -fsanitize=thread $(WIN32_LIBS) +LDFLAGS_TSAN = -lm -lstdc++ -lpthread -lz $(TSAN_SANITIZE) $(WIN32_LIBS) # ── Source files ───────────────────────────────────────────────── @@ -480,13 +481,17 @@ GRAMMAR_CFLAGS = -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -I$(TS_INCLUDE) GRAMMAR_CFLAGS_TEST = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \ $(SANITIZE) GRAMMAR_CFLAGS_TSAN = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \ - -fsanitize=thread -fno-omit-frame-pointer + $(TSAN_SANITIZE) # Object files for grammars + ts_runtime + lsp_all + preprocessor GRAMMAR_OBJS_TEST = $(patsubst $(CBM_DIR)/%.c,$(BUILD_DIR)/%.o,$(GRAMMAR_SRCS)) TS_RUNTIME_OBJ_TEST = $(BUILD_DIR)/ts_runtime.o LSP_OBJ_TEST = $(BUILD_DIR)/lsp_all.o PP_OBJ_TEST = $(BUILD_DIR)/preprocessor.o +GRAMMAR_OBJS_TSAN = $(patsubst $(CBM_DIR)/%.c,$(BUILD_DIR)/tsan_%.o,$(GRAMMAR_SRCS)) +TS_RUNTIME_OBJ_TSAN = $(BUILD_DIR)/tsan_ts_runtime.o +LSP_OBJ_TSAN = $(BUILD_DIR)/tsan_lsp_all.o +PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o # ── Targets ────────────────────────────────────────────────────── @@ -517,53 +522,86 @@ $(BUILD_DIR)/lsp_all.o: $(LSP_UNITY_DEPS) | $(BUILD_DIR) $(BUILD_DIR)/preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR) $(CXX) $(CXXFLAGS_TEST) -w -I$(CBM_DIR)/vendored -c -o $@ $< +$(BUILD_DIR)/tsan_%.o: $(CBM_DIR)/%.c | $(BUILD_DIR) + $(CC) $(GRAMMAR_CFLAGS_TSAN) -c -o $@ $< + +$(BUILD_DIR)/tsan_ts_runtime.o: $(CBM_DIR)/ts_runtime.c | $(BUILD_DIR) + $(CC) $(GRAMMAR_CFLAGS_TSAN) -c -o $@ $< + +$(BUILD_DIR)/tsan_lsp_all.o: $(LSP_UNITY_DEPS) | $(BUILD_DIR) + $(CC) $(GRAMMAR_CFLAGS_TSAN) -c -o $@ $< + +$(BUILD_DIR)/tsan_preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR) + $(CXX) $(CXXFLAGS_TSAN) -w -I$(CBM_DIR)/vendored -c -o $@ $< + # ── Full test binary ───────────────────────────────────────────── # mimalloc object files MIMALLOC_OBJ_TEST = $(BUILD_DIR)/mimalloc.o +MIMALLOC_OBJ_TSAN = $(BUILD_DIR)/tsan_mimalloc.o MIMALLOC_OBJ_PROD = $(BUILD_DIR)/prod_mimalloc.o $(BUILD_DIR)/mimalloc.o: $(MIMALLOC_SRC) | $(BUILD_DIR) $(CC) $(MIMALLOC_CFLAGS_TEST) -c -o $@ $< +$(BUILD_DIR)/tsan_mimalloc.o: $(MIMALLOC_SRC) | $(BUILD_DIR) + $(CC) $(MIMALLOC_CFLAGS_TEST) $(TSAN_SANITIZE) -c -o $@ $< + $(BUILD_DIR)/prod_mimalloc.o: $(MIMALLOC_SRC) | $(BUILD_DIR) $(CC) $(MIMALLOC_CFLAGS) -c -o $@ $< # sqlite3 object files (vendored amalgamation) SQLITE3_OBJ_TEST = $(BUILD_DIR)/sqlite3.o +SQLITE3_OBJ_TSAN = $(BUILD_DIR)/tsan_sqlite3.o SQLITE3_OBJ_PROD = $(BUILD_DIR)/prod_sqlite3.o $(BUILD_DIR)/sqlite3.o: $(SQLITE3_SRC) | $(BUILD_DIR) $(CC) $(SQLITE3_CFLAGS_TEST) $(SANITIZE) -c -o $@ $< +$(BUILD_DIR)/tsan_sqlite3.o: $(SQLITE3_SRC) | $(BUILD_DIR) + $(CC) $(SQLITE3_CFLAGS_TEST) $(TSAN_SANITIZE) -c -o $@ $< + $(BUILD_DIR)/prod_sqlite3.o: $(SQLITE3_SRC) | $(BUILD_DIR) $(CC) $(SQLITE3_CFLAGS) -c -o $@ $< # TRE regex (only compiled on Windows — POSIX uses system ) TRE_OBJ_TEST := +TRE_OBJ_TSAN := TRE_OBJ_PROD := ifeq ($(IS_MINGW),yes) TRE_OBJ_TEST = $(BUILD_DIR)/tre.o +TRE_OBJ_TSAN = $(BUILD_DIR)/tsan_tre.o TRE_OBJ_PROD = $(BUILD_DIR)/prod_tre.o $(BUILD_DIR)/tre.o: $(TRE_SRC) | $(BUILD_DIR) $(CC) $(TRE_CFLAGS) $(SANITIZE) -fno-sanitize=alignment -c -o $@ $< +$(BUILD_DIR)/tsan_tre.o: $(TRE_SRC) | $(BUILD_DIR) + $(CC) $(TRE_CFLAGS) $(TSAN_SANITIZE) -c -o $@ $< + $(BUILD_DIR)/prod_tre.o: $(TRE_SRC) | $(BUILD_DIR) $(CC) $(TRE_CFLAGS) -O2 -c -o $@ $< endif # Vendored LZ4 (test build) LZ4_OBJ_TEST = $(BUILD_DIR)/test_lz4.o $(BUILD_DIR)/test_lz4hc.o +LZ4_OBJ_TSAN = $(BUILD_DIR)/tsan_lz4.o $(BUILD_DIR)/tsan_lz4hc.o $(BUILD_DIR)/test_lz4.o: $(CBM_DIR)/vendored/lz4/lz4.c | $(BUILD_DIR) $(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(SANITIZE) -w -I$(CBM_DIR) -c -o $@ $< $(BUILD_DIR)/test_lz4hc.o: $(CBM_DIR)/vendored/lz4/lz4hc.c | $(BUILD_DIR) $(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(SANITIZE) -w -I$(CBM_DIR)/vendored/lz4 -c -o $@ $< +$(BUILD_DIR)/tsan_lz4.o: $(CBM_DIR)/vendored/lz4/lz4.c | $(BUILD_DIR) + $(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(TSAN_SANITIZE) -w -I$(CBM_DIR) -c -o $@ $< +$(BUILD_DIR)/tsan_lz4hc.o: $(CBM_DIR)/vendored/lz4/lz4hc.c | $(BUILD_DIR) + $(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(TSAN_SANITIZE) -w -I$(CBM_DIR)/vendored/lz4 -c -o $@ $< # Vendored zstd (test build) ZSTD_OBJ_TEST = $(BUILD_DIR)/test_zstd.o +ZSTD_OBJ_TSAN = $(BUILD_DIR)/tsan_zstd.o $(BUILD_DIR)/test_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c | $(BUILD_DIR) $(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(SANITIZE) -w -I$(CBM_DIR)/vendored/zstd -c -o $@ $< +$(BUILD_DIR)/tsan_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c | $(BUILD_DIR) + $(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(TSAN_SANITIZE) -w -I$(CBM_DIR)/vendored/zstd -c -o $@ $< # nomic-embed-code pretrained vector blob UNIXCODER_OBJ = $(BUILD_DIR)/unixcoder_blob.o @@ -571,6 +609,7 @@ $(UNIXCODER_OBJ): $(UNIXCODER_BLOB_SRC) vendored/nomic/code_vectors.bin | $(BUIL $(CC) -c -o $@ $< OBJS_VENDORED_TEST = $(MIMALLOC_OBJ_TEST) $(SQLITE3_OBJ_TEST) $(TRE_OBJ_TEST) $(GRAMMAR_OBJS_TEST) $(TS_RUNTIME_OBJ_TEST) $(LSP_OBJ_TEST) $(PP_OBJ_TEST) $(LZ4_OBJ_TEST) $(ZSTD_OBJ_TEST) $(UNIXCODER_OBJ) +OBJS_VENDORED_TSAN = $(MIMALLOC_OBJ_TSAN) $(SQLITE3_OBJ_TSAN) $(TRE_OBJ_TSAN) $(GRAMMAR_OBJS_TSAN) $(TS_RUNTIME_OBJ_TSAN) $(LSP_OBJ_TSAN) $(PP_OBJ_TSAN) $(LZ4_OBJ_TSAN) $(ZSTD_OBJ_TSAN) $(UNIXCODER_OBJ) $(BUILD_DIR)/test-runner: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) | $(BUILD_DIR) $(CC) $(CFLAGS_TEST) -Itests -Itests/repro -o $@ \ @@ -598,8 +637,18 @@ test-repro: $(BUILD_DIR)/test-repro-runner # ── TSan full test ─────────────────────────────────────────────── -test-tsan: - @echo "TSan not yet wired for full extraction tests" +TEST_TSAN_SUITES ?= mem slab_alloc parallel +TSAN_OPTIONS ?= halt_on_error=1 + +$(BUILD_DIR)/test-runner-tsan: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TSAN) | $(BUILD_DIR) + $(CC) $(CFLAGS_TSAN) -Itests -Itests/repro -o $@ \ + $(ALL_TEST_SRCS) $(PROD_SRCS) \ + $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \ + $(OBJS_VENDORED_TSAN) \ + $(LDFLAGS_TSAN) + +test-tsan: $(BUILD_DIR)/test-runner-tsan + cd $(CURDIR) && TSAN_OPTIONS="$(TSAN_OPTIONS)" $(BUILD_DIR)/test-runner-tsan $(TEST_TSAN_SUITES) # ── Production binary ────────────────────────────────────────────