radlink: perf + feature series — ICF/ICFSTATIC, GCTYPES, header-units, link-time + memory (rebased on dev, taken commits dropped)

build.bat

@@ -52,13 +52,15 @@ if "%pgo%"=="1" (
   where llvm-profdata /q || echo llvm-profdata is not in the PATH || exit /b 1 
   if "%clang%"=="1" (
     if "%pgo_run%" == "1" (
-      call llvm-profdata merge %LLVM_PROFILE_FILE% -output=%~dp0build\build.profdata || exit /b 1
+      call llvm-profdata merge %~dp0build\pgo_raw\*.profraw -output=%~dp0build\build.profdata || exit /b 1
       set auto_compile_flags=%auto_compile_flags% -fprofile-use=%~dp0build\build.profdata
       set pgo_run=0
     ) else (
       echo [pgo enabled]
       set auto_compile_flags=%auto_compile_flags% -fprofile-generate -mllvm -vp-counters-per-site=5
-      set LLVM_PROFILE_FILE=%~dp0build\build.profraw
+      if not exist %~dp0build\pgo_raw mkdir %~dp0build\pgo_raw
+      del /q %~dp0build\pgo_raw\*.profraw 2>nul
+      set LLVM_PROFILE_FILE=%~dp0build\pgo_raw\build.%%p.profraw
       set pgo_run=1
     )
   ) else (
@@ -142,7 +144,14 @@ pushd build
 if "%raddbg%"=="1"                     set didbuild=1 && %compile% ..\src\raddbg\raddbg_main.c                               %compile_link% %link_icon% %out%raddbg.exe || exit /b 1
 if "%raddbg_non_graphical%"=="1"       set didbuild=1 && %compile% -DWM_STUB=1 -DR_BACKEND=R_BACKEND_STUB ..\src\raddbg\raddbg_main.c %compile_link% %link_icon% %out%raddbg_non_graphical.exe || exit /b 1
 if "%com_shim%"=="1"                   set didbuild=1 && %compile% ..\src\com_shim\com_shim_main.c                           %compile_link% %out%com_shim.exe || exit /b 1
-if "%radlink%"=="1"                    set didbuild=1 && %compile% ..\src\linker\lnk.c                                       %compile_link% %linker% /NOIMPLIB %linker% /NATVIS:"%~dp0\src\linker\linker.natvis" %out%radlink.exe || exit /b 1
+:: NOTE: -DBLAKE3_ATOMICS=1 makes BLAKE3 use C11 _Atomic (plain atomic load) for
+:: get_cpu_features instead of MSVC's _InterlockedOr `lock or` barrier on every
+:: compress dispatch (was a ~5.5s main-thread hot spot). MSVC C11 atomics require
+:: /std:c11 /experimental:c11atomics. Kept external so the vendored blake3 source
+:: stays pristine.
+set radlink_msvc_flags=
+if "%msvc%"=="1" set radlink_msvc_flags=/std:c11 /experimental:c11atomics -DBLAKE3_ATOMICS=1
+if "%radlink%"=="1"                    set didbuild=1 && %compile% %radlink_msvc_flags% ..\src\linker\lnk.c                    %compile_link% %linker% /NOIMPLIB %linker% /NATVIS:"%~dp0\src\linker\linker.natvis" %out%radlink.exe || exit /b 1
 if "%radbin%"=="1"                     set didbuild=1 && %compile% ..\src\radbin\radbin_main.c                               %compile_link% %out%radbin.exe || exit /b 1
 if "%raddump%"=="1"                    set didbuild=1 && %compile% ..\src\raddump\raddump_main.c                             %compile_link% %out%raddump.exe || exit /b 1
 if "%ryan_scratch%"=="1"               set didbuild=1 && %compile% ..\src\scratch\ryan_scratch.c                             %compile_link% %out%ryan_scratch.exe || exit /b 1
@@ -183,6 +192,10 @@ if "%pgo_run%"=="1" (
   if "%radlink%"=="1" (
     pushd local\lyra_pgo
     call %~dp0build\radlink @lyra.rsp || exit /b 1
+    rem lyra.rsp trains with /OPT:NOREF /OPT:NOICF; run a second link with REF+ICF on
+    rem (later switches win) so the profile also covers the /OPT:REF walk and the ICF
+    rem refinement -- otherwise PGO marks them cold and the build regresses on real links
+    call %~dp0build\radlink @lyra.rsp /OPT:REF /OPT:ICF || exit /b 1
     popd
   )
   goto restart

src/base/base_arena.c

@@ -49,16 +49,32 @@ arena_alloc_(ArenaParams *params)
       commit_size  = AlignPow2(commit_size,  get_system_info()->page_size);
     }
 
+    B32 commit_ok = 0;
     if(params->flags & ArenaFlag_LargePages)
     {
       base = reserve_memory_large(reserve_size);
-      commit_memory_large(base, commit_size);
+      commit_ok = (base != 0) && (commit_size == 0 || commit_memory_large(base, commit_size));
     }
     else
     {
       base = reserve_memory(reserve_size);
-      commit_memory(base, commit_size);
+      commit_ok = (base != 0) && (commit_size == 0 || commit_memory(base, commit_size));
     }
+
+    // rjf: panic on arena creation failure (previously unchecked in
+    // non-graphical builds: writing the arena header into an unreserved or
+    // uncommitted base address turned an out-of-memory condition into an
+    // access violation)
+    if(Unlikely(base == 0 || !commit_ok))
+    {
+#if OS_FEATURE_GRAPHICAL
+      wm_graphical_message(1, str8_lit("Fatal Allocation Failure"), str8_lit("Unexpected memory allocation failure."));
+#else
+      fprintf(stderr, "fatal: out of memory: unable to allocate a memory block (reserve %llu bytes, commit %llu bytes)\n", (unsigned long long)reserve_size, (unsigned long long)commit_size);
+#endif
+      abort_self(1);
+    }
+
     AsanPoisonMemoryRegion(base, commit_size);
     // TODO(rjf): we need to reintroduce this later when we have the ability to remove annotations...
     // raddbg_annotate_vaddr_range(base, reserve_size, "arena %s:%i", params->allocation_site_file, params->allocation_site_line);
@@ -68,15 +84,6 @@ arena_alloc_(ArenaParams *params)
     AsanPoisonMemoryRegion(base, params->reserve_size);
   }
 
-  // rjf: panic on arena creation failure
-#if OS_FEATURE_GRAPHICAL
-  if(Unlikely(base == 0))
-  {
-    wm_graphical_message(1, str8_lit("Fatal Allocation Failure"), str8_lit("Unexpected memory allocation failure."));
-    abort_self(1);
-  }
-#endif
-
   // rjf: extract arena header & fill
   AsanUnpoisonMemoryRegion(base, ARENA_HEADER_SIZE);
   Arena *arena                = base;
@@ -263,16 +270,23 @@ arena_push(Arena *arena, U64 size, U64 align, B32 zero)
     U64 cmt_pst_clamped = ClampTop(cmt_pst_aligned, current->res);
     U64 cmt_size = cmt_pst_clamped - current->cmt;
     U8 *cmt_ptr = (U8 *)current + current->cmt;
+    B32 commit_ok;
     if(current->flags & ArenaFlag_LargePages)
     {
-      commit_memory_large(cmt_ptr, cmt_size);
+      commit_ok = commit_memory_large(cmt_ptr, cmt_size);
     }
     else
     {
-      commit_memory(cmt_ptr, cmt_size);
+      commit_ok = commit_memory(cmt_ptr, cmt_size);
+    }
+    // rjf: a failed commit must not advance current->cmt: the pages are not
+    // backed, and pretending they are turns out-of-memory into an access
+    // violation on first touch (the result == 0 panic below reports it cleanly)
+    if(Likely(commit_ok))
+    {
+      AsanPoisonMemoryRegion(cmt_ptr, cmt_size);
+      current->cmt = cmt_pst_clamped;
     }
-    AsanPoisonMemoryRegion(cmt_ptr, cmt_size);
-    current->cmt = cmt_pst_clamped;
   }
 
   // rjf: push onto current block
@@ -294,15 +308,18 @@ arena_push(Arena *arena, U64 size, U64 align, B32 zero)
   }
 #endif
 
-  // rjf: panic on failure
-#if OS_FEATURE_GRAPHICAL
+  // rjf: panic on failure (a failed commit above leaves result == 0; returning
+  // it would turn out-of-memory into an access violation at the use site)
   if(Unlikely(result == 0))
   {
+#if OS_FEATURE_GRAPHICAL
     wm_graphical_message(1, str8_lit("Fatal Allocation Failure"), str8_lit("Unexpected memory allocation failure."));
+#else
+    fprintf(stderr, "fatal: out of memory: unable to allocate %llu bytes\n", (unsigned long long)size);
+#endif
     abort_self(1);
   }
-#endif
-
+
   return result;
 }
 
@@ -353,6 +370,54 @@ arena_pop_to(Arena *arena, U64 pos)
 
 }
 
+//- rjf: arena decommit of unused (rewound/free) pages
+
+internal void
+arena_decommit_unused(Arena *arena)
+{
+  // NOTE(perf): decommit committed-but-unused pages so they stop counting against
+  // working set, while keeping the reservation. Only touches pages strictly above
+  // the live `pos` high-water of each block in the active chain, and the unused
+  // bodies of free-list blocks. Live data (<= pos) is never touched. The push path
+  // re-commits on demand (arena_push grows `cmt`), so reuse is transparent.
+  if(arena->flags & ArenaFlag_LargePages)
+  {
+    // large pages cannot be partially decommitted safely; skip.
+    return;
+  }
+  U64 page_size = get_system_info()->page_size;
+
+  // rjf: active chain -- decommit committed region above each block's live pos
+  for(Arena *n = arena->current; n != 0; n = n->prev)
+  {
+    U64 pos_aligned = AlignPow2(n->pos, page_size);
+    if(pos_aligned < n->cmt)
+    {
+      U8 *decommit_ptr = (U8 *)n + pos_aligned;
+      U64 decommit_size = n->cmt - pos_aligned;
+      AsanPoisonMemoryRegion(decommit_ptr, decommit_size);
+      decommit_memory(decommit_ptr, decommit_size);
+      n->cmt = pos_aligned;
+    }
+  }
+
+#if ARENA_FREE_LIST
+  // rjf: free chain -- decommit everything above the first (header) page
+  for(Arena *n = arena->free_last; n != 0; n = n->prev)
+  {
+    U64 keep = AlignPow2(ARENA_HEADER_SIZE, page_size);
+    if(keep < n->cmt)
+    {
+      U8 *decommit_ptr = (U8 *)n + keep;
+      U64 decommit_size = n->cmt - keep;
+      AsanPoisonMemoryRegion(decommit_ptr, decommit_size);
+      decommit_memory(decommit_ptr, decommit_size);
+      n->cmt = keep;
+    }
+  }
+#endif
+}
+
 //- rjf: arena push/pop helpers
 
 internal void

src/base/base_arena.h

@@ -76,6 +76,9 @@ internal void *arena_push(Arena *arena, U64 size, U64 align, B32 zero);
 internal U64   arena_pos(Arena *arena);
 internal void  arena_pop_to(Arena *arena, U64 pos);
 
+//- rjf: arena decommit of unused (rewound/free) pages
+internal void arena_decommit_unused(Arena *arena);
+
 //- rjf: arena push/pop helpers
 internal void arena_clear(Arena *arena);
 internal void arena_pop(Arena *arena, U64 amt);

src/base/base_thread_context.c

@@ -17,15 +17,21 @@ C_LINKAGE thread_static TCTX *tctx_thread_local = 0;
 internal TCTX *
 tctx_alloc(void)
 {
+  // 2MB commit quantum for scratch arenas (vs the 64KB default): scratch takes
+  // heavy churn on every thread (the linker pushes tens of GB through these);
+  // the larger quantum cuts VirtualAlloc(MEM_COMMIT) calls -- all serialized on
+  // the process address-space lock -- ~32x. Slack is <= 2MB per scratch arena
+  // past its high-water mark (2 arenas per thread), and arena_decommit_unused
+  // still trims page-granular, independent of the commit quantum.
 #if PROFILE_TELEMETRY
   thread_static static char name[2][1024];
   raddbg_snprintf(name[0], sizeof(name[0]), "Scratch/0[TID:%u]", tid());
   raddbg_snprintf(name[1], sizeof(name[1]), "Scratch/1[TID:%u]", tid());
-  Arena *arena_0 = arena_alloc(.name = name[0]);
-  Arena *arena_1 = arena_alloc(.name = name[1]);
+  Arena *arena_0 = arena_alloc(.commit_size = MB(2), .name = name[0]);
+  Arena *arena_1 = arena_alloc(.commit_size = MB(2), .name = name[1]);
 #else
-  Arena *arena_0 = arena_alloc();
-  Arena *arena_1 = arena_alloc();
+  Arena *arena_0 = arena_alloc(.commit_size = MB(2));
+  Arena *arena_1 = arena_alloc(.commit_size = MB(2));
 #endif
   TCTX *tctx = push_array(arena_0, TCTX, 1);
   tctx->arenas[0] = arena_0;
@@ -82,6 +88,21 @@ tctx_get_scratch(Arena **conflicts, U64 count)
   return result;
 }
 
+//- rjf: scratch decommit (release committed-but-unused scratch pages back to OS)
+
+internal void
+tctx_scratch_decommit(void)
+{
+  TCTX *tctx = tctx_selected();
+  for(U64 i = 0; i < ArrayCount(tctx->arenas); i += 1)
+  {
+    if(tctx->arenas[i] != 0)
+    {
+      arena_decommit_unused(tctx->arenas[i]);
+    }
+  }
+}
+
 //- rjf: lane metadata
 
 internal LaneCtx

src/base/base_thread_context.h

@@ -90,6 +90,7 @@ internal TCTX *tctx_selected(void);
 
 //- rjf: scratch arenas
 internal Arena *tctx_get_scratch(Arena **conflicts, U64 count);
+internal void tctx_scratch_decommit(void);
 #define scratch_begin(conflicts, count) temp_begin(tctx_get_scratch((conflicts), (count)))
 #define scratch_end(scratch) temp_end(scratch)
 

src/base/base_threads.h

@@ -128,8 +128,12 @@ internal void      semaphore_release(Semaphore semaphore);
 internal Semaphore semaphore_open(String8 name);
 internal void      semaphore_close(Semaphore semaphore);
 internal B32       semaphore_take(Semaphore semaphore, U64 endt_us);
+internal B32       semaphore_take_n(Semaphore semaphore, U32 count, U64 endt_us); // blocking acquire of `count` permits (off hot path)
 internal void      semaphore_drop(Semaphore semaphore);
 internal void      semaphore_drop_count(Semaphore semaphore, U64 drop_count);
+internal void      semaphore_drop_if_room(Semaphore semaphore); // best-effort post; no-op if already at max
+internal void      semaphore_drop_n(Semaphore semaphore, U32 count); // release `count` permits in one syscall
+internal B32       semaphore_drop_prev(Semaphore semaphore, U32 *prev_count_out); // release 1 permit + report the pre-release count (win32: exact, from ReleaseSemaphore; posix: best-effort sem_getvalue)
 
 //- rjf: barriers
 internal Barrier barrier_alloc(U64 count);

src/codeview/codeview.h

@@ -2701,6 +2701,32 @@ struct CV_TypeIndexInfoList
   CV_TypeIndexInfo *last;
 };
 
+typedef struct CV_TiOff CV_TiOff;
+struct CV_TiOff
+{
+  CV_TypeIndexSource source;
+  U32                offset;
+};
+
+// Flat, allocation-free view over a record's type-index sites. For any record kind at
+// most one of the two parts is populated:
+//  - `arr`: fixed-shape kinds point at a static per-kind table (zero allocation);
+//    member-walk kinds (FIELDLIST/METHODLIST/inlinee lines) point at an arena-
+//    materialized array.
+//  - homogeneous run (count-stride kinds: ARGLIST, SUBSTR_LIST, BUILDINFO, VFTPATH,
+//    CALLERS/CALLEES/INLINEES): offset(i) = run_base + i*sizeof(CV_TypeIndex).
+// Emission order (arr order, then ascending run) matches the legacy
+// CV_TypeIndexInfoList push order exactly; hash streams depend on it.
+typedef struct CV_TiOffsets CV_TiOffsets;
+struct CV_TiOffsets
+{
+  const CV_TiOff    *arr;
+  U32                arr_count;
+  CV_TypeIndexSource run_source;
+  U32                run_base;
+  U32                run_count;
+};
+
 typedef struct CV_TypeIndexArray CV_TypeIndexArray;
 struct CV_TypeIndexArray
 {