feat(hrr): strip build info from stored code objects, blank GPU identifiers

projects/clr/hipamd/src/hrr/CMakeLists.txt

@@ -21,6 +21,7 @@ endif()
 target_sources(amdhip64 PRIVATE
   ${CMAKE_CURRENT_SOURCE_DIR}/hip_capture.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/hip_capture_metadata.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/hip_capture_scrub.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/hip_capture_writer.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/hip_capture_generated.cpp
 )

projects/clr/hipamd/src/hrr/hip_capture.cpp

@@ -31,6 +31,7 @@
 
 #include "hip_capture.h"
 #include "hip_capture_metadata.h"
+#include "hip_capture_scrub.h"
 #include "hip_capture_writer.h"
 
 // hrr_api_args.h — for hrr_args_* struct types and hrr_api_id_t enum
@@ -1671,6 +1672,15 @@ static size_t compute_bundle_size(const void* blob) {
   return static_cast<size_t>(end);
 }
 
+// Stores a fat binary with the .comment and debug sections of its code
+// objects zeroed; the application's own copy is left alone.
+static hrr_cap::Hash128 write_fat_binary_blob(const void* blob, size_t blob_size) {
+  std::vector<uint8_t> scrubbed(static_cast<const uint8_t*>(blob),
+                                static_cast<const uint8_t*>(blob) + blob_size);
+  hrr_cap::scrub::scrub_code_object(scrubbed.data(), scrubbed.size());
+  return hrr_cap::writer::write_blob(scrubbed.data(), scrubbed.size());
+}
+
 void** capture___hipRegisterFatBinary(const void* data) {
   void** r = g_real_compiler_table.__hipRegisterFatBinary_fn(data);
   // Shim is only installed when capture is active — no hip_capture_enabled() check needed.
@@ -1699,7 +1709,7 @@ void** capture___hipRegisterFatBinary(const void* data) {
   a.ret      = reinterpret_cast<uint64_t>(r);
   a.blob_size = static_cast<uint64_t>(blob_size);
   if (blob && blob_size > 0) {
-    auto h = hrr_cap::writer::write_blob(blob, blob_size);
+    auto h = write_fat_binary_blob(blob, blob_size);
     a.blob_hash_lo = h.lo;
     a.blob_hash_hi = h.hi;
   }
@@ -2900,7 +2910,7 @@ static void record_fat_binary_blob(const void* blob_ptr) {
   hrr_args___hipRegisterFatBinary a{};
   a.ret      = 0;  // handle not meaningful at init time
   a.blob_size = static_cast<uint64_t>(blob_size);
-  auto h = hrr_cap::writer::write_blob(blob_ptr, blob_size);
+  auto h = write_fat_binary_blob(blob_ptr, blob_size);
   a.blob_hash_lo = h.lo;
   a.blob_hash_hi = h.hi;
   hrr_cap::writer::write_event_raw(HRR_API_HIPREGISTERFATBINARY, &a.hdr, sizeof(a));

projects/clr/hipamd/src/hrr/hip_capture_generated.cpp

@@ -46,6 +46,7 @@
 #include "utils/debug.hpp"
 
 #include <atomic>
+#include <cstddef>
 #include <cstdint>
 #include <cstdio>
 #include <cstring>
@@ -207,6 +208,11 @@ static hipError_t capture_hipChooseDevice(int* device, const hipDeviceProp_tR060
     a.prop = reinterpret_cast<uint64_t>(prop);
     if (prop) {
       std::memcpy(a.prop_bytes, prop, sizeof(hipDeviceProp_t));
+      std::memset(a.prop_bytes + offsetof(hipDeviceProp_t, uuid), 0, sizeof(prop->uuid));
+      std::memset(a.prop_bytes + offsetof(hipDeviceProp_t, luid), 0, sizeof(prop->luid));
+      std::memset(a.prop_bytes + offsetof(hipDeviceProp_t, pciDomainID), 0, sizeof(prop->pciDomainID));
+      std::memset(a.prop_bytes + offsetof(hipDeviceProp_t, pciBusID), 0, sizeof(prop->pciBusID));
+      std::memset(a.prop_bytes + offsetof(hipDeviceProp_t, pciDeviceID), 0, sizeof(prop->pciDeviceID));
       a.prop_present = 1;
     }
     hrr_cap::writer::write_event_raw(HRR_API_HIPCHOOSEDEVICE, &a.hdr, sizeof(a));
@@ -224,6 +230,9 @@ static hipError_t capture_hipChooseDeviceR0000(int* device, const hipDeviceProp_
     a.properties = reinterpret_cast<uint64_t>(properties);
     if (properties) {
       std::memcpy(a.properties_bytes, properties, sizeof(hipDeviceProp_tR0000));
+      std::memset(a.properties_bytes + offsetof(hipDeviceProp_tR0000, pciDomainID), 0, sizeof(properties->pciDomainID));
+      std::memset(a.properties_bytes + offsetof(hipDeviceProp_tR0000, pciBusID), 0, sizeof(properties->pciBusID));
+      std::memset(a.properties_bytes + offsetof(hipDeviceProp_tR0000, pciDeviceID), 0, sizeof(properties->pciDeviceID));
       a.properties_present = 1;
     }
     hrr_cap::writer::write_event_raw(HRR_API_HIPCHOOSEDEVICER0000, &a.hdr, sizeof(a));

projects/clr/hipamd/src/hrr/hip_capture_metadata.cpp

@@ -88,22 +88,6 @@ std::string version_string_from_int(int version) {
   return std::to_string(major) + "." + std::to_string(minor) + "." + std::to_string(patch);
 }
 
-std::string bytes_to_hex(const char* bytes, size_t len) {
-  static constexpr char kHex[] = "0123456789abcdef";
-  std::string result;
-  result.resize(len * 2);
-  for (size_t i = 0; i < len; ++i) {
-    const auto v = static_cast<unsigned char>(bytes[i]);
-    result[2 * i] = kHex[v >> 4];
-    result[2 * i + 1] = kHex[v & 0xf];
-  }
-  return result;
-}
-
-std::string uuid_to_hex(const hipUUID& uuid) {
-  return bytes_to_hex(uuid.bytes, sizeof(uuid.bytes));
-}
-
 void append_prop_fields(std::ostringstream& os, const hipDeviceProp_t& prop) {
   os << "      \"properties\": {\n"
      << "        \"name\": " << quote(bounded_string(prop.name, sizeof(prop.name))) << ",\n"
@@ -123,11 +107,11 @@ void append_prop_fields(std::ostringstream& os, const hipDeviceProp_t& prop) {
      << "        \"compute_mode\": " << prop.computeMode << ",\n"
      << "        \"compute_capability\": " << quote(std::to_string(prop.major) + "." +
                                                   std::to_string(prop.minor)) << ",\n"
-     << "        \"pci\": " << quote(std::to_string(prop.pciDomainID) + ":" +
-                                    std::to_string(prop.pciBusID) + ":" +
-                                    std::to_string(prop.pciDeviceID)) << ",\n"
-     << "        \"uuid\": " << quote(uuid_to_hex(prop.uuid)) << ",\n"
-     << "        \"luid\": " << quote(bytes_to_hex(prop.luid, sizeof(prop.luid))) << ",\n"
+     // Left blank: they name the individual accelerator, and with it the machine
+     // that captured, and replay needs none of them.
+     << "        \"pci\": \"\",\n"
+     << "        \"uuid\": \"\",\n"
+     << "        \"luid\": \"\",\n"
      << "        \"luid_device_node_mask\": " << prop.luidDeviceNodeMask << "\n"
      << "      }";
 }

projects/clr/hipamd/src/hrr/hip_capture_scrub.cpp

@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "hip_capture_scrub.h"
+
+#include <cstring>
+
+// Field offsets are spelled out rather than taken from <elf.h>, which Windows
+// does not have and whose Elf64_* typedefs collide with CLR's own ELF headers.
+// The image is the application's memory, so every read is bounds-checked and
+// copied out with memcpy, whatever its alignment.
+
+namespace hrr_cap {
+namespace scrub {
+
+namespace {
+
+constexpr size_t kElfHeaderSize = 64;
+constexpr size_t kSectionHeaderSize = 64;
+constexpr uint8_t kElfClass64 = 2;
+constexpr uint8_t kElfDataLsb = 1;
+constexpr uint32_t kShtNobits = 8;
+constexpr uint64_t kShfAlloc = 0x2;
+
+constexpr char kBundleMagic[] = "__CLANG_OFFLOAD_BUNDLE__";
+constexpr size_t kBundleMagicLen = sizeof(kBundleMagic) - 1;
+constexpr size_t kBundleEntryHeaderSize = 3 * sizeof(uint64_t);
+
+template <typename T>
+T read(const uint8_t* p, size_t at) {
+  T v;
+  std::memcpy(&v, p + at, sizeof(v));
+  return v;
+}
+
+// [offset, offset + size) lies inside [0, limit).
+bool in_range(uint64_t offset, uint64_t size, size_t limit) {
+  return offset <= limit && size <= limit - offset;
+}
+
+bool has_prefix(const char* s, size_t len, const char* prefix) {
+  const size_t n = std::strlen(prefix);
+  return len >= n && std::memcmp(s, prefix, n) == 0;
+}
+
+bool is_scrubbed_section(const char* name, size_t len) {
+  return (len == 8 && std::memcmp(name, ".comment", 8) == 0) || has_prefix(name, len, ".debug");
+}
+
+size_t scrub_elf(uint8_t* image, size_t size) {
+  if (size < kElfHeaderSize || std::memcmp(image, "\x7f" "ELF", 4) != 0 ||
+      image[4] != kElfClass64 || image[5] != kElfDataLsb)
+    return 0;
+  const uint64_t shoff = read<uint64_t>(image, 0x28);
+  const uint16_t shentsize = read<uint16_t>(image, 0x3a);
+  const uint16_t shnum = read<uint16_t>(image, 0x3c);
+  const uint16_t shstrndx = read<uint16_t>(image, 0x3e);
+  // shnum == 0 also covers extended section numbering, which code objects never need.
+  if (shentsize != kSectionHeaderSize || shnum == 0 || shstrndx >= shnum ||
+      !in_range(shoff, uint64_t{shnum} * kSectionHeaderSize, size))
+    return 0;
+
+  const uint8_t* shstr_hdr = image + shoff + size_t{shstrndx} * kSectionHeaderSize;
+  const uint64_t names_off = read<uint64_t>(shstr_hdr, 24);
+  const uint64_t names_size = read<uint64_t>(shstr_hdr, 32);
+  if (!in_range(names_off, names_size, size)) return 0;
+  const char* names = reinterpret_cast<const char*>(image + names_off);
+
+  size_t zeroed = 0;
+  for (uint16_t i = 0; i < shnum; ++i) {
+    uint8_t* sh = image + shoff + size_t{i} * kSectionHeaderSize;
+    const uint32_t name = read<uint32_t>(sh, 0);
+    const uint32_t type = read<uint32_t>(sh, 4);
+    const uint64_t flags = read<uint64_t>(sh, 8);
+    const uint64_t offset = read<uint64_t>(sh, 24);
+    const uint64_t sec_size = read<uint64_t>(sh, 32);
+    if (name >= names_size || type == kShtNobits || (flags & kShfAlloc) != 0) continue;
+    const size_t name_len = strnlen(names + name, names_size - name);
+    if (name_len == names_size - name) continue;  // no terminator inside the table
+    if (!is_scrubbed_section(names + name, name_len)) continue;
+    if (sec_size == 0 || !in_range(offset, sec_size, size)) continue;
+    std::memset(image + offset, 0, sec_size);
+    // SHT_NOBITS makes DWARF readers see the section as absent rather than as
+    // malformed zeros. It is not SHF_ALLOC, so the loader maps nothing from it.
+    std::memcpy(sh + 4, &kShtNobits, sizeof(kShtNobits));
+    ++zeroed;
+  }
+  return zeroed;
+}
+
+size_t scrub_bundle(uint8_t* image, size_t size) {
+  size_t pos = kBundleMagicLen;
+  if (size < pos + sizeof(uint64_t)) return 0;
+  const uint64_t count = read<uint64_t>(image, pos);
+  pos += sizeof(uint64_t);
+  size_t zeroed = 0;
+  for (uint64_t i = 0; i < count; ++i) {
+    if (size - pos < kBundleEntryHeaderSize) break;
+    const uint64_t offset = read<uint64_t>(image, pos);
+    const uint64_t entry_size = read<uint64_t>(image, pos + 8);
+    const uint64_t id_size = read<uint64_t>(image, pos + 16);
+    pos += kBundleEntryHeaderSize;
+    if (id_size > size - pos) break;
+    pos += id_size;
+    if (in_range(offset, entry_size, size)) zeroed += scrub_elf(image + offset, entry_size);
+  }
+  return zeroed;
+}
+
+}  // namespace
+
+size_t scrub_code_object(uint8_t* image, size_t size) {
+  if (!image) return 0;
+  if (size >= kBundleMagicLen && std::memcmp(image, kBundleMagic, kBundleMagicLen) == 0)
+    return scrub_bundle(image, size);
+  return scrub_elf(image, size);
+}
+
+}  // namespace scrub
+}  // namespace hrr_cap

projects/clr/hipamd/src/hrr/hip_capture_scrub.h

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
+ *
+ * SPDX-License-Identifier: MIT
+ */
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+namespace hrr_cap {
+namespace scrub {
+
+// Zeroes, in place, the contents of the .comment and .debug* sections of a code
+// object about to be stored: an ELF64 image, or every entry of an uncompressed
+// clang offload bundle. Those sections carry the compiler build, absolute build
+// paths and source file names; the GPU never loads them, so replay is unchanged.
+// A string the GPU does load, such as the file name in a device assert(), stays.
+// Each zeroed section is retyped SHT_NOBITS, so debuggers and profilers see it as
+// absent. Sizes and offsets stay as they are, so a bundle's entry table stays valid.
+// A compressed bundle, or anything that does not parse, is left as it is.
+// Returns how many sections were zeroed.
+size_t scrub_code_object(uint8_t* image, size_t size);
+
+}  // namespace scrub
+}  // namespace hrr_cap

projects/clr/hipamd/src/hrr/hip_capture_writer.cpp

@@ -45,6 +45,7 @@
 #include "hip_capture_writer.h"
 #include "hip_capture.h"
 #include "hip_capture_metadata.h"
+#include "hip_capture_scrub.h"
 
 #include "os/os.hpp"           // amd::Os::timeNanos()
 #include "utils/debug.hpp"     // LogPrintfError, LogPrintfWarning, LogPrintfInfo
@@ -1772,7 +1773,13 @@ Hash128 write_code_object(const void* image, size_t image_size) {
     if (g_events_fd < 0 || g_events_finalized) return {};  // not open, or past the trailer
   }
 
-  Hash128 h = hash_buffer(image, image_size);
+  // The stored bytes, and so the hash, are the image with its .comment and
+  // debug sections zeroed (see hip_capture_scrub.h).
+  std::vector<uint8_t> scrubbed(static_cast<const uint8_t*>(image),
+                                static_cast<const uint8_t*>(image) + image_size);
+  scrub::scrub_code_object(scrubbed.data(), scrubbed.size());
+
+  Hash128 h = hash_buffer(scrubbed.data(), image_size);
   char hex[33];
   hash_hex(h, hex);
   std::string key = std::string("co:") + hex;  // namespace to match playback load_code_object key
@@ -1792,7 +1799,7 @@ Hash128 write_code_object(const void* image, size_t image_size) {
   ClaimedWrite claim(key);
 
   const bool ok = atomic_write_file(g_code_objects_fd.load(), g_output_dir + "/code_objects",
-                                    std::string(hex) + ".hsaco", image, image_size);
+                                    std::string(hex) + ".hsaco", scrubbed.data(), image_size);
   if (!ok) LogPrintfWarning("[HRR capture] Failed to write code object %s", hex);
   claim.done(ok);
   return h;

projects/hrr/PRIVACY.md

@@ -48,15 +48,16 @@ every capture is a contents capture.
 | Sequence of successful HIP calls, with sizes, flags, device addresses, handles, thread ids | `events.bin` | both | — kept, replay needs it |
 | Kernel names, mangled but read back with `c++filt`; they describe the model's architecture and precision | `events.bin` | both | — kept, triage names the failing kernel |
 | Event timestamps, from a monotonic clock (not the date or time of day) | `events.bin` | both | — kept |
-| Device properties passed to `hipChooseDevice`, the whole `hipDeviceProp_t`: device and architecture names, PCI ids and, in the current layout, the UUID and LUID | `events.bin` | both | — kept, the call's answer depends on them |
+| Device properties passed to `hipChooseDevice`, the whole `hipDeviceProp_t` with the device and architecture names; its UUID, LUID and PCI ids are written as zeros | `events.bin` | both | [AIRUNTIME-2881](https://amd-hub.atlassian.net/browse/AIRUNTIME-2881): UUID, LUID and PCI ids blanked; the rest is kept, the call's answer depends on it |
 | Other structures passed into a HIP call, copied whole, such as copy, memset, graph node, allocation, pool and launch-attribute parameters. Padding, reserved and unused union bytes are copied as the caller left them | `events.bin` | both | — kept, replay needs it |
 | Host buffers copied to or from the device, byte for byte, unencrypted. For a serving stack this includes prompts and generated output as token ids, which the model's public tokeniser reverses, and the model weights | `blobs/` | contents | [AIRUNTIME-2880](https://amd-hub.atlassian.net/browse/AIRUNTIME-2880) |
 | Host memory registered with `hipHostRegister`: the whole range, byte for byte, as it was at registration, whether or not it is ever copied | `blobs/` | contents | [AIRUNTIME-2880](https://amd-hub.atlassian.net/browse/AIRUNTIME-2880) |
 | Kernel argument bytes | `events.bin` | contents | [AIRUNTIME-2880](https://amd-hub.atlassian.net/browse/AIRUNTIME-2880) |
 | Code objects the process loaded (the application's own GPU binary code) | `code_objects/`, `blobs/` | contents | [AIRUNTIME-2880](https://amd-hub.atlassian.net/browse/AIRUNTIME-2880) |
-| Linker input buffers passed to `hipLinkAddData`, every byte, with their names, including inputs the call rejected and inputs never linked or loaded | `blobs/` | contents | [AIRUNTIME-2880](https://amd-hub.atlassian.net/browse/AIRUNTIME-2880) |
+| Linker input buffers passed to `hipLinkAddData`, every byte, with their names, including inputs the call rejected and inputs never linked or loaded. A SPIR-V or LLVM bitcode input keeps its compiler identification and, built with `-g`, its source and build directories; capture does not scrub them | `blobs/` | contents | [AIRUNTIME-2880](https://amd-hub.atlassian.net/browse/AIRUNTIME-2880) |
 | Whole fat binary bundles, including targets HIP never loads | `blobs/` | contents | [AIRUNTIME-2858](https://amd-hub.atlassian.net/browse/AIRUNTIME-2858): only the code objects HIP loads |
-| Debug and comment sections of code objects: absolute build paths, source file names, the exact compiler build | `code_objects/`, `blobs/` | contents | [AIRUNTIME-2881](https://amd-hub.atlassian.net/browse/AIRUNTIME-2881): stripped |
+| Debug and comment sections of code objects: absolute build paths, source file names, the exact compiler build | `code_objects/`, `blobs/` | contents | [AIRUNTIME-2881](https://amd-hub.atlassian.net/browse/AIRUNTIME-2881): zeroed in code objects and uncompressed fat binaries; kept in compressed (`CCOB`) fat binaries |
+| Strings in the code the GPU loads, such as the absolute source path a device `assert()` prints | `code_objects/`, `blobs/` | contents | — kept, the kernel uses them; `verify` counts code objects holding an absolute path |
 | Framework allocator layout, when a region producer ran beside capture | `regions/` | both | — kept, written only by an opt-in producer |
 | Marker that capture is running: on Linux the boot id and the process start time in clock ticks since boot; on Windows the process creation date and time. A normal exit removes it; a process that crashes or is killed leaves it | `active` | both | — kept, region producers read it |
 
@@ -69,10 +70,11 @@ The per-process `manifest.json` also holds `pid`, `parent_pid`, `complete`, `eve
 | `total_global_mem`, `multi_processor_count`, `warp_size`, `max_threads_per_block` | yes | — |
 | `clock_rate_khz`, `memory_clock_rate_khz`, `memory_bus_width`, `l2_cache_size` | yes | — |
 | `integrated`, `managed_memory`, `memory_pools_supported`, `compute_mode` | yes | — |
-| `uuid`, `pci`, `luid`, `luid_device_node_mask`: identify the individual GPU and its slot | no | [AIRUNTIME-2881](https://amd-hub.atlassian.net/browse/AIRUNTIME-2881): blanked |
+| `luid_device_node_mask`, a node mask rather than an identifier | yes | — |
+| `uuid`, `pci`, `luid`: identify the individual GPU and its slot | written blank | [AIRUNTIME-2881](https://amd-hub.atlassian.net/browse/AIRUNTIME-2881) |
 
-This table covers `manifest.json` only. A `hipChooseDevice` call can carry the same
-identifiers in `events.bin`, as listed above.
+This table covers `manifest.json` only. The copy of the device properties a
+`hipChooseDevice` call keeps in `events.bin` has the same identifiers blanked, as listed above.
 
 ## Never recorded
 

projects/hrr/README.md

@@ -236,7 +236,16 @@ capture.hrr/
 - **code_objects/** — code objects, content-addressed by hash (device images
   extracted by the runtime, for `hipModuleLoad` too); a launch
   records the hash so kernels sharing a name (Triton emits many `triton_`)
-  still resolve to the code object they came from
+  still resolve to the code object they came from. Their `.comment` and
+  `.debug*` sections, which hold the compiler build and the build paths of
+  debug information, are zeroed and retyped `SHT_NOBITS` before they are
+  stored, as are those of uncompressed fat binaries, so a debugger on a
+  replay sees no debug information. What the GPU loads is not touched:
+  kernel names stay, and so does a string a kernel keeps, such as the
+  source path a device `assert()` prints. Compressed (`CCOB`) fat binaries
+  and `hipLinkAddData` inputs are stored as they were passed.
+- **manifest.json** — counts, versions and device properties; the GPU's PCI
+  address, UUID and LUID are written blank
 - **Complete: NO** — original run crashed before clean shutdown; reader still recovers complete events
 
 Code objects read back from the runtime must start with `\x7fELF`,