[3.15] gh-154060: Preserve replay duration and rate (GH-154101)

(cherry picked from commit 7f0ccd6)

Co-authored-by: Pablo Galindo Salgado <Pablogsal@gmail.com>

Author: pablogsal

Include/internal/pycore_global_objects_fini_generated.h

@@ -445,6 +445,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(dont_inherit)
         STRUCT_FOR_ID(dst)
         STRUCT_FOR_ID(dst_dir_fd)
+        STRUCT_FOR_ID(duration_sec)
         STRUCT_FOR_ID(eager_start)
         STRUCT_FOR_ID(effective_ids)
         STRUCT_FOR_ID(element_factory)
@@ -458,6 +459,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(endpos)
         STRUCT_FOR_ID(entrypoint)
         STRUCT_FOR_ID(env)
+        STRUCT_FOR_ID(error_rate)
         STRUCT_FOR_ID(errors)
         STRUCT_FOR_ID(event)
         STRUCT_FOR_ID(eventmask)
@@ -641,6 +643,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(milliseconds)
         STRUCT_FOR_ID(minute)
         STRUCT_FOR_ID(minutes)
+        STRUCT_FOR_ID(missed_samples)
         STRUCT_FOR_ID(mod)
         STRUCT_FOR_ID(mode)
         STRUCT_FOR_ID(module)
@@ -768,6 +771,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(rounding)
         STRUCT_FOR_ID(salt)
         STRUCT_FOR_ID(sample_interval_us)
+        STRUCT_FOR_ID(sample_rate)
         STRUCT_FOR_ID(sched_priority)
         STRUCT_FOR_ID(scheduler)
         STRUCT_FOR_ID(script)

Include/internal/pycore_global_strings.h

@@ -34,7 +34,7 @@ by 10-50x compared to text formats while also enabling faster I/O.
 
 ## File Layout
 
-The file consists of five sections:
+The file consists of five required sections and one optional extension:
 
 ```
 +------------------+  Offset 0
@@ -47,6 +47,8 @@ The file consists of five sections:
 |   String Table   |  Variable size
 +------------------+  frame_table_offset
 |   Frame Table    |  Variable size
++------------------+  file_size - 64 (when stats are present)
+| Profile Stats    |  32 bytes (optional)
 +------------------+  file_size - 32
 |     Footer       |  32 bytes (fixed)
 +------------------+  file_size
@@ -354,6 +356,38 @@ location. Zigzag encoding ensures these small negative values encode
 efficiently (−1 becomes 1, which is one byte) rather than requiring the
 maximum varint length.
 
+## Profile Statistics
+
+New files can store measured duration, sampling rate, error rate, and missed
+sample percentage in an optional 56-byte extension immediately before the
+footer. Older readers ignore these
+bytes after parsing the declared number of frame-table entries, and newer
+readers treat a missing extension as unavailable statistics.
+
+```
+ Offset   Size   Type      Description
++--------+------+---------+----------------------------------------+
+|    0   |  8   | double  | Measured duration (seconds)            |
+|    8   |  8   | double  | Measured sample rate (samples/second)  |
+|   16   |  8   | double  | Failed sample percentage               |
+|   24   |  8   | double  | Missed sample percentage               |
+|   32   |  4   | uint32  | Optional field presence flags          |
+|   36   |  4   | uint32  | Reserved                               |
+|   40   |  8   | bytes   | Signature ("TACHSTAT")                 |
+|   48   |  4   | uint32  | Extension version (1)                  |
+|   52   |  4   | uint32  | Extension size (56)                    |
++--------+------+---------+----------------------------------------+
+```
+
+Putting the signature, version, and size at the end lets readers discover
+the extension from its fixed position relative to the footer while allowing
+future versions to add fields before that trailer. Multi-byte values use the
+same native byte order as the rest of the file and are byte-swapped by
+cross-endian readers.
+
+Readers also accept the original 32-byte extension, which only contains the
+duration and sampling rate.
+
 ## Footer
 
 ```
@@ -448,8 +482,9 @@ compress less; higher levels (6+) compress more but slow down writing. Level
 4. Flush remaining buffered data and finalize compression
 5. Write the string table (length-prefixed strings in index order)
 6. Write the frame table (varint-encoded entries in index order)
-7. Write the footer with final counts
-8. Seek to offset 0 and write the header with actual values
+7. Write measured profile statistics, when available
+8. Write the footer with final counts
+9. Seek to offset 0 and write the header with actual values
 
 The writer maintains two dictionaries: one mapping strings to indices, one
 mapping (filename_idx, funcname_idx, lineno) tuples to frame indices. These
@@ -461,12 +496,13 @@ enable O(1) lookup during interning.
    if the magic appears byte-swapped)
 2. Validate version and read remaining header fields (byte-swapping if needed)
 3. Seek to end − 32 and read the footer (byte-swapping counts if needed)
-4. Allocate string array of `string_count` elements
-5. Parse the string table, populating the array
-6. Allocate frame array of `frame_count * 3` uint32 elements
-7. Parse the frame table, populating the array
-8. If compressed, decompress the sample data region
-9. Iterate through samples, resolving indices to strings/frames
+4. Read measured profile statistics when the optional extension is present
+5. Allocate string array of `string_count` elements
+6. Parse the string table, populating the array
+7. Allocate the frame array
+8. Parse the frame table, populating the array
+9. If compressed, decompress the sample data region
+10. Iterate through samples, resolving indices to strings/frames
    (byte-swapping thread_id and interpreter_id if needed)
 
 The reader builds lookup arrays rather than dictionaries since it only needs
@@ -530,11 +566,10 @@ one write() call (or feeds through the compression stream).
 
 ## Future Considerations
 
-The format reserves space for future extensions. The 12 reserved bytes in
-the header could hold additional metadata. The 16-byte checksum field in
-the footer is currently unused. The version field allows incompatible
-changes with graceful rejection. New compression types could be added
-(compression_type > 1).
+The optional profile-statistics block provides an extensible metadata area.
+The 16-byte checksum field in the footer is currently unused. The version
+field allows incompatible changes with graceful rejection. New compression
+types could be added (compression_type > 1).
 
 Any changes that alter the meaning of existing fields or the parsing logic
 should increment the version number to prevent older readers from

Include/internal/pycore_runtime_init_generated.h

@@ -87,6 +87,13 @@ def collect_failed_sample(self):
         """Record a failed sample attempt (no-op for binary format)."""
         pass
 
+    def set_stats(self, sample_interval_usec, duration_sec, sample_rate,
+                  error_rate=None, missed_samples=None, **kwargs):
+        """Persist measured statistics for later replay."""
+        self._writer.set_stats(
+            duration_sec, sample_rate, error_rate, missed_samples
+        )
+
     def export(self, filename=None):
         """Finalize and close the binary file.
 

Include/internal/pycore_unicodeobject_generated.h

@@ -125,6 +125,8 @@ def convert_binary_to_format(input_file, output_file, output_format,
 
         # Replay samples through collector
         count = reader.replay_samples(collector, progress_callback)
+        if hasattr(collector, "set_replay_stats"):
+            collector.set_replay_stats(info)
 
         # Export to target format
         collector.export(output_file)

InternalDocs/profiling_binary_format.md

@@ -776,6 +776,8 @@ def progress_callback(current, total):
             )
 
     count = reader.replay_samples(collector, progress_callback)
+    if hasattr(collector, "set_replay_stats"):
+        collector.set_replay_stats(info)
     print()
 
     if args.format == "pstats":

Lib/profiling/sampling/binary_collector.py

@@ -103,7 +103,6 @@ def collect(self, stack_frames, timestamps_us=None):
         """Override to track thread status statistics before processing frames."""
         # Weight is number of timestamps (samples with identical stack)
         weight = len(timestamps_us) if timestamps_us else 1
-
         # Increment sample count by weight
         self._sample_count += weight
 
@@ -148,6 +147,21 @@ def set_stats(self, sample_interval_usec, duration_sec, sample_rate,
             "mode": mode
         }
 
+    def set_replay_stats(self, info):
+        """Restore measured statistics stored in a binary profile."""
+        duration_sec = info.get("duration_sec")
+        sample_rate = info.get("sample_rate")
+        if duration_sec is None or sample_rate is None:
+            return
+        self.set_stats(
+            self.sample_interval_usec,
+            duration_sec,
+            sample_rate,
+            error_rate=info.get("error_rate"),
+            missed_samples=info.get("missed_samples"),
+            mode=self.stats.get("mode"),
+        )
+
     def export(self, filename):
         flamegraph_data = self._convert_to_flamegraph_format()