Add CPython 3.15 support to mixed-mode debugging

Python/Product/Common/Parsing/PythonLanguageVersion.cs

@@ -46,6 +46,7 @@ public enum PythonLanguageVersion {
         V312 = 0x030c,
         V313 = 0x030d,
         V314 = 0x030e, // Added for Python 3.14 support
+        V315 = 0x030f, // Added for Python 3.15 support
     }
 
     public static class PythonLanguageVersionExtensions {

Python/Product/Debugger.Concord/ExpressionEvaluator.cs

@@ -385,8 +385,8 @@ public void GetFrameLocals(DkmInspectionContext inspectionContext, DkmWorkList w
             // misaligned address and throws, which aborts the entire locals enumeration. Object
             // pointers are 8-byte aligned, so the mask is a no-op for untagged slots; it is 0 for
             // versions before 3.14, leaving their behavior byte-for-byte identical.
-            ulong localsPlusTagMask =
-                pythonFrame.Process.GetPythonRuntimeInfo().LanguageVersion >= PythonLanguageVersion.V314 ? 0x3ul : 0ul;
+            ulong localsPlusTagMask = PyInterpreterFrame.GetStackReferenceTagMask(
+                pythonFrame.Process.GetPythonRuntimeInfo().LanguageVersion);
 
             // Process cellvars and freevars first, because function arguments can appear in both cellvars and varnames if the argument is captured by a closure,
             // in which case we want to use the cellvar because the regular var slot will then be unused by Python (and in Python 3.4+, nulled out).

Python/Product/Debugger.Concord/Proxies/PointerProxy.cs

@@ -62,6 +62,10 @@ public PointerProxy<TProxy> ReinterpretCast<TProxy>(bool polymorphic = true)
             where TProxy : IDataProxy {
             return new PointerProxy<TProxy>(Process, Address, polymorphic);
         }
+
+        internal static ulong RemoveTagBits(ulong value, ulong tagMask) {
+            return value & ~tagMask;
+        }
     }
 
     [DebuggerDisplay("& {TryRead()}")]
@@ -122,7 +126,7 @@ public PointerProxy Raw {
 
         // The pointer value stored at Address, with any tag bits stripped.
         private ulong ReadTarget() {
-            return Raw.Read() & ~_tagMask;
+            return PointerProxy.RemoveTagBits(Raw.Read(), _tagMask);
         }
 
         public TProxy Read() {

Python/Product/Debugger.Concord/Proxies/Structs/DebugOffsetsFieldProvider.cs

@@ -33,11 +33,13 @@ internal interface IStructFieldOffsetProvider {
     }
 
     /// <summary>
-    /// Resolves a curated set of hot-path struct fields from CPython 3.14's <c>_Py_DebugOffsets</c>
-    /// table (see <see cref="PyDebugOffsets"/>) instead of the PDB. Coverage is intentionally limited
-    /// to the frame / code-object / thread-state fields that the mixed-mode stack walk and in-process
-    /// line-number computation depend on; every other field still resolves via the PDB. This is what
-    /// automatically tracks the free-threaded build's shifted object layouts.
+    /// Resolves a curated set of hot-path struct fields from CPython's <c>_Py_DebugOffsets</c>
+    /// table (see <see cref="PyDebugOffsets"/>) instead of the PDB, for the interpreter versions the
+    /// reader understands (currently 3.14 and 3.15). Coverage is intentionally limited to the frame /
+    /// code-object / thread-state fields that the mixed-mode stack walk and in-process line-number
+    /// computation depend on; every other field still resolves via the PDB. This is what
+    /// automatically tracks the free-threaded build's shifted object layouts. Every group/field this
+    /// map references exists in both the 3.14 and 3.15 layouts, so no per-version map is needed.
     ///
     /// The map keys are CPython struct/field names (matching <see cref="StructProxy"/>'s field names
     /// and each proxy's <c>StructName</c>); the values are the corresponding

Python/Product/Debugger.Concord/Proxies/Structs/PyInterpreterFrame.cs

@@ -54,6 +54,10 @@ internal class Fields {
         // the real PyObject* back.
         private const ulong StackRefTagMask = 0x3;
 
+        internal static ulong GetStackReferenceTagMask(PythonLanguageVersion version) {
+            return version >= PythonLanguageVersion.V314 ? StackRefTagMask : 0;
+        }
+
         private readonly Fields _fields;
 
         public PyInterpreterFrame(DkmProcess process, ulong address)
@@ -69,13 +73,14 @@ public PointerProxy<PyCodeObject> f_code {
                 }
 
                 var executable = GetFieldProxy(_fields.f_executable);
-                if (Process.GetPythonRuntimeInfo().LanguageVersion >= PythonLanguageVersion.V314) {
+                var tagMask = GetStackReferenceTagMask(Process.GetPythonRuntimeInfo().LanguageVersion);
+                if (tagMask != 0) {
                     // In 3.14, f_executable is a _PyStackRef rather than a plain PyObject*. Its two
                     // low bits are a reference tag (Py_TAG_BITS, i.e. mask 0x3; set for deferred/
                     // immortal references such as frozen-module code objects), so strip them to
                     // recover the PyCodeObject pointer. This mirrors CPython's own out-of-process
                     // reader (CLEAR_PTR_TAG in _remote_debugging_module.c).
-                    executable = executable.WithTagMask(StackRefTagMask);
+                    executable = executable.WithTagMask(tagMask);
                 }
                 return executable;
             }

Python/Product/Debugger.Concord/PythonRuntimeInfo.cs

@@ -24,7 +24,7 @@
 namespace Microsoft.PythonTools.Debugger.Concord {
 
     internal class PythonDLLs {
-        private static readonly Regex pythonName = new Regex(@"^python(3\d+)(?:_d)?\.dll$");
+        private static readonly Regex pythonName = new Regex(@"^python(3\d+)(t)?(?:_d)?\.dll$");
 
         public static readonly string[] DebuggerHelperNames = {
             "Microsoft.PythonTools.Debugger.Helper.x86.dll",
@@ -60,28 +60,38 @@ public DkmNativeModuleInstance Python {
         public DkmNativeModuleInstance CTypes { get; set; }
 
         public static PythonLanguageVersion GetPythonLanguageVersion(DkmNativeModuleInstance moduleInstance) {
-            var m = pythonName.Match(moduleInstance.Name);
+            return GetPythonLanguageVersion(moduleInstance.Name);
+        }
+
+        internal static PythonLanguageVersion GetPythonLanguageVersion(string moduleName) {
+            var m = pythonName.Match(moduleName);
             if (!m.Success) {
                 return PythonLanguageVersion.None;
             }
 
             var ver = m.Groups[1].Value;
+            PythonLanguageVersion version;
             switch (ver) {
-                case "27": return PythonLanguageVersion.V27;
-                case "33": return PythonLanguageVersion.V33;
-                case "34": return PythonLanguageVersion.V34;
-                case "35": return PythonLanguageVersion.V35;
-                case "36": return PythonLanguageVersion.V36;
-                case "37": return PythonLanguageVersion.V37;
-                case "38": return PythonLanguageVersion.V38;
-                case "39": return PythonLanguageVersion.V39;
-                case "310": return PythonLanguageVersion.V310;
-                case "311": return PythonLanguageVersion.V311;
-                case "312": return PythonLanguageVersion.V312;
-                case "313": return PythonLanguageVersion.V313;
-                case "314": return PythonLanguageVersion.V314;
+                case "27": version = PythonLanguageVersion.V27; break;
+                case "33": version = PythonLanguageVersion.V33; break;
+                case "34": version = PythonLanguageVersion.V34; break;
+                case "35": version = PythonLanguageVersion.V35; break;
+                case "36": version = PythonLanguageVersion.V36; break;
+                case "37": version = PythonLanguageVersion.V37; break;
+                case "38": version = PythonLanguageVersion.V38; break;
+                case "39": version = PythonLanguageVersion.V39; break;
+                case "310": version = PythonLanguageVersion.V310; break;
+                case "311": version = PythonLanguageVersion.V311; break;
+                case "312": version = PythonLanguageVersion.V312; break;
+                case "313": version = PythonLanguageVersion.V313; break;
+                case "314": version = PythonLanguageVersion.V314; break;
+                case "315": version = PythonLanguageVersion.V315; break;
                 default: return PythonLanguageVersion.None;
             }
+
+            return !m.Groups[2].Success || version >= PythonLanguageVersion.V313
+                ? version
+                : PythonLanguageVersion.None;
         }
     }
 
@@ -123,17 +133,18 @@ public Proxies.Structs.PyDebugOffsets DebugOffsets {
 
         /// <summary>
         /// Offset source that <see cref="Proxies.StructProxy"/> consults before falling back to the
-        /// interpreter PDB. Non-null only for CPython 3.14, where the <c>_Py_DebugOffsets</c> table
-        /// authoritatively describes the (potentially free-threaded-shifted) layout of the mixed-mode
-        /// hot-path structs. Older interpreters return null and resolve every field via the PDB exactly
-        /// as before, so this cannot regress them.
+        /// interpreter PDB. Non-null only for CPython versions whose <c>_Py_DebugOffsets</c> layout this
+        /// reader understands (3.14 and 3.15), where the table authoritatively describes the (potentially
+        /// free-threaded-shifted) layout of the mixed-mode hot-path structs. Older interpreters, and any
+        /// newer version this reader hasn't been taught yet, return null and resolve every field via the
+        /// PDB exactly as before, so this cannot regress them.
         /// </summary>
         public Proxies.Structs.IStructFieldOffsetProvider StructFieldOffsetProvider {
             get {
                 if (!_offsetProviderProbed) {
                     _offsetProviderProbed = true;
                     var offsets = DebugOffsets;
-                    if (offsets != null && offsets.Is314) {
+                    if (offsets != null && offsets.IsSupported) {
                         _offsetProvider = new Proxies.Structs.DebugOffsetsFieldProvider(offsets);
                     }
                 }

Python/Tests/DebuggerTests/DebuggerTests.csproj

@@ -72,6 +72,7 @@
     <Compile Include="PyDebugOffsetsTests.cs" />
     <Compile Include="PyDebugOffsetsProviderTests.cs" />
     <Compile Include="PyLineTableTests.cs" />
+    <Compile Include="PythonRuntimeInfoTests.cs" />
     <Compile Include="TaskExtensions.cs" />
     <Compile Include="MiniDumpWriter.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
@@ -114,4 +115,4 @@
     <WCFMetadata Include="Service References\" />
   </ItemGroup>
   <Import Project="..\TestProjectAfter.settings" />
-</Project>
+</Project>

Python/Tests/DebuggerTests/PyDebugOffsetsProviderTests.cs

@@ -21,10 +21,12 @@
 namespace DebuggerTests {
     /// <summary>
     /// Verifies the hot-path offset provider (<see cref="DebugOffsetsFieldProvider"/>) that lets
-    /// <c>StructProxy</c> source CPython 3.14 frame / code-object / thread-state field offsets from the
-    /// self-describing <c>_Py_DebugOffsets</c> table instead of the PDB. Everything is exercised against
-    /// the same real 3.14.6 vectors used by <see cref="PyDebugOffsetsTests"/>, mapping the CPython
-    /// struct/field names that <c>StructProxy</c> passes in to the values the table reports.
+    /// <c>StructProxy</c> source CPython frame / code-object / thread-state field offsets from the
+    /// self-describing <c>_Py_DebugOffsets</c> table instead of the PDB. The cases run against the real
+    /// 3.14.6 and 3.15.0rc1 vectors used by <see cref="PyDebugOffsetsTests"/> (standard and free-threaded),
+    /// plus a synthetic 3.15 blob that pins the mapping to the exact ordinals of the (grown) 3.15 layout.
+    /// Each case maps the CPython struct/field names that <c>StructProxy</c> passes in to the values the
+    /// table reports.
     /// </summary>
     [TestClass]
     public class DebugOffsetsFieldProviderTests {
@@ -117,5 +119,88 @@ public void ReturnsFalse_ForUnmappedFieldsAndStructs() {
             Assert.IsFalse(provider.TryGetFieldOffset(null, "next", out offset));
             Assert.IsFalse(provider.TryGetFieldOffset("_ts", null, out offset));
         }
+
+        [TestMethod, Priority(0)]
+        public void Standard315_MapsInterpreterFrameHotPath() {
+            var provider = Provider(PyDebugOffsetsTests.RawV315);
+            Assert.AreEqual(8L, Offset(provider, "_PyInterpreterFrame", "previous"));
+            Assert.AreEqual(0L, Offset(provider, "_PyInterpreterFrame", "f_executable"));
+            Assert.AreEqual(56L, Offset(provider, "_PyInterpreterFrame", "instr_ptr"));
+            Assert.AreEqual(80L, Offset(provider, "_PyInterpreterFrame", "localsplus"));
+            Assert.AreEqual(74L, Offset(provider, "_PyInterpreterFrame", "owner"));
+        }
+
+        [TestMethod, Priority(0)]
+        public void Standard315_MapsCodeObjectHotPath() {
+            var provider = Provider(PyDebugOffsetsTests.RawV315);
+            Assert.AreEqual(112L, Offset(provider, "PyCodeObject", "co_filename"));
+            Assert.AreEqual(120L, Offset(provider, "PyCodeObject", "co_name"));
+            Assert.AreEqual(68L, Offset(provider, "PyCodeObject", "co_firstlineno"));
+            Assert.AreEqual(96L, Offset(provider, "PyCodeObject", "co_localsplusnames"));
+            Assert.AreEqual(104L, Offset(provider, "PyCodeObject", "co_localspluskinds"));
+            Assert.AreEqual(208L, Offset(provider, "PyCodeObject", "co_code_adaptive"));
+            Assert.AreEqual(136L, Offset(provider, "PyCodeObject", "co_linetable"));
+        }
+
+        [TestMethod, Priority(0)]
+        public void Standard315_MapsThreadStateHotPath() {
+            var provider = Provider(PyDebugOffsetsTests.RawV315);
+            Assert.AreEqual(8L, Offset(provider, "_ts", "next"));
+            Assert.AreEqual(16L, Offset(provider, "_ts", "interp"));
+            Assert.AreEqual(176L, Offset(provider, "_ts", "thread_id"));
+            Assert.AreEqual(72L, Offset(provider, "_ts", "current_frame"));
+        }
+
+        [TestMethod, Priority(0)]
+        public void FreeThreaded315_TracksShiftedLayout() {
+            var standard = Provider(PyDebugOffsetsTests.RawV315);
+            var freeThreaded = Provider(PyDebugOffsetsTests.RawV315T);
+
+            // Same story as 3.14: the free-threaded build shifts these code-object fields and the
+            // provider surfaces the shifted offsets with no code change (co_linetable 136->152,
+            // co_firstlineno 68->84).
+            Assert.AreEqual(136L, Offset(standard, "PyCodeObject", "co_linetable"));
+            Assert.AreEqual(152L, Offset(freeThreaded, "PyCodeObject", "co_linetable"));
+            Assert.AreEqual(68L, Offset(standard, "PyCodeObject", "co_firstlineno"));
+            Assert.AreEqual(84L, Offset(freeThreaded, "PyCodeObject", "co_firstlineno"));
+
+            // Fields that don't move stay put across builds.
+            Assert.AreEqual(56L, Offset(freeThreaded, "_PyInterpreterFrame", "instr_ptr"));
+            Assert.AreEqual(72L, Offset(freeThreaded, "_ts", "current_frame"));
+        }
+
+        // Builds a provider over a synthetic 3.15 table where field i holds value i, so the hot-path
+        // mappings must resolve to each field's ordinal in the 3.15 layout (see PyDebugOffsetsTests for
+        // how the ordinals are derived from the CPython 3.15 header).
+        private static DebugOffsetsFieldProvider Synthetic315Provider() {
+            PyDebugOffsets offsets;
+            string error;
+            Assert.IsTrue(PyDebugOffsets.TryParse(PyDebugOffsetsTests.BuildSynthetic315(), out offsets, out error), error);
+            Assert.IsTrue(offsets.Is315);
+            return new DebugOffsetsFieldProvider(offsets);
+        }
+
+        [TestMethod, Priority(0)]
+        public void Synthetic315_MapsHotPathAgainstGrownLayout() {
+            var provider = Synthetic315Provider();
+
+            // interpreter_frame hot path (ordinals shifted by the 3.15 insertions).
+            Assert.AreEqual(37L, Offset(provider, "_PyInterpreterFrame", "previous"));
+            Assert.AreEqual(38L, Offset(provider, "_PyInterpreterFrame", "f_executable"));
+            Assert.AreEqual(39L, Offset(provider, "_PyInterpreterFrame", "instr_ptr"));
+            Assert.AreEqual(40L, Offset(provider, "_PyInterpreterFrame", "localsplus"));
+            Assert.AreEqual(41L, Offset(provider, "_PyInterpreterFrame", "owner"));
+
+            // code_object hot path.
+            Assert.AreEqual(48L, Offset(provider, "PyCodeObject", "co_linetable"));
+            Assert.AreEqual(49L, Offset(provider, "PyCodeObject", "co_firstlineno"));
+            Assert.AreEqual(53L, Offset(provider, "PyCodeObject", "co_code_adaptive"));
+
+            // thread_state hot path (current_frame keeps ordinal 23; interp/next precede it).
+            Assert.AreEqual(21L, Offset(provider, "_ts", "next"));
+            Assert.AreEqual(22L, Offset(provider, "_ts", "interp"));
+            Assert.AreEqual(23L, Offset(provider, "_ts", "current_frame"));
+            Assert.AreEqual(27L, Offset(provider, "_ts", "thread_id"));
+        }
     }
 }