Add CPython 3.14 support to mixed-mode debugging

Python/Product/Debugger.Concord/Debugger.Concord.csproj

@@ -126,6 +126,8 @@
     <Compile Include="Proxies\Structs\PyFrameObject310.cs" />
     <Compile Include="Proxies\Structs\PyFrameObject311.cs" />
     <Compile Include="Proxies\Structs\PyFunctionObject.cs" />
+    <Compile Include="Proxies\Structs\PyDebugOffsets.cs" />
+    <Compile Include="Proxies\Structs\DebugOffsetsFieldProvider.cs" />
     <Compile Include="Proxies\Structs\PyInterpreterFrame.cs" />
     <Compile Include="Proxies\Structs\PyLineTable.cs" />
     <Compile Include="Proxies\Structs\PyRuntimeState.cs" />

Python/Product/Debugger.Concord/DkmExtensions.cs

@@ -162,6 +162,19 @@ public static ulong GetExportedStaticVariableAddress(this DkmNativeModuleInstanc
             return moduleInstance.BaseAddress + addr.RVA;
         }
 
+        /// <summary>
+        /// Like <see cref="GetExportedStaticVariableAddress"/>, but returns 0 instead of throwing
+        /// when the export is not present. Useful for optional symbols (e.g. <c>_PyRuntime</c>)
+        /// that we want to probe for without a PDB and without asserting when absent.
+        /// </summary>
+        public static ulong TryGetExportedStaticVariableAddress(this DkmNativeModuleInstance moduleInstance, string name) {
+            var addr = moduleInstance.FindExportName(name, false);
+            if (addr == null) {
+                return 0;
+            }
+            return moduleInstance.BaseAddress + addr.RVA;
+        }
+
         public static TProxy GetExportedStaticVariable<TProxy>(this DkmNativeModuleInstance moduleInstance, string name)
             where TProxy : IDataProxy {
             ulong address = GetExportedStaticVariableAddress(moduleInstance, name);

Python/Product/Debugger.Concord/ExpressionEvaluator.cs

@@ -377,43 +377,62 @@ public void GetFrameLocals(DkmInspectionContext inspectionContext, DkmWorkList w
             var f_code = pythonFrame.f_code.Read();
             var f_localsplus = pythonFrame.f_localsplus;
 
+            // In CPython 3.14, the frame's localsplus slots (locals, cells, and free vars) are
+            // _PyStackRef values rather than plain PyObject*. Their low bits carry a deferred/immortal
+            // reference tag (Py_TAG_BITS), so strip them to recover the real object pointer; this
+            // mirrors CPython's own PyStackRef_AsPyObjectBorrow. Without this, any local holding a
+            // deferred object (None, booleans, small ints, interned strings, ...) reads at a
+            // misaligned address and throws, which aborts the entire locals enumeration. Object
+            // pointers are 8-byte aligned, so the mask is a no-op for untagged slots; it is 0 for
+            // versions before 3.14, leaving their behavior byte-for-byte identical.
+            ulong localsPlusTagMask =
+                pythonFrame.Process.GetPythonRuntimeInfo().LanguageVersion >= PythonLanguageVersion.V314 ? 0x3ul : 0ul;
+
             // Process cellvars and freevars first, because function arguments can appear in both cellvars and varnames if the argument is captured by a closure,
             // in which case we want to use the cellvar because the regular var slot will then be unused by Python (and in Python 3.4+, nulled out).
             var namesSeen = new HashSet<string>();
             var cellNames = f_code.co_cellvars.Read().ReadElements().Concat(f_code.co_freevars.Read().ReadElements());
             var cellSlots = f_localsplus.Skip(f_code.co_nlocals.Read());
             foreach (var pair in cellNames.Zip(cellSlots, (nameObj, cellSlot) => new { nameObj, cellSlot = cellSlot })) {
                 var nameObj = pair.nameObj;
-                var cellSlot = pair.cellSlot;
+                var cellSlot = pair.cellSlot.WithTagMask(localsPlusTagMask);
 
                 var name = (nameObj.Read() as IPyBaseStringObject).ToStringOrNull();
                 if (name == null) {
                     continue;
                 }
                 namesSeen.Add(name);
 
-                if (cellSlot.IsNull) {
-                    continue;
-                }
+                try {
+                    if (cellSlot.IsNull) {
+                        continue;
+                    }
 
-                var cell = cellSlot.Read() as PyCellObject;
-                if (cell == null) {
-                    continue;
-                }
+                    var cell = cellSlot.Read() as PyCellObject;
+                    if (cell == null) {
+                        continue;
+                    }
 
-                var localPtr = cell.ob_ref;
-                if (localPtr.IsNull) {
-                    continue;
-                }
+                    var localPtr = cell.ob_ref;
+                    if (localPtr.IsNull) {
+                        continue;
+                    }
 
-                var evalResult = CreatePyObjectEvaluationResult(inspectionContext, stackFrame, null, new PythonEvaluationResult(localPtr, name), cppEval);
-                evalResults.Add(evalResult);
+                    var evalResult = CreatePyObjectEvaluationResult(inspectionContext, stackFrame, null, new PythonEvaluationResult(localPtr, name), cppEval);
+                    evalResults.Add(evalResult);
+                } catch (Exception ex) {
+                    // A single unreadable local must never blank out the entire Locals window.
+                    // Surface it as a failed entry (which also aids diagnosis) and keep going.
+                    evalResults.Add(DkmFailedEvaluationResult.Create(
+                        inspectionContext, stackFrame, name, name,
+                        ex.Message, DkmEvaluationResultFlags.Invalid, null));
+                }
             }
 
             PyTupleObject co_varnames = f_code.co_varnames.Read();
             foreach (var pair in co_varnames.ReadElements().Zip(f_localsplus, (nameObj, varSlot) => new { nameObj, cellSlot = varSlot })) {
                 var nameObj = pair.nameObj;
-                var varSlot = pair.cellSlot;
+                var varSlot = pair.cellSlot.WithTagMask(localsPlusTagMask);
 
                 var name = (nameObj.Read() as IPyBaseStringObject).ToStringOrNull();
                 if (name == null) {
@@ -425,12 +444,19 @@ public void GetFrameLocals(DkmInspectionContext inspectionContext, DkmWorkList w
                     continue;
                 }
 
-                if (varSlot.IsNull) {
-                    continue;
-                }
+                try {
+                    if (varSlot.IsNull) {
+                        continue;
+                    }
 
-                var evalResult = CreatePyObjectEvaluationResult(inspectionContext, stackFrame, null, new PythonEvaluationResult(varSlot, name), cppEval);
-                evalResults.Add(evalResult);
+                    var evalResult = CreatePyObjectEvaluationResult(inspectionContext, stackFrame, null, new PythonEvaluationResult(varSlot, name), cppEval);
+                    evalResults.Add(evalResult);
+                } catch (Exception ex) {
+                    // A single unreadable local must never blank out the entire Locals window.
+                    evalResults.Add(DkmFailedEvaluationResult.Create(
+                        inspectionContext, stackFrame, name, name,
+                        ex.Message, DkmEvaluationResultFlags.Invalid, null));
+                }
             }
 
             var globals = pythonFrame.f_globals.TryRead();

Python/Product/Debugger.Concord/Proxies/PointerProxy.cs

@@ -73,24 +73,44 @@ internal struct PointerProxy<TProxy> : IWritableDataProxy<TProxy>
 
         private readonly bool _polymorphic;
 
+        // Low bits to strip from the stored pointer value before dereferencing. Used for CPython
+        // 3.14 _PyStackRef fields (e.g. _PyInterpreterFrame.f_executable), whose low bits carry a
+        // tag (Py_TAG_BITS). Zero (the default) leaves the pointer untouched, so every existing
+        // construction is unaffected. Masking aligned pointers is a no-op, so it is always safe.
+        private readonly ulong _tagMask;
+
         public PointerProxy(DkmProcess process, ulong address)
             : this(process, address, true) {
         }
 
         public PointerProxy(DkmProcess process, ulong address, bool polymorphic)
+            : this(process, address, polymorphic, 0) {
+        }
+
+        private PointerProxy(DkmProcess process, ulong address, bool polymorphic, ulong tagMask)
             : this() {
             Debug.Assert(process != null && address != 0);
             Process = process;
             Address = address;
             _polymorphic = polymorphic;
+            _tagMask = tagMask;
+        }
+
+        /// <summary>
+        /// Returns a copy of this pointer that strips <paramref name="tagMask"/> from the stored
+        /// value before dereferencing. Used to read CPython 3.14 <c>_PyStackRef</c> fields, whose
+        /// low bits are a tag rather than part of the object address.
+        /// </summary>
+        public PointerProxy<TProxy> WithTagMask(ulong tagMask) {
+            return new PointerProxy<TProxy>(Process, Address, _polymorphic, tagMask);
         }
 
         public long ObjectSize {
             get { return Process.GetPointerSize(); }
         }
 
         public bool IsNull {
-            get { return Raw.IsNull; }
+            get { return ReadTarget() == 0; }
         }
 
         /// <summary>
@@ -100,25 +120,30 @@ public PointerProxy Raw {
             get { return new PointerProxy(Process, Address); }
         }
 
+        // The pointer value stored at Address, with any tag bits stripped.
+        private ulong ReadTarget() {
+            return Raw.Read() & ~_tagMask;
+        }
+
         public TProxy Read() {
-            if (IsNull) {
+            var ptr = ReadTarget();
+            if (ptr == 0) {
                 Debug.Fail("Trying to dereference a null PointerProxy.");
                 throw new InvalidOperationException();
             }
 
-            var ptr = Raw.Read();
             return DataProxy.Create<TProxy>(Process, ptr, _polymorphic);
         }
 
         /// <summary>
         /// Like <see cref="Read"/>, but returns default(<see cref="TProxy"/>) if pointer is null.
         /// </summary>
         public TProxy TryRead() {
-            if (IsNull) {
+            var ptr = ReadTarget();
+            if (ptr == 0) {
                 return default(TProxy);
             }
 
-            var ptr = Raw.Read();
             return DataProxy.Create<TProxy>(Process, ptr, _polymorphic);
         }
 

Python/Product/Debugger.Concord/Proxies/StructProxy.cs

@@ -195,7 +195,9 @@ private static TFields GetStructFields<TFields>(StructMetadata metadata)
                 return (TFields)metadata.Fields;
             }
 
-            var pyVersion = metadata.Process.GetPythonRuntimeInfo().LanguageVersion;
+            var pyrtInfo = metadata.Process.GetPythonRuntimeInfo();
+            var pyVersion = pyrtInfo.LanguageVersion;
+            var offsetProvider = pyrtInfo.StructFieldOffsetProvider;
 
             var fields = new TFields();
             foreach (var fieldInfo in typeof(TFields).GetFields()) {
@@ -208,7 +210,15 @@ private static TFields GetStructFields<TFields>(StructMetadata metadata)
                         continue;
                     }
 
-                    long offset = metadata.Symbol.GetFieldOffset(name);
+                    // Prefer an authoritative offset from the interpreter's self-describing table
+                    // (CPython 3.14 _Py_DebugOffsets, hot-path fields only) when one is available;
+                    // otherwise fall back to the PDB. The provider is null for every other version,
+                    // so this path is byte-for-byte identical to before there.
+                    long offset;
+                    if (offsetProvider == null || !offsetProvider.TryGetFieldOffset(metadata.Name, name, out offset)) {
+                        offset = metadata.Symbol.GetFieldOffset(name);
+                    }
+
                     var field = (IStructField)Activator.CreateInstance(fieldType);
                     field.Process = metadata.Process;
                     field.Offset = offset;

Python/Product/Debugger.Concord/Proxies/Structs/DebugOffsetsFieldProvider.cs

@@ -0,0 +1,97 @@
+// Python Tools for Visual Studio
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Collections.Generic;
+
+namespace Microsoft.PythonTools.Debugger.Concord.Proxies.Structs {
+    /// <summary>
+    /// Supplies the byte offset of a struct field by some means other than the interpreter's PDB.
+    /// <see cref="StructProxy"/> consults this before falling back to DIA/PDB symbol lookup, letting
+    /// us source authoritative offsets from CPython's self-describing tables when available.
+    /// </summary>
+    internal interface IStructFieldOffsetProvider {
+        /// <summary>
+        /// Returns true and sets <paramref name="offset"/> if this provider knows the offset of
+        /// <paramref name="fieldName"/> within <paramref name="structName"/>; otherwise returns false
+        /// so the caller falls back to its default (PDB) resolution.
+        /// </summary>
+        bool TryGetFieldOffset(string structName, string fieldName, out long offset);
+    }
+
+    /// <summary>
+    /// Resolves a curated set of hot-path struct fields from CPython 3.14's <c>_Py_DebugOffsets</c>
+    /// table (see <see cref="PyDebugOffsets"/>) instead of the PDB. Coverage is intentionally limited
+    /// to the frame / code-object / thread-state fields that the mixed-mode stack walk and in-process
+    /// line-number computation depend on; every other field still resolves via the PDB. This is what
+    /// automatically tracks the free-threaded build's shifted object layouts.
+    ///
+    /// The map keys are CPython struct/field names (matching <see cref="StructProxy"/>'s field names
+    /// and each proxy's <c>StructName</c>); the values are the corresponding
+    /// <see cref="PyDebugOffsets"/> (group, field) entries.
+    /// </summary>
+    internal sealed class DebugOffsetsFieldProvider : IStructFieldOffsetProvider {
+        private static readonly Dictionary<string, KeyValuePair<string, string>> Map =
+            new Dictionary<string, KeyValuePair<string, string>>(StringComparer.Ordinal) {
+                // _PyInterpreterFrame (StructName "_PyInterpreterFrame").
+                { "_PyInterpreterFrame.previous", Entry("interpreter_frame", "previous") },
+                { "_PyInterpreterFrame.f_executable", Entry("interpreter_frame", "executable") },
+                { "_PyInterpreterFrame.instr_ptr", Entry("interpreter_frame", "instr_ptr") },
+                { "_PyInterpreterFrame.localsplus", Entry("interpreter_frame", "localsplus") },
+                { "_PyInterpreterFrame.owner", Entry("interpreter_frame", "owner") },
+
+                // PyCodeObject (StructName "PyCodeObject").
+                { "PyCodeObject.co_filename", Entry("code_object", "filename") },
+                { "PyCodeObject.co_name", Entry("code_object", "name") },
+                { "PyCodeObject.co_firstlineno", Entry("code_object", "firstlineno") },
+                { "PyCodeObject.co_localsplusnames", Entry("code_object", "localsplusnames") },
+                { "PyCodeObject.co_localspluskinds", Entry("code_object", "localspluskinds") },
+                { "PyCodeObject.co_code_adaptive", Entry("code_object", "co_code_adaptive") },
+                { "PyCodeObject.co_linetable", Entry("code_object", "linetable") },
+
+                // PyThreadState (StructName "_ts").
+                { "_ts.next", Entry("thread_state", "next") },
+                { "_ts.interp", Entry("thread_state", "interp") },
+                { "_ts.thread_id", Entry("thread_state", "thread_id") },
+                { "_ts.current_frame", Entry("thread_state", "current_frame") },
+            };
+
+        private readonly PyDebugOffsets _offsets;
+
+        public DebugOffsetsFieldProvider(PyDebugOffsets offsets) {
+            _offsets = offsets ?? throw new ArgumentNullException(nameof(offsets));
+        }
+
+        public bool TryGetFieldOffset(string structName, string fieldName, out long offset) {
+            offset = 0;
+            if (structName == null || fieldName == null) {
+                return false;
+            }
+
+            KeyValuePair<string, string> entry;
+            if (!Map.TryGetValue(structName + "." + fieldName, out entry)) {
+                return false;
+            }
+
+            offset = (long)_offsets.Offset(entry.Key, entry.Value);
+            return true;
+        }
+
+        private static KeyValuePair<string, string> Entry(string group, string field) {
+            return new KeyValuePair<string, string>(group, field);
+        }
+    }
+}