From 96ce9a2302cb6f40842e4e4b49f4bcc3a4b643d9 Mon Sep 17 00:00:00 2001 From: Emmett Butler Date: Mon, 22 Sep 2025 10:35:45 -0700 Subject: [PATCH 1/9] update bytecode support for python 3.14 --- ddtrace/debugging/_expressions.py | 8 +- ddtrace/internal/assembly.py | 5 +- .../internal/bytecode_injection/__init__.py | 14 +- ddtrace/internal/coverage/instrumentation.py | 4 +- .../coverage/instrumentation_py3_14.py | 21 +++ ddtrace/internal/symbol_db/symbols.py | 3 +- ddtrace/internal/wrapping/__init__.py | 13 +- ddtrace/internal/wrapping/asyncs.py | 128 +++++++++++++++++- ddtrace/internal/wrapping/context.py | 49 ++++++- ddtrace/internal/wrapping/generators.py | 81 ++++++++++- 10 files changed, 315 insertions(+), 11 deletions(-) create mode 100644 ddtrace/internal/coverage/instrumentation_py3_14.py diff --git a/ddtrace/debugging/_expressions.py b/ddtrace/debugging/_expressions.py index 3b3d1ef6b56..a26ad964e15 100644 --- a/ddtrace/debugging/_expressions.py +++ b/ddtrace/debugging/_expressions.py @@ -39,6 +39,7 @@ from typing import Tuple from typing import Union +from bytecode import BinaryOp from bytecode import Bytecode from bytecode import Compare from bytecode import Instr @@ -290,7 +291,12 @@ def _compile_arg_operation(self, ast: DDASTType) -> Optional[List[Instr]]: raise ValueError("Invalid argument: %r" % a) if cb is None: raise ValueError("Invalid argument: %r" % b) - return cv + ca + cb + [Instr("BUILD_SLICE", 2), Instr("BINARY_SUBSCR")] + + if PY >= (3, 14): + subscr_instruction = Instr("BINARY_OP", BinaryOp.SUBSCR) + else: + subscr_instruction = Instr("BINARY_SUBSCR") + return cv + ca + cb + [Instr("BUILD_SLICE", 2), subscr_instruction] if _type == "filter": a, b = args diff --git a/ddtrace/internal/assembly.py b/ddtrace/internal/assembly.py index c1740192540..1402f64c658 100644 --- a/ddtrace/internal/assembly.py +++ b/ddtrace/internal/assembly.py @@ -41,7 +41,10 @@ def relocate(instrs: bc.Bytecode, lineno: int) -> bc.Bytecode: def transform_instruction(opcode: str, arg: t.Any) -> t.Tuple[str, t.Any]: # Handle pseudo-instructions - if sys.version_info >= (3, 12): + if sys.version_info >= (3, 14): + if opcode.upper() == "LOAD_ATTR" and not isinstance(arg, tuple): + arg = (True, arg) + elif sys.version_info >= (3, 12): if opcode.upper() == "LOAD_METHOD": opcode = "LOAD_ATTR" arg = (True, arg) diff --git a/ddtrace/internal/bytecode_injection/__init__.py b/ddtrace/internal/bytecode_injection/__init__.py index b31e52e3140..a112a3843c7 100644 --- a/ddtrace/internal/bytecode_injection/__init__.py +++ b/ddtrace/internal/bytecode_injection/__init__.py @@ -30,8 +30,18 @@ class InvalidLine(Exception): # the stack to the state prior to the call. INJECTION_ASSEMBLY = Assembly() -if PY >= (3, 14): - raise NotImplementedError("Python >= 3.14 is not supported yet") +if PY >= (3, 15): + raise NotImplementedError("Python >= 3.15 is not supported yet") +elif PY >= (3, 14): + INJECTION_ASSEMBLY.parse( + r""" + load_const {hook} + push_null + load_const {arg} + call 1 + pop_top + """ + ) elif PY >= (3, 13): INJECTION_ASSEMBLY.parse( r""" diff --git a/ddtrace/internal/coverage/instrumentation.py b/ddtrace/internal/coverage/instrumentation.py index 3e24f66239d..16447dad8d3 100644 --- a/ddtrace/internal/coverage/instrumentation.py +++ b/ddtrace/internal/coverage/instrumentation.py @@ -2,7 +2,9 @@ # Import are noqa'd otherwise some formatters will helpfully remove them -if sys.version_info >= (3, 13): +if sys.version_info >= (3, 14): + from ddtrace.internal.coverage.instrumentation_py3_14 import instrument_all_lines # noqa +elif sys.version_info >= (3, 13): from ddtrace.internal.coverage.instrumentation_py3_13 import instrument_all_lines # noqa elif sys.version_info >= (3, 12): from ddtrace.internal.coverage.instrumentation_py3_12 import instrument_all_lines # noqa diff --git a/ddtrace/internal/coverage/instrumentation_py3_14.py b/ddtrace/internal/coverage/instrumentation_py3_14.py new file mode 100644 index 00000000000..abec791ef22 --- /dev/null +++ b/ddtrace/internal/coverage/instrumentation_py3_14.py @@ -0,0 +1,21 @@ +import dis +import sys +from types import CodeType +import typing as t + +from ddtrace.internal.bytecode_injection import HookType +from ddtrace.internal.test_visibility.coverage_lines import CoverageLines + + +# This is primarily to make mypy happy without having to nest the rest of this module behind a version check +assert sys.version_info >= (3, 14) # nosec + +EXTENDED_ARG = dis.EXTENDED_ARG +IMPORT_NAME = dis.opmap["IMPORT_NAME"] +IMPORT_FROM = dis.opmap["IMPORT_FROM"] +RESUME = dis.opmap["RESUME"] + + +def instrument_all_lines(code: CodeType, hook: HookType, path: str, package: str) -> t.Tuple[CodeType, CoverageLines]: + # No-op + return code, CoverageLines() diff --git a/ddtrace/internal/symbol_db/symbols.py b/ddtrace/internal/symbol_db/symbols.py index 803ce59cd11..0e39e37b913 100644 --- a/ddtrace/internal/symbol_db/symbols.py +++ b/ddtrace/internal/symbol_db/symbols.py @@ -96,7 +96,8 @@ def get_fields(cls: type) -> t.Set[str]: return { code.co_names[b.arg] for a, b in zip(*(islice(t, i, None) for i, t in enumerate(tee(dis.get_instructions(code), 2)))) - if a.opname == "LOAD_FAST" and a.arg == 0 and b.opname == "STORE_ATTR" + # Python 3.14 changed this to LOAD_FAST_BORROW + if a.opname.startswith("LOAD_FAST") and a.arg == 0 and b.opname == "STORE_ATTR" } except AttributeError: return set() diff --git a/ddtrace/internal/wrapping/__init__.py b/ddtrace/internal/wrapping/__init__.py index 852c99dc151..cf71616f279 100644 --- a/ddtrace/internal/wrapping/__init__.py +++ b/ddtrace/internal/wrapping/__init__.py @@ -40,7 +40,18 @@ def _add(lineno): UPDATE_MAP = Assembly() -if PY >= (3, 12): +if PY >= (3, 14): + UPDATE_MAP.parse( + r""" + copy 1 + load_attr $update + load_fast {varkwargsname} + call 1 + pop_top + """ + ) + +elif PY >= (3, 12): UPDATE_MAP.parse( r""" copy 1 diff --git a/ddtrace/internal/wrapping/asyncs.py b/ddtrace/internal/wrapping/asyncs.py index 66341657ca4..e6288eb9175 100644 --- a/ddtrace/internal/wrapping/asyncs.py +++ b/ddtrace/internal/wrapping/asyncs.py @@ -34,7 +34,133 @@ ASYNC_GEN_ASSEMBLY = Assembly() ASYNC_HEAD_ASSEMBLY = None -if PY >= (3, 12): +if PY >= (3, 14): + ASYNC_HEAD_ASSEMBLY = Assembly() + ASYNC_HEAD_ASSEMBLY.parse( + r""" + return_generator + pop_top + """ + ) + + COROUTINE_ASSEMBLY.parse( + r""" + get_awaitable 0 + load_const None + + presend: + send @send + yield_value 2 + resume 3 + jump_backward_no_interrupt @presend + send: + end_send + """ + ) + + ASYNC_GEN_ASSEMBLY.parse( + r""" + try @stopiter + copy 1 + store_fast $__ddgen + load_attr (False, 'asend') + store_fast $__ddgensend + load_fast $__ddgen + load_attr (True, '__anext__') + call 0 + + loop: + get_awaitable 0 + load_const None + presend0: + send @send0 + tried + + try @genexit lasti + yield_value 3 + resume 3 + jump_backward_no_interrupt @loop + send0: + end_send + + yield: + call_intrinsic_1 asm.Intrinsic1Op.INTRINSIC_ASYNC_GEN_WRAP + yield_value 3 + resume 1 + push_null + swap 2 + load_fast $__ddgensend + swap 2 + call 1 + jump_backward @loop + tried + + genexit: + try @stopiter + push_exc_info + load_const GeneratorExit + check_exc_match + pop_jump_if_false @exc + pop_top + load_fast $__ddgen + load_attr (True, 'aclose') + call 0 + get_awaitable 0 + load_const None + + presend1: + send @send1 + yield_value 4 + resume 3 + jump_backward_no_interrupt @presend1 + send1: + end_send + pop_top + pop_except + load_const None + return_value + + exc: + pop_top + push_null + load_fast $__ddgen + load_attr (False, 'athrow') + push_null + load_const sys.exc_info + call 0 + call_function_ex + get_awaitable 0 + load_const None + + presend2: + send @send2 + yield_value 4 + resume 3 + jump_backward_no_interrupt @presend2 + send2: + end_send + swap 2 + pop_except + jump_backward @yield + tried + + stopiter: + push_exc_info + load_const StopAsyncIteration + check_exc_match + pop_jump_if_false @propagate + pop_top + pop_except + load_const None + return_value + + propagate: + reraise 0 + """ + ) + + +elif PY >= (3, 12): ASYNC_HEAD_ASSEMBLY = Assembly() ASYNC_HEAD_ASSEMBLY.parse( r""" diff --git a/ddtrace/internal/wrapping/context.py b/ddtrace/internal/wrapping/context.py index 87ce3430855..bed85ae11a9 100644 --- a/ddtrace/internal/wrapping/context.py +++ b/ddtrace/internal/wrapping/context.py @@ -67,8 +67,53 @@ CONTEXT_RETURN = Assembly() CONTEXT_FOOT = Assembly() -if sys.version_info >= (3, 14): - raise NotImplementedError("Python >= 3.14 is not supported yet") +if sys.version_info >= (3, 15): + raise NotImplementedError("Python >= 3.15 is not supported yet") +elif sys.version_info >= (3, 14): + CONTEXT_HEAD.parse( + r""" + load_const {context_enter} + push_null + call 0 + pop_top + """ + ) + CONTEXT_RETURN.parse( + r""" + push_null + load_const {context_return} + swap 3 + call 1 + """ + ) + + CONTEXT_RETURN_CONST = Assembly() + CONTEXT_RETURN_CONST.parse( + r""" + load_const {context_return} + push_null + load_const {value} + call 1 + """ + ) + + CONTEXT_FOOT.parse( + r""" + try @_except lasti + push_exc_info + load_const {context_exit} + push_null + call 0 + pop_top + reraise 2 + tried + + _except: + copy 3 + pop_except + reraise 1 + """ + ) elif sys.version_info >= (3, 13): CONTEXT_HEAD.parse( r""" diff --git a/ddtrace/internal/wrapping/generators.py b/ddtrace/internal/wrapping/generators.py index 91cbed49962..8ab0df95bde 100644 --- a/ddtrace/internal/wrapping/generators.py +++ b/ddtrace/internal/wrapping/generators.py @@ -30,7 +30,86 @@ GENERATOR_ASSEMBLY = Assembly() GENERATOR_HEAD_ASSEMBLY = None -if PY >= (3, 12): +if PY >= (3, 14): + GENERATOR_HEAD_ASSEMBLY = Assembly() + GENERATOR_HEAD_ASSEMBLY.parse( + r""" + return_generator + pop_top + """ + ) + + GENERATOR_ASSEMBLY.parse( + r""" + try @stopiter + copy 1 + store_fast $__ddgen + load_attr $send + store_fast $__ddgensend + push_null + load_const next + load_fast $__ddgen + + loop: + call 1 + tried + + yield: + try @genexit lasti + yield_value 3 + resume 1 + push_null + swap 2 + load_fast $__ddgensend + swap 2 + jump_backward @loop + tried + + genexit: + try @stopiter + push_exc_info + load_const GeneratorExit + check_exc_match + pop_jump_if_false @exc + pop_top + load_fast $__ddgen + load_attr $close + call 0 + swap 2 + pop_except + load_const None + return_value + + exc: + pop_top + push_null + load_fast $__ddgen + load_attr $throw + push_null + load_const sys.exc_info + call 0 + call_function_ex + swap 2 + pop_except + jump_backward @yield + tried + + stopiter: + push_exc_info + load_const StopIteration + check_exc_match + pop_jump_if_false @propagate + pop_top + pop_except + load_const None + return_value + + propagate: + reraise 0 + """ + ) + +elif PY >= (3, 12): GENERATOR_HEAD_ASSEMBLY = Assembly() GENERATOR_HEAD_ASSEMBLY.parse( r""" From 5bb4cf6de6a9b9a2b7d73fe5fe5ad9f1cf146801 Mon Sep 17 00:00:00 2001 From: Emmett Butler Date: Mon, 22 Sep 2025 11:10:14 -0700 Subject: [PATCH 2/9] undo stuff that only works on 3.14 --- ddtrace/internal/coverage/instrumentation.py | 4 +--- .../coverage/instrumentation_py3_14.py | 21 ------------------- 2 files changed, 1 insertion(+), 24 deletions(-) delete mode 100644 ddtrace/internal/coverage/instrumentation_py3_14.py diff --git a/ddtrace/internal/coverage/instrumentation.py b/ddtrace/internal/coverage/instrumentation.py index 16447dad8d3..3e24f66239d 100644 --- a/ddtrace/internal/coverage/instrumentation.py +++ b/ddtrace/internal/coverage/instrumentation.py @@ -2,9 +2,7 @@ # Import are noqa'd otherwise some formatters will helpfully remove them -if sys.version_info >= (3, 14): - from ddtrace.internal.coverage.instrumentation_py3_14 import instrument_all_lines # noqa -elif sys.version_info >= (3, 13): +if sys.version_info >= (3, 13): from ddtrace.internal.coverage.instrumentation_py3_13 import instrument_all_lines # noqa elif sys.version_info >= (3, 12): from ddtrace.internal.coverage.instrumentation_py3_12 import instrument_all_lines # noqa diff --git a/ddtrace/internal/coverage/instrumentation_py3_14.py b/ddtrace/internal/coverage/instrumentation_py3_14.py deleted file mode 100644 index abec791ef22..00000000000 --- a/ddtrace/internal/coverage/instrumentation_py3_14.py +++ /dev/null @@ -1,21 +0,0 @@ -import dis -import sys -from types import CodeType -import typing as t - -from ddtrace.internal.bytecode_injection import HookType -from ddtrace.internal.test_visibility.coverage_lines import CoverageLines - - -# This is primarily to make mypy happy without having to nest the rest of this module behind a version check -assert sys.version_info >= (3, 14) # nosec - -EXTENDED_ARG = dis.EXTENDED_ARG -IMPORT_NAME = dis.opmap["IMPORT_NAME"] -IMPORT_FROM = dis.opmap["IMPORT_FROM"] -RESUME = dis.opmap["RESUME"] - - -def instrument_all_lines(code: CodeType, hook: HookType, path: str, package: str) -> t.Tuple[CodeType, CoverageLines]: - # No-op - return code, CoverageLines() From e9efe96718186a199e7c4e6befc6e6e3150687be Mon Sep 17 00:00:00 2001 From: Emmett Butler <723615+emmettbutler@users.noreply.github.com> Date: Tue, 23 Sep 2025 06:26:41 -0700 Subject: [PATCH 3/9] Update ddtrace/internal/symbol_db/symbols.py Co-authored-by: Gabriele N. Tornetta --- ddtrace/internal/symbol_db/symbols.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/internal/symbol_db/symbols.py b/ddtrace/internal/symbol_db/symbols.py index 0e39e37b913..050e9655902 100644 --- a/ddtrace/internal/symbol_db/symbols.py +++ b/ddtrace/internal/symbol_db/symbols.py @@ -97,7 +97,7 @@ def get_fields(cls: type) -> t.Set[str]: code.co_names[b.arg] for a, b in zip(*(islice(t, i, None) for i, t in enumerate(tee(dis.get_instructions(code), 2)))) # Python 3.14 changed this to LOAD_FAST_BORROW - if a.opname.startswith("LOAD_FAST") and a.arg == 0 and b.opname == "STORE_ATTR" + if a.opname.startswith("LOAD_FAST") and a.arg & 15 == 0 and b.opname == "STORE_ATTR" } except AttributeError: return set() From 6fe6edea12d98f920a87e35fa57cea2c28736efa Mon Sep 17 00:00:00 2001 From: Emmett Butler <723615+emmettbutler@users.noreply.github.com> Date: Tue, 23 Sep 2025 06:27:03 -0700 Subject: [PATCH 4/9] Update ddtrace/internal/bytecode_injection/__init__.py Co-authored-by: Gabriele N. Tornetta --- ddtrace/internal/bytecode_injection/__init__.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/ddtrace/internal/bytecode_injection/__init__.py b/ddtrace/internal/bytecode_injection/__init__.py index a112a3843c7..a151fbea654 100644 --- a/ddtrace/internal/bytecode_injection/__init__.py +++ b/ddtrace/internal/bytecode_injection/__init__.py @@ -32,16 +32,6 @@ class InvalidLine(Exception): INJECTION_ASSEMBLY = Assembly() if PY >= (3, 15): raise NotImplementedError("Python >= 3.15 is not supported yet") -elif PY >= (3, 14): - INJECTION_ASSEMBLY.parse( - r""" - load_const {hook} - push_null - load_const {arg} - call 1 - pop_top - """ - ) elif PY >= (3, 13): INJECTION_ASSEMBLY.parse( r""" From 3c713610a60ef2f3849676526b5592182bca7c57 Mon Sep 17 00:00:00 2001 From: Emmett Butler Date: Tue, 23 Sep 2025 08:35:27 -0700 Subject: [PATCH 5/9] more correct pseudo-instruction handling for 3.14 --- ddtrace/internal/assembly.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ddtrace/internal/assembly.py b/ddtrace/internal/assembly.py index 1402f64c658..fdeb32755c3 100644 --- a/ddtrace/internal/assembly.py +++ b/ddtrace/internal/assembly.py @@ -41,15 +41,12 @@ def relocate(instrs: bc.Bytecode, lineno: int) -> bc.Bytecode: def transform_instruction(opcode: str, arg: t.Any) -> t.Tuple[str, t.Any]: # Handle pseudo-instructions - if sys.version_info >= (3, 14): - if opcode.upper() == "LOAD_ATTR" and not isinstance(arg, tuple): - arg = (True, arg) - elif sys.version_info >= (3, 12): + if sys.version_info >= (3, 12): if opcode.upper() == "LOAD_METHOD": opcode = "LOAD_ATTR" arg = (True, arg) elif opcode.upper() == "LOAD_ATTR" and not isinstance(arg, tuple): - arg = (False, arg) + arg = (sys.version_info >= (3, 14), arg) return opcode, arg From 0777b85395d1cc462318249fa9a0048eb538494f Mon Sep 17 00:00:00 2001 From: Emmett Butler Date: Tue, 23 Sep 2025 08:45:27 -0700 Subject: [PATCH 6/9] more correct pseudo-instruction handling for 3.14 --- ddtrace/internal/wrapping/__init__.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/ddtrace/internal/wrapping/__init__.py b/ddtrace/internal/wrapping/__init__.py index cf71616f279..852c99dc151 100644 --- a/ddtrace/internal/wrapping/__init__.py +++ b/ddtrace/internal/wrapping/__init__.py @@ -40,18 +40,7 @@ def _add(lineno): UPDATE_MAP = Assembly() -if PY >= (3, 14): - UPDATE_MAP.parse( - r""" - copy 1 - load_attr $update - load_fast {varkwargsname} - call 1 - pop_top - """ - ) - -elif PY >= (3, 12): +if PY >= (3, 12): UPDATE_MAP.parse( r""" copy 1 From 8e46e581d2580e396b314e265820edb7d18d7d61 Mon Sep 17 00:00:00 2001 From: Emmett Butler Date: Tue, 23 Sep 2025 11:41:32 -0700 Subject: [PATCH 7/9] handle LOAD_METHOD --- ddtrace/internal/assembly.py | 5 +++++ ddtrace/internal/wrapping/generators.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ddtrace/internal/assembly.py b/ddtrace/internal/assembly.py index fdeb32755c3..0d099ed9af2 100644 --- a/ddtrace/internal/assembly.py +++ b/ddtrace/internal/assembly.py @@ -157,6 +157,11 @@ def parse_try_end(self, line: str) -> t.Optional[bc.TryEnd]: def parse_opcode(self, text: str) -> str: opcode = text.upper() + + # `dis` doesn't include `LOAD_METHOD` in 3.14.0rc1 + if sys.version_info >= (3, 14) and opcode == "LOAD_METHOD": + return opcode + if opcode not in dis.opmap: raise ValueError("unknown opcode %s" % opcode) diff --git a/ddtrace/internal/wrapping/generators.py b/ddtrace/internal/wrapping/generators.py index 8ab0df95bde..a31b1f51849 100644 --- a/ddtrace/internal/wrapping/generators.py +++ b/ddtrace/internal/wrapping/generators.py @@ -73,7 +73,7 @@ pop_jump_if_false @exc pop_top load_fast $__ddgen - load_attr $close + load_method $close call 0 swap 2 pop_except From 1d2bd21619839f5ae5e55e736eaa7a02eed6ae9d Mon Sep 17 00:00:00 2001 From: Emmett Butler Date: Tue, 23 Sep 2025 11:54:22 -0700 Subject: [PATCH 8/9] remove unnecessary block --- ddtrace/internal/wrapping/context.py | 45 ---------------------------- 1 file changed, 45 deletions(-) diff --git a/ddtrace/internal/wrapping/context.py b/ddtrace/internal/wrapping/context.py index bed85ae11a9..2e5be4b1013 100644 --- a/ddtrace/internal/wrapping/context.py +++ b/ddtrace/internal/wrapping/context.py @@ -69,51 +69,6 @@ if sys.version_info >= (3, 15): raise NotImplementedError("Python >= 3.15 is not supported yet") -elif sys.version_info >= (3, 14): - CONTEXT_HEAD.parse( - r""" - load_const {context_enter} - push_null - call 0 - pop_top - """ - ) - CONTEXT_RETURN.parse( - r""" - push_null - load_const {context_return} - swap 3 - call 1 - """ - ) - - CONTEXT_RETURN_CONST = Assembly() - CONTEXT_RETURN_CONST.parse( - r""" - load_const {context_return} - push_null - load_const {value} - call 1 - """ - ) - - CONTEXT_FOOT.parse( - r""" - try @_except lasti - push_exc_info - load_const {context_exit} - push_null - call 0 - pop_top - reraise 2 - tried - - _except: - copy 3 - pop_except - reraise 1 - """ - ) elif sys.version_info >= (3, 13): CONTEXT_HEAD.parse( r""" From 8294b68e7ff4d9c866c8e59423250bc00b503d7d Mon Sep 17 00:00:00 2001 From: Emmett Butler Date: Wed, 24 Sep 2025 08:32:37 -0700 Subject: [PATCH 9/9] remove unnecessary instruction --- ddtrace/internal/wrapping/generators.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ddtrace/internal/wrapping/generators.py b/ddtrace/internal/wrapping/generators.py index a31b1f51849..37b762b64e5 100644 --- a/ddtrace/internal/wrapping/generators.py +++ b/ddtrace/internal/wrapping/generators.py @@ -77,7 +77,6 @@ call 0 swap 2 pop_except - load_const None return_value exc: