ci: use Python version specific CARGO_TARGET_DIR (#14953)

## Description

Use Python version specific `CARGO_TARGET_DIR` for src/native, Rust
extension module. This has a few benefits

1. Parallelizing build_base_venv jobs. While working on #13984, I
noticed that since build_base_venv jobs on GitLab run in the same
dd-trace-py directory and use the same `CARGO_TARGET_DIR`, and there's
cargo lock, only one of them could run at a time.
2. Re-use Rust build artifacts as much as possible. Consider a scenario
one builds dd-trace-py using multiple different Python versions in a
row. In that case, the build had to spend extra time after switching to
a different Python version, as src/native has to be re-linked and
re-built with different Python binaries.

<!-- Provide an overview of the change and motivation for the change -->

## Testing

<!-- Describe your testing strategy or note what tests are included -->

## Risks

<!-- Note any risks associated with this change, or "None" if no risks
-->

## Additional Notes

<!-- Any other information that would be helpful for reviewers -->

Author: taegyunkim

.gitignore

@@ -198,3 +198,6 @@ tests/appsec/iast/fixtures/taint_sinks/not_exists.txt
 
 *.debug
 *.dSYM/
+
+# Rust build artifacts
+src/native/target*

ddtrace/internal/datadog/profiling/cmake/FindLibNative.cmake

@@ -23,7 +23,8 @@ message(WARNING "LIBRARY_NAME: ${LIBRARY_NAME}")
 
 # We expect the native extension to be built and installed the headers in the following directory. It is configured in
 # setup.py by setting CARGO_TARGET_DIR environment variable.
-set(SOURCE_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/../../../../../src/native/target/include)
+set(SOURCE_INCLUDE_DIR
+    ${CMAKE_SOURCE_DIR}/../../../../../src/native/target${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/include)
 
 set(DEST_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR})
 set(DEST_INCLUDE_DIR ${DEST_LIB_DIR}/include)

ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt

@@ -69,8 +69,11 @@ if(APPLE)
 elseif(UNIX)
     set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..")
 endif()
-target_include_directories(${EXTENSION_NAME} PRIVATE ../dd_wrapper/include ../../../../../src/native/target/include/
-                                                     ${Datadog_INCLUDE_DIRS} ${Python3_INCLUDE_DIRS})
+target_include_directories(
+    ${EXTENSION_NAME}
+    PRIVATE ../dd_wrapper/include
+            ../../../../../src/native/target${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}/include/
+            ${Datadog_INCLUDE_DIRS} ${Python3_INCLUDE_DIRS})
 
 target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper)
 

setup.py

@@ -92,6 +92,7 @@
 DDUP_DIR = HERE / "ddtrace" / "internal" / "datadog" / "profiling" / "ddup"
 STACK_V2_DIR = HERE / "ddtrace" / "internal" / "datadog" / "profiling" / "stack_v2"
 NATIVE_CRATE = HERE / "src" / "native"
+CARGO_TARGET_DIR = NATIVE_CRATE.absolute() / f"target{sys.version_info.major}.{sys.version_info.minor}"
 
 BUILD_PROFILING_NATIVE_TESTS = os.getenv("DD_PROFILING_NATIVE_TESTS", "0").lower() in ("1", "yes", "on", "true")
 
@@ -208,7 +209,7 @@ def __init__(self, attrs=None):
         # but at the same time dropped support for Python 3.8. So we'd need to
         # make sure that this env var is set to install the ffi headers in the
         # right place.
-        os.environ["CARGO_TARGET_DIR"] = str(NATIVE_CRATE.absolute() / "target")
+        os.environ["CARGO_TARGET_DIR"] = str(CARGO_TARGET_DIR)
         self.rust_extensions = [
             RustExtension(
                 # The Python import path of your extension:
@@ -235,7 +236,9 @@ def run(self):
                     sources = [
                         _
                         for _ in source_path.glob("**/*")
-                        if _.is_file() and _.relative_to(source_path).parts[0] != "target"
+                        if _.is_file()
+                        and _.relative_to(source_path).parts[0]
+                        != f"target{sys.version_info.major}.{sys.version_info.minor}"
                     ]
                 else:
                     sources = [Path(_) for _ in ext.sources]
@@ -318,7 +321,7 @@ def run(self):
             dedup_env["PATH"] = cargo_bin + os.pathsep + os.environ["PATH"]
 
             # Run dedup_headers on the generated headers
-            include_dir = NATIVE_CRATE.absolute() / "target" / "include" / "datadog"
+            include_dir = CARGO_TARGET_DIR / "include" / "datadog"
             if include_dir.exists():
                 subprocess.run(
                     ["dedup_headers", "common.h", "profiling.h"],
@@ -499,8 +502,7 @@ def remove_artifacts():
     @staticmethod
     def remove_rust():
         """Clean the Rust crate using cargo clean."""
-        target_dir = NATIVE_CRATE / "target"
-        if target_dir.exists():
+        if CARGO_TARGET_DIR.exists():
             subprocess.run(
                 ["cargo", "clean"],
                 cwd=str(NATIVE_CRATE),