Experimental tracepoint support

rust/bindings/bindings_helper.h

@@ -15,6 +15,7 @@
 #include <linux/refcount.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
+#include <linux/tracepoint.h>
 #include <linux/wait.h>
 #include <linux/workqueue.h>
 

rust/bindings/lib.rs

@@ -48,3 +48,18 @@ mod bindings_helper {
 }
 
 pub use bindings_raw::*;
+
+/// Rust version of the C macro `rcu_dereference_raw`.
+///
+/// The rust helper only works with void pointers, but this wrapper method makes it work with any
+/// pointer type using pointer casts.
+///
+/// # Safety
+///
+/// This method has the same safety requirements as the C macro of the same name.
+#[inline(always)]
+pub unsafe fn rcu_dereference_raw<T>(p: *const *mut T) -> *mut T {
+    // SAFETY: This helper calls into the C macro, so the caller promises to uphold the safety
+    // requirements.
+    unsafe { __rcu_dereference_raw(p as *mut *mut _) as *mut T }
+}

rust/helpers.c

@@ -165,6 +165,24 @@ rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags)
 }
 EXPORT_SYMBOL_GPL(rust_helper_krealloc);
 
+void rust_helper_preempt_enable_notrace(void)
+{
+	preempt_enable_notrace();
+}
+EXPORT_SYMBOL_GPL(rust_helper_preempt_enable_notrace);
+
+void rust_helper_preempt_disable_notrace(void)
+{
+	preempt_disable_notrace();
+}
+EXPORT_SYMBOL_GPL(rust_helper_preempt_disable_notrace);
+
+void *rust_helper___rcu_dereference_raw(void **p)
+{
+	return rcu_dereference_raw(p);
+}
+EXPORT_SYMBOL_GPL(rust_helper___rcu_dereference_raw);
+
 /*
  * `bindgen` binds the C `size_t` type as the Rust `usize` type, so we can
  * use it in contexts where Rust expects a `usize` like slice (array) indices.

rust/kernel/lib.rs

@@ -38,12 +38,15 @@ pub mod net;
 pub mod prelude;
 pub mod print;
 mod static_assert;
+pub mod static_call;
+pub mod static_key;
 #[doc(hidden)]
 pub mod std_vendor;
 pub mod str;
 pub mod sync;
 pub mod task;
 pub mod time;
+pub mod tracepoint;
 pub mod types;
 pub mod workqueue;
 

rust/kernel/static_call.rs

@@ -0,0 +1,90 @@
+// SPDX-License-Identifier: GPL-2.0
+
+// Copyright (C) 2024 Google LLC.
+
+//! Logic for static calls.
+
+#[macro_export]
+#[doc(hidden)]
+macro_rules! ty_underscore_for {
+    ($arg:expr) => {
+        _
+    };
+}
+
+#[doc(hidden)]
+#[repr(transparent)]
+pub struct AddressableStaticCallKey {
+    _ptr: *const bindings::static_call_key,
+}
+unsafe impl Sync for AddressableStaticCallKey {}
+impl AddressableStaticCallKey {
+    pub const fn new(ptr: *const bindings::static_call_key) -> Self {
+        Self { _ptr: ptr }
+    }
+}
+
+#[cfg(CONFIG_HAVE_STATIC_CALL)]
+#[doc(hidden)]
+#[macro_export]
+macro_rules! _static_call {
+    ($name:ident($($args:expr),* $(,)?)) => {{
+        // Symbol mangling will give this symbol a unique name.
+        #[cfg(CONFIG_HAVE_STATIC_CALL_INLINE)]
+        #[link_section = ".discard.addressable"]
+        #[used]
+        static __ADDRESSABLE: $crate::static_call::AddressableStaticCallKey = unsafe {
+            $crate::static_call::AddressableStaticCallKey::new(::core::ptr::addr_of!(
+                $crate::macros::paste! { $crate::bindings:: [<__SCK__ $name >]; }
+            ))
+        };
+
+        let fn_ptr: unsafe extern "C" fn($($crate::static_call::ty_underscore_for!($args)),*) -> _ =
+            $crate::macros::paste! { $crate::bindings:: [<__SCT__ $name >]; };
+        (fn_ptr)($($args),*)
+    }};
+}
+
+#[cfg(not(CONFIG_HAVE_STATIC_CALL))]
+#[doc(hidden)]
+#[macro_export]
+macro_rules! _static_call {
+    ($name:ident($($args:expr),* $(,)?)) => {{
+        let void_ptr_fn: *mut ::core::ffi::c_void = $crate::macros::paste! { $crate::bindings:: [<__SCK__ $name >]; }.func;
+
+        let fn_ptr: unsafe extern "C" fn($($crate::static_call::ty_underscore_for!($args)),*) -> _ = if true {
+            ::core::mem::transmute(void_ptr_fn)
+        } else {
+            // This is dead code, but it influences type inference on `fn_ptr` so that we transmute
+            // the function pointer to the right type.
+            $crate::macros::paste! { $crate::bindings:: [<__SCT__ $name >]; }
+        };
+
+        (fn_ptr)($($args),*)
+    }};
+}
+
+/// Statically call a global function.
+///
+/// # Safety
+///
+/// This macro will call the provided function. It is up to the caller to uphold the safety
+/// guarantees of the function.
+///
+/// # Examples
+///
+/// ```ignore
+/// fn call_static() {
+///     unsafe {
+///         static_call! { your_static_call() };
+///     }
+/// }
+/// ```
+#[macro_export]
+macro_rules! static_call {
+    // Forward to the real implementation. Separated like this so that we don't have to duplicate
+    // the documentation.
+    ($($args:tt)*) => { $crate::static_call::_static_call! { $($args)* } };
+}
+
+pub use {_static_call, static_call, ty_underscore_for};

rust/kernel/static_key.rs

@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: GPL-2.0
+
+// Copyright (C) 2024 Google LLC.
+
+//! Logic for static keys.
+
+use crate::bindings::*;
+
+#[doc(hidden)]
+#[macro_export]
+#[cfg(target_arch = "x86_64")]
+#[cfg(not(CONFIG_HAVE_RUST_ASM_GOTO))]
+macro_rules! _static_key_false {
+    ($key:path, $keytyp:ty, $field:ident) => {{
+        let mut output = 1u32;
+
+        core::arch::asm!(
+            r#"
+            1: .byte 0x0f,0x1f,0x44,0x00,0x00
+
+            .pushsection __jump_table,  "aw"
+            .balign 8
+            .long 1b - .
+            .long 3f - .
+            .quad {0} + {1} - .
+            .popsection
+
+            2: mov {2:e}, 0
+            3:
+            "#,
+            sym $key,
+            const ::core::mem::offset_of!($keytyp, $field),
+            inout(reg) output,
+        );
+
+        output != 0
+    }};
+}
+
+#[doc(hidden)]
+#[macro_export]
+#[cfg(target_arch = "x86_64")]
+#[cfg(CONFIG_HAVE_RUST_ASM_GOTO)]
+macro_rules! _static_key_false {
+    ($key:path, $keytyp:ty, $field:ident) => {'my_label: {
+        core::arch::asm!(
+            r#"
+            1: .byte 0x0f,0x1f,0x44,0x00,0x00
+
+            .pushsection __jump_table,  "aw"
+            .balign 8
+            .long 1b - .
+            .long {0} - .
+            .quad {1} + {2} - .
+            .popsection
+            "#,
+            label {
+                break 'my_label true;
+            },
+            sym $key,
+            const ::core::mem::offset_of!($keytyp, $field),
+        );
+
+        break 'my_label false;
+    }};
+}
+
+#[doc(hidden)]
+#[macro_export]
+#[cfg(target_arch = "aarch64")]
+#[cfg(not(CONFIG_HAVE_RUST_ASM_GOTO))]
+macro_rules! _static_key_false {
+    ($key:path, $keytyp:ty, $field:ident) => {{
+        let mut output = 1u32;
+
+        core::arch::asm!(
+            r#"
+            1: nop
+
+            .pushsection __jump_table,  "aw"
+            .align 3
+            .long 1b - ., 3f - .
+            .quad {0} + {1} - .
+            .popsection
+
+            2: mov {2:w}, 0
+            3:
+            "#,
+            sym $key,
+            const ::core::mem::offset_of!($keytyp, $field),
+            inout(reg) output
+        );
+
+        output != 0
+    }};
+}
+
+#[doc(hidden)]
+#[macro_export]
+#[cfg(target_arch = "aarch64")]
+#[cfg(CONFIG_HAVE_RUST_ASM_GOTO)]
+macro_rules! _static_key_false {
+    ($key:path, $keytyp:ty, $field:ident) => {'my_label: {
+        core::arch::asm!(
+            r#"
+            1: nop
+
+            .pushsection __jump_table,  "aw"
+            .align 3
+            .long 1b - ., {0} - .
+            .quad {1} + {2} - .
+            .popsection
+            "#,
+            label {
+                break 'my_label true;
+            },
+            sym $key,
+            const ::core::mem::offset_of!($keytyp, $field),
+        );
+
+        break 'my_label false;
+    }};
+}
+
+/// Branch based on a static key.
+///
+/// Takes three arguments:
+///
+/// * `key` - the path to the static variable containing the `static_key`.
+/// * `keytyp` - the type of `key`.
+/// * `field` - the name of the field of `key` that contains the `static_key`.
+#[macro_export]
+macro_rules! static_key_false {
+    // Forward to the real implementation. Separated like this so that we don't have to duplicate
+    // the documentation.
+    ($key:path, $keytyp:ty, $field:ident) => {{
+        // Assert that `$key` has type `$keytyp` and that `$key.$field` has type `static_key`.
+        //
+        // SAFETY: We know that `$key` is a static because otherwise the inline assembly will not
+        // compile. The raw pointers created in this block are in-bounds of `$key`.
+        static _TY_ASSERT: () = unsafe {
+            let key: *const $keytyp = ::core::ptr::addr_of!($key);
+            let _: *const $crate::bindings::static_key = ::core::ptr::addr_of!((*key).$field);
+        };
+
+        $crate::static_key::_static_key_false! { $key, $keytyp, $field }
+    }};
+}
+
+pub use {_static_key_false, static_key_false};

rust/kernel/tracepoint.rs

@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: GPL-2.0
+
+// Copyright (C) 2024 Google LLC.
+
+//! Logic for tracepoints.
+
+/// Declare the Rust entry point for a tracepoint.
+#[macro_export]
+macro_rules! declare_trace {
+    ($($(#[$attr:meta])* $pub:vis fn $name:ident($($argname:ident : $argtyp:ty),* $(,)?);)*) => {$(
+        $( #[$attr] )*
+        #[inline(always)]
+        $pub unsafe fn $name($($argname : $argtyp),*) {
+            #[cfg(CONFIG_TRACEPOINTS)]
+            {
+                use $crate::bindings::*;
+
+                // SAFETY: This macro only compiles if $name is a real tracepoint, and if it is a
+                // real tracepoint, then it is okay to query the static key.
+                let should_trace = unsafe {
+                    $crate::macros::paste! {
+                        $crate::static_key::static_key_false!(
+                            [< __tracepoint_ $name >],
+                            $crate::bindings::tracepoint,
+                            key
+                        )
+                    }
+                };
+
+                if should_trace {
+                    // TODO: cpu_online(raw_smp_processor_id())
+                    let cond = true;
+                    $crate::tracepoint::do_trace!($name($($argname : $argtyp),*), cond);
+                }
+            }
+
+            #[cfg(not(CONFIG_TRACEPOINTS))]
+            {
+                // If tracepoints are disabled, insert a trivial use of each argument
+                // to avoid unused argument warnings.
+                $( let _unused = $argname; )*
+            }
+        }
+    )*}
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! do_trace {
+    ($name:ident($($argname:ident : $argtyp:ty),* $(,)?), $cond:expr) => {{
+        if !$cond {
+            return;
+        }
+
+        // SAFETY: This call is balanced with the call below.
+        unsafe { $crate::bindings::preempt_disable_notrace() };
+
+        // SAFETY: This calls the tracepoint with the provided arguments. The caller of the Rust
+        // wrapper guarantees that this is okay.
+        #[cfg(CONFIG_HAVE_STATIC_CALL)]
+        unsafe {
+            let it_func_ptr: *mut $crate::bindings::tracepoint_func =
+                $crate::bindings::rcu_dereference_raw(
+                    ::core::ptr::addr_of!(
+                        $crate::macros::concat_idents!(__tracepoint_, $name).funcs
+                    )
+                );
+
+            if !it_func_ptr.is_null() {
+                let __data = (*it_func_ptr).data;
+                $crate::macros::paste! {
+                    $crate::static_call::static_call! {
+                        [< tp_func_ $name >] (__data, $($argname),*)
+                    };
+                }
+            }
+        }
+
+        // SAFETY: This calls the tracepoint with the provided arguments. The caller of the Rust
+        // wrapper guarantees that this is okay.
+        #[cfg(not(CONFIG_HAVE_STATIC_CALL))]
+        unsafe {
+            $crate::macros::concat_idents!(__traceiter_, $name)(
+                ::core::ptr::null_mut(),
+                $($argname),*
+            );
+        }
+
+        // SAFETY: This call is balanced with the call above.
+        unsafe { $crate::bindings::preempt_enable_notrace() };
+    }}
+}
+
+pub use {declare_trace, do_trace};

scripts/Makefile.build

@@ -263,7 +263,7 @@ $(obj)/%.lst: $(obj)/%.c FORCE
 # Compile Rust sources (.rs)
 # ---------------------------------------------------------------------------
 
-rust_allowed_features := new_uninit
+rust_allowed_features := asm_const,asm_goto,new_uninit
 
 # `--out-dir` is required to avoid temporaries being created by `rustc` in the
 # current working directory, which may be not accessible in the out-of-tree