cleanup: simd runtime detection (#132)

Author: AaronO

src/simd/avx2.rs

@@ -1,29 +1,24 @@
 use crate::iter::Bytes;
 
-pub enum Scan {
-    /// Returned when an implementation finds a noteworthy token.
-    Found,
-    /// Returned when an implementation couldn't keep running because the input was too short.
-    TooShort,
-}
-
 #[cfg(target_arch = "x86")]
-unsafe fn parse_uri_batch_32(_: &[u8]) -> usize {
+pub unsafe fn match_uri_vectored(_: &mut Bytes) {
     unreachable!("AVX2 detection should be disabled for x86");
 }
 
+#[inline]
 #[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avx2")]
-pub unsafe fn parse_uri_batch_32(bytes: &mut Bytes) -> Scan {
+#[target_feature(enable = "avx2", enable = "sse4.2")]
+pub unsafe fn match_uri_vectored(bytes: &mut Bytes) {
     while bytes.as_ref().len() >= 32 {
         let advance = match_url_char_32_avx(bytes.as_ref());
         bytes.advance(advance);
 
         if advance != 32 {
-            return Scan::Found;
+            return;
         }
     }
-    Scan::TooShort
+    // do both, since avx2 only works when bytes.len() >= 32
+    super::sse42::match_uri_vectored(bytes)
 }
 
 #[inline(always)]
@@ -69,22 +64,23 @@ unsafe fn match_url_char_32_avx(buf: &[u8]) -> usize {
 }
 
 #[cfg(target_arch = "x86")]
-unsafe fn match_header_value_batch_32(_: &[u8]) -> usize {
+pub unsafe fn match_header_value_vectored(_: &mut Bytes) {
     unreachable!("AVX2 detection should be disabled for x86");
 }
 
 #[cfg(target_arch = "x86_64")]
-#[target_feature(enable = "avx2")]
-pub unsafe fn match_header_value_batch_32(bytes: &mut Bytes) -> Scan {
+#[target_feature(enable = "avx2", enable = "sse4.2")]
+pub unsafe fn match_header_value_vectored(bytes: &mut Bytes) {
     while bytes.as_ref().len() >= 32 {
         let advance = match_header_value_char_32_avx(bytes.as_ref());
         bytes.advance(advance);
 
         if advance != 32 {
-            return Scan::Found;
+            return;
         }
     }
-    Scan::TooShort
+    // do both, since avx2 only works when bytes.len() >= 32
+    super::sse42::match_header_value_vectored(bytes)
 }
 
 #[inline(always)]
@@ -120,17 +116,16 @@ unsafe fn match_header_value_char_32_avx(buf: &[u8]) -> usize {
 
 #[test]
 fn avx2_code_matches_uri_chars_table() {
-    match super::detect() {
-        super::AVX_2 | super::AVX_2_AND_SSE_42 => {},
-        _ => return,
+    if !is_x86_feature_detected!("avx2") {
+        return;
     }
 
     unsafe {
-        assert!(byte_is_allowed(b'_', parse_uri_batch_32));
+        assert!(byte_is_allowed(b'_', match_uri_vectored));
 
         for (b, allowed) in crate::URI_MAP.iter().cloned().enumerate() {
             assert_eq!(
-                byte_is_allowed(b as u8, parse_uri_batch_32), allowed,
+                byte_is_allowed(b as u8, match_uri_vectored), allowed,
                 "byte_is_allowed({:?}) should be {:?}", b, allowed,
             );
         }
@@ -139,25 +134,24 @@ fn avx2_code_matches_uri_chars_table() {
 
 #[test]
 fn avx2_code_matches_header_value_chars_table() {
-    match super::detect() {
-        super::AVX_2 | super::AVX_2_AND_SSE_42 => {},
-        _ => return,
+    if !is_x86_feature_detected!("avx2") {
+        return;
     }
 
     unsafe {
-        assert!(byte_is_allowed(b'_', match_header_value_batch_32));
+        assert!(byte_is_allowed(b'_', match_header_value_vectored));
 
         for (b, allowed) in crate::HEADER_VALUE_MAP.iter().cloned().enumerate() {
             assert_eq!(
-                byte_is_allowed(b as u8, match_header_value_batch_32), allowed,
+                byte_is_allowed(b as u8, match_header_value_vectored), allowed,
                 "byte_is_allowed({:?}) should be {:?}", b, allowed,
             );
         }
     }
 }
 
 #[cfg(test)]
-unsafe fn byte_is_allowed(byte: u8, f: unsafe fn(bytes: &mut Bytes<'_>) -> Scan) -> bool {
+unsafe fn byte_is_allowed(byte: u8, f: unsafe fn(bytes: &mut Bytes<'_>)) -> bool {
     let slice = [
         b'_', b'_', b'_', b'_',
         b'_', b'_', b'_', b'_',

src/simd/mod.rs

@@ -38,45 +38,6 @@ mod sse42;
 ))]
 mod avx2;
 
-#[cfg(all(
-    httparse_simd,
-    any(
-        target_arch = "x86",
-        target_arch = "x86_64",
-    ),
-))]
-pub const SSE_42: usize = 1;
-#[cfg(all(
-    httparse_simd,
-    any(not(httparse_simd_target_feature_sse42), httparse_simd_target_feature_avx2),
-    any(
-        target_arch = "x86",
-        target_arch = "x86_64",
-    ),
-))]
-pub const AVX_2: usize = 2;
-#[cfg(all(
-    httparse_simd,
-    any(
-        not(httparse_simd_target_feature_sse42),
-        httparse_simd_target_feature_avx2,
-        test,
-    ),
-    any(
-        target_arch = "x86",
-        target_arch = "x86_64",
-    ),
-))]
-pub const AVX_2_AND_SSE_42: usize = 3;
-
-#[cfg(all(
-    httparse_simd,
-    any(
-        target_arch = "x86",
-        target_arch = "x86_64",
-    ),
-))]
-const NONE: usize = std::usize::MAX;
 #[cfg(all(
     httparse_simd,
     not(any(
@@ -88,77 +49,7 @@ const NONE: usize = std::usize::MAX;
         target_arch = "x86_64",
     ),
 ))]
-mod runtime {
-    //! Runtime detection of simd features. Used when the build script
-    //! doesn't notice any target features at build time.
-    //!
-    //! While `is_x86_feature_detected!` has it's own caching built-in,
-    //! at least in 1.27.0, the functions don't inline, leaving using it
-    //! actually *slower* than just using the scalar fallback.
-
-    use core::sync::atomic::{AtomicUsize, Ordering};
-
-    static FEATURE: AtomicUsize = AtomicUsize::new(0);
-
-    const INIT: usize = 0;
-
-    pub fn detect() -> usize {
-        let feat = FEATURE.load(Ordering::Relaxed);
-        if feat == INIT {
-            if cfg!(target_arch = "x86_64") && is_x86_feature_detected!("avx2") {
-                if is_x86_feature_detected!("sse4.2") {
-                    FEATURE.store(super::AVX_2_AND_SSE_42, Ordering::Relaxed);
-                    return super::AVX_2_AND_SSE_42;
-                } else {
-                    FEATURE.store(super::AVX_2, Ordering::Relaxed);
-                    return super::AVX_2;
-                }
-            } else if is_x86_feature_detected!("sse4.2") {
-                FEATURE.store(super::SSE_42, Ordering::Relaxed);
-                return super::SSE_42;
-            } else {
-                FEATURE.store(super::NONE, Ordering::Relaxed);
-            }
-        }
-        feat
-    }
-
-    pub fn match_uri_vectored(bytes: &mut crate::iter::Bytes) {
-        unsafe {
-            match detect() {
-                super::SSE_42 => super::sse42::parse_uri_batch_16(bytes),
-                super::AVX_2 => { super::avx2::parse_uri_batch_32(bytes); },
-                super::AVX_2_AND_SSE_42 => {
-                    if let super::avx2::Scan::Found = super::avx2::parse_uri_batch_32(bytes) {
-                        return;
-                    }
-                    super::sse42::parse_uri_batch_16(bytes)
-                },
-                _ => ()
-            }
-        }
-
-        // else do nothing
-    }
-
-    pub fn match_header_value_vectored(bytes: &mut crate::iter::Bytes) {
-        unsafe {
-            match detect() {
-                super::SSE_42 => super::sse42::match_header_value_batch_16(bytes),
-                super::AVX_2 => { super::avx2::match_header_value_batch_32(bytes); },
-                super::AVX_2_AND_SSE_42 => {
-                    if let super::avx2::Scan::Found = super::avx2::match_header_value_batch_32(bytes) {
-                        return;
-                    }
-                    super::sse42::match_header_value_batch_16(bytes)
-                },
-                _ => ()
-            }
-        }
-
-        // else do nothing
-    }
-}
+mod runtime;
 
 #[cfg(all(
     httparse_simd,
@@ -183,32 +74,16 @@ pub use self::runtime::*;
     ),
 ))]
 mod sse42_compile_time {
-    pub fn match_uri_vectored(bytes: &mut crate::iter::Bytes) {
-        if detect() == super::SSE_42 {
-            unsafe {
-                super::sse42::parse_uri_batch_16(bytes);
-            }
-        }
-
-        // else do nothing
+    #[inline(always)]
+    pub fn match_uri_vectored(b: &mut crate::iter::Bytes<'_>) {
+        // SAFETY: calls are guarded by a compile time feature check
+        unsafe { crate::simd::sse42::match_uri_vectored(b) }
     }
-
-    pub fn match_header_value_vectored(bytes: &mut crate::iter::Bytes) {
-        if detect() == super::SSE_42 {
-            unsafe {
-                super::sse42::match_header_value_batch_16(bytes);
-            }
-        }
-
-        // else do nothing
-    }
-
-    pub fn detect() -> usize {
-        if is_x86_feature_detected!("sse4.2") {
-            super::SSE_42
-        } else {
-            super::NONE
-        }
+    
+    #[inline(always)]
+    pub fn match_header_value_vectored(b: &mut crate::iter::Bytes<'_>) {
+        // SAFETY: calls are guarded by a compile time feature check
+        unsafe { crate::simd::sse42::match_header_value_vectored(b) }
     }
 }
 
@@ -232,51 +107,16 @@ pub use self::sse42_compile_time::*;
     ),
 ))]
 mod avx2_compile_time {
-    pub fn match_uri_vectored(bytes: &mut crate::iter::Bytes) {
-        // do both, since avx2 only works when bytes.len() >= 32
-        if detect() == super::AVX_2_AND_SSE_42 {
-            unsafe {
-                super::avx2::parse_uri_batch_32(bytes);
-            }
-
-        } 
-        if detect() == super::SSE_42 {
-            unsafe {
-                super::sse42::parse_uri_batch_16(bytes);
-            }
-        }
-
-        // else do nothing
+    #[inline(always)]
+    pub fn match_uri_vectored(b: &mut crate::iter::Bytes<'_>) {
+        // SAFETY: calls are guarded by a compile time feature check
+        unsafe { crate::simd::avx2::match_uri_vectored(b) }
     }
-
-    pub fn match_header_value_vectored(bytes: &mut crate::iter::Bytes) {
-        // do both, since avx2 only works when bytes.len() >= 32
-        if detect() == super::AVX_2_AND_SSE_42 {
-            let scanned = unsafe {
-                super::avx2::match_header_value_batch_32(bytes)
-            };
-
-            if let super::avx2::Scan::Found = scanned {
-                return;
-            }
-        }
-        if detect() == super::SSE_42 {
-            unsafe {
-                super::sse42::match_header_value_batch_16(bytes);
-            }
-        }
-
-        // else do nothing
-    }
-
-    pub fn detect() -> usize {
-        if cfg!(target_arch = "x86_64") && is_x86_feature_detected!("avx2") {
-            super::AVX_2_AND_SSE_42
-        } else if is_x86_feature_detected!("sse4.2") {
-            super::SSE_42
-        } else {
-            super::NONE
-        }
+    
+    #[inline(always)]
+    pub fn match_header_value_vectored(b: &mut crate::iter::Bytes<'_>) {
+        // SAFETY: calls are guarded by a compile time feature check
+        unsafe { crate::simd::avx2::match_header_value_vectored(b) }
     }
 }