Fix benchmark variances (#246)

benchmark/main.cpp

@@ -39,11 +39,21 @@ using test_result = ddwaf::benchmark::runner::test_result;
 using generator_type = benchmark::object_generator::generator_type;
 
 std::map<std::string, benchmark::object_generator::settings> default_tests = {
-    {"random.any", {.type = generator_type::random}},
-    {"random.long_strings", {.string_length = {512, 1024}, .type = generator_type::random}},
-    {"random.deep_containers", {.container_depth = {5, 10}, .type = generator_type::random}},
+    {"random.1x1.128", {.string_length = 128, .type = generator_type::random}},
+    {"random.1x1.512", {.string_length = 512, .type = generator_type::random}},
+    {"random.1x1.1024", {.string_length = 1024, .type = generator_type::random}},
+    {"random.1x1.4096", {.string_length = 1024, .type = generator_type::random}},
+
+    {"random.1x16.128", {.container_size = 16, .type = generator_type::random}},
+    {"random.1x64.128", {.container_size = 64, .type = generator_type::random}},
+    {"random.1x128.128", {.container_size = 128, .type = generator_type::random}},
+    {"random.1x256.128", {.container_size = 256, .type = generator_type::random}},
+
+    {"random.5x1.128", {.container_depth = 5, .type = generator_type::random}},
+    {"random.10x1.128", {.container_depth = 10, .type = generator_type::random}},
+    {"random.20x1.128", {.container_depth = 20, .type = generator_type::random}},
+
     {"valid", {.type = generator_type::valid}},
-    {"mixed", {.type = generator_type::mixed}},
 };
 
 // NOLINTNEXTLINE(bugprone-easily-swappable-parameters)

benchmark/object_generator.cpp

@@ -19,22 +19,20 @@ namespace {
 using settings = object_generator::settings;
 
 void generate_object(
-    ddwaf_object &o, const settings &l, std::size_t &max_elements, std::size_t depth = 0);
+    ddwaf_object &o, const settings &l, std::size_t depth = 0);
 
 char *generate_random_string(const settings &l, std::size_t *length)
 {
     static const auto &charset = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
                                  "`¬|\\|,<.>/?;:'@#~[{]}=+-_)(*&^%$£\"!";
 
-    std::size_t numchars = l.string_length.min + random::get() % l.string_length.range();
-
     // NOLINTNEXTLINE
-    char *str = (char *)malloc(numchars + 1);
-    for (std::size_t i = 0; i < numchars; i++) {
+    char *str = (char *)malloc(l.string_length + 1);
+    for (std::size_t i = 0; i < l.string_length; i++) {
         str[i] = charset[random::get() % (sizeof(charset) - 2)];
     }
-    str[numchars] = '\0';
-    *length = numchars;
+    str[l.string_length] = '\0';
+    *length = l.string_length;
 
     return str;
 }
@@ -48,60 +46,48 @@ void generate_string_object(ddwaf_object &o, const settings &l)
 
 // NOLINTNEXTLINE(misc-no-recursion)
 void generate_map_object(
-    ddwaf_object &o, const settings &l, std::size_t &max_elements, std::size_t depth)
+    ddwaf_object &o, const settings &l, std::size_t depth)
 
 {
     ddwaf_object_map(&o);
 
-    std::size_t n = l.container_size.min + random::get() % l.container_size.range();
-
-    n = std::min(n, max_elements);
-
-    for (std::size_t i = 0; i < n; i++) {
+    for (std::size_t i = 0; i < l.container_size; i++) {
         std::size_t length = 0;
         char *key = generate_random_string(l, &length);
 
         ddwaf_object value;
-        generate_object(value, l, max_elements, depth + 1);
+        generate_object(value, l, depth + 1);
         ddwaf_object_map_addl_nc(&o, key, length, &value);
     }
 }
 
 // NOLINTNEXTLINE(misc-no-recursion)
 void generate_array_object(
-    ddwaf_object &o, const settings &l, std::size_t &max_elements, std::size_t depth)
+    ddwaf_object &o, const settings &l, std::size_t depth)
 {
     ddwaf_object_array(&o);
 
-    std::size_t n = l.container_size.min + random::get() % l.container_size.range();
-
-    n = std::min(n, max_elements);
-
-    for (std::size_t i = 0; i < n; i++) {
+    for (std::size_t i = 0; i < l.container_size; i++) {
         ddwaf_object value;
-        generate_object(value, l, max_elements, depth + 1);
+        generate_object(value, l, depth + 1);
         ddwaf_object_array_add(&o, &value);
     }
 }
 
 // NOLINTNEXTLINE(misc-no-recursion)
 void generate_object(
-    ddwaf_object &o, const settings &l, std::size_t &max_elements, std::size_t depth)
+    ddwaf_object &o, const settings &l, std::size_t depth)
 {
-    if (max_elements > 0) {
-        max_elements--;
-    }
-
-    if (depth >= l.container_depth.max) {
+    if (depth >= l.container_depth) {
         generate_string_object(o, l);
         return;
     }
 
-    if (depth < l.container_depth.min) {
+    if (depth < l.container_depth) {
         if (random::get_bool()) {
-            generate_map_object(o, l, max_elements, depth);
+            generate_map_object(o, l, depth);
         } else {
-            generate_array_object(o, l, max_elements, depth);
+            generate_array_object(o, l, depth);
         }
         return;
     }
@@ -112,10 +98,10 @@ void generate_object(
         generate_string_object(o, l);
         break;
     case 1: // Map
-        generate_map_object(o, l, max_elements, depth);
+        generate_map_object(o, l, depth);
         break;
     case 2: // Array
-        generate_array_object(o, l, max_elements, depth);
+        generate_array_object(o, l, depth);
         break;
     }
 }
@@ -133,12 +119,10 @@ object_generator::object_generator(
     }
 
     for (auto it = test_vectors.begin(); it != test_vectors.end(); ++it) {
-        auto first_entry = it->begin();
-        auto key = first_entry->first.as<std::string>();
-
+        auto key = it->first.as<std::string>();
         auto &current_values = addresses_[key];
 
-        auto array = first_entry->second;
+        auto array = it->second;
         for (auto value_it = array.begin(); value_it != array.end(); ++value_it) {
             auto vector = value_it->as<ddwaf_object>();
             objects_.push_back(vector);
@@ -164,31 +148,26 @@ std::vector<ddwaf_object> object_generator::operator()(
         for (const auto &[addr, valid_values] : addresses_) {
             ddwaf_object value;
 
-            std::size_t max_elements = l.elements.max / addresses_.size();
-            if (max_elements == 0) {
-                max_elements = 1;
-            }
-
             generator_type type = l.type;
-            if (valid_values.empty()) {
-                type = generator_type::random;
-            } else {
-                if (type == generator_type::mixed) {
-                    type = static_cast<generator_type>(random::get() % 2);
+            if (type == generator_type::valid) {
+                if (valid_values.empty()) {
+                    continue;
                 }
-            }
 
-            if (type == generator_type::random) {
-                generate_object(value, l, max_elements);
-            } else {
                 std::size_t index = random::get() % valid_values.size();
                 value = utils::object_dup(valid_values[index]);
+            } else {
+                generate_object(value, l);
             }
 
             ddwaf_object_map_add(&root, addr.data(), &value);
         }
     }
 
+    if (output.empty() && l.type == generator_type::valid) {
+        throw std::runtime_error("No valid values available");
+    }
+
     return output;
 }
 

benchmark/object_generator.hpp

@@ -23,24 +23,16 @@ class object_generator {
     enum class generator_type : unsigned {
         valid = 1,
         random = 2,
-        mixed = valid | random,
     };
 
     struct settings {
-        struct limit_type {
-            std::size_t min, max;
-            [[nodiscard]] std::size_t range() const { return max - min; }
-        };
+        static constexpr std::size_t default_depth = 1;
+        static constexpr std::size_t default_size = 1;
+        static constexpr std::size_t default_length = 128;
 
-        static constexpr std::size_t max_depth = 20;
-        static constexpr std::size_t max_size = 256;
-        static constexpr std::size_t max_length = 1024;
-        static constexpr std::size_t max_elements = 512;
-
-        limit_type container_depth{0, max_depth};
-        limit_type container_size{0, max_size};
-        limit_type string_length{0, max_length};
-        limit_type elements{0, max_elements};
+        std::size_t container_depth = default_depth;
+        std::size_t container_size = default_size;
+        std::size_t string_length = default_length;
         generator_type type{generator_type::random};
     };
 

benchmark/scenarios/float_equals_matcher.json

@@ -27,7 +27,7 @@
         ]
     },
     "vectors": {
-        "waf.context.processor.extract-schema": [
+        "waf.context.duration": [
             5.23
         ]
     }

benchmark/scenarios/ruleset_1.10.0.json

@@ -10579,6 +10579,12 @@
             {
                 "xml": "<?xml version=\"1.0\" encoding=\"utf-8\"?><!DOCTYPE attack [<!ENTITY a SYSTEM \"file:///etc/passwd\">]><attack>&a;</attack>"
             }
+        ],
+        "server.response.status": [ "404" ],
+        "waf.context.processor": [
+            {
+                "extract-schema": true
+            }
         ]
     }
 }

benchmark/scenarios/signed_equals_matcher.json

@@ -27,11 +27,8 @@
         ]
     },
     "vectors": {
-        "waf.context.processor.extract-schema": [
+        "waf.context.events.length": [
             -2123
         ]
     }
 }
-
-
-

benchmark/scenarios/unsigned_equals_matcher.json

@@ -27,7 +27,7 @@
         ]
     },
     "vectors": {
-        "waf.context.processor.extract-schema": [
+        "waf.context.events.length": [
             2123
         ]
     }