Allow to run system benchmarks with more than one input

docs/howto/system_benchmarking.md

@@ -62,7 +62,7 @@ resources using selected cloud provider and use them for testing (e.g. observe a
 
 Sample `main.tf` definition:
 
-```
+```hcl
 variable "TEST_RUN_ID" {
   default = "detached"
 }
@@ -97,7 +97,7 @@ For example, if a `SQS queue` is configured in terraform and if the `queue_url`
 
 Sample Terraform definition
 
-```
+```hcl
 resource "aws_sqs_queue" "test" {
 
 }
@@ -109,7 +109,7 @@ output "queue_url"{
 
 Sample system test config
 
-``` yaml
+```yaml
 data_stream:
   vars:
     period: 5m
@@ -122,7 +122,7 @@ data_stream:
 
 For complex outputs from terraform you can use `{{TF_OUTPUT_root_key.nested_key}}`
 
-```
+```hcl
 output "root_key"{
   value = someoutput.nested_key_value
 }
@@ -143,7 +143,7 @@ output "root_key"{
   }
 }
 ```
-``` yaml
+```yaml
 data_stream:
   vars:
     queue_url: '{{TF_OUTPUT_root_key.nested_key}}'
@@ -303,9 +303,13 @@ Placeholders used in the `<scenario>.yml` must be enclosed in `{{` and `}}` deli
 
 Once the configuration is defined as described in the previous section, you are ready to run system benchmarks for a package.
 
-First you must deploy the Elastic Stack. 
+First, build your package (if you have local changes) and deploy the Elastic Stack.
+
+```shell
+# Build the package first if you've made local modifications.
+# This step is required before starting the Elastic stack when testing local changes.
+elastic-package build
 
-```
 elastic-package stack up -d
 ```
 
@@ -388,9 +392,23 @@ elastic-package benchmark system --benchmark logs-benchmark -v
 Done
 ```
 
+To re-run the system benchmark with local changes, follow these steps:
+1. Uninstall the package to ensure your latest local changes are used:
+```shell
+elastic-package uninstall
+```
+2. Build the package with your changes:
+```shell
+elastic-package build
+```
+3. Restart the Package Registry service to load the updated package:
+```shell
+elastic-package stack up -d --services package-registry
+```
+
 Finally, when you are done running the benchmark, bring down the Elastic Stack. 
 
-```
+```shell
 elastic-package stack down
 ```
 
@@ -421,6 +439,6 @@ The collected metrics include the following node stats: `nodes.*.breakers`, `nod
 
 Ingest pipelines metrics are only collected at the end since its own collection would affect the benchmark results.
 
-You can see a sample collected metric [here](./sample_metric.json)
+You can see a sample collected metric [here](./sample_metric.json).
 
 Additionally, if the `reindex-to-metricstore` flag is used, the data generated during the benchmark will be sent to the metricstore into an index called `bench-reindex-{datastream}-{testRunID}` for further analysis. The events will be enriched with metadata related to the benchmark run.

internal/benchrunner/runners/system/runner.go

@@ -42,6 +42,8 @@ const (
 
 	// BenchType defining system benchmark
 	BenchType benchrunner.Type = "system"
+
+	defaultNamespace = "ep"
 )
 
 type runner struct {
@@ -169,14 +171,25 @@ func (r *runner) setUp(ctx context.Context) error {
 		return fmt.Errorf("reading package manifest failed: %w", err)
 	}
 
-	policy, err := r.createBenchmarkPolicy(ctx, pkgManifest)
+	// Set default values for scenario fields from package manifest if not set
+	if r.scenario.Version == "" {
+		r.scenario.Version = pkgManifest.Version
+	}
+
+	if r.scenario.Package == "" {
+		r.scenario.Package = pkgManifest.Name
+	}
+
+	if r.scenario.PolicyTemplate == "" {
+		r.scenario.PolicyTemplate = pkgManifest.PolicyTemplates[0].Name
+	}
+
+	policy, err := r.createBenchmarkPolicy(ctx, pkgManifest, defaultNamespace)
 	if err != nil {
 		return err
 	}
 	r.benchPolicy = policy
 
-	// Delete old data
-	logger.Debug("deleting old data in data stream...")
 	dataStreamManifest, err := packages.ReadDataStreamManifest(
 		filepath.Join(
 			common.DataStreamPath(r.options.PackageRootPath, r.scenario.DataStream.Name),
@@ -210,6 +223,7 @@ func (r *runner) setUp(ctx context.Context) error {
 		return nil
 	}
 
+	logger.Debug("deleting old data in data stream...")
 	if err := r.deleteDataStreamDocs(ctx, r.runtimeDataStream); err != nil {
 		return fmt.Errorf("error deleting old data in data stream: %s: %w", r.runtimeDataStream, err)
 	}
@@ -367,14 +381,14 @@ func (r *runner) deleteDataStreamDocs(ctx context.Context, dataStream string) er
 	return nil
 }
 
-func (r *runner) createBenchmarkPolicy(ctx context.Context, pkgManifest *packages.PackageManifest) (*kibana.Policy, error) {
+func (r *runner) createBenchmarkPolicy(ctx context.Context, pkgManifest *packages.PackageManifest, namespace string) (*kibana.Policy, error) {
 	// Configure package (single data stream) via Ingest Manager APIs.
 	logger.Debug("creating benchmark policy...")
 	benchTime := time.Now().Format("20060102T15:04:05Z")
 	p := kibana.Policy{
 		Name:              fmt.Sprintf("ep-bench-%s-%s", r.options.BenchName, benchTime),
 		Description:       fmt.Sprintf("policy created by elastic-package for benchmark %s", r.options.BenchName),
-		Namespace:         "ep",
+		Namespace:         namespace,
 		MonitoringEnabled: []string{"logs", "metrics"},
 	}
 
@@ -388,67 +402,64 @@ func (r *runner) createBenchmarkPolicy(ctx context.Context, pkgManifest *package
 		return nil, err
 	}
 
-	packagePolicy, err := r.createPackagePolicy(ctx, pkgManifest, policy)
-	if err != nil {
-		return nil, err
-	}
-
 	r.deletePolicyHandler = func(ctx context.Context) error {
-		var merr multierror.Error
-
-		logger.Debug("deleting benchmark package policy...")
-		if err := r.options.KibanaClient.DeletePackagePolicy(ctx, *packagePolicy); err != nil {
-			merr = append(merr, fmt.Errorf("error cleaning up benchmark package policy: %w", err))
-		}
-
+		// Package policy deletion is handled when deleting this policy.
+		// Setting here the deletion handler ensures that if package policy creation fails,
+		// no orphaned package policies are left behind.
 		logger.Debug("deleting benchmark policy...")
 		if err := r.options.KibanaClient.DeletePolicy(ctx, policy.ID); err != nil {
-			merr = append(merr, fmt.Errorf("error cleaning up benchmark policy: %w", err))
-		}
-
-		if len(merr) > 0 {
-			return merr
+			return fmt.Errorf("error cleaning up benchmark policy: %w", err)
 		}
 
 		return nil
 	}
 
+	_, err = r.createPackagePolicy(ctx, pkgManifest, policy)
+	if err != nil {
+		return nil, err
+	}
+
 	return policy, nil
 }
 
 func (r *runner) createPackagePolicy(ctx context.Context, pkgManifest *packages.PackageManifest, p *kibana.Policy) (*kibana.PackagePolicy, error) {
 	logger.Debug("creating package policy...")
 
-	if r.scenario.Version == "" {
-		r.scenario.Version = pkgManifest.Version
-	}
-
-	if r.scenario.Package == "" {
-		r.scenario.Package = pkgManifest.Name
-	}
-
-	if r.scenario.PolicyTemplate == "" {
-		r.scenario.PolicyTemplate = pkgManifest.PolicyTemplates[0].Name
-	}
-
 	pp := kibana.PackagePolicy{
-		Namespace: "ep",
+		Namespace: p.Namespace,
 		PolicyID:  p.ID,
 		Force:     true,
 		Inputs: map[string]kibana.PackagePolicyInput{
 			fmt.Sprintf("%s-%s", r.scenario.PolicyTemplate, r.scenario.Input): {
 				Enabled: true,
 				Vars:    r.scenario.Vars,
 				Streams: map[string]kibana.PackagePolicyStream{
-					fmt.Sprintf("%s.%s", pkgManifest.Name, r.scenario.DataStream.Name): {
+					fmt.Sprintf("%s.%s", r.scenario.Package, r.scenario.DataStream.Name): {
 						Enabled: true,
 						Vars:    r.scenario.DataStream.Vars,
 					},
 				},
 			},
 		},
 	}
-	pp.Package.Name = pkgManifest.Name
+
+	// By default, all policy templates are enabled when creating a package policy.
+	// This could lead to errors if other policy templates have required variables.
+	// Therefore, all other policy templates and inputs must be disabled since here
+	// just the variables for the current input are set.
+	// NOTE: This data is retrieved from the local package manifest.
+	for _, policyTemplate := range pkgManifest.PolicyTemplates {
+		for _, input := range policyTemplate.Inputs {
+			if policyTemplate.Name == r.scenario.PolicyTemplate && input.Type == r.scenario.Input {
+				continue
+			}
+			pp.Inputs[fmt.Sprintf("%s-%s", policyTemplate.Name, input.Type)] = kibana.PackagePolicyInput{
+				Enabled: false,
+			}
+		}
+	}
+
+	pp.Package.Name = r.scenario.Package
 	pp.Package.Version = r.scenario.Version
 
 	policy, err := r.options.KibanaClient.CreatePackagePolicy(ctx, pp)

scripts/test-check-packages.sh

@@ -71,8 +71,17 @@ export SERVERLESS=${SERVERLESS:-"false"}
 run_system_benchmark() {
   local package_name="$1"
   local package_path="$2"
-  echo "--- Run system benchmarks for package ${package_name}"
-  elastic-package benchmark system -C "$package_path" --benchmark logs-benchmark -v --defer-cleanup 1s
+
+  local benchmark_file_path=""
+  local benchmark_filename=""
+  local benchmark_name=""
+
+  while IFS= read -r -d '' benchmark_file_path; do
+      benchmark_filename="$(basename "${benchmark_file_path}")"
+      benchmark_name="${benchmark_filename%.*}"
+      echo "--- Run system benchmarks for package ${package_name} - ${benchmark_name}"
+      elastic-package benchmark system -C "$package_path" --benchmark "${benchmark_name}" -v --defer-cleanup 1s
+  done < <(find "${package_path}/_dev/benchmark/system/" -maxdepth 1 -mindepth 1 -type f -name "*.yml" -print0)
 }
 
 run_serverless_tests() {
@@ -162,14 +171,21 @@ for d in test/packages/${PACKAGE_TEST_TYPE}/${PACKAGE_UNDER_TEST}/; do
   package_to_test=$(basename "${d}")
 
   if [ "${PACKAGE_TEST_TYPE}" == "benchmarks" ]; then
-    # FIXME: There are other packages in test/packages/benchmarks folder that are not tested like rally_benchmark
     case "${package_to_test}" in
       pipeline_benchmark|use_pipeline_tests)
         run_pipeline_benchmark "${package_to_test}" "$d"
         ;;
-      system_benchmark)
+      system_benchmark*)
         run_system_benchmark "${package_to_test}" "$d"
         ;;
+      rally_benchmark)
+        # FIXME: There are other packages in test/packages/benchmarks folder that are not tested like rally_benchmark
+        # rally benchmarks require to install esrally tool in the host. 
+        echo "Skipping rally_benchmark tests. esrally tool needs to be installed in the system."
+        ;;
+      *)
+        echo "Unknown benchmark package: ${package_to_test}"
+        exit 1
     esac
     continue
   fi

test/packages/benchmarks/system_benchmark_sentinel_one/_dev/benchmark/system/alert-benchmark.yml

@@ -0,0 +1,26 @@
+---
+description: Benchmark 100000 alert events ingested
+input: httpjson
+# This field was not available, fixed in https://github.com/elastic/package-spec/pull/1004
+# policy_template: sentinel_one
+vars:
+  url: "http://svc-sentinel_one:8080/"
+  api_token: xxxx
+  enable_request_tracer: true
+data_stream:
+  name: alert
+  vars:
+    preserve_original_event: true
+warmup_time_period: 2s
+corpora:
+  input_service:
+    name: sentinel_one
+  generator:
+    total_events: 100000
+    template:
+      path: ./alert-benchmark/template.ndjson
+      type: gotext
+    config:
+      path: ./alert-benchmark/config.yml
+    fields:
+      path: ./alert-benchmark/fields.yml