migrate benchmarks to k6

Signed-off-by: Sahil Yeole <[email protected]>

Author: beelchester

.github/workflows/bench.yml

@@ -25,14 +25,19 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
 
-      - name: Build devcontainer and run benchmarks
+      - name: Build devcontainer
         uses: devcontainers/[email protected]
         with:
           imageName: graphql-benchmarks
           push: never
-          runCmd: |
-            bash ./setup.sh
-            bash ./run_benchmarks.sh
+
+      - name: Setup k6
+        uses: grafana/setup-k6-action@v1
+
+      - name: Run benchmarks
+        run: |
+          bash ./setup.sh
+          bash ./run_benchmarks.sh
 
       - name: Print benchmark results
         run: cat ./results.md

k6/bench.js

@@ -0,0 +1,52 @@
+import http from 'k6/http';
+import { check } from 'k6';
+
+const whichBenchmark = Number(__ENV.BENCHMARK);
+
+export const options = {
+  scenarios: {
+    posts: {
+      executor: 'constant-vus',
+      duration: whichBenchmark === 2 ? '30s' : '10s',
+      gracefulStop: '0s',
+      vus: 100,
+    }
+  },
+  cloud: {
+    name: __ENV.TEST_NAME + '-' + whichBenchmark,
+  },
+};
+
+const url = 'http://localhost:8000/graphql';
+const params = {
+  headers: {
+    'Connection': 'keep-alive',
+    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
+    'Content-Type': 'application/json',
+  },
+};
+
+export default function() {
+  const payload = JSON.stringify({
+    operationName: null,
+    variables: {},
+    query: whichBenchmark === 2 ? '{posts{id,userId,title,user{id,name,email}}}' : '{posts{title}}',
+  });
+
+  const res = http.post(url, payload, params);
+  check(res, {
+    'status is 200': (r) => r.status === 200,
+  });
+}
+
+export function handleSummary(data) {
+  const med_request_count = data.metrics.http_reqs.values.count;
+  const avg_latency = data.metrics.http_req_duration.values.avg;
+  const trimmed_avg_latency = Math.round(avg_latency * 100) / 100;
+  const request_count_message = `Requests/sec: ${med_request_count}\n`;
+  const latency_message = `Latency: ${trimmed_avg_latency} ms\n`;
+
+  return {
+    stdout: request_count_message + latency_message,
+  };
+}

k6/bench.sh

@@ -0,0 +1,5 @@
+test_name=$1
+benchmark=$2
+
+# k6 run k6/bench.js --quiet --out cloud --env TEST_NAME=$test_name --env BENCHMARK=$benchmark
+k6 run k6/bench.js --quiet --env TEST_NAME=$test_name --env BENCHMARK=$benchmark

k6/warmup.sh

@@ -0,0 +1 @@
+k6 run k6/bench.js --quiet

run_benchmarks.sh

@@ -20,15 +20,17 @@ sh nginx/run.sh
 function runBenchmark() {
     killServerOnPort 8000
     sleep 5
-    local serviceScript="$1"
+    local service="$1"
     local benchmarks=(1 2)
+    local serviceScript="graphql/${service}/run.sh"
 
     bash "$serviceScript" &   # Run in daemon mode
     sleep 15   # Give some time for the service to start up
 
     for bench in "${benchmarks[@]}"; do
-        local benchmarkScript="wrk/bench${bench}.sh"
-        
+        local benchmarkScript="tests/test.sh"
+        local warmupScript="tests/warmup.sh"
+
         # Replace / with _
         local sanitizedServiceScriptName=$(echo "$serviceScript" | tr '/' '_')
 
@@ -37,18 +39,18 @@ function runBenchmark() {
         bash "test_query${bench}.sh"
 
         # Warmup run
-        bash "$benchmarkScript" > /dev/null
+        bash "$warmupScript" > /dev/null
         sleep 1   # Give some time for apps to finish in-flight requests from warmup
-        bash "$benchmarkScript" > /dev/null
+        bash "$warmupScript" > /dev/null
         sleep 1
-        bash "$benchmarkScript" > /dev/null
+        bash "$warmupScript" > /dev/null
         sleep 1
 
 
         # 3 benchmark runs
         for resultFile in "${resultFiles[@]}"; do
             echo "Running benchmark $bench for $serviceScript"
-            bash "$benchmarkScript" > "bench${bench}_${resultFile}"
+            bash "$benchmarkScript" "$service" "$bench" > "bench${bench}_${resultFile}"
             if [ "$bench" == "1" ]; then
                 bench1Results+=("bench1_${resultFile}")
             else
@@ -61,7 +63,7 @@ function runBenchmark() {
 rm "results.md"
 
 for service in "apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "async_graphql"; do
-    runBenchmark "graphql/${service}/run.sh"
+    runBenchmark "$service"
     if [ "$service" == "apollo_server" ]; then
         cd graphql/apollo_server/
         npm stop