Merge remote-tracking branch 'upstream/main' into k6_migration

Author: beelchester

.github/workflows/bench.yml

@@ -1,17 +1,14 @@
 name: "Run benchmark"
-
 on:
   pull_request_target:
     types: [assigned, opened, synchronize, reopened, edited]
   push:
     branches:
       - main
-
 permissions:
   contents: write
   pull-requests: write
   issues: write
-
 jobs:
   build:
     runs-on: benchmarking-runner
@@ -22,6 +19,19 @@ jobs:
       INFLUXDB_TOKEN: ${{ secrets.INFLUXDB_TOKEN }}
       INFLUXDB_ORG: ${{ secrets.INFLUXDB_ORG }}
       INFLUXDB_URL: ${{ secrets.INFLUXDB_URL }}
+    strategy:
+      matrix:
+        service:
+          [
+            apollo_server,
+            caliban,
+            netflix_dgs,
+            gqlgen,
+            tailcall,
+            async_graphql,
+            hasura,
+            graphql_jit,
+          ]
     steps:
       - name: Checkout (GitHub)
         uses: actions/checkout@v4
@@ -53,9 +63,37 @@ jobs:
 
       - name: Run benchmarks
         run: |
-          bash ./setup.sh
-          bash ./run_benchmarks.sh
+          bash ./graphql/${{ matrix.service }}/setup.sh
+          bash run_benchmarks.sh ${{ matrix.service }}
+      - name: List benchmark files
+        run: |
+          ls -la bench*.txt || echo "No matching files found"
 
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v3
+        with:
+          name: benchmark-results
+          path: bench*.txt
+
+  analyze:
+    needs: build
+    runs-on: benchmarking-runner
+    steps:
+      - name: Checkout (GitHub)
+        uses: actions/checkout@v4
+
+      - name: Download all benchmark results
+        uses: actions/download-artifact@v3
+        with:
+          name: benchmark-results
+          path: .
+
+      - name: List downloaded artifacts
+        run: ls -la bench*.txt || echo "No matching files found"
+
+      - name: Analyze results
+        run: |
+          bash run_analyze_script.sh
       - name: Print benchmark results
         run: cat ./results.md
 
@@ -73,4 +111,4 @@ jobs:
         with:
           branch: main
           commit_author: Author <[email protected]>
-          commit_message: "Update performance results in README.md"
+          commit_message: "[ci skip] update performance results in README.md"

README.md

@@ -58,32 +58,32 @@ Get started with the benchmarks:
 | Query | Server | Requests/sec | Latency (ms) | Relative |
 |-------:|--------:|--------------:|--------------:|---------:|
 | 1 | `{ posts { id userId title user { id name email }}}` |
-|| [Tailcall] | `29,496.60` | `3.38` | `196.74x` |
-|| [async-graphql] | `1,805.45` | `55.70` | `12.04x` |
-|| [Caliban] | `1,547.91` | `64.33` | `10.32x` |
-|| [GraphQL JIT] | `1,335.26` | `74.60` | `8.91x` |
-|| [Gqlgen] | `772.80` | `128.41` | `5.15x` |
-|| [Netflix DGS] | `358.06` | `177.42` | `2.39x` |
-|| [Apollo GraphQL] | `268.25` | `366.16` | `1.79x` |
-|| [Hasura] | `149.93` | `549.07` | `1.00x` |
+|| [Tailcall] | `29,173.60` | `3.41` | `211.81x` |
+|| [async-graphql] | `2,019.59` | `49.49` | `14.66x` |
+|| [Caliban] | `1,744.80` | `57.28` | `12.67x` |
+|| [GraphQL JIT] | `1,342.93` | `74.16` | `9.75x` |
+|| [Gqlgen] | `797.87` | `124.41` | `5.79x` |
+|| [Netflix DGS] | `366.18` | `181.49` | `2.66x` |
+|| [Apollo GraphQL] | `263.39` | `372.49` | `1.91x` |
+|| [Hasura] | `137.74` | `587.80` | `1.00x` |
 | 2 | `{ posts { title }}` |
-|| [Tailcall] | `58,939.60` | `1.69` | `67.77x` |
-|| [Caliban] | `9,238.76` | `11.16` | `10.62x` |
-|| [async-graphql] | `9,138.76` | `11.34` | `10.51x` |
-|| [Gqlgen] | `2,136.19` | `48.48` | `2.46x` |
-|| [Apollo GraphQL] | `1,775.87` | `56.26` | `2.04x` |
-|| [Netflix DGS] | `1,593.80` | `69.82` | `1.83x` |
-|| [GraphQL JIT] | `1,399.68` | `71.35` | `1.61x` |
-|| [Hasura] | `869.69` | `114.81` | `1.00x` |
+|| [Tailcall] | `58,485.30` | `1.70` | `67.37x` |
+|| [Caliban] | `9,751.71` | `10.62` | `11.23x` |
+|| [async-graphql] | `9,444.13` | `10.73` | `10.88x` |
+|| [Gqlgen] | `2,181.26` | `47.48` | `2.51x` |
+|| [Apollo GraphQL] | `1,760.69` | `56.72` | `2.03x` |
+|| [Netflix DGS] | `1,598.09` | `70.06` | `1.84x` |
+|| [GraphQL JIT] | `1,392.45` | `71.71` | `1.60x` |
+|| [Hasura] | `868.17` | `114.99` | `1.00x` |
 | 3 | `{ greet }` |
-|| [Caliban] | `67,637.00` | `1.07` | `25.83x` |
-|| [Tailcall] | `59,557.80` | `1.69` | `22.75x` |
-|| [Gqlgen] | `47,525.20` | `5.21` | `18.15x` |
-|| [async-graphql] | `47,150.60` | `2.21` | `18.01x` |
-|| [Netflix DGS] | `8,261.44` | `15.02` | `3.16x` |
-|| [Apollo GraphQL] | `8,186.12` | `12.54` | `3.13x` |
-|| [GraphQL JIT] | `5,273.45` | `18.93` | `2.01x` |
-|| [Hasura] | `2,618.18` | `38.13` | `1.00x` |
+|| [Caliban] | `67,512.70` | `1.07` | `26.49x` |
+|| [Tailcall] | `59,015.70` | `1.71` | `23.16x` |
+|| [Gqlgen] | `47,378.10` | `5.25` | `18.59x` |
+|| [async-graphql] | `47,361.00` | `2.25` | `18.58x` |
+|| [Netflix DGS] | `8,297.54` | `14.39` | `3.26x` |
+|| [Apollo GraphQL] | `8,128.40` | `12.43` | `3.19x` |
+|| [GraphQL JIT] | `5,241.04` | `19.05` | `2.06x` |
+|| [Hasura] | `2,548.54` | `39.16` | `1.00x` |
 
 <!-- PERFORMANCE_RESULTS_END -->
 

analyze.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Install gnuplot
+sudo apt-get update && sudo apt-get install -y gnuplot
+
 function extractMetric() {
   local file="$1"
   local metric="$2"
@@ -128,4 +131,4 @@ fi
 # Delete the result TXT files
 for file in "${resultFiles[@]}"; do
   rm "$file"
-done
+done

results.md

@@ -3,31 +3,31 @@
 | Query | Server | Requests/sec | Latency (ms) | Relative |
 |-------:|--------:|--------------:|--------------:|---------:|
 | 1 | `{ posts { id userId title user { id name email }}}` |
-|| [Tailcall] | `29,496.60` | `3.38` | `196.74x` |
-|| [async-graphql] | `1,805.45` | `55.70` | `12.04x` |
-|| [Caliban] | `1,547.91` | `64.33` | `10.32x` |
-|| [GraphQL JIT] | `1,335.26` | `74.60` | `8.91x` |
-|| [Gqlgen] | `772.80` | `128.41` | `5.15x` |
-|| [Netflix DGS] | `358.06` | `177.42` | `2.39x` |
-|| [Apollo GraphQL] | `268.25` | `366.16` | `1.79x` |
-|| [Hasura] | `149.93` | `549.07` | `1.00x` |
+|| [Tailcall] | `29,173.60` | `3.41` | `211.81x` |
+|| [async-graphql] | `2,019.59` | `49.49` | `14.66x` |
+|| [Caliban] | `1,744.80` | `57.28` | `12.67x` |
+|| [GraphQL JIT] | `1,342.93` | `74.16` | `9.75x` |
+|| [Gqlgen] | `797.87` | `124.41` | `5.79x` |
+|| [Netflix DGS] | `366.18` | `181.49` | `2.66x` |
+|| [Apollo GraphQL] | `263.39` | `372.49` | `1.91x` |
+|| [Hasura] | `137.74` | `587.80` | `1.00x` |
 | 2 | `{ posts { title }}` |
-|| [Tailcall] | `58,939.60` | `1.69` | `67.77x` |
-|| [Caliban] | `9,238.76` | `11.16` | `10.62x` |
-|| [async-graphql] | `9,138.76` | `11.34` | `10.51x` |
-|| [Gqlgen] | `2,136.19` | `48.48` | `2.46x` |
-|| [Apollo GraphQL] | `1,775.87` | `56.26` | `2.04x` |
-|| [Netflix DGS] | `1,593.80` | `69.82` | `1.83x` |
-|| [GraphQL JIT] | `1,399.68` | `71.35` | `1.61x` |
-|| [Hasura] | `869.69` | `114.81` | `1.00x` |
+|| [Tailcall] | `58,485.30` | `1.70` | `67.37x` |
+|| [Caliban] | `9,751.71` | `10.62` | `11.23x` |
+|| [async-graphql] | `9,444.13` | `10.73` | `10.88x` |
+|| [Gqlgen] | `2,181.26` | `47.48` | `2.51x` |
+|| [Apollo GraphQL] | `1,760.69` | `56.72` | `2.03x` |
+|| [Netflix DGS] | `1,598.09` | `70.06` | `1.84x` |
+|| [GraphQL JIT] | `1,392.45` | `71.71` | `1.60x` |
+|| [Hasura] | `868.17` | `114.99` | `1.00x` |
 | 3 | `{ greet }` |
-|| [Caliban] | `67,637.00` | `1.07` | `25.83x` |
-|| [Tailcall] | `59,557.80` | `1.69` | `22.75x` |
-|| [Gqlgen] | `47,525.20` | `5.21` | `18.15x` |
-|| [async-graphql] | `47,150.60` | `2.21` | `18.01x` |
-|| [Netflix DGS] | `8,261.44` | `15.02` | `3.16x` |
-|| [Apollo GraphQL] | `8,186.12` | `12.54` | `3.13x` |
-|| [GraphQL JIT] | `5,273.45` | `18.93` | `2.01x` |
-|| [Hasura] | `2,618.18` | `38.13` | `1.00x` |
+|| [Caliban] | `67,512.70` | `1.07` | `26.49x` |
+|| [Tailcall] | `59,015.70` | `1.71` | `23.16x` |
+|| [Gqlgen] | `47,378.10` | `5.25` | `18.59x` |
+|| [async-graphql] | `47,361.00` | `2.25` | `18.58x` |
+|| [Netflix DGS] | `8,297.54` | `14.39` | `3.26x` |
+|| [Apollo GraphQL] | `8,128.40` | `12.43` | `3.19x` |
+|| [GraphQL JIT] | `5,241.04` | `19.05` | `2.06x` |
+|| [Hasura] | `2,548.54` | `39.16` | `1.00x` |
 
 <!-- PERFORMANCE_RESULTS_END -->

run_analyze_script.sh

@@ -1,13 +1,36 @@
 #!/bin/bash
 
-rm results.md
+# Update and install gnuplot
+sudo apt-get update && sudo apt-get install -y gnuplot
 
+# Remove existing results file
+rm -f results.md
+
+services=("apollo" "caliban" "netflixdgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit")
+
+# Loop through each benchmark (1, 2, 3)
 for bench in 1 2 3; do
-    if ls bench${bench}*.txt &> /dev/null; then
-        echo "Processing files for bench${bench}:"
-        bash analyze.sh bench${bench}*.txt
-        echo "Files processed: $(ls bench${bench}*.txt)"
-    else
-        echo "No matching files found for bench${bench}*.txt"
-    fi
+    echo "Processing files for bench${bench}:"
+    
+    # Construct the command for each benchmark
+    cmd="bash analyze.sh"
+    
+    # Loop through each service 
+    for service in "${services[@]}"; do
+        # Convert service name to match file naming convention 
+        case $service in
+            "apollo") file_service="apollo_server" ;;
+            "netflixdgs") file_service="netflix_dgs" ;;
+            *) file_service=$service ;;
+        esac
+        
+        # Add files for current service to the command
+        for run in 1 2 3; do
+            cmd+=" bench${bench}_result${run}_graphql_${file_service}_run.sh.txt"
+        done
+    done
+    
+    # Execute the command
+    echo "Executing: $cmd"
+    eval $cmd
 done

run_benchmarks.sh

@@ -12,6 +12,7 @@ function killServerOnPort() {
     echo "No process found running on port $port"
   fi
 }
+
 bench1Results=()
 bench2Results=()
 bench3Results=()
@@ -73,16 +74,32 @@ function runBenchmark() {
 
 rm "results.md"
 
-for service in "apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit"; do
-  runBenchmark "$service"
-  if [ "$service" == "apollo_server" ]; then
+# Main script
+if [ $# -eq 0 ]; then
+    echo "Usage: $0 <service_name>"
+    echo "Available services: apollo_server, caliban, netflix_dgs, gqlgen, tailcall, async_graphql, hasura, graphql_jit"
+    exit 1
+fi  
+
+service="$1"
+valid_services=("apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit")
+
+if [[ ! " ${valid_services[@]} " =~ " ${service} " ]]; then
+    echo "Invalid service name. Available services: ${valid_services[*]}"
+    exit 1
+fi
+
+
+
+runBenchmark "graphql/${service}/run.sh"
+    
+if [ "$service" == "apollo_server" ]; then
     cd graphql/apollo_server/
     npm stop
     cd ../../
-  elif [ "$service" == "hasura" ]; then
+elif [ "$service" == "hasura" ]; then
     bash "graphql/hasura/kill.sh"
-  fi
-done
+fi
 
 bash analyze.sh "${bench1Results[@]}"
 bash analyze.sh "${bench2Results[@]}"
@@ -101,4 +118,4 @@ if [[ "$UPLOAD_TO_CLOUD" == "true" ]]; then
   curl -o assets/posts_latency.png -H "Authorization: Bearer $GRAFANA_API_KEY" "https://tailcall.grafana.net/render/d-solo/cdqucydulbfggb?tab=queries&from=$from&to=$now&panelId=panel-6&__feature.dashboardSceneSolo&width=1000&height=500&tz=Asia%2FCalcutta" --connect-timeout 120
   curl -o assets/greet_req.png -H "Authorization: Bearer $GRAFANA_API_KEY" "https://tailcall.grafana.net/render/d-solo/cdqucydulbfggb?tab=queries&from=$from&to=$now&panelId=panel-8&__feature.dashboardSceneSolo&width=1000&height=500&tz=Asia%2FCalcutta" --connect-timeout 120
   curl -o assets/greet_latency.png -H "Authorization: Bearer $GRAFANA_API_KEY" "https://tailcall.grafana.net/render/d-solo/cdqucydulbfggb?tab=queries&from=$from&to=$now&panelId=panel-9&__feature.dashboardSceneSolo&width=1000&height=500&tz=Asia%2FCalcutta" --connect-timeout 120
-fi
+fi