ion-java-performance-regression-detector.yml

# This workflow will build a Java project with Maven
# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven

name: Ion Java performance regression detector

on:
  pull_request:
    paths:
      - 'src/**'

permissions:
  contents: read

env:
  data_size: '50000'
  test_data_id: 'generated-test-data'

jobs:
  generate-test-data:
    name: Generate Test Data
    runs-on: ubuntu-latest
    steps:
      - name: Checkout ion-data-generator
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: amazon-ion/ion-data-generator
          ref: main

      - name: Build ion-data-generator
        run: mvn clean install

      - name: Generate test data
        env:
          jar_file: target/ion-data-generator-1.0-SNAPSHOT.jar
          schema_dir: tst/com/amazon/ion/workflow
        run: |
          mkdir -p testData
          for test in nestedStruct nestedList sexp realWorldDataSchema01 realWorldDataSchema02 realWorldDataSchema03
          do
          java -jar $jar_file generate -S ${{env.data_size}} --input-ion-schema $schema_dir/${test}.isl testData/${test}.10n
          done

      - name: Upload test Ion Data to artifacts
        uses: actions/upload-artifact@v4
        with:
          name: ${{env.test_data_id}}
          path: testData

  select-runner-type:
    # If you want to use codebuild runners for your personal fork, follow the instructions to set
    # up a codebuild project. https://docs.aws.amazon.com/codebuild/latest/userguide/action-runner.html
    # Then, create a repository variable for your fork named `CODEBUILD_PROJECT_NAME` with the name
    # of the project you created.
    name: Select Runner Type
    runs-on: ubuntu-latest
    # We want to run on external PRs, but not on internal ones as push automatically builds
    # H/T: https://github.com/Dart-Code/Dart-Code/commit/612732d5879730608baa9622bf7f5e5b7b51ae65
    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != 'amazon-ion/ion-java'
    env:
      use-codebuild: ${{ vars.CODEBUILD_PROJECT_NAME != '' || github.repository_owner == 'amazon-ion' }}
      codebuild-project-name: ${{ vars.CODEBUILD_PROJECT_NAME != '' && vars.CODEBUILD_PROJECT_NAME || 'ion-java' }}
      al2-arm: "codebuild-${{ vars.CODEBUILD_PROJECT_NAME != '' && vars.CODEBUILD_PROJECT_NAME || 'ion-java' }}-${{ github.run_id }}-${{ github.run_attempt }}-arm-3.0-large"
    outputs:
      runner-type: ${{ env.use-codebuild && env.al2_arm || 'ubuntu-latest' }}
    steps:
      - name: Dump Config
        run: echo '${{ toJSON(env) }}'

  detect-regression:
    name: Detect Regression
    runs-on: ${{ needs.select-runner-type.outputs.runner-type }}
    needs: [ generate-test-data, select-runner-type ]
    strategy:
      matrix:
        test-data: ['nestedStruct', 'nestedList', 'sexp', 'realWorldDataSchema01', 'realWorldDataSchema02', 'realWorldDataSchema03']
        usage-patterns: [' read --mode AverageTime --forks 3 --warmups 2 --iterations 10 --api streaming ',' read --mode AverageTime --forks 3 --warmups 2 --iterations 10 --api dom ', ' write --mode AverageTime --forks 3 --warmups 2 --iterations 10 --api streaming --io-type buffer --ion-length-preallocation 1 ', ' write --mode AverageTime --forks 3 --warmups 2 --iterations 10 --api dom --io-type buffer --ion-length-preallocation 1 ']
      fail-fast: false

    steps:
      - name: Set up JDK 17
        uses: actions/setup-java@v4.7.0
        with:
          distribution: 'corretto'
          java-version: 17

      - name: Checkout ion-java-benchmark-cli
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: amazon-ion/ion-java-benchmark-cli
          ref: master
          path: ion-java-benchmark-cli

      - name: Checkout ion-java from the previous commit
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          ref: ${{ github.base_ref }}
          submodules: recursive
          path: baseline

      - name: Checkout ion-java from the new commit.
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.head_ref }}
          submodules: recursive
          path: new

      - name: Download test Ion Data from artifacts
        uses: actions/download-artifact@v4
        with:
          name: generated-test-data
          path: testData

      # Get the benchmark results from the base commit
      - name: Build ion-java from the base commit
        working-directory: baseline
        run: |
          ./gradlew clean publishToMavenLocal

      - name: Benchmark ion-java from the base commit
        working-directory: ion-java-benchmark-cli
        run: |
          mvn clean install
          mkdir /home/runner/work/ion-java/ion-java/benchmarkresults
          java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar ${{matrix.usage-patterns}} -r ion -o /home/runner/work/ion-java/ion-java/benchmarkresults/before.ion /home/runner/work/ion-java/ion-java/testData/${{matrix.test-data}}.10n 

      - name: Build ion-java from the new commit
        working-directory: new
        run: |
          ./gradlew clean publishToMavenLocal

      - name: Benchmark ion-java from the new commit
        working-directory: ion-java-benchmark-cli
        run: |
          mvn clean install
          java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar ${{matrix.usage-patterns}} -r ion -o /home/runner/work/ion-java/ion-java/benchmarkresults/after.ion /home/runner/work/ion-java/ion-java/testData/${{matrix.test-data}}.10n

      #Detect regression
      - name: Detect regression
        working-directory: ion-java-benchmark-cli
        run: |
          a=$(java -jar target/ion-java-benchmark-cli-0.0.1-SNAPSHOT-jar-with-dependencies.jar compare --benchmark-result-previous /home/runner/work/ion-java/ion-java/benchmarkresults/before.ion --benchmark-result-new /home/runner/work/ion-java/ion-java/benchmarkresults/after.ion)
          if [ ! -z "$a" ]; then echo "${a}" >> $GITHUB_STEP_SUMMARY; exit 1; fi