diff --git a/.github/workflows/measure_latency_difference.yml b/.github/workflows/measure_latency_difference.yml new file mode 100644 index 00000000000..49cd51fa632 --- /dev/null +++ b/.github/workflows/measure_latency_difference.yml @@ -0,0 +1,244 @@ +name: Measure latency difference +on: + pull_request: + types: [labeled] + +jobs: + trigger-latency-measurement: + if: github.event.label.name == 'measure-latency-difference' + runs-on: ubuntu-latest + permissions: + pull-requests: write + contents: read + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Determine commits + id: commits + run: | + COMMIT="${{ github.event.pull_request.head.sha }}" + BASE_COMMIT=$(git merge-base HEAD origin/master) + echo "commit=$COMMIT" >> "$GITHUB_OUTPUT" + echo "base_commit=$BASE_COMMIT" >> "$GITHUB_OUTPUT" + echo "Commit: $COMMIT" + echo "Base commit: $BASE_COMMIT" + + - name: Trigger Bitrise build + id: trigger + env: + BITRISE_APP_SLUG: ${{ secrets.BITRISE_APP_SLUG }} + BITRISE_API_TOKEN: ${{ secrets.BITRISE_API_TOKEN }} + COMMIT: ${{ steps.commits.outputs.commit }} + BASE_COMMIT: ${{ steps.commits.outputs.base_commit }} + run: | + REQUEST_BODY=$(jq -n \ + --arg commit "$COMMIT" \ + --arg base_commit "$BASE_COMMIT" \ + '{ + hook_info: { type: "bitrise" }, + build_params: { + workflow_id: "run-paymentsheet-latency-difference", + commit_hash: $commit, + environments: [ + { mapped_to: "COMMIT", value: $commit, is_expand: false }, + { mapped_to: "BASE_COMMIT", value: $base_commit, is_expand: false } + ] + } + }') + + RESPONSE=$(curl --silent --fail-with-body \ + --request POST \ + --url "https://api.bitrise.io/v0.1/apps/${BITRISE_APP_SLUG}/builds" \ + --header "Authorization: ${BITRISE_API_TOKEN}" \ + --header "Content-Type: application/json" \ + --data "$REQUEST_BODY") + + BUILD_SLUG=$(echo "$RESPONSE" | jq -r '.build_slug // empty') + BUILD_URL=$(echo "$RESPONSE" | jq -r '.build_url // empty') + + if [ -z "$BUILD_SLUG" ]; then + echo "Failed to get build_slug from Bitrise API. Response:" + echo "$RESPONSE" + exit 1 + fi + + echo "build_slug=$BUILD_SLUG" >> "$GITHUB_OUTPUT" + echo "build_url=$BUILD_URL" >> "$GITHUB_OUTPUT" + echo "Triggered Bitrise build: $BUILD_SLUG" + echo "Build URL: $BUILD_URL" + + - name: Post pending comment on PR + id: pending_comment + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 + with: + issue-number: ${{ github.event.pull_request.number }} + token: ${{ secrets.GITHUB_TOKEN }} + body: | + ## Latency Difference Measurement + + ⏳ Measuring latency difference between commits... + + - **Commit:** `${{ steps.commits.outputs.commit }}` + - **Base commit:** `${{ steps.commits.outputs.base_commit }}` + - **Bitrise build:** ${{ steps.trigger.outputs.build_url }} + + This comment will be updated once the measurement is complete. + + - name: Poll Bitrise build until complete + id: poll + env: + BITRISE_APP_SLUG: ${{ secrets.BITRISE_APP_SLUG }} + BITRISE_API_TOKEN: ${{ secrets.BITRISE_API_TOKEN }} + BUILD_SLUG: ${{ steps.trigger.outputs.build_slug }} + run: | + echo "Polling build: $BUILD_SLUG" + BUILD_STATUS="" + + for i in $(seq 1 120); do + sleep 60 + + # Use || true so a transient network error doesn't crash the loop + RESPONSE=$(curl --silent \ + --request GET \ + --url "https://api.bitrise.io/v0.1/apps/${BITRISE_APP_SLUG}/builds/${BUILD_SLUG}" \ + --header "Authorization: ${BITRISE_API_TOKEN}") || true + + # Use // empty so jq returns "" rather than "null" when the field is missing, + # and || STATUS="" so a parse error on a bad response retries instead of crashing + STATUS=$(echo "$RESPONSE" | jq -r '.data.status_text // empty' 2>/dev/null) || STATUS="" + + echo "Attempt $i/120 — build status: ${STATUS:-}" + + case "$STATUS" in + success|failed|aborted|error) + BUILD_STATUS="$STATUS" + break + ;; + esac + done + + if [ -z "$BUILD_STATUS" ]; then + BUILD_STATUS="timeout" + fi + + echo "build_status=$BUILD_STATUS" >> "$GITHUB_OUTPUT" + + - name: Fetch and extract build results + id: results + if: > + steps.poll.outputs.build_status == 'success' || + steps.poll.outputs.build_status == 'failed' + env: + BITRISE_APP_SLUG: ${{ secrets.BITRISE_APP_SLUG }} + BITRISE_API_TOKEN: ${{ secrets.BITRISE_API_TOKEN }} + BUILD_SLUG: ${{ steps.trigger.outputs.build_slug }} + run: | + LOG_RESPONSE=$(curl --silent --fail \ + --request GET \ + --url "https://api.bitrise.io/v0.1/apps/${BITRISE_APP_SLUG}/builds/${BUILD_SLUG}/log" \ + --header "Authorization: ${BITRISE_API_TOKEN}") + + IS_ARCHIVED=$(echo "$LOG_RESPONSE" | jq -r '.is_archived') + + if [ "$IS_ARCHIVED" = "true" ]; then + EXPIRING_URL=$(echo "$LOG_RESPONSE" | jq -r '.expiring_raw_log_url') + FULL_LOG=$(curl --silent "$EXPIRING_URL") + else + FULL_LOG=$(echo "$LOG_RESPONSE" | jq -r '[.log_chunks[].chunk] | join("")') + fi + + # Extract latency delta report section + DELTA_REPORT=$(echo "$FULL_LOG" \ + | awk '/LATENCY DELTA REPORT/{ found=1 } found{ print } /^$/ && found && NR>1 { exit }' \ + | head -80) + + # Extract raw measurements section + RAW_REPORT=$(echo "$FULL_LOG" \ + | awk '/RAW LATENCY MEASUREMENTS/{ found=1 } found{ print } /^={3}/ && found && NR>1 { exit }' \ + | head -80) + + # Extract benchmark duration + DURATION=$(echo "$FULL_LOG" | grep 'BENCHMARK_DURATION:' | tail -1) + + # Write extracted sections to files + printf '%s\n' "$DELTA_REPORT" > /tmp/delta_report.txt + printf '%s\n' "$RAW_REPORT" > /tmp/raw_report.txt + printf '%s\n' "$DURATION" > /tmp/duration.txt + + - name: Build result comment body + id: comment_body + env: + BUILD_STATUS: ${{ steps.poll.outputs.build_status }} + BUILD_URL: ${{ steps.trigger.outputs.build_url }} + COMMIT: ${{ steps.commits.outputs.commit }} + BASE_COMMIT: ${{ steps.commits.outputs.base_commit }} + run: | + case "$BUILD_STATUS" in + success) ICON="✅" ; STATUS_MSG="Measurement complete" ;; + failed) ICON="❌" ; STATUS_MSG="Measurement failed" ;; + timeout) ICON="⏰" ; STATUS_MSG="Measurement timed out (> 2 hours)" ;; + *) ICON="⚠️" ; STATUS_MSG="Measurement ended with status: $BUILD_STATUS" ;; + esac + + { + echo "## Latency Difference Measurement" + echo "" + echo "$ICON $STATUS_MSG" + echo "" + echo "- **Commit:** \`$COMMIT\`" + echo "- **Base commit:** \`$BASE_COMMIT\`" + echo "- **Bitrise build:** $BUILD_URL" + } > /tmp/comment.md + + if [ -s /tmp/delta_report.txt ]; then + { + echo "" + echo "
" + echo "Latency Delta Report" + echo "" + echo '```' + cat /tmp/delta_report.txt + echo '```' + echo "" + echo "
" + } >> /tmp/comment.md + fi + + if [ -s /tmp/raw_report.txt ]; then + { + echo "" + echo "
" + echo "Raw Measurements" + echo "" + echo '```' + cat /tmp/raw_report.txt + echo '```' + echo "" + echo "
" + } >> /tmp/comment.md + fi + + if [ -s /tmp/duration.txt ]; then + { + echo "" + echo "_$(cat /tmp/duration.txt)_" + } >> /tmp/comment.md + fi + + # Expose as output (multiline) + { + echo "body<> "$GITHUB_OUTPUT" + + - name: Update PR comment with results + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 + with: + comment-id: ${{ steps.pending_comment.outputs.comment-id }} + edit-mode: replace + issue-number: ${{ github.event.pull_request.number }} + token: ${{ secrets.GITHUB_TOKEN }} + body: ${{ steps.comment_body.outputs.body }} diff --git a/bitrise.yml b/bitrise.yml index 81a8de31fad..2494bff8a83 100644 --- a/bitrise.yml +++ b/bitrise.yml @@ -366,6 +366,36 @@ workflows: title: Export PaymentSheet E2E test results inputs: - test_title: PaymentSheet E2E tests + run-paymentsheet-latency-difference: + before_run: + - start_emulator + - prepare_all + after_run: + - conclude_all + steps: + - script@1: + title: Install Ruby via asdf + inputs: + - content: asdf install ruby 3.2.4 + - wait-for-android-emulator@1: + inputs: + - boot_timeout: 600 + - script@1: + title: Measure PaymentSheet latency difference + timeout: 7200 + inputs: + - content: | + ruby scripts/measure_latency_difference.rb \ + --base-commit "$BASE_COMMIT" \ + --commit "$COMMIT" + - bundle::export_instrumentation_test_results: + title: Export PaymentSheet latency difference results + inputs: + - test_title: PaymentSheet latency difference + meta: + bitrise.io: + stack: linux-docker-android-22.04 + machine_type_id: g2.linux.x-large run-paymentsheet-latency-synthetics: before_run: - start_emulator