Retry only failed Android E2E flows

Author: cipolleschi

.github/actions/maestro-android/action.yml

@@ -26,6 +26,13 @@ inputs:
     required: false
     default: x86
     description: The architecture of the emulator to run
+  test-state-path:
+    required: false
+    default: /tmp/maestro-android-state/results.json
+    description: The path used to persist per-flow test results between retries
+  test-state-artifact-name:
+    required: true
+    description: The artifact used to pass per-flow test results to retry jobs
 
 runs:
   using: composite
@@ -64,7 +71,7 @@ runs:
         cores: '4'
         disable-animations: false
         avd-name: e2e_emulator
-        script: node .github/workflow-scripts/maestro-android.js ${{ inputs.app-path }} ${{ inputs.app-id }} ${{ inputs.maestro-flow }} ${{ inputs.flavor }} ${{ inputs.working-directory }}
+        script: node .github/workflow-scripts/maestro-android.js ${{ inputs.app-path }} ${{ inputs.app-id }} ${{ inputs.maestro-flow }} ${{ inputs.flavor }} ${{ inputs.working-directory }} ${{ inputs.test-state-path }}
     - name: Normalize APP_ID
       id: normalize-app-id
       shell: bash
@@ -81,6 +88,14 @@ runs:
         path: |
           report.xml
           screen.mp4
+    - name: Store per-flow test state
+      uses: actions/upload-artifact@v6
+      if: always()
+      with:
+        name: ${{ inputs.test-state-artifact-name }}
+        overwrite: true
+        if-no-files-found: warn
+        path: ${{ inputs.test-state-path }}
     - name: Store Logs
       if: steps.run-tests.outcome == 'failure'
       uses: actions/upload-artifact@v6

.github/workflow-scripts/__tests__/maestro-android-test.js

@@ -0,0 +1,87 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const {
+  collectFlows,
+  executeFlowSuite,
+  loadState,
+} = require('../maestro-android');
+
+describe('Maestro Android runner', () => {
+  let temporaryDirectory;
+
+  beforeEach(() => {
+    temporaryDirectory = fs.mkdtempSync(
+      path.join(os.tmpdir(), 'maestro-android-test-'),
+    );
+  });
+
+  afterEach(() => {
+    fs.rmSync(temporaryDirectory, {recursive: true, force: true});
+  });
+
+  it('collects flows recursively in a stable order', () => {
+    const nestedDirectory = path.join(temporaryDirectory, 'nested');
+    fs.mkdirSync(nestedDirectory);
+    fs.writeFileSync(path.join(temporaryDirectory, 'second.yaml'), 'appId: x');
+    fs.writeFileSync(path.join(temporaryDirectory, 'image.png'), 'not a flow');
+    fs.writeFileSync(path.join(nestedDirectory, 'first.yml'), 'appId: x');
+
+    expect(collectFlows(temporaryDirectory)).toEqual([
+      path.join(nestedDirectory, 'first.yml'),
+      path.join(temporaryDirectory, 'second.yaml'),
+    ]);
+  });
+
+  it('runs every flow and retries only flows that have not passed', () => {
+    const flows = ['first.yml', 'second.yml', 'third.yml'].map(file =>
+      path.join(temporaryDirectory, file),
+    );
+    const statePath = path.join(temporaryDirectory, 'state', 'results.json');
+    const firstAttempt = jest.fn(flow => {
+      if (flow.endsWith('second.yml')) {
+        throw new Error('failed assertion');
+      }
+    });
+
+    expect(() =>
+      executeFlowSuite({
+        flows,
+        appId: 'com.example',
+        state: loadState(statePath),
+        statePath,
+        executeFlow: firstAttempt,
+      }),
+    ).toThrow('1 Maestro flow(s) failed');
+    expect(firstAttempt).toHaveBeenCalledTimes(3);
+
+    const retry = jest.fn();
+    executeFlowSuite({
+      flows,
+      appId: 'com.example',
+      state: loadState(statePath),
+      statePath,
+      executeFlow: retry,
+    });
+
+    expect(retry).toHaveBeenCalledTimes(1);
+    expect(retry.mock.calls[0][0]).toBe(flows[1]);
+
+    const finalState = loadState(statePath);
+    expect(Object.values(finalState.flows)).toEqual([
+      {status: 'passed', attempts: 1},
+      {status: 'passed', attempts: 2},
+      {status: 'passed', attempts: 1},
+    ]);
+  });
+});