diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 6967674..1deb7b6 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -394,6 +394,30 @@ jobs: expected: failure actual: ${{ steps.sad_path_timeout.outcome }} + - name: sad-path (timeout persistent) + id: sad_path_timeout_persistent + uses: ./ + continue-on-error: true + with: + timeout_seconds: 15 + max_attempts: 2 + command: | + node -e " + process.on('SIGTERM', () => console.log('SIGTERM ignored')); + setInterval(() => console.log('still running'), 1000); + setTimeout(() => { + console.log('Process stopping after lack of timeout'); + }, 300000); + " + - uses: nick-invision/assert-action@v1 + with: + expected: 2 + actual: ${{ steps.sad_path_timeout_persistent.outputs.total_attempts }} + - uses: nick-invision/assert-action@v1 + with: + expected: failure + actual: ${{ steps.sad_path_timeout_persistent.outcome }} + ci_integration_timeout_retry_on_timeout: name: Run Integration Timeout Tests (retry_on timeout) runs-on: ubuntu-latest diff --git a/src/index.ts b/src/index.ts index aa5642e..65c84bb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,6 +13,8 @@ const OUTPUT_EXIT_ERROR_KEY = 'exit_error'; let exit: number; let done: boolean; +let processExited: boolean; +let processFinished: boolean; function getExecutable(inputs: Inputs): string { if (!inputs.shell) { @@ -73,6 +75,8 @@ async function runCmd(attempt: number, inputs: Inputs) { exit = 0; done = false; + processExited = false; + processFinished = false; let timeout = false; debug(`Running command ${inputs.command} on ${OS} using shell ${executable}`); @@ -91,9 +95,10 @@ async function runCmd(attempt: number, inputs: Inputs) { child.on('exit', (code, signal) => { debug(`Code: ${code}`); debug(`Signal: ${signal}`); + processExited = true; // timeouts are killed manually - if (signal === 'SIGTERM') { + if (signal === 'SIGTERM' || signal === 'SIGINT' || signal === 'SIGKILL') { return; } @@ -109,14 +114,28 @@ async function runCmd(attempt: number, inputs: Inputs) { done = true; }); + child.on('close', () => { + // Occurs on closing of streams and IPC channels. + debug(`Process streams closed.`); + processFinished = true; + }); + do { await wait(ms.seconds(inputs.polling_interval_seconds)); } while (Date.now() < end_time && !done); if (!done && child.pid) { timeout = true; - kill(child.pid); + kill(child.pid, "SIGTERM"); await retryWait(ms.seconds(inputs.retry_wait_seconds)); + // If still not done, send SIGINT followed by SIGKILL + if (!processExited) { + kill(child.pid, "SIGINT"); + await wait(3000); + if (!processFinished){ + kill(child.pid, "SIGKILL"); + } + } throw new Error(`Timeout of ${getTimeout(inputs)}ms hit`); } else if (exit > 0) { await retryWait(ms.seconds(inputs.retry_wait_seconds));