diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..b3d06ce --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,89 @@ +name: Bug Report +description: File a bug report to help us improve +title: "[Bug]: " +labels: ["bug"] +assignees: [] + +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + + - type: textarea + id: what-happened + attributes: + label: What happened? + description: A clear and concise description of what the bug is. + placeholder: Tell us what you see! + validations: + required: true + + - type: textarea + id: expected + attributes: + label: Expected behavior + description: A clear and concise description of what you expected to happen. + validations: + required: true + + - type: textarea + id: reproduce + attributes: + label: Steps to reproduce + description: Steps to reproduce the behavior + placeholder: | + 1. Go to '...' + 2. Click on '....' + 3. Scroll down to '....' + 4. See error + validations: + required: true + + - type: textarea + id: code + attributes: + label: Minimal code example + description: If applicable, add a minimal code example that reproduces the issue + render: rust + + - type: dropdown + id: version + attributes: + label: Version + description: What version of apalis-sqlite are you running? + options: + - 1.0.0-alpha.1 + - main branch + - Other (specify in additional context) + validations: + required: true + + - type: textarea + id: environment + attributes: + label: Environment + description: | + Please provide information about your environment: + - OS: [e.g. Ubuntu 20.04, macOS 12.0, Windows 11] + - Rust version: [e.g. 1.70.0] + - Cargo version: [e.g. 1.70.0] + value: | + - OS: + - Rust version: + - Cargo version: + validations: + required: true + + - type: textarea + id: logs + attributes: + label: Relevant log output + description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + render: shell + + - type: textarea + id: context + attributes: + label: Additional context + description: Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..412b715 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,57 @@ +name: Feature Request +description: Suggest an idea for this project +title: "[Feature]: " +labels: ["enhancement"] +assignees: [] + +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to suggest a new feature! + + - type: textarea + id: problem + attributes: + label: Is your feature request related to a problem? + description: A clear and concise description of what the problem is. + placeholder: I'm always frustrated when... + validations: + required: true + + - type: textarea + id: solution + attributes: + label: Describe the solution you'd like + description: A clear and concise description of what you want to happen. + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: Describe alternatives you've considered + description: A clear and concise description of any alternative solutions or features you've considered. + + - type: textarea + id: use-case + attributes: + label: Use case + description: Describe your specific use case and how this feature would help. + validations: + required: true + + - type: textarea + id: implementation + attributes: + label: Additional context + description: Add any other context, screenshots, or implementation ideas about the feature request here. + + - type: checkboxes + id: terms + attributes: + label: Code of Conduct + description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/apalis-dev/apalis-sqlite/blob/main/CODE_OF_CONDUCT.md) + options: + - label: I agree to follow this project's Code of Conduct + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..bd6b9c0 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,32 @@ +## Description + +Brief description of what this PR does. + +## Type of Change + +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] Documentation update +- [ ] Performance improvement +- [ ] Refactoring (no functional changes) + +## Testing + +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] I have run the existing tests and they pass +- [ ] I have run `cargo fmt` and `cargo clippy` + +## Checklist + +- [ ] My code follows the code style of this project +- [ ] I have performed a self-review of my own code +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] I have made corresponding changes to the documentation +- [ ] My changes generate no new warnings +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] New and existing unit tests pass locally with my changes + +## Additional Notes + +Any additional information, context, or screenshots about the pull request here. \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..876c9ab --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,39 @@ +version: 2 +updates: + # Enable version updates for Cargo + - package-ecosystem: "cargo" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "06:00" + open-pull-requests-limit: 10 + reviewers: + - "geofmureithi" + assignees: + - "geofmureithi" + commit-message: + prefix: "deps" + include: "scope" + labels: + - "dependencies" + - "rust" + + # Enable version updates for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + time: "06:00" + open-pull-requests-limit: 5 + reviewers: + - "geofmureithi" + assignees: + - "geofmureithi" + commit-message: + prefix: "ci" + include: "scope" + labels: + - "dependencies" + - "github-actions" diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..afdabe7 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,80 @@ +name: Benchmarks + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +env: + CARGO_TERM_COLOR: always + +jobs: + benchmark: + name: Performance Benchmarks + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-bench-${{ hashFiles('**/Cargo.lock') }} + + - name: Run benchmarks + run: | + if find . -name "*.rs" -exec grep -l "#\[bench\]" {} \; | grep -q .; then + echo "Found benchmark tests, running cargo bench" + cargo bench + else + echo "No benchmark tests found, skipping" + fi + + - name: Install cargo-criterion (if criterion benchmarks exist) + run: | + if find . -name "*.rs" -exec grep -l "criterion::" {} \; | grep -q .; then + echo "Found criterion benchmarks, installing cargo-criterion" + cargo install cargo-criterion + cargo criterion + else + echo "No criterion benchmarks found, skipping" + fi + + msrv: + name: Minimum Supported Rust Version + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Get MSRV from Cargo.toml + id: msrv + run: | + MSRV=$(grep "rust-version" Cargo.toml | sed 's/.*rust-version.*=.*"\(.*\)".*/\1/' || echo "1.70.0") + echo "msrv=$MSRV" >> $GITHUB_OUTPUT + echo "Detected MSRV: $MSRV" + + - name: Install MSRV Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ steps.msrv.outputs.msrv }} + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-msrv-${{ steps.msrv.outputs.msrv }}-${{ hashFiles('**/Cargo.lock') }} + + - name: Test with MSRV + run: cargo test --all-features diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..fdd040a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,192 @@ +name: CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + +jobs: + test: + name: Test Suite + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + rust: [stable, beta] + include: + - os: ubuntu-latest + rust: nightly + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ matrix.rust }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-${{ matrix.rust }}- + ${{ runner.os }}-cargo- + + - name: Run cargo test + run: cargo test --verbose + + - name: Run cargo test with all features + run: cargo test --all-features --verbose + + features: + name: Feature Matrix Tests + runs-on: ubuntu-latest + strategy: + matrix: + features: + - "" + - "--features migrate" + - "--features async-std-comp" + - "--features async-std-comp-native-tls" + - "--features tokio-comp" + - "--features tokio-comp-native-tls" + - "--all-features" + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-features-${{ hashFiles('**/Cargo.lock') }} + + - name: Test with features + run: cargo test ${{ matrix.features }} --verbose + + fmt: + name: Rustfmt + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + + - name: Run cargo fmt + run: cargo fmt --all -- --check + + clippy: + name: Clippy + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: clippy + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }} + + - name: Run cargo clippy + run: cargo clippy --all-targets --all-features + + docs: + name: Documentation + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-docs-${{ hashFiles('**/Cargo.lock') }} + + - name: Build documentation + run: cargo doc --all-features --no-deps --document-private-items + env: + RUSTDOCFLAGS: "-Dwarnings" + + security: + name: Security Audit + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install cargo-audit + run: cargo install cargo-audit + + - name: Run cargo audit + run: cargo audit + + coverage: + name: Code Coverage + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install cargo-tarpaulin + run: cargo install cargo-tarpaulin + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-coverage-${{ hashFiles('**/Cargo.lock') }} + + - name: Generate code coverage + run: cargo tarpaulin --all-features --verbose --workspace --timeout 120 --out xml + + - name: Upload to codecov.io + uses: codecov/codecov-action@v3 + with: + file: cobertura.xml + fail_ci_if_error: false diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..a6a6290 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,36 @@ +name: Documentation + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +env: + CARGO_TERM_COLOR: always + +jobs: + docs: + name: Build and Test Documentation + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-docs-${{ hashFiles('**/Cargo.lock') }} + + - name: Build documentation + run: | + cargo doc --all-features --no-deps --document-private-items + env: + RUSTDOCFLAGS: "--cfg docsrs -Dwarnings" diff --git a/.github/workflows/pr-preview.yml b/.github/workflows/pr-preview.yml new file mode 100644 index 0000000..fde1d61 --- /dev/null +++ b/.github/workflows/pr-preview.yml @@ -0,0 +1,117 @@ +name: PR Preview + +on: + pull_request: + types: [opened, synchronize, reopened] + +env: + CARGO_TERM_COLOR: always + +jobs: + quick-check: + name: Quick Check + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-quick-${{ hashFiles('**/Cargo.lock') }} + + - name: Check format + run: cargo fmt --all -- --check + + - name: Run clippy + run: cargo clippy --all-targets --all-features + + - name: Check build + run: cargo check --all-features + + - name: Run quick tests + run: cargo test --lib + + changes: + name: Detect Changes + runs-on: ubuntu-latest + outputs: + src: ${{ steps.changes.outputs.src }} + docs: ${{ steps.changes.outputs.docs }} + workflows: ${{ steps.changes.outputs.workflows }} + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - uses: dorny/paths-filter@v2 + id: changes + with: + filters: | + src: + - 'src/**' + - 'Cargo.toml' + - 'Cargo.lock' + docs: + - '**.md' + - 'docs/**' + workflows: + - '.github/workflows/**' + + test-if-needed: + name: Test if Source Changed + runs-on: ubuntu-latest + needs: changes + if: needs.changes.outputs.src == 'true' + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-test-${{ hashFiles('**/Cargo.lock') }} + + - name: Run full test suite + run: cargo test --all-features + + summary: + name: PR Preview Summary + runs-on: ubuntu-latest + needs: [quick-check, changes, test-if-needed] + if: always() + steps: + - name: Summary + run: | + echo "## PR Preview Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **Quick Check**: ${{ needs.quick-check.result }}" >> $GITHUB_STEP_SUMMARY + if [[ "${{ needs.changes.outputs.src }}" == "true" ]]; then + echo "- **Full Tests**: ${{ needs.test-if-needed.result }}" >> $GITHUB_STEP_SUMMARY + else + echo "- **Full Tests**: Skipped (no source changes)" >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Changed Files" >> $GITHUB_STEP_SUMMARY + if [[ "${{ needs.changes.outputs.src }}" == "true" ]]; then + echo "- ✅ Source code" >> $GITHUB_STEP_SUMMARY + fi + if [[ "${{ needs.changes.outputs.docs }}" == "true" ]]; then + echo "- 📚 Documentation" >> $GITHUB_STEP_SUMMARY + fi + if [[ "${{ needs.changes.outputs.workflows }}" == "true" ]]; then + echo "- ⚙️ GitHub Actions workflows" >> $GITHUB_STEP_SUMMARY + fi \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..ea14cd6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,132 @@ +name: Release and Publish + +on: + push: + tags: + - 'v*' + workflow_dispatch: + inputs: + version: + description: 'Version to publish (without v prefix)' + required: true + type: string + +env: + CARGO_TERM_COLOR: always + +jobs: + validate: + name: Validate Release + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-release-${{ hashFiles('**/Cargo.lock') }} + + - name: Extract version + id: version + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + VERSION="${{ github.event.inputs.version }}" + else + VERSION=${GITHUB_REF#refs/tags/v} + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Extracted version: $VERSION" + + - name: Verify version matches Cargo.toml + run: | + CARGO_VERSION=$(grep '^version = ' Cargo.toml | head -1 | sed 's/version = "\(.*\)"/\1/') + if [[ "$CARGO_VERSION" != "${{ steps.version.outputs.version }}" ]]; then + echo "Version mismatch: Cargo.toml has $CARGO_VERSION, but release is ${{ steps.version.outputs.version }}" + exit 1 + fi + + - name: Run tests + run: cargo test --all-features + + - name: Check format + run: cargo fmt --all -- --check + + - name: Run clippy + run: cargo clippy --all-targets --all-features + + - name: Build docs + run: cargo doc --all-features --no-deps + + publish: + name: Publish to crates.io + runs-on: ubuntu-latest + needs: validate + environment: release + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-publish-${{ hashFiles('**/Cargo.lock') }} + + - name: Login to crates.io + run: cargo login ${{ secrets.CRATES_IO_TOKEN }} + + - name: Publish to crates.io + run: cargo publish + + github-release: + name: Create GitHub Release + runs-on: ubuntu-latest + needs: [validate, publish] + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Generate changelog + id: changelog + run: | + if [[ -f CHANGELOG.md ]]; then + # Extract changelog for this version + VERSION="v${{ needs.validate.outputs.version }}" + sed -n "/^## \[$VERSION\]/,/^## \[/p" CHANGELOG.md | sed '$d' > release_notes.md + if [[ -s release_notes.md ]]; then + echo "Found changelog entry for $VERSION" + else + echo "## Changes" > release_notes.md + echo "See commit history for detailed changes." >> release_notes.md + fi + else + echo "## Changes" > release_notes.md + echo "See commit history for detailed changes." >> release_notes.md + fi + + - name: Create GitHub Release + uses: softprops/action-gh-release@v1 + with: + name: Release v${{ needs.validate.outputs.version }} + body_path: release_notes.md + draft: false + prerelease: ${{ contains(needs.validate.outputs.version, 'alpha') || contains(needs.validate.outputs.version, 'beta') || contains(needs.validate.outputs.version, 'rc') }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/rust-security.yml b/.github/workflows/rust-security.yml new file mode 100644 index 0000000..e2f0bb8 --- /dev/null +++ b/.github/workflows/rust-security.yml @@ -0,0 +1,68 @@ +name: Rust Security + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + schedule: + - cron: '0 3 * * 1' # Run weekly on Monday at 3 AM UTC + +env: + CARGO_TERM_COLOR: always + +jobs: + cargo-deny: + name: Cargo Deny + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install cargo-deny + uses: taiki-e/install-action@cargo-deny + + - name: Run cargo deny + run: cargo deny check + + supply-chain: + name: Supply Chain Security + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install cargo-audit + uses: taiki-e/install-action@cargo-audit + + - name: Audit dependencies + run: cargo audit + + - name: Install cargo-outdated + uses: taiki-e/install-action@cargo-outdated + + - name: Check for outdated dependencies + run: cargo outdated --exit-code 1 + continue-on-error: true + + unused-deps: + name: Unused Dependencies + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust nightly toolchain + uses: dtolnay/rust-toolchain@nightly + + - name: Install cargo-udeps + uses: taiki-e/install-action@cargo-udeps + + - name: Check for unused dependencies + run: cargo +nightly udeps --all-targets --all-features diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..7f501f7 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,83 @@ +name: Security and Dependencies + +on: + schedule: + # Run security audit daily at 02:00 UTC + - cron: '0 2 * * *' + push: + branches: [ main ] + paths: + - '**/Cargo.toml' + - '**/Cargo.lock' + pull_request: + paths: + - '**/Cargo.toml' + - '**/Cargo.lock' + +env: + CARGO_TERM_COLOR: always + +jobs: + security-audit: + name: Security Audit + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install cargo-audit + uses: taiki-e/install-action@cargo-audit + + - name: Run cargo audit + run: cargo audit + + - name: Run cargo audit (JSON output) + run: cargo audit --json > audit-results.json + continue-on-error: true + + - name: Upload audit results + uses: actions/upload-artifact@v4 + if: always() + with: + name: security-audit-results + path: audit-results.json + + dependency-review: + name: Dependency Review + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Dependency Review + uses: actions/dependency-review-action@v3 + + license-check: + name: License Check + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install cargo-license + uses: taiki-e/install-action@cargo-license + + - name: Check licenses + run: | + echo "## License Report" > license-report.md + echo "" >> license-report.md + cargo license --json | jq -r '.[] | "- **\(.name)** (\(.version)): \(.license // "Unknown")"' >> license-report.md + cat license-report.md + + - name: Upload license report + uses: actions/upload-artifact@v4 + with: + name: license-report + path: license-report.md diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..9aff8d0 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,38 @@ +name: Mark stale issues and pull requests + +on: + schedule: + - cron: '0 0 * * *' # Run daily at midnight + workflow_dispatch: + +jobs: + stale: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v8 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: | + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs within 7 days. + Thank you for your contributions. + stale-pr-message: | + This pull request has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs within 7 days. + Thank you for your contributions. + close-issue-message: | + This issue has been automatically closed due to inactivity. + If you believe this is still relevant, please reopen it or create a new issue. + close-pr-message: | + This pull request has been automatically closed due to inactivity. + If you believe this is still relevant, please reopen it or create a new pull request. + days-before-stale: 60 + days-before-close: 7 + stale-issue-label: 'stale' + stale-pr-label: 'stale' + exempt-issue-labels: 'pinned,security,good first issue' + exempt-pr-labels: 'pinned,security' + operations-per-run: 50 \ No newline at end of file diff --git a/.gitignore b/.gitignore index ad67955..1a7bd7d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ # will have compiled files and executables debug target +Cargo.lock # These are backup files generated by rustfmt **/*.rs.bk @@ -19,3 +20,11 @@ target # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# dotenv environment variables file +.env + +# SQLite database files +test.db +test.db-shm +test.db-wal diff --git a/.sqlx/query-0c51f68a6dc3a43da8d955530e02acf90e2a3710690f398f9ffaa71e180f1d9e.json b/.sqlx/query-0c51f68a6dc3a43da8d955530e02acf90e2a3710690f398f9ffaa71e180f1d9e.json new file mode 100644 index 0000000..c9b9a22 --- /dev/null +++ b/.sqlx/query-0c51f68a6dc3a43da8d955530e02acf90e2a3710690f398f9ffaa71e180f1d9e.json @@ -0,0 +1,38 @@ +{ + "db_name": "SQLite", + "query": "SELECT\n 1 AS priority,\n 'Number' AS type,\n 'RUNNING_JOBS' AS statistic,\n CAST(\n SUM(\n CASE\n WHEN status = 'Running' THEN 1\n ELSE 0\n END\n ) AS REAL\n ) AS value\nFROM\n Jobs\nUNION\nALL\nSELECT\n 1,\n 'Number',\n 'PENDING_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Pending' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 2,\n 'Number',\n 'FAILED_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Failed' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 2,\n 'Number',\n 'ACTIVE_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status IN ('Pending', 'Running', 'Queued') THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 2,\n 'Number',\n 'STALE_RUNNING_JOBS',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n status = 'Running'\n AND run_at < strftime('%s', 'now', '-1 hour')\nUNION\nALL\nSELECT\n 2,\n 'Percentage',\n 'KILL_RATE',\n CAST(\n ROUND(\n 100.0 * SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) / NULLIF(COUNT(*), 0),\n 2\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 3,\n 'Number',\n 'JOBS_PAST_HOUR',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n run_at >= strftime('%s', 'now', '-1 hour')\nUNION\nALL\nSELECT\n 3,\n 'Number',\n 'JOBS_TODAY',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n date(run_at, 'unixepoch') = date('now')\nUNION\nALL\nSELECT\n 3,\n 'Number',\n 'KILLED_JOBS_TODAY',\n CAST(\n SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n date(run_at, 'unixepoch') = date('now')\nUNION\nALL\nSELECT\n 3,\n 'Decimal',\n 'AVG_JOBS_PER_MINUTE_PAST_HOUR',\n CAST(ROUND(COUNT(*) / 60.0, 2) AS REAL)\nFROM\n Jobs\nWHERE\n run_at >= strftime('%s', 'now', '-1 hour')\nUNION\nALL\nSELECT\n 4,\n 'Number',\n 'TOTAL_JOBS',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nUNION\nALL\nSELECT\n 4,\n 'Number',\n 'DONE_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Done' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 4,\n 'Number',\n 'COMPLETED_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status IN ('Done', 'Failed', 'Killed') THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 4,\n 'Number',\n 'KILLED_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 4,\n 'Percentage',\n 'SUCCESS_RATE',\n CAST(\n ROUND(\n 100.0 * SUM(\n CASE\n WHEN status = 'Done' THEN 1\n ELSE 0\n END\n ) / NULLIF(COUNT(*), 0),\n 2\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 5,\n 'Decimal',\n 'AVG_JOB_DURATION_MINS',\n CAST(ROUND(AVG((done_at - run_at) / 60.0), 2) AS REAL)\nFROM\n Jobs\nWHERE\n status IN ('Done', 'Failed', 'Killed')\n AND done_at IS NOT NULL\nUNION\nALL\nSELECT\n 5,\n 'Decimal',\n 'LONGEST_RUNNING_JOB_MINS',\n CAST(\n ROUND(\n MAX(\n CASE\n WHEN status = 'Running' THEN (strftime('%s', 'now') - run_at) / 60.0\n ELSE 0\n END\n ),\n 2\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 5,\n 'Number',\n 'QUEUE_BACKLOG',\n CAST(\n SUM(\n CASE\n WHEN status = 'Pending'\n AND run_at <= strftime('%s', 'now') THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nUNION\nALL\nSELECT\n 6,\n 'Number',\n 'JOBS_PAST_24_HOURS',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n run_at >= strftime('%s', 'now', '-1 day')\nUNION\nALL\nSELECT\n 6,\n 'Number',\n 'JOBS_PAST_7_DAYS',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n run_at >= strftime('%s', 'now', '-7 days')\nUNION\nALL\nSELECT\n 6,\n 'Number',\n 'KILLED_JOBS_PAST_7_DAYS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n run_at >= strftime('%s', 'now', '-7 days')\nUNION\nALL\nSELECT\n 6,\n 'Percentage',\n 'SUCCESS_RATE_PAST_24H',\n CAST(\n ROUND(\n 100.0 * SUM(\n CASE\n WHEN status = 'Done' THEN 1\n ELSE 0\n END\n ) / NULLIF(COUNT(*), 0),\n 2\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n run_at >= strftime('%s', 'now', '-1 day')\nUNION\nALL\nSELECT\n 7,\n 'Decimal',\n 'AVG_JOBS_PER_HOUR_PAST_24H',\n CAST(ROUND(COUNT(*) / 24.0, 2) AS REAL)\nFROM\n Jobs\nWHERE\n run_at >= strftime('%s', 'now', '-1 day')\nUNION\nALL\nSELECT\n 7,\n 'Decimal',\n 'AVG_JOBS_PER_DAY_PAST_7D',\n CAST(ROUND(COUNT(*) / 7.0, 2) AS REAL)\nFROM\n Jobs\nWHERE\n run_at >= strftime('%s', 'now', '-7 days')\nUNION\nALL\nSELECT\n 8,\n 'Timestamp',\n 'MOST_RECENT_JOB',\n CAST(MAX(run_at) AS REAL)\nFROM\n Jobs\nUNION\nALL\nSELECT\n 8,\n 'Timestamp',\n 'OLDEST_PENDING_JOB',\n CAST(MIN(run_at) AS REAL)\nFROM\n Jobs\nWHERE\n status = 'Pending'\n AND run_at <= strftime('%s', 'now')\nUNION\nALL\nSELECT\n 8,\n 'Number',\n 'PEAK_HOUR_JOBS',\n CAST(MAX(hourly_count) AS REAL)\nFROM\n (\n SELECT\n COUNT(*) as hourly_count\n FROM\n Jobs\n WHERE\n run_at >= strftime('%s', 'now', '-1 day')\n GROUP BY\n strftime('%H', run_at, 'unixepoch')\n )\nUNION\nALL\nSELECT\n 9,\n 'Number',\n 'DB_PAGE_SIZE',\n CAST(page_size AS REAL)\nFROM\n pragma_page_size()\nUNION\nALL\nSELECT\n 9,\n 'Number',\n 'DB_PAGE_COUNT',\n CAST(page_count AS REAL)\nFROM\n pragma_page_count()\nUNION\nALL\nSELECT\n 9,\n 'Number',\n 'DB_SIZE',\n CAST(page_size * page_count AS REAL)\nFROM\n pragma_page_size(),\n pragma_page_count()\nORDER BY\n priority,\n statistic;\n", + "describe": { + "columns": [ + { + "name": "priority", + "ordinal": 0, + "type_info": "Integer" + }, + { + "name": "type", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "statistic", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "value", + "ordinal": 3, + "type_info": "Float" + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "0c51f68a6dc3a43da8d955530e02acf90e2a3710690f398f9ffaa71e180f1d9e" +} diff --git a/.sqlx/query-1cd760004a2341bbded38a4fa431eaa74232f6f6f3121c7086a0b138195e9b0d.json b/.sqlx/query-1cd760004a2341bbded38a4fa431eaa74232f6f6f3121c7086a0b138195e9b0d.json new file mode 100644 index 0000000..92182f9 --- /dev/null +++ b/.sqlx/query-1cd760004a2341bbded38a4fa431eaa74232f6f6f3121c7086a0b138195e9b0d.json @@ -0,0 +1,86 @@ +{ + "db_name": "SQLite", + "query": "SELECT\n *\nFROM\n Jobs\nWHERE\n status = ?1\n AND job_type = ?2\nORDER BY\n done_at DESC,\n run_at DESC\nLIMIT\n ?3 OFFSET ?4\n", + "describe": { + "columns": [ + { + "name": "job", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "id", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "job_type", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "status", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "attempts", + "ordinal": 4, + "type_info": "Integer" + }, + { + "name": "max_attempts", + "ordinal": 5, + "type_info": "Integer" + }, + { + "name": "run_at", + "ordinal": 6, + "type_info": "Integer" + }, + { + "name": "last_result", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "lock_at", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "lock_by", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "done_at", + "ordinal": 10, + "type_info": "Integer" + }, + { + "name": "priority", + "ordinal": 11, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 4 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + false + ] + }, + "hash": "1cd760004a2341bbded38a4fa431eaa74232f6f6f3121c7086a0b138195e9b0d" +} diff --git a/.sqlx/query-40a420986a37c9db2fc35b161092d6063f186242ca1808cfae9d6477c7d8687b.json b/.sqlx/query-40a420986a37c9db2fc35b161092d6063f186242ca1808cfae9d6477c7d8687b.json new file mode 100644 index 0000000..bab3c7a --- /dev/null +++ b/.sqlx/query-40a420986a37c9db2fc35b161092d6063f186242ca1808cfae9d6477c7d8687b.json @@ -0,0 +1,86 @@ +{ + "db_name": "SQLite", + "query": "SELECT\n *\nFROM\n Jobs\nWHERE\n status = ?1\nORDER BY\n done_at DESC,\n run_at DESC\nLIMIT\n ?2 OFFSET ?3\n", + "describe": { + "columns": [ + { + "name": "job", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "id", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "job_type", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "status", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "attempts", + "ordinal": 4, + "type_info": "Integer" + }, + { + "name": "max_attempts", + "ordinal": 5, + "type_info": "Integer" + }, + { + "name": "run_at", + "ordinal": 6, + "type_info": "Integer" + }, + { + "name": "last_result", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "lock_at", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "lock_by", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "done_at", + "ordinal": 10, + "type_info": "Integer" + }, + { + "name": "priority", + "ordinal": 11, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 3 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + false + ] + }, + "hash": "40a420986a37c9db2fc35b161092d6063f186242ca1808cfae9d6477c7d8687b" +} diff --git a/.sqlx/query-41198057d67f5386d366f332c349af4519adf779487e4d2e8b7137b79db1b5e7.json b/.sqlx/query-41198057d67f5386d366f332c349af4519adf779487e4d2e8b7137b79db1b5e7.json new file mode 100644 index 0000000..f0c561c --- /dev/null +++ b/.sqlx/query-41198057d67f5386d366f332c349af4519adf779487e4d2e8b7137b79db1b5e7.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "UPDATE\n Jobs\nSET\n status = 'Running',\n lock_at = strftime('%s', 'now'),\n lock_by = ?2\nWHERE\n id = ?1\n AND (\n status = 'Queued'\n OR status = 'Pending'\n OR (\n status = 'Failed'\n AND attempts < max_attempts\n )\n )\n", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + }, + "hash": "41198057d67f5386d366f332c349af4519adf779487e4d2e8b7137b79db1b5e7" +} diff --git a/.sqlx/query-41ce7e67750c304b39bb88def9c4dd6e8a3e7ea31ef8d622534a9fa0093c31e2.json b/.sqlx/query-41ce7e67750c304b39bb88def9c4dd6e8a3e7ea31ef8d622534a9fa0093c31e2.json new file mode 100644 index 0000000..5c528f8 --- /dev/null +++ b/.sqlx/query-41ce7e67750c304b39bb88def9c4dd6e8a3e7ea31ef8d622534a9fa0093c31e2.json @@ -0,0 +1,50 @@ +{ + "db_name": "SQLite", + "query": "SELECT\n *\nFROM\n Workers\nORDER BY\n last_seen DESC\nLIMIT\n ?1 OFFSET ?2\n", + "describe": { + "columns": [ + { + "name": "id", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "worker_type", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "storage_name", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "layers", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "last_seen", + "ordinal": 4, + "type_info": "Integer" + }, + { + "name": "started_at", + "ordinal": 5, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 2 + }, + "nullable": [ + false, + false, + false, + true, + false, + true + ] + }, + "hash": "41ce7e67750c304b39bb88def9c4dd6e8a3e7ea31ef8d622534a9fa0093c31e2" +} diff --git a/.sqlx/query-42042a81a76b1f366a6f421acc329a245d0bb2b84595f87a883c60d6f120cf63.json b/.sqlx/query-42042a81a76b1f366a6f421acc329a245d0bb2b84595f87a883c60d6f120cf63.json new file mode 100644 index 0000000..9aef9ee --- /dev/null +++ b/.sqlx/query-42042a81a76b1f366a6f421acc329a245d0bb2b84595f87a883c60d6f120cf63.json @@ -0,0 +1,38 @@ +{ + "db_name": "SQLite", + "query": "WITH queue_stats AS (\n SELECT\n job_type,\n json_group_array(\n json_object(\n 'title',\n statistic,\n 'stat_type',\n type,\n 'value',\n value,\n 'priority',\n priority\n )\n ) as stats\n FROM\n (\n SELECT\n job_type,\n 1 AS priority,\n 'Number' AS type,\n 'RUNNING_JOBS' AS statistic,\n CAST(\n SUM(\n CASE\n WHEN status = 'Running' THEN 1\n ELSE 0\n END\n ) AS TEXT\n ) AS value\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 1,\n 'Number',\n 'PENDING_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Pending' THEN 1\n ELSE 0\n END\n ) AS TEXT\n )\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 1,\n 'Number',\n 'FAILED_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Failed' THEN 1\n ELSE 0\n END\n ) AS TEXT\n )\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 2,\n 'Number',\n 'ACTIVE_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status IN ('Pending', 'Queued', 'Running') THEN 1\n ELSE 0\n END\n ) AS TEXT\n )\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 2,\n 'Number',\n 'STALE_RUNNING_JOBS',\n CAST(COUNT(*) AS TEXT)\n FROM\n Jobs\n WHERE\n status = 'Running'\n AND run_at < strftime('%s', 'now', '-1 hour')\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 2,\n 'Percentage',\n 'KILL_RATE',\n CAST(\n ROUND(\n 100.0 * SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) / NULLIF(COUNT(*), 0),\n 2\n ) AS TEXT\n )\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 3,\n 'Number',\n 'JOBS_PAST_HOUR',\n CAST(COUNT(*) AS TEXT)\n FROM\n Jobs\n WHERE\n run_at >= strftime('%s', 'now', '-1 hour')\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 3,\n 'Number',\n 'JOBS_TODAY',\n CAST(COUNT(*) AS TEXT)\n FROM\n Jobs\n WHERE\n date(run_at, 'unixepoch') = date('now')\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 3,\n 'Number',\n 'KILLED_JOBS_TODAY',\n CAST(\n SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) AS TEXT\n )\n FROM\n Jobs\n WHERE\n date(run_at, 'unixepoch') = date('now')\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 3,\n 'Decimal',\n 'AVG_JOBS_PER_MINUTE_PAST_HOUR',\n CAST(ROUND(COUNT(*) / 60.0, 2) AS TEXT)\n FROM\n Jobs\n WHERE\n run_at >= strftime('%s', 'now', '-1 hour')\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 4,\n 'Number',\n 'TOTAL_JOBS',\n CAST(COUNT(*) AS TEXT)\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 4,\n 'Number',\n 'DONE_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Done' THEN 1\n ELSE 0\n END\n ) AS TEXT\n )\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 4,\n 'Number',\n 'KILLED_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) AS TEXT\n )\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 4,\n 'Percentage',\n 'SUCCESS_RATE',\n CAST(\n ROUND(\n 100.0 * SUM(\n CASE\n WHEN status = 'Done' THEN 1\n ELSE 0\n END\n ) / NULLIF(COUNT(*), 0),\n 2\n ) AS TEXT\n )\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 5,\n 'Decimal',\n 'AVG_JOB_DURATION_MINS',\n CAST(ROUND(AVG((done_at - run_at) / 60.0), 2) AS TEXT)\n FROM\n Jobs\n WHERE\n status IN ('Done', 'Failed', 'Killed')\n AND done_at IS NOT NULL\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 5,\n 'Decimal',\n 'LONGEST_RUNNING_JOB_MINS',\n CAST(\n ROUND(\n MAX(\n CASE\n WHEN status = 'Running' THEN (strftime('%s', 'now') - run_at) / 60.0\n ELSE 0\n END\n ),\n 2\n ) AS TEXT\n )\n FROM\n Jobs\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 6,\n 'Number',\n 'JOBS_PAST_7_DAYS',\n CAST(COUNT(*) AS TEXT)\n FROM\n Jobs\n WHERE\n run_at >= strftime('%s', 'now', '-7 days')\n GROUP BY\n job_type\n UNION\n ALL\n SELECT\n job_type,\n 8,\n 'Timestamp',\n 'MOST_RECENT_JOB',\n CAST(MAX(run_at) AS TEXT)\n FROM\n Jobs\n GROUP BY\n job_type\n ORDER BY\n job_type,\n priority,\n statistic\n )\n GROUP BY\n job_type\n),\nall_job_types AS (\n SELECT\n worker_type AS job_type\n FROM\n Workers\n UNION\n SELECT\n job_type\n FROM\n Jobs\n)\nSELECT\n jt.job_type as name,\n COALESCE(qs.stats, '[]') as stats,\n COALESCE(\n (\n SELECT\n json_group_array(DISTINCT lock_by)\n FROM\n Jobs\n WHERE\n job_type = jt.job_type\n AND lock_by IS NOT NULL\n ),\n '[]'\n ) as workers,\n COALESCE(\n (\n SELECT\n json_group_array(daily_count)\n FROM\n (\n SELECT\n COUNT(*) as daily_count\n FROM\n Jobs\n WHERE\n job_type = jt.job_type\n AND run_at >= strftime('%s', 'now', '-7 days')\n GROUP BY\n date(run_at, 'unixepoch')\n ORDER BY\n date(run_at, 'unixepoch')\n )\n ),\n '[]'\n ) as activity\nFROM\n all_job_types jt\n LEFT JOIN queue_stats qs ON jt.job_type = qs.job_type\nORDER BY\n name;\n", + "describe": { + "columns": [ + { + "name": "name", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "stats", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "workers", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "activity", + "ordinal": 3, + "type_info": "Text" + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "42042a81a76b1f366a6f421acc329a245d0bb2b84595f87a883c60d6f120cf63" +} diff --git a/.sqlx/query-4ead10dafe0d2d024654403f289c4db57308e5cf88f44efaf23b05c585626465.json b/.sqlx/query-4ead10dafe0d2d024654403f289c4db57308e5cf88f44efaf23b05c585626465.json new file mode 100644 index 0000000..e2ab422 --- /dev/null +++ b/.sqlx/query-4ead10dafe0d2d024654403f289c4db57308e5cf88f44efaf23b05c585626465.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "INSERT INTO\n Jobs\nVALUES\n (\n ?1,\n ?2,\n ?3,\n 'Pending',\n 0,\n ?4,\n ?5,\n NULL,\n NULL,\n NULL,\n NULL,\n ?6\n )\n", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + }, + "hash": "4ead10dafe0d2d024654403f289c4db57308e5cf88f44efaf23b05c585626465" +} diff --git a/.sqlx/query-61843a18bffdee192cd01f1537f0f03d75403970fc8347d0f017f04d746b2b98.json b/.sqlx/query-61843a18bffdee192cd01f1537f0f03d75403970fc8347d0f017f04d746b2b98.json new file mode 100644 index 0000000..194ba6c --- /dev/null +++ b/.sqlx/query-61843a18bffdee192cd01f1537f0f03d75403970fc8347d0f017f04d746b2b98.json @@ -0,0 +1,86 @@ +{ + "db_name": "SQLite", + "query": "UPDATE Jobs\nSET\n status = 'Queued',\n lock_at = strftime('%s', 'now')\nWHERE ROWID IN (\n SELECT ROWID\n FROM Jobs\n WHERE job_type IN (\n SELECT value FROM json_each(?1)\n )\n AND status = 'Pending'\n AND lock_by IS NULL\n AND (\n run_at IS NULL\n OR run_at <= strftime('%s', 'now')\n )\n AND ROWID IN (\n SELECT value FROM json_each(?2)\n )\n ORDER BY\n priority DESC,\n run_at ASC,\n id ASC\n LIMIT ?3\n)\nRETURNING *;\n", + "describe": { + "columns": [ + { + "name": "job", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "id", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "job_type", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "status", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "attempts", + "ordinal": 4, + "type_info": "Integer" + }, + { + "name": "max_attempts", + "ordinal": 5, + "type_info": "Integer" + }, + { + "name": "run_at", + "ordinal": 6, + "type_info": "Integer" + }, + { + "name": "last_result", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "lock_at", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "lock_by", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "done_at", + "ordinal": 10, + "type_info": "Integer" + }, + { + "name": "priority", + "ordinal": 11, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 3 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + false + ] + }, + "hash": "61843a18bffdee192cd01f1537f0f03d75403970fc8347d0f017f04d746b2b98" +} diff --git a/.sqlx/query-75c53a93669a7ac2141b1bb62af8edce306564754651c4098f8dfe693195f6ca.json b/.sqlx/query-75c53a93669a7ac2141b1bb62af8edce306564754651c4098f8dfe693195f6ca.json new file mode 100644 index 0000000..2d8ca07 --- /dev/null +++ b/.sqlx/query-75c53a93669a7ac2141b1bb62af8edce306564754651c4098f8dfe693195f6ca.json @@ -0,0 +1,38 @@ +{ + "db_name": "SQLite", + "query": "SELECT\n 1 AS priority,\n 'Number' AS type,\n 'RUNNING_JOBS' AS statistic,\n CAST(\n SUM(\n CASE\n WHEN status = 'Running' THEN 1\n ELSE 0\n END\n ) AS REAL\n ) AS value\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 1,\n 'Number',\n 'PENDING_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Pending' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 2,\n 'Number',\n 'FAILED_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Failed' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 2,\n 'Number',\n 'ACTIVE_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status IN ('Pending', 'Queued', 'Running') THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 2,\n 'Number',\n 'STALE_RUNNING_JOBS',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND status = 'Running'\n AND run_at < strftime('%s', 'now', '-1 hour')\nUNION\nALL\nSELECT\n 2,\n 'Percentage',\n 'KILL_RATE',\n CAST(\n ROUND(\n 100.0 * SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) / NULLIF(COUNT(*), 0),\n 2\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 3,\n 'Number',\n 'JOBS_PAST_HOUR',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND run_at >= strftime('%s', 'now', '-1 hour')\nUNION\nALL\nSELECT\n 3,\n 'Number',\n 'JOBS_TODAY',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND date(run_at, 'unixepoch') = date('now')\nUNION\nALL\nSELECT\n 3,\n 'Number',\n 'KILLED_JOBS_TODAY',\n CAST(\n SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND date(run_at, 'unixepoch') = date('now')\nUNION\nALL\nSELECT\n 3,\n 'Decimal',\n 'AVG_JOBS_PER_MINUTE_PAST_HOUR',\n CAST(ROUND(COUNT(*) / 60.0, 2) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND run_at >= strftime('%s', 'now', '-1 hour')\nUNION\nALL\nSELECT\n 4,\n 'Number',\n 'TOTAL_JOBS',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 4,\n 'Number',\n 'DONE_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Done' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 4,\n 'Number',\n 'COMPLETED_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status IN ('Done', 'Failed', 'Killed') THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 4,\n 'Number',\n 'KILLED_JOBS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 4,\n 'Percentage',\n 'SUCCESS_RATE',\n CAST(\n ROUND(\n 100.0 * SUM(\n CASE\n WHEN status = 'Done' THEN 1\n ELSE 0\n END\n ) / NULLIF(COUNT(*), 0),\n 2\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 5,\n 'Decimal',\n 'AVG_JOB_DURATION_MINS',\n CAST(ROUND(AVG((done_at - run_at) / 60.0), 2) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND status IN ('Done', 'Failed', 'Killed')\n AND done_at IS NOT NULL\nUNION\nALL\nSELECT\n 5,\n 'Decimal',\n 'LONGEST_RUNNING_JOB_MINS',\n CAST(\n ROUND(\n MAX(\n CASE\n WHEN status = 'Running' THEN (strftime('%s', 'now') - run_at) / 60.0\n ELSE 0\n END\n ),\n 2\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 5,\n 'Number',\n 'QUEUE_BACKLOG',\n CAST(\n SUM(\n CASE\n WHEN status = 'Pending'\n AND run_at <= strftime('%s', 'now') THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 6,\n 'Number',\n 'JOBS_PAST_24_HOURS',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND run_at >= strftime('%s', 'now', '-1 day')\nUNION\nALL\nSELECT\n 6,\n 'Number',\n 'JOBS_PAST_7_DAYS',\n CAST(COUNT(*) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND run_at >= strftime('%s', 'now', '-7 days')\nUNION\nALL\nSELECT\n 6,\n 'Number',\n 'KILLED_JOBS_PAST_7_DAYS',\n CAST(\n SUM(\n CASE\n WHEN status = 'Killed' THEN 1\n ELSE 0\n END\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND run_at >= strftime('%s', 'now', '-7 days')\nUNION\nALL\nSELECT\n 6,\n 'Percentage',\n 'SUCCESS_RATE_PAST_24H',\n CAST(\n ROUND(\n 100.0 * SUM(\n CASE\n WHEN status = 'Done' THEN 1\n ELSE 0\n END\n ) / NULLIF(COUNT(*), 0),\n 2\n ) AS REAL\n )\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND run_at >= strftime('%s', 'now', '-1 day')\nUNION\nALL\nSELECT\n 7,\n 'Decimal',\n 'AVG_JOBS_PER_HOUR_PAST_24H',\n CAST(ROUND(COUNT(*) / 24.0, 2) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND run_at >= strftime('%s', 'now', '-1 day')\nUNION\nALL\nSELECT\n 7,\n 'Decimal',\n 'AVG_JOBS_PER_DAY_PAST_7D',\n CAST(ROUND(COUNT(*) / 7.0, 2) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND run_at >= strftime('%s', 'now', '-7 days')\nUNION\nALL\nSELECT\n 8,\n 'Timestamp',\n 'MOST_RECENT_JOB',\n CAST(MAX(run_at) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\nUNION\nALL\nSELECT\n 8,\n 'Timestamp',\n 'OLDEST_PENDING_JOB',\n CAST(MIN(run_at) AS REAL)\nFROM\n Jobs\nWHERE\n job_type = ?1\n AND status = 'Pending'\n AND run_at <= strftime('%s', 'now')\nUNION\nALL\nSELECT\n 8,\n 'Number',\n 'PEAK_HOUR_JOBS',\n CAST(MAX(hourly_count) AS REAL)\nFROM\n (\n SELECT\n COUNT(*) as hourly_count\n FROM\n Jobs\n WHERE\n job_type = ?1\n AND run_at >= strftime('%s', 'now', '-1 day')\n GROUP BY\n strftime('%H', run_at, 'unixepoch')\n )\nORDER BY\n priority,\n statistic;\n", + "describe": { + "columns": [ + { + "name": "priority", + "ordinal": 0, + "type_info": "Integer" + }, + { + "name": "type", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "statistic", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "value", + "ordinal": 3, + "type_info": "Float" + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "75c53a93669a7ac2141b1bb62af8edce306564754651c4098f8dfe693195f6ca" +} diff --git a/.sqlx/query-7bd7de027caa72c93f7f89bd24a5a78685df4c262c2a13eafd9803c3672dcd10.json b/.sqlx/query-7bd7de027caa72c93f7f89bd24a5a78685df4c262c2a13eafd9803c3672dcd10.json new file mode 100644 index 0000000..8be601e --- /dev/null +++ b/.sqlx/query-7bd7de027caa72c93f7f89bd24a5a78685df4c262c2a13eafd9803c3672dcd10.json @@ -0,0 +1,50 @@ +{ + "db_name": "SQLite", + "query": "SELECT\n *\nFROM\n Workers\nWHERE\n worker_type = ?1\nORDER BY\n last_seen DESC\nLIMIT\n ?2 OFFSET ?3\n", + "describe": { + "columns": [ + { + "name": "id", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "worker_type", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "storage_name", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "layers", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "last_seen", + "ordinal": 4, + "type_info": "Integer" + }, + { + "name": "started_at", + "ordinal": 5, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 3 + }, + "nullable": [ + false, + false, + false, + true, + false, + true + ] + }, + "hash": "7bd7de027caa72c93f7f89bd24a5a78685df4c262c2a13eafd9803c3672dcd10" +} diff --git a/.sqlx/query-90406ca41cb79e68f66a300730f85ad7b0a3ddc713341adcbbc959021082c8e9.json b/.sqlx/query-90406ca41cb79e68f66a300730f85ad7b0a3ddc713341adcbbc959021082c8e9.json new file mode 100644 index 0000000..6dbad87 --- /dev/null +++ b/.sqlx/query-90406ca41cb79e68f66a300730f85ad7b0a3ddc713341adcbbc959021082c8e9.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "UPDATE\n Jobs\nSET\n status = ?4,\n attempts = ?2,\n last_result = ?3,\n done_at = strftime('%s', 'now')\nWHERE\n id = ?1\n AND lock_by = ?5\n", + "describe": { + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + }, + "hash": "90406ca41cb79e68f66a300730f85ad7b0a3ddc713341adcbbc959021082c8e9" +} diff --git a/.sqlx/query-a43052e877930a522d6a63789935653f4fe06d10361c555f8ade4b65589520bc.json b/.sqlx/query-a43052e877930a522d6a63789935653f4fe06d10361c555f8ade4b65589520bc.json new file mode 100644 index 0000000..cb5c3e6 --- /dev/null +++ b/.sqlx/query-a43052e877930a522d6a63789935653f4fe06d10361c555f8ade4b65589520bc.json @@ -0,0 +1,86 @@ +{ + "db_name": "SQLite", + "query": "SELECT\n *\nFROM\n Jobs\nWHERE\n id = ?1\nLIMIT\n 1;\n", + "describe": { + "columns": [ + { + "name": "job", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "id", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "job_type", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "status", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "attempts", + "ordinal": 4, + "type_info": "Integer" + }, + { + "name": "max_attempts", + "ordinal": 5, + "type_info": "Integer" + }, + { + "name": "run_at", + "ordinal": 6, + "type_info": "Integer" + }, + { + "name": "last_result", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "lock_at", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "lock_by", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "done_at", + "ordinal": 10, + "type_info": "Integer" + }, + { + "name": "priority", + "ordinal": 11, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + false + ] + }, + "hash": "a43052e877930a522d6a63789935653f4fe06d10361c555f8ade4b65589520bc" +} diff --git a/.sqlx/query-b44b1058179869ec68916630635a9f1318e707f1ee1acd8662952d0e0e67200b.json b/.sqlx/query-b44b1058179869ec68916630635a9f1318e707f1ee1acd8662952d0e0e67200b.json new file mode 100644 index 0000000..a656bce --- /dev/null +++ b/.sqlx/query-b44b1058179869ec68916630635a9f1318e707f1ee1acd8662952d0e0e67200b.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "INSERT INTO\n Workers (id, worker_type, storage_name, layers, last_seen, started_at)\nSELECT\n ?1,\n ?2,\n ?3,\n ?4,\n strftime('%s', 'now'),\n strftime('%s', 'now')\nWHERE\n NOT EXISTS (\n SELECT\n 1\n FROM\n Workers\n WHERE\n id = ?1\n AND strftime('%s', 'now') - last_seen >= ?5\n );\n", + "describe": { + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + }, + "hash": "b44b1058179869ec68916630635a9f1318e707f1ee1acd8662952d0e0e67200b" +} diff --git a/.sqlx/query-d2a2123b8ede2014f106836cf41b7ec3470a737fa5174e7a2aa16dcf16fc5576.json b/.sqlx/query-d2a2123b8ede2014f106836cf41b7ec3470a737fa5174e7a2aa16dcf16fc5576.json new file mode 100644 index 0000000..d2a9814 --- /dev/null +++ b/.sqlx/query-d2a2123b8ede2014f106836cf41b7ec3470a737fa5174e7a2aa16dcf16fc5576.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "UPDATE\n Jobs\nSET\n status = \"Pending\",\n done_at = NULL,\n lock_by = NULL,\n lock_at = NULL,\n attempts = attempts + 1,\n last_result = '{\"Err\": \"Re-enqueued due to worker heartbeat timeout.\"}'\nWHERE\n id IN (\n SELECT\n Jobs.id\n FROM\n Jobs\n INNER JOIN Workers ON lock_by = Workers.id\n WHERE\n (\n status = \"Running\"\n OR status = \"Queued\"\n )\n AND strftime('%s', 'now') - Workers.last_seen >= ?1\n AND Workers.worker_type = ?2\n );\n", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + }, + "hash": "d2a2123b8ede2014f106836cf41b7ec3470a737fa5174e7a2aa16dcf16fc5576" +} diff --git a/.sqlx/query-d7aefe54cd7388c208fff5b946390f217b575f0ca464a5faddd0fe2d51793983.json b/.sqlx/query-d7aefe54cd7388c208fff5b946390f217b575f0ca464a5faddd0fe2d51793983.json new file mode 100644 index 0000000..d5f2525 --- /dev/null +++ b/.sqlx/query-d7aefe54cd7388c208fff5b946390f217b575f0ca464a5faddd0fe2d51793983.json @@ -0,0 +1,86 @@ +{ + "db_name": "SQLite", + "query": "UPDATE Jobs\nSET\n status = 'Queued',\n lock_by = ?1,\n lock_at = strftime('%s', 'now')\nWHERE\n ROWID IN (\n SELECT ROWID\n FROM Jobs\n WHERE job_type = ?2\n AND (\n (status = 'Pending' AND lock_by IS NULL) \n OR \n (status = 'Failed' AND attempts < max_attempts)\n )\n AND (run_at IS NULL OR run_at <= strftime('%s', 'now'))\n ORDER BY priority DESC, run_at ASC, id ASC\n LIMIT ?3\n )\nRETURNING *\n", + "describe": { + "columns": [ + { + "name": "job", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "id", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "job_type", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "status", + "ordinal": 3, + "type_info": "Text" + }, + { + "name": "attempts", + "ordinal": 4, + "type_info": "Integer" + }, + { + "name": "max_attempts", + "ordinal": 5, + "type_info": "Integer" + }, + { + "name": "run_at", + "ordinal": 6, + "type_info": "Integer" + }, + { + "name": "last_result", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "lock_at", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "lock_by", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "done_at", + "ordinal": 10, + "type_info": "Integer" + }, + { + "name": "priority", + "ordinal": 11, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 3 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + false + ] + }, + "hash": "d7aefe54cd7388c208fff5b946390f217b575f0ca464a5faddd0fe2d51793983" +} diff --git a/.sqlx/query-f4e22035b9e6b6c274b326c7a25b72909f4e44f49d068fb64c0b40a0b017f22e.json b/.sqlx/query-f4e22035b9e6b6c274b326c7a25b72909f4e44f49d068fb64c0b40a0b017f22e.json new file mode 100644 index 0000000..44b9bdb --- /dev/null +++ b/.sqlx/query-f4e22035b9e6b6c274b326c7a25b72909f4e44f49d068fb64c0b40a0b017f22e.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "UPDATE\n Workers\nSET\n last_seen = strftime('%s', 'now')\nWHERE\n id = $1 AND worker_type = $2;\n", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + }, + "hash": "f4e22035b9e6b6c274b326c7a25b72909f4e44f49d068fb64c0b40a0b017f22e" +} diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..61082ba --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,54 @@ +[package] +name = "apalis-sqlite" +version = "1.0.0-alpha.1" +authors = ["Njuguna Mureithi "] +readme = "README.md" +edition = "2024" +repository = "https://github.com/apalis-dev/apalis-sqlite" +license = "MIT" +description = "Background task processing for rust using apalis and sqlite" + +[features] +default = ["migrate", "json"] +migrate = ["sqlx/migrate", "sqlx/macros"] +async-std-comp = ["async-std", "sqlx/runtime-async-std-rustls"] +async-std-comp-native-tls = ["async-std", "sqlx/runtime-async-std-native-tls"] +tokio-comp = ["tokio", "sqlx/runtime-tokio-rustls"] +tokio-comp-native-tls = ["tokio", "sqlx/runtime-tokio-native-tls"] +bytes = [] +json = ["sqlx/json"] + +[dependencies.sqlx] +version = "0.8.6" +default-features = false +features = ["chrono", "sqlite"] + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1" } +apalis-core = { version = "1.0.0-alpha.4", default-features = false, features = [ + "sleep", + "json", +] } +log = "0.4.21" +futures = "0.3.30" +tokio = { version = "1", features = ["rt", "net"], optional = true } +async-std = { version = "1.13.0", optional = true } +chrono = { version = "0.4", features = ["serde"] } +thiserror = "2.0.0" +pin-project = "1.1.10" +libsqlite3-sys = "0.30.1" +ulid = { version = "1", features = ["serde"] } +tower-layer = "0.3.3" +tower-service = "0.3.3" +bytes = "1.1.0" + +[dev-dependencies] +tokio = { version = "1", features = ["macros", "rt-multi-thread"] } +apalis-core = { version = "1.0.0-alpha.4", features = ["test-utils"] } +apalis-sqlite = { path = ".", features = ["migrate", "tokio-comp"] } + +[package.metadata.docs.rs] +# defines the configuration attribute `docsrs` +rustdoc-args = ["--cfg", "docsrs"] +all-features = true diff --git a/README.md b/README.md index 11e3580..f0166c3 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,117 @@ # apalis-sqlite -Background task processing for rust using apalis and sqlite + +Background task processing for Rust using Apalis and SQLite. + +## Features + +- **Reliable job queue** using SQLite as the backend. +- **Multiple storage types**: standard polling and event-driven (hooked) storage. +- **Custom codecs** for serializing/deserializing job arguments as bytes and json. +- **Heartbeat and orphaned job re-enqueueing** for robust job processing. +- **Integration with Apalis workers and middleware.** + +## Storage Types + +- [`SqliteStorage`]: Standard polling-based storage. +- [`SqliteStorageWithHook`]: Event-driven storage using SQLite update hooks for low-latency job fetching. +- [`SharedSqliteStorage`]: Shared storage for multiple job types. + +The naming is designed to clearly indicate the storage mechanism and its capabilities, but under the hood the result is the `SqliteStorage` struct with different configurations. + +## Examples + +### Basic Worker Example + +```rust,no_run +#[tokio::main] +async fn main() { + let pool = SqlitePool::connect(":memory:").await.unwrap(); + SqliteStorage::setup(&pool).await.unwrap(); + let mut backend = SqliteStorage::new(&pool); // With default config + + let mut start = 0; + let mut items = stream::repeat_with(move || { + start += 1; + let task = Task::builder(start) + .run_after(Duration::from_secs(1)) + .with_ctx(SqliteContext::new().with_priority(1)) + .build(); + Ok(task) + }) + .take(10); + backend.send_all(&mut items).await.unwrap(); + + async fn send_reminder(item: usize, wrk: WorkerContext) -> Result<(), BoxDynError> { + Ok(()) + } + + let worker = apalis_core::worker::builder::WorkerBuilder::new("worker-1") + .backend(backend) + .build(send_reminder); + worker.run().await.unwrap(); +} +``` + +### Hooked Worker Example (Event-driven) + +```rust,no_run + +#[tokio::main] +async fn main() { + let pool = SqlitePool::connect(":memory:").await.unwrap(); + SqliteStorage::setup(&pool).await.unwrap(); + + let lazy_strategy = StrategyBuilder::new() + .apply(IntervalStrategy::new(Duration::from_secs(5))) + .build(); + let config = Config::new("queue") + .with_poll_interval(lazy_strategy) + .set_buffer_size(5); + let backend = SqliteStorage::new_with_callback(&pool, &config).await; + + tokio::spawn({ + let pool = pool.clone(); + let config = config.clone(); + async move { + tokio::time::sleep(Duration::from_secs(2)).await; + let mut start = 0; + let items = stream::repeat_with(move || { + start += 1; + Task::builder(serde_json::to_value(&start).unwrap()) + .run_after(Duration::from_secs(1)) + .with_ctx(SqliteContext::new().with_priority(start)) + .build() + }) + .take(20) + .collect::>() + .await; + apalis_sqlite::sink::push_tasks(pool, config, items).await.unwrap(); + } + }); + + async fn send_reminder(item: usize, wrk: WorkerContext) -> Result<(), BoxDynError> { + Ok(()) + } + + let worker = apalis_core::worker::builder::WorkerBuilder::new("worker-2") + .backend(backend) + .build(send_reminder); + worker.run().await.unwrap(); +} +``` + +## Migrations + +If the `migrate` feature is enabled, you can run built-in migrations with: + +```rust,no_run +use sqlx::SqlitePool; +#[tokio::main] async fn main() { + let pool = SqlitePool::connect(":memory:").await.unwrap(); + apalis_sqlite::SqliteStorage::setup(&pool).await.unwrap(); +} +``` + +## License + +Licensed under either of Apache License, Version 2.0 or MIT license at your option. diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..3a8149e --- /dev/null +++ b/build.rs @@ -0,0 +1,3 @@ +fn main() { + println!("cargo:rerun-if-changed=migrations"); +} diff --git a/migrations/bytes/20251017141210_initial_bytes_storage.sql b/migrations/bytes/20251017141210_initial_bytes_storage.sql new file mode 100644 index 0000000..97836d8 --- /dev/null +++ b/migrations/bytes/20251017141210_initial_bytes_storage.sql @@ -0,0 +1,63 @@ +CREATE TABLE IF NOT EXISTS Workers ( + id TEXT NOT NULL UNIQUE, + worker_type TEXT NOT NULL, + storage_name TEXT NOT NULL, + layers TEXT, + last_seen INTEGER NOT NULL DEFAULT (strftime('%s', 'now')) +); + +CREATE INDEX IF NOT EXISTS Idx ON Workers(id); + +CREATE INDEX IF NOT EXISTS WTIdx ON Workers(worker_type); + +CREATE INDEX IF NOT EXISTS LSIdx ON Workers(last_seen); + +CREATE TABLE IF NOT EXISTS Jobs ( + job BLOB NOT NULL, + id TEXT NOT NULL UNIQUE, + job_type TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'Pending', + attempts INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 25, + run_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')), + last_result TEXT, + lock_at INTEGER, + lock_by TEXT, + done_at INTEGER, + FOREIGN KEY(lock_by) REFERENCES Workers(id) +); + +CREATE INDEX IF NOT EXISTS TIdx ON Jobs(id); + +CREATE INDEX IF NOT EXISTS SIdx ON Jobs(status); + +CREATE INDEX IF NOT EXISTS LIdx ON Jobs(lock_by); + +CREATE INDEX IF NOT EXISTS JTIdx ON Jobs(job_type); + + +ALTER TABLE Jobs +ADD priority INTEGER NOT NULL DEFAULT 0; + +CREATE INDEX IF NOT EXISTS idx_jobs_job_type_status_run_at ON Jobs(job_type, status, run_at); + +CREATE INDEX IF NOT EXISTS idx_jobs_status_run_at ON Jobs(status, run_at); + +CREATE INDEX IF NOT EXISTS idx_jobs_run_at_status ON Jobs(run_at, status); + +CREATE INDEX IF NOT EXISTS idx_jobs_completed_done_at ON Jobs(status, done_at, run_at) +WHERE + status IN ('Done', 'Failed', 'Killed') + AND done_at IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_jobs_pending ON Jobs(run_at) +WHERE + status = 'Pending'; + +CREATE INDEX IF NOT EXISTS idx_jobs_running ON Jobs(run_at) +WHERE + status = 'Running'; + +CREATE INDEX IF NOT EXISTS idx_jobs_job_type_run_at ON Jobs(job_type, run_at); + +CREATE INDEX IF NOT EXISTS idx_jobs_job_type_covering ON Jobs(job_type, status, run_at, done_at); diff --git a/migrations/bytes/20251017162501_worker_started_at.sql b/migrations/bytes/20251017162501_worker_started_at.sql new file mode 100644 index 0000000..1ef26e5 --- /dev/null +++ b/migrations/bytes/20251017162501_worker_started_at.sql @@ -0,0 +1,4 @@ +ALTER TABLE + Workers +ADD + COLUMN started_at INTEGER; diff --git a/migrations/json/20220530084123_jobs_workers.sql b/migrations/json/20220530084123_jobs_workers.sql new file mode 100644 index 0000000..5883b48 --- /dev/null +++ b/migrations/json/20220530084123_jobs_workers.sql @@ -0,0 +1,36 @@ +CREATE TABLE IF NOT EXISTS Workers ( + id TEXT NOT NULL UNIQUE, + worker_type TEXT NOT NULL, + storage_name TEXT NOT NULL, + layers TEXT, + last_seen INTEGER NOT NULL DEFAULT (strftime('%s', 'now')) +); + +CREATE INDEX IF NOT EXISTS Idx ON Workers(id); + +CREATE INDEX IF NOT EXISTS WTIdx ON Workers(worker_type); + +CREATE INDEX IF NOT EXISTS LSIdx ON Workers(last_seen); + +CREATE TABLE IF NOT EXISTS Jobs ( + job TEXT NOT NULL, + id TEXT NOT NULL UNIQUE, + job_type TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'Pending', + attempts INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 25, + run_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')), + last_error TEXT, + lock_at INTEGER, + lock_by TEXT, + done_at INTEGER, + FOREIGN KEY(lock_by) REFERENCES Workers(id) +); + +CREATE INDEX IF NOT EXISTS TIdx ON Jobs(id); + +CREATE INDEX IF NOT EXISTS SIdx ON Jobs(status); + +CREATE INDEX IF NOT EXISTS LIdx ON Jobs(lock_by); + +CREATE INDEX IF NOT EXISTS JTIdx ON Jobs(job_type); \ No newline at end of file diff --git a/migrations/json/20250313213411_add_job_priority.sql b/migrations/json/20250313213411_add_job_priority.sql new file mode 100644 index 0000000..030fda7 --- /dev/null +++ b/migrations/json/20250313213411_add_job_priority.sql @@ -0,0 +1,2 @@ +ALTER TABLE Jobs +ADD priority INTEGER NOT NULL DEFAULT 0; diff --git a/migrations/json/20251013233016_stats_indexes.sql b/migrations/json/20251013233016_stats_indexes.sql new file mode 100644 index 0000000..9c029b9 --- /dev/null +++ b/migrations/json/20251013233016_stats_indexes.sql @@ -0,0 +1,22 @@ +CREATE INDEX IF NOT EXISTS idx_jobs_job_type_status_run_at ON Jobs(job_type, status, run_at); + +CREATE INDEX IF NOT EXISTS idx_jobs_status_run_at ON Jobs(status, run_at); + +CREATE INDEX IF NOT EXISTS idx_jobs_run_at_status ON Jobs(run_at, status); + +CREATE INDEX IF NOT EXISTS idx_jobs_completed_done_at ON Jobs(status, done_at, run_at) +WHERE + status IN ('Done', 'Failed', 'Killed') + AND done_at IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_jobs_pending ON Jobs(run_at) +WHERE + status = 'Pending'; + +CREATE INDEX IF NOT EXISTS idx_jobs_running ON Jobs(run_at) +WHERE + status = 'Running'; + +CREATE INDEX IF NOT EXISTS idx_jobs_job_type_run_at ON Jobs(job_type, run_at); + +CREATE INDEX IF NOT EXISTS idx_jobs_job_type_covering ON Jobs(job_type, status, run_at, done_at); diff --git a/migrations/json/20251017150712_rename_last_error.sql b/migrations/json/20251017150712_rename_last_error.sql new file mode 100644 index 0000000..f04d95b --- /dev/null +++ b/migrations/json/20251017150712_rename_last_error.sql @@ -0,0 +1 @@ +ALTER TABLE Jobs RENAME COLUMN last_error TO last_result; diff --git a/migrations/json/20251017162501_worker_started_at.sql b/migrations/json/20251017162501_worker_started_at.sql new file mode 100644 index 0000000..1ef26e5 --- /dev/null +++ b/migrations/json/20251017162501_worker_started_at.sql @@ -0,0 +1,4 @@ +ALTER TABLE + Workers +ADD + COLUMN started_at INTEGER; diff --git a/queries/backend/fetch_next.sql b/queries/backend/fetch_next.sql new file mode 100644 index 0000000..f2552f9 --- /dev/null +++ b/queries/backend/fetch_next.sql @@ -0,0 +1,20 @@ +UPDATE Jobs +SET + status = 'Queued', + lock_by = ?1, + lock_at = strftime('%s', 'now') +WHERE + ROWID IN ( + SELECT ROWID + FROM Jobs + WHERE job_type = ?2 + AND ( + (status = 'Pending' AND lock_by IS NULL) + OR + (status = 'Failed' AND attempts < max_attempts) + ) + AND (run_at IS NULL OR run_at <= strftime('%s', 'now')) + ORDER BY priority DESC, run_at ASC, id ASC + LIMIT ?3 + ) +RETURNING * diff --git a/queries/backend/fetch_next_shared.sql b/queries/backend/fetch_next_shared.sql new file mode 100644 index 0000000..80fd356 --- /dev/null +++ b/queries/backend/fetch_next_shared.sql @@ -0,0 +1,26 @@ +UPDATE Jobs +SET + status = 'Queued', + lock_at = strftime('%s', 'now') +WHERE ROWID IN ( + SELECT ROWID + FROM Jobs + WHERE job_type IN ( + SELECT value FROM json_each(?1) + ) + AND status = 'Pending' + AND lock_by IS NULL + AND ( + run_at IS NULL + OR run_at <= strftime('%s', 'now') + ) + AND ROWID IN ( + SELECT value FROM json_each(?2) + ) + ORDER BY + priority DESC, + run_at ASC, + id ASC + LIMIT ?3 +) +RETURNING *; diff --git a/queries/backend/keep_alive.sql b/queries/backend/keep_alive.sql new file mode 100644 index 0000000..a7984a9 --- /dev/null +++ b/queries/backend/keep_alive.sql @@ -0,0 +1,6 @@ +UPDATE + Workers +SET + last_seen = strftime('%s', 'now') +WHERE + id = $1 AND worker_type = $2; diff --git a/queries/backend/list_all_jobs.sql b/queries/backend/list_all_jobs.sql new file mode 100644 index 0000000..ad6c050 --- /dev/null +++ b/queries/backend/list_all_jobs.sql @@ -0,0 +1,11 @@ +SELECT + * +FROM + Jobs +WHERE + status = ?1 +ORDER BY + done_at DESC, + run_at DESC +LIMIT + ?2 OFFSET ?3 diff --git a/queries/backend/list_all_workers.sql b/queries/backend/list_all_workers.sql new file mode 100644 index 0000000..6215402 --- /dev/null +++ b/queries/backend/list_all_workers.sql @@ -0,0 +1,8 @@ +SELECT + * +FROM + Workers +ORDER BY + last_seen DESC +LIMIT + ?1 OFFSET ?2 diff --git a/queries/backend/list_jobs.sql b/queries/backend/list_jobs.sql new file mode 100644 index 0000000..bf0a3f5 --- /dev/null +++ b/queries/backend/list_jobs.sql @@ -0,0 +1,12 @@ +SELECT + * +FROM + Jobs +WHERE + status = ?1 + AND job_type = ?2 +ORDER BY + done_at DESC, + run_at DESC +LIMIT + ?3 OFFSET ?4 diff --git a/queries/backend/list_queues.sql b/queries/backend/list_queues.sql new file mode 100644 index 0000000..5515254 --- /dev/null +++ b/queries/backend/list_queues.sql @@ -0,0 +1,386 @@ +WITH queue_stats AS ( + SELECT + job_type, + json_group_array( + json_object( + 'title', + statistic, + 'stat_type', + type, + 'value', + value, + 'priority', + priority + ) + ) as stats + FROM + ( + SELECT + job_type, + 1 AS priority, + 'Number' AS type, + 'RUNNING_JOBS' AS statistic, + CAST( + SUM( + CASE + WHEN status = 'Running' THEN 1 + ELSE 0 + END + ) AS TEXT + ) AS value + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 1, + 'Number', + 'PENDING_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Pending' THEN 1 + ELSE 0 + END + ) AS TEXT + ) + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 1, + 'Number', + 'FAILED_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Failed' THEN 1 + ELSE 0 + END + ) AS TEXT + ) + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 2, + 'Number', + 'ACTIVE_JOBS', + CAST( + SUM( + CASE + WHEN status IN ('Pending', 'Queued', 'Running') THEN 1 + ELSE 0 + END + ) AS TEXT + ) + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 2, + 'Number', + 'STALE_RUNNING_JOBS', + CAST(COUNT(*) AS TEXT) + FROM + Jobs + WHERE + status = 'Running' + AND run_at < strftime('%s', 'now', '-1 hour') + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 2, + 'Percentage', + 'KILL_RATE', + CAST( + ROUND( + 100.0 * SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) / NULLIF(COUNT(*), 0), + 2 + ) AS TEXT + ) + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 3, + 'Number', + 'JOBS_PAST_HOUR', + CAST(COUNT(*) AS TEXT) + FROM + Jobs + WHERE + run_at >= strftime('%s', 'now', '-1 hour') + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 3, + 'Number', + 'JOBS_TODAY', + CAST(COUNT(*) AS TEXT) + FROM + Jobs + WHERE + date(run_at, 'unixepoch') = date('now') + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 3, + 'Number', + 'KILLED_JOBS_TODAY', + CAST( + SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) AS TEXT + ) + FROM + Jobs + WHERE + date(run_at, 'unixepoch') = date('now') + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 3, + 'Decimal', + 'AVG_JOBS_PER_MINUTE_PAST_HOUR', + CAST(ROUND(COUNT(*) / 60.0, 2) AS TEXT) + FROM + Jobs + WHERE + run_at >= strftime('%s', 'now', '-1 hour') + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 4, + 'Number', + 'TOTAL_JOBS', + CAST(COUNT(*) AS TEXT) + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 4, + 'Number', + 'DONE_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Done' THEN 1 + ELSE 0 + END + ) AS TEXT + ) + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 4, + 'Number', + 'KILLED_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) AS TEXT + ) + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 4, + 'Percentage', + 'SUCCESS_RATE', + CAST( + ROUND( + 100.0 * SUM( + CASE + WHEN status = 'Done' THEN 1 + ELSE 0 + END + ) / NULLIF(COUNT(*), 0), + 2 + ) AS TEXT + ) + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 5, + 'Decimal', + 'AVG_JOB_DURATION_MINS', + CAST(ROUND(AVG((done_at - run_at) / 60.0), 2) AS TEXT) + FROM + Jobs + WHERE + status IN ('Done', 'Failed', 'Killed') + AND done_at IS NOT NULL + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 5, + 'Decimal', + 'LONGEST_RUNNING_JOB_MINS', + CAST( + ROUND( + MAX( + CASE + WHEN status = 'Running' THEN (strftime('%s', 'now') - run_at) / 60.0 + ELSE 0 + END + ), + 2 + ) AS TEXT + ) + FROM + Jobs + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 6, + 'Number', + 'JOBS_PAST_7_DAYS', + CAST(COUNT(*) AS TEXT) + FROM + Jobs + WHERE + run_at >= strftime('%s', 'now', '-7 days') + GROUP BY + job_type + UNION + ALL + SELECT + job_type, + 8, + 'Timestamp', + 'MOST_RECENT_JOB', + CAST(MAX(run_at) AS TEXT) + FROM + Jobs + GROUP BY + job_type + ORDER BY + job_type, + priority, + statistic + ) + GROUP BY + job_type +), +all_job_types AS ( + SELECT + worker_type AS job_type + FROM + Workers + UNION + SELECT + job_type + FROM + Jobs +) +SELECT + jt.job_type as name, + COALESCE(qs.stats, '[]') as stats, + COALESCE( + ( + SELECT + json_group_array(DISTINCT lock_by) + FROM + Jobs + WHERE + job_type = jt.job_type + AND lock_by IS NOT NULL + ), + '[]' + ) as workers, + COALESCE( + ( + SELECT + json_group_array(daily_count) + FROM + ( + SELECT + COUNT(*) as daily_count + FROM + Jobs + WHERE + job_type = jt.job_type + AND run_at >= strftime('%s', 'now', '-7 days') + GROUP BY + date(run_at, 'unixepoch') + ORDER BY + date(run_at, 'unixepoch') + ) + ), + '[]' + ) as activity +FROM + all_job_types jt + LEFT JOIN queue_stats qs ON jt.job_type = qs.job_type +ORDER BY + name; diff --git a/queries/backend/list_workers.sql b/queries/backend/list_workers.sql new file mode 100644 index 0000000..27558c7 --- /dev/null +++ b/queries/backend/list_workers.sql @@ -0,0 +1,10 @@ +SELECT + * +FROM + Workers +WHERE + worker_type = ?1 +ORDER BY + last_seen DESC +LIMIT + ?2 OFFSET ?3 diff --git a/queries/backend/overview.sql b/queries/backend/overview.sql new file mode 100644 index 0000000..6963dc7 --- /dev/null +++ b/queries/backend/overview.sql @@ -0,0 +1,421 @@ +SELECT + 1 AS priority, + 'Number' AS type, + 'RUNNING_JOBS' AS statistic, + CAST( + SUM( + CASE + WHEN status = 'Running' THEN 1 + ELSE 0 + END + ) AS REAL + ) AS value +FROM + Jobs +UNION +ALL +SELECT + 1, + 'Number', + 'PENDING_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Pending' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 2, + 'Number', + 'FAILED_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Failed' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 2, + 'Number', + 'ACTIVE_JOBS', + CAST( + SUM( + CASE + WHEN status IN ('Pending', 'Running', 'Queued') THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 2, + 'Number', + 'STALE_RUNNING_JOBS', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + status = 'Running' + AND run_at < strftime('%s', 'now', '-1 hour') +UNION +ALL +SELECT + 2, + 'Percentage', + 'KILL_RATE', + CAST( + ROUND( + 100.0 * SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) / NULLIF(COUNT(*), 0), + 2 + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 3, + 'Number', + 'JOBS_PAST_HOUR', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + run_at >= strftime('%s', 'now', '-1 hour') +UNION +ALL +SELECT + 3, + 'Number', + 'JOBS_TODAY', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + date(run_at, 'unixepoch') = date('now') +UNION +ALL +SELECT + 3, + 'Number', + 'KILLED_JOBS_TODAY', + CAST( + SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + date(run_at, 'unixepoch') = date('now') +UNION +ALL +SELECT + 3, + 'Decimal', + 'AVG_JOBS_PER_MINUTE_PAST_HOUR', + CAST(ROUND(COUNT(*) / 60.0, 2) AS REAL) +FROM + Jobs +WHERE + run_at >= strftime('%s', 'now', '-1 hour') +UNION +ALL +SELECT + 4, + 'Number', + 'TOTAL_JOBS', + CAST(COUNT(*) AS REAL) +FROM + Jobs +UNION +ALL +SELECT + 4, + 'Number', + 'DONE_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Done' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 4, + 'Number', + 'COMPLETED_JOBS', + CAST( + SUM( + CASE + WHEN status IN ('Done', 'Failed', 'Killed') THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 4, + 'Number', + 'KILLED_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 4, + 'Percentage', + 'SUCCESS_RATE', + CAST( + ROUND( + 100.0 * SUM( + CASE + WHEN status = 'Done' THEN 1 + ELSE 0 + END + ) / NULLIF(COUNT(*), 0), + 2 + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 5, + 'Decimal', + 'AVG_JOB_DURATION_MINS', + CAST(ROUND(AVG((done_at - run_at) / 60.0), 2) AS REAL) +FROM + Jobs +WHERE + status IN ('Done', 'Failed', 'Killed') + AND done_at IS NOT NULL +UNION +ALL +SELECT + 5, + 'Decimal', + 'LONGEST_RUNNING_JOB_MINS', + CAST( + ROUND( + MAX( + CASE + WHEN status = 'Running' THEN (strftime('%s', 'now') - run_at) / 60.0 + ELSE 0 + END + ), + 2 + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 5, + 'Number', + 'QUEUE_BACKLOG', + CAST( + SUM( + CASE + WHEN status = 'Pending' + AND run_at <= strftime('%s', 'now') THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +UNION +ALL +SELECT + 6, + 'Number', + 'JOBS_PAST_24_HOURS', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + run_at >= strftime('%s', 'now', '-1 day') +UNION +ALL +SELECT + 6, + 'Number', + 'JOBS_PAST_7_DAYS', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + run_at >= strftime('%s', 'now', '-7 days') +UNION +ALL +SELECT + 6, + 'Number', + 'KILLED_JOBS_PAST_7_DAYS', + CAST( + SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + run_at >= strftime('%s', 'now', '-7 days') +UNION +ALL +SELECT + 6, + 'Percentage', + 'SUCCESS_RATE_PAST_24H', + CAST( + ROUND( + 100.0 * SUM( + CASE + WHEN status = 'Done' THEN 1 + ELSE 0 + END + ) / NULLIF(COUNT(*), 0), + 2 + ) AS REAL + ) +FROM + Jobs +WHERE + run_at >= strftime('%s', 'now', '-1 day') +UNION +ALL +SELECT + 7, + 'Decimal', + 'AVG_JOBS_PER_HOUR_PAST_24H', + CAST(ROUND(COUNT(*) / 24.0, 2) AS REAL) +FROM + Jobs +WHERE + run_at >= strftime('%s', 'now', '-1 day') +UNION +ALL +SELECT + 7, + 'Decimal', + 'AVG_JOBS_PER_DAY_PAST_7D', + CAST(ROUND(COUNT(*) / 7.0, 2) AS REAL) +FROM + Jobs +WHERE + run_at >= strftime('%s', 'now', '-7 days') +UNION +ALL +SELECT + 8, + 'Timestamp', + 'MOST_RECENT_JOB', + CAST(MAX(run_at) AS REAL) +FROM + Jobs +UNION +ALL +SELECT + 8, + 'Timestamp', + 'OLDEST_PENDING_JOB', + CAST(MIN(run_at) AS REAL) +FROM + Jobs +WHERE + status = 'Pending' + AND run_at <= strftime('%s', 'now') +UNION +ALL +SELECT + 8, + 'Number', + 'PEAK_HOUR_JOBS', + CAST(MAX(hourly_count) AS REAL) +FROM + ( + SELECT + COUNT(*) as hourly_count + FROM + Jobs + WHERE + run_at >= strftime('%s', 'now', '-1 day') + GROUP BY + strftime('%H', run_at, 'unixepoch') + ) +UNION +ALL +SELECT + 9, + 'Number', + 'DB_PAGE_SIZE', + CAST(page_size AS REAL) +FROM + pragma_page_size() +UNION +ALL +SELECT + 9, + 'Number', + 'DB_PAGE_COUNT', + CAST(page_count AS REAL) +FROM + pragma_page_count() +UNION +ALL +SELECT + 9, + 'Number', + 'DB_SIZE', + CAST(page_size * page_count AS REAL) +FROM + pragma_page_size(), + pragma_page_count() +ORDER BY + priority, + statistic; diff --git a/queries/backend/overview_by_queue.sql b/queries/backend/overview_by_queue.sql new file mode 100644 index 0000000..8252712 --- /dev/null +++ b/queries/backend/overview_by_queue.sql @@ -0,0 +1,433 @@ +SELECT + 1 AS priority, + 'Number' AS type, + 'RUNNING_JOBS' AS statistic, + CAST( + SUM( + CASE + WHEN status = 'Running' THEN 1 + ELSE 0 + END + ) AS REAL + ) AS value +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 1, + 'Number', + 'PENDING_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Pending' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 2, + 'Number', + 'FAILED_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Failed' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 2, + 'Number', + 'ACTIVE_JOBS', + CAST( + SUM( + CASE + WHEN status IN ('Pending', 'Queued', 'Running') THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 2, + 'Number', + 'STALE_RUNNING_JOBS', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND status = 'Running' + AND run_at < strftime('%s', 'now', '-1 hour') +UNION +ALL +SELECT + 2, + 'Percentage', + 'KILL_RATE', + CAST( + ROUND( + 100.0 * SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) / NULLIF(COUNT(*), 0), + 2 + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 3, + 'Number', + 'JOBS_PAST_HOUR', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND run_at >= strftime('%s', 'now', '-1 hour') +UNION +ALL +SELECT + 3, + 'Number', + 'JOBS_TODAY', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND date(run_at, 'unixepoch') = date('now') +UNION +ALL +SELECT + 3, + 'Number', + 'KILLED_JOBS_TODAY', + CAST( + SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 + AND date(run_at, 'unixepoch') = date('now') +UNION +ALL +SELECT + 3, + 'Decimal', + 'AVG_JOBS_PER_MINUTE_PAST_HOUR', + CAST(ROUND(COUNT(*) / 60.0, 2) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND run_at >= strftime('%s', 'now', '-1 hour') +UNION +ALL +SELECT + 4, + 'Number', + 'TOTAL_JOBS', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 4, + 'Number', + 'DONE_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Done' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 4, + 'Number', + 'COMPLETED_JOBS', + CAST( + SUM( + CASE + WHEN status IN ('Done', 'Failed', 'Killed') THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 4, + 'Number', + 'KILLED_JOBS', + CAST( + SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 4, + 'Percentage', + 'SUCCESS_RATE', + CAST( + ROUND( + 100.0 * SUM( + CASE + WHEN status = 'Done' THEN 1 + ELSE 0 + END + ) / NULLIF(COUNT(*), 0), + 2 + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 5, + 'Decimal', + 'AVG_JOB_DURATION_MINS', + CAST(ROUND(AVG((done_at - run_at) / 60.0), 2) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND status IN ('Done', 'Failed', 'Killed') + AND done_at IS NOT NULL +UNION +ALL +SELECT + 5, + 'Decimal', + 'LONGEST_RUNNING_JOB_MINS', + CAST( + ROUND( + MAX( + CASE + WHEN status = 'Running' THEN (strftime('%s', 'now') - run_at) / 60.0 + ELSE 0 + END + ), + 2 + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 5, + 'Number', + 'QUEUE_BACKLOG', + CAST( + SUM( + CASE + WHEN status = 'Pending' + AND run_at <= strftime('%s', 'now') THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 6, + 'Number', + 'JOBS_PAST_24_HOURS', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND run_at >= strftime('%s', 'now', '-1 day') +UNION +ALL +SELECT + 6, + 'Number', + 'JOBS_PAST_7_DAYS', + CAST(COUNT(*) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND run_at >= strftime('%s', 'now', '-7 days') +UNION +ALL +SELECT + 6, + 'Number', + 'KILLED_JOBS_PAST_7_DAYS', + CAST( + SUM( + CASE + WHEN status = 'Killed' THEN 1 + ELSE 0 + END + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 + AND run_at >= strftime('%s', 'now', '-7 days') +UNION +ALL +SELECT + 6, + 'Percentage', + 'SUCCESS_RATE_PAST_24H', + CAST( + ROUND( + 100.0 * SUM( + CASE + WHEN status = 'Done' THEN 1 + ELSE 0 + END + ) / NULLIF(COUNT(*), 0), + 2 + ) AS REAL + ) +FROM + Jobs +WHERE + job_type = ?1 + AND run_at >= strftime('%s', 'now', '-1 day') +UNION +ALL +SELECT + 7, + 'Decimal', + 'AVG_JOBS_PER_HOUR_PAST_24H', + CAST(ROUND(COUNT(*) / 24.0, 2) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND run_at >= strftime('%s', 'now', '-1 day') +UNION +ALL +SELECT + 7, + 'Decimal', + 'AVG_JOBS_PER_DAY_PAST_7D', + CAST(ROUND(COUNT(*) / 7.0, 2) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND run_at >= strftime('%s', 'now', '-7 days') +UNION +ALL +SELECT + 8, + 'Timestamp', + 'MOST_RECENT_JOB', + CAST(MAX(run_at) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 +UNION +ALL +SELECT + 8, + 'Timestamp', + 'OLDEST_PENDING_JOB', + CAST(MIN(run_at) AS REAL) +FROM + Jobs +WHERE + job_type = ?1 + AND status = 'Pending' + AND run_at <= strftime('%s', 'now') +UNION +ALL +SELECT + 8, + 'Number', + 'PEAK_HOUR_JOBS', + CAST(MAX(hourly_count) AS REAL) +FROM + ( + SELECT + COUNT(*) as hourly_count + FROM + Jobs + WHERE + job_type = ?1 + AND run_at >= strftime('%s', 'now', '-1 day') + GROUP BY + strftime('%H', run_at, 'unixepoch') + ) +ORDER BY + priority, + statistic; diff --git a/queries/backend/queue_length.sql b/queries/backend/queue_length.sql new file mode 100644 index 0000000..0dd2c31 --- /dev/null +++ b/queries/backend/queue_length.sql @@ -0,0 +1,12 @@ +Select + COUNT(*) AS count +FROM + Jobs +WHERE + ( + status = 'Pending' + OR ( + status = 'Failed' + AND attempts < max_attempts + ) + ) diff --git a/queries/backend/reenqueue_orphaned.sql b/queries/backend/reenqueue_orphaned.sql new file mode 100644 index 0000000..3e09593 --- /dev/null +++ b/queries/backend/reenqueue_orphaned.sql @@ -0,0 +1,24 @@ +UPDATE + Jobs +SET + status = "Pending", + done_at = NULL, + lock_by = NULL, + lock_at = NULL, + attempts = attempts + 1, + last_result = '{"Err": "Re-enqueued due to worker heartbeat timeout."}' +WHERE + id IN ( + SELECT + Jobs.id + FROM + Jobs + INNER JOIN Workers ON lock_by = Workers.id + WHERE + ( + status = "Running" + OR status = "Queued" + ) + AND strftime('%s', 'now') - Workers.last_seen >= ?1 + AND Workers.worker_type = ?2 + ); diff --git a/queries/backend/register_worker.sql b/queries/backend/register_worker.sql new file mode 100644 index 0000000..cfe59ab --- /dev/null +++ b/queries/backend/register_worker.sql @@ -0,0 +1,19 @@ +INSERT INTO + Workers (id, worker_type, storage_name, layers, last_seen, started_at) +SELECT + ?1, + ?2, + ?3, + ?4, + strftime('%s', 'now'), + strftime('%s', 'now') +WHERE + NOT EXISTS ( + SELECT + 1 + FROM + Workers + WHERE + id = ?1 + AND strftime('%s', 'now') - last_seen >= ?5 + ); diff --git a/queries/backend/stats.sql b/queries/backend/stats.sql new file mode 100644 index 0000000..5ee7796 --- /dev/null +++ b/queries/backend/stats.sql @@ -0,0 +1,11 @@ +SELECT + CAST(COUNT(*) AS INTEGER) AS total, + CAST(SUM(CASE WHEN status = 'Pending' THEN 1 ELSE 0 END) AS INTEGER) AS pending, + CAST(SUM(CASE WHEN status = 'Running' THEN 1 ELSE 0 END) AS INTEGER) AS running, + CAST(SUM(CASE WHEN status = 'Done' THEN 1 ELSE 0 END) AS INTEGER) AS done, + CAST(SUM(CASE WHEN status = 'Failed' THEN 1 ELSE 0 END) AS INTEGER) AS failed, + CAST(SUM(CASE WHEN status = 'Killed' THEN 1 ELSE 0 END) AS INTEGER) AS killed, + CAST(SUM(CASE WHEN status IN ('Done', 'Failed', 'Killed') THEN 1 ELSE 0 END) AS INTEGER) AS completed, + CAST(SUM(CASE WHEN status IN ('Pending', 'Running') THEN 1 ELSE 0 END) AS INTEGER) AS active +FROM Jobs +WHERE job_type = ?1 diff --git a/queries/backend/vacuum.sql b/queries/backend/vacuum.sql new file mode 100644 index 0000000..94050db --- /dev/null +++ b/queries/backend/vacuum.sql @@ -0,0 +1,11 @@ +Delete from + Jobs +where + status = 'Done' + OR status = 'Killed' + OR ( + status = 'Failed' + AND max_attempts <= attempts + ); + +VACUUM; diff --git a/queries/task/ack.sql b/queries/task/ack.sql new file mode 100644 index 0000000..3089f74 --- /dev/null +++ b/queries/task/ack.sql @@ -0,0 +1,10 @@ +UPDATE + Jobs +SET + status = ?4, + attempts = ?2, + last_result = ?3, + done_at = strftime('%s', 'now') +WHERE + id = ?1 + AND lock_by = ?5 diff --git a/queries/task/find_by_id.sql b/queries/task/find_by_id.sql new file mode 100644 index 0000000..dbbcaff --- /dev/null +++ b/queries/task/find_by_id.sql @@ -0,0 +1,8 @@ +SELECT + * +FROM + Jobs +WHERE + id = ?1 +LIMIT + 1; diff --git a/queries/task/kill.sql b/queries/task/kill.sql new file mode 100644 index 0000000..7bb6483 --- /dev/null +++ b/queries/task/kill.sql @@ -0,0 +1,9 @@ +UPDATE + Jobs +SET + status = 'Killed', + done_at = strftime('%s', 'now'), + last_result = ?3 +WHERE + id = ?1 + AND lock_by = ?2 diff --git a/queries/task/lock.sql b/queries/task/lock.sql new file mode 100644 index 0000000..d2807b6 --- /dev/null +++ b/queries/task/lock.sql @@ -0,0 +1,16 @@ +UPDATE + Jobs +SET + status = 'Running', + lock_at = strftime('%s', 'now'), + lock_by = ?2 +WHERE + id = ?1 + AND ( + status = 'Queued' + OR status = 'Pending' + OR ( + status = 'Failed' + AND attempts < max_attempts + ) + ) diff --git a/queries/task/reschedule.sql b/queries/task/reschedule.sql new file mode 100644 index 0000000..6870a91 --- /dev/null +++ b/queries/task/reschedule.sql @@ -0,0 +1,11 @@ +UPDATE + Jobs +SET + status = 'Failed', + attempts = attempts + 1, + done_at = NULL, + lock_by = NULL, + lock_at = NULL, + run_at = ?2 +WHERE + id = ?1 diff --git a/queries/task/retry.sql b/queries/task/retry.sql new file mode 100644 index 0000000..f46aaef --- /dev/null +++ b/queries/task/retry.sql @@ -0,0 +1,11 @@ +UPDATE + Jobs +SET + status = 'Pending', + attempt = attempt + 1, + run_at = ?1, + done_at = NULL, + lock_by = NULL +WHERE + id = ?2 + AND lock_by = ?3 diff --git a/queries/task/sink.sql b/queries/task/sink.sql new file mode 100644 index 0000000..ee42376 --- /dev/null +++ b/queries/task/sink.sql @@ -0,0 +1,17 @@ +INSERT INTO + Jobs +VALUES + ( + ?1, + ?2, + ?3, + 'Pending', + 0, + ?4, + ?5, + NULL, + NULL, + NULL, + NULL, + ?6 + ) diff --git a/queries/task/update_by_id.sql b/queries/task/update_by_id.sql new file mode 100644 index 0000000..15418d9 --- /dev/null +++ b/queries/task/update_by_id.sql @@ -0,0 +1,12 @@ +UPDATE + Jobs +SET + status = ?1, + attempts = ?2, + done_at = ?3, + lock_by = ?4, + lock_at = ?5, + last_result = ?6, + priority = ?7 +WHERE + id = ?8 diff --git a/src/ack.rs b/src/ack.rs new file mode 100644 index 0000000..875e0c1 --- /dev/null +++ b/src/ack.rs @@ -0,0 +1,169 @@ +use apalis_core::{ + error::{AbortError, BoxDynError}, + task::{Parts, status::Status}, + worker::{context::WorkerContext, ext::ack::Acknowledge}, +}; +use futures::{FutureExt, future::BoxFuture}; +use serde::Serialize; +use sqlx::SqlitePool; +use tower_layer::Layer; +use tower_service::Service; +use ulid::Ulid; + +use crate::{SqliteTask, context::SqliteContext}; + +#[derive(Clone)] +pub struct SqliteAck { + pool: SqlitePool, +} +impl SqliteAck { + pub fn new(pool: SqlitePool) -> Self { + Self { pool } + } +} + +impl Acknowledge for SqliteAck { + type Error = sqlx::Error; + type Future = BoxFuture<'static, Result<(), Self::Error>>; + fn ack( + &mut self, + res: &Result, + parts: &Parts, + ) -> Self::Future { + let task_id = parts.task_id; + let worker_id = parts.ctx.lock_by().clone(); + + let response = serde_json::to_string(&res.as_ref().map_err(|e| e.to_string())); + let status = calculate_status(parts, res); + parts.status.store(status.clone()); + let attempt = parts.attempt.current() as i32; + let pool = self.pool.clone(); + let res = response.map_err(|e| sqlx::Error::Decode(e.into())); + let status = status.to_string(); + async move { + let task_id = task_id + .ok_or(sqlx::Error::ColumnNotFound("TASK_ID_FOR_ACK".to_owned()))? + .to_string(); + let worker_id = + worker_id.ok_or(sqlx::Error::ColumnNotFound("WORKER_ID_LOCK_BY".to_owned()))?; + let res_ok = res?; + let res = sqlx::query_file!( + "queries/task/ack.sql", + task_id, + attempt, + res_ok, + status, + worker_id + ) + .execute(&pool) + .await?; + + if res.rows_affected() == 0 { + return Err(sqlx::Error::RowNotFound); + } + Ok(()) + } + .boxed() + } +} + +pub fn calculate_status( + parts: &Parts, + res: &Result, +) -> Status { + match &res { + Ok(_) => Status::Done, + Err(e) => match e { + _ if parts.ctx.max_attempts() as usize <= parts.attempt.current() => Status::Killed, + e if e.downcast_ref::().is_some() => Status::Killed, + _ => Status::Failed, + }, + } +} + +pub async fn lock_task( + pool: &SqlitePool, + task_id: &Ulid, + worker_id: &str, +) -> Result<(), sqlx::Error> { + let task_id = task_id.to_string(); + let res = sqlx::query_file!("queries/task/lock.sql", task_id, worker_id) + .execute(pool) + .await?; + + if res.rows_affected() == 0 { + return Err(sqlx::Error::RowNotFound); + } + Ok(()) +} + +pub struct LockTaskLayer { + pool: SqlitePool, +} + +impl LockTaskLayer { + pub fn new(pool: SqlitePool) -> Self { + Self { pool } + } +} + +impl Layer for LockTaskLayer { + type Service = LockTaskService; + + fn layer(&self, inner: S) -> Self::Service { + LockTaskService { + inner, + pool: self.pool.clone(), + } + } +} + +pub struct LockTaskService { + inner: S, + pool: SqlitePool, +} + +impl Service> for LockTaskService +where + S: Service> + Send + 'static, + S::Future: Send + 'static, + S::Error: Into, + Args: Send + 'static, +{ + type Response = S::Response; + type Error = BoxDynError; + type Future = BoxFuture<'static, Result>; + + fn poll_ready( + &mut self, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.inner.poll_ready(cx).map_err(|e| e.into()) + } + + fn call(&mut self, req: SqliteTask) -> Self::Future { + let pool = self.pool.clone(); + let worker_id = req + .parts + .data + .get::() + .map(|w| w.name().to_owned()) + .unwrap(); + let parts = &req.parts; + let task_id = match &parts.task_id { + Some(id) => id.inner().clone(), + None => { + return async { + Err(sqlx::Error::ColumnNotFound("TASK_ID_FOR_LOCK".to_owned()).into()) + } + .boxed(); + } + }; + let fut = self.inner.call(req); + async move { + lock_task(&pool, &task_id, &worker_id).await?; + fut.await.map_err(|e| e.into()) + } + .boxed() + } +} diff --git a/src/callback.rs b/src/callback.rs new file mode 100644 index 0000000..eeecc63 --- /dev/null +++ b/src/callback.rs @@ -0,0 +1,93 @@ +use futures::channel::mpsc::{self, UnboundedReceiver}; +use futures::{Stream, StreamExt}; +use std::ffi::{CStr, c_void}; +use std::os::raw::{c_char, c_int}; +use std::pin::Pin; +use std::task::{Context, Poll}; + +#[derive(Debug)] +pub struct DbEvent { + op: &'static str, + db_name: String, + table_name: String, + rowid: i64, +} + +impl DbEvent { + pub fn operation(&self) -> &'static str { + self.op + } + + pub fn db_name(&self) -> &str { + &self.db_name + } + + pub fn table_name(&self) -> &str { + &self.table_name + } + + pub fn rowid(&self) -> i64 { + self.rowid + } +} + +// Callback for SQLite update hook +pub(crate) extern "C" fn update_hook_callback( + arg: *mut c_void, + op: c_int, + db_name: *const c_char, + table_name: *const c_char, + rowid: i64, +) { + let op_str = match op { + libsqlite3_sys::SQLITE_INSERT => "INSERT", + libsqlite3_sys::SQLITE_UPDATE => "UPDATE", + libsqlite3_sys::SQLITE_DELETE => "DELETE", + _ => "UNKNOWN", + }; + + unsafe { + let db = CStr::from_ptr(db_name).to_string_lossy().to_string(); + let table = CStr::from_ptr(table_name).to_string_lossy().to_string(); + + log::debug!( + "DB Event - Operation: {}, DB: {}, Table: {}, RowID: {}", + op_str, + db, + table, + rowid + ); + + // Recover sender from raw pointer + let tx = &mut *(arg as *mut mpsc::UnboundedSender); + + // Ignore send errors (receiver closed) + let _ = tx.start_send(DbEvent { + op: op_str, + db_name: db, + table_name: table, + rowid, + }); + } +} + +#[derive(Debug, Clone)] +pub struct HookCallbackListener; + +#[derive(Debug)] +pub struct CallbackListener { + rx: UnboundedReceiver, +} +impl CallbackListener { + pub fn new(rx: UnboundedReceiver) -> Self { + Self { rx } + } +} + +impl Stream for CallbackListener { + type Item = DbEvent; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.rx.poll_next_unpin(cx) + } +} diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..fa8aeb2 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,145 @@ +use std::time::Duration; + +use apalis_core::backend::{ + Backend, ConfigExt, + poll_strategy::{BackoffConfig, IntervalStrategy, MultiStrategy, StrategyBuilder}, + queue::Queue, +}; +use ulid::Ulid; + +use crate::{CompactType, SqliteContext, SqliteStorage}; + +#[derive(Debug, Clone)] +pub struct Config { + keep_alive: Duration, + buffer_size: usize, + poll_strategy: MultiStrategy, + reenqueue_orphaned_after: Duration, + queue: Queue, + ack: bool, +} + +impl Default for Config { + fn default() -> Self { + Self { + keep_alive: Duration::from_secs(30), + buffer_size: 10, + poll_strategy: StrategyBuilder::new() + .apply( + IntervalStrategy::new(Duration::from_millis(100)) + .with_backoff(BackoffConfig::default()), + ) + .build(), + reenqueue_orphaned_after: Duration::from_secs(300), // 5 minutes + queue: Queue::from("default"), + ack: true, + } + } +} + +impl Config { + /// Create a new config with a jobs queue + pub fn new(queue: &str) -> Self { + Config { + queue: Queue::from(queue), + ..Default::default() + } + } + + /// Interval between database poll queries + /// + /// Defaults to 100ms + pub fn with_poll_interval(mut self, strategy: MultiStrategy) -> Self { + self.poll_strategy = strategy; + self + } + + /// Interval between worker keep-alive database updates + /// + /// Defaults to 30s + pub fn set_keep_alive(mut self, keep_alive: Duration) -> Self { + self.keep_alive = keep_alive; + self + } + + /// Buffer size to use when querying for jobs + /// + /// Defaults to 10 + pub fn set_buffer_size(mut self, buffer_size: usize) -> Self { + self.buffer_size = buffer_size; + self + } + + /// Gets a reference to the keep_alive duration. + pub fn keep_alive(&self) -> &Duration { + &self.keep_alive + } + + /// Gets a mutable reference to the keep_alive duration. + pub fn keep_alive_mut(&mut self) -> &mut Duration { + &mut self.keep_alive + } + + /// Gets the buffer size. + pub fn buffer_size(&self) -> usize { + self.buffer_size + } + + /// Gets a reference to the poll_strategy. + pub fn poll_strategy(&self) -> &MultiStrategy { + &self.poll_strategy + } + + /// Gets a mutable reference to the poll_strategy. + pub fn poll_strategy_mut(&mut self) -> &mut MultiStrategy { + &mut self.poll_strategy + } + + /// Gets a reference to the queue. + pub fn queue(&self) -> &Queue { + &self.queue + } + + /// Gets a mutable reference to the queue. + pub fn queue_mut(&mut self) -> &mut Queue { + &mut self.queue + } + + /// Gets the reenqueue_orphaned_after duration. + pub fn reenqueue_orphaned_after(&self) -> Duration { + self.reenqueue_orphaned_after + } + + /// Gets a mutable reference to the reenqueue_orphaned_after. + pub fn reenqueue_orphaned_after_mut(&mut self) -> &mut Duration { + &mut self.reenqueue_orphaned_after + } + + /// Occasionally some workers die, or abandon jobs because of panics. + /// This is the time a task takes before its back to the queue + /// + /// Defaults to 5 minutes + pub fn set_reenqueue_orphaned_after(mut self, after: Duration) -> Self { + self.reenqueue_orphaned_after = after; + self + } + + pub fn ack(&self) -> bool { + self.ack + } + + pub fn set_ack(mut self, auto_ack: bool) -> Self { + self.ack = auto_ack; + self + } +} + +impl ConfigExt for SqliteStorage +where + SqliteStorage: + Backend, +{ + fn get_queue(&self) -> Queue { + self.config.queue.clone() + } +} diff --git a/src/context.rs b/src/context.rs new file mode 100644 index 0000000..c1c22ff --- /dev/null +++ b/src/context.rs @@ -0,0 +1,121 @@ +use std::convert::Infallible; + +use apalis_core::task_fn::FromRequest; + +use serde::{Deserialize, Serialize}; + +use crate::SqliteTask; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SqliteContext { + max_attempts: i32, + last_result: Option, + lock_at: Option, + lock_by: Option, + done_at: Option, + priority: i32, + queue: Option, +} + +impl Default for SqliteContext { + fn default() -> Self { + Self::new() + } +} + +impl SqliteContext { + /// Build a new context with defaults + pub fn new() -> Self { + SqliteContext { + lock_at: None, + done_at: None, + max_attempts: 5, + last_result: None, + lock_by: None, + priority: 0, + queue: None, + } + } + + /// Set the number of attempts + pub fn with_max_attempts(mut self, max_attempts: i32) -> Self { + self.max_attempts = max_attempts; + self + } + + /// Gets the maximum attempts for a job. Default 25 + pub fn max_attempts(&self) -> i32 { + self.max_attempts + } + + /// Get the time a job was done + pub fn done_at(&self) -> &Option { + &self.done_at + } + + /// Set the time a job was done + pub fn with_done_at(mut self, done_at: Option) -> Self { + self.done_at = done_at; + self + } + + /// Get the time a job was locked + pub fn lock_at(&self) -> &Option { + &self.lock_at + } + + /// Set the lock_at value + pub fn with_lock_at(mut self, lock_at: Option) -> Self { + self.lock_at = lock_at; + self + } + + /// Get the time a job was locked + pub fn lock_by(&self) -> &Option { + &self.lock_by + } + + /// Set `lock_by` + pub fn with_lock_by(mut self, lock_by: Option) -> Self { + self.lock_by = lock_by; + self + } + + /// Get the time a job was locked + pub fn last_result(&self) -> &Option { + &self.last_result + } + + /// Set the last result + pub fn with_last_result(mut self, result: Option) -> Self { + self.last_result = result; + self + } + + /// Set the job priority. Larger values will run sooner. Default is 0. + pub fn with_priority(mut self, priority: i32) -> Self { + self.priority = priority; + self + } + + /// Get the job priority + pub fn priority(&self) -> i32 { + self.priority + } + + pub fn queue(&self) -> &Option { + &self.queue + } + + pub fn with_queue(mut self, queue: String) -> Self { + self.queue = Some(queue); + self + } +} + +impl FromRequest> for SqliteContext { + type Error = Infallible; + async fn from_request(req: &SqliteTask) -> Result { + Ok(req.parts.ctx.clone()) + } +} diff --git a/src/fetcher.rs b/src/fetcher.rs new file mode 100644 index 0000000..3490005 --- /dev/null +++ b/src/fetcher.rs @@ -0,0 +1,207 @@ +use std::{ + collections::VecDeque, + marker::PhantomData, + pin::Pin, + sync::{Arc, atomic::AtomicUsize}, + task::{Context, Poll}, +}; + +use apalis_core::{ + backend::{ + codec::{Codec, }, + poll_strategy::{PollContext, PollStrategyExt}, + }, + task::Task, + worker::context::WorkerContext, +}; +use futures::{FutureExt, future::BoxFuture, stream::Stream}; +use pin_project::pin_project; +use sqlx::{Pool, Sqlite, SqlitePool}; +use ulid::Ulid; + +use crate::{CompactType, SqliteTask, config::Config, context::SqliteContext, from_row::TaskRow}; + +pub async fn fetch_next>( + pool: SqlitePool, + config: Config, + worker: WorkerContext, +) -> Result>, sqlx::Error> +where + D::Error: std::error::Error + Send + Sync + 'static, +{ + let job_type = config.queue().to_string(); + let buffer_size = config.buffer_size() as i32; + let worker = worker.name().to_string(); + sqlx::query_file_as!( + TaskRow, + "queries/backend/fetch_next.sql", + worker, + job_type, + buffer_size + ) + .fetch_all(&pool) + .await? + .into_iter() + .map(|r| r.try_into_task::()) + .collect() +} + +enum StreamState { + Ready, + Delay, + Fetch(BoxFuture<'static, Result>, sqlx::Error>>), + Buffered(VecDeque>), + Empty, +} + +/// Dispatcher for fetching tasks from a SQLite backend via [SqlitePollFetcher] +#[derive(Clone, Debug)] +pub struct SqliteFetcher { + pub _marker: PhantomData<(Args, Compact, Decode)>, +} + +/// Polling-based fetcher for retrieving tasks from a SQLite backend +#[pin_project] +pub struct SqlitePollFetcher { + pool: SqlitePool, + config: Config, + wrk: WorkerContext, + _marker: PhantomData<(Compact, Decode)>, + #[pin] + state: StreamState, + + #[pin] + delay_stream: Option + Send>>>, + + prev_count: Arc, +} + +impl Clone for SqlitePollFetcher { + fn clone(&self) -> Self { + Self { + pool: self.pool.clone(), + config: self.config.clone(), + wrk: self.wrk.clone(), + _marker: PhantomData, + state: StreamState::Ready, + delay_stream: None, + prev_count: Arc::new(AtomicUsize::new(0)), + } + } +} + +impl SqlitePollFetcher { + pub fn new(pool: &Pool, config: &Config, wrk: &WorkerContext) -> Self + where + Decode: Codec + 'static, + Decode::Error: std::error::Error + Send + Sync + 'static, + { + Self { + pool: pool.clone(), + config: config.clone(), + wrk: wrk.clone(), + _marker: PhantomData, + state: StreamState::Ready, + delay_stream: None, + prev_count: Arc::new(AtomicUsize::new(0)), + } + } +} + +impl Stream for SqlitePollFetcher +where + Decode::Error: std::error::Error + Send + Sync + 'static, + Args: Send + 'static + Unpin, + Decode: Codec + 'static, +{ + type Item = Result>, sqlx::Error>; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + if this.delay_stream.is_none() { + let strategy = this + .config + .poll_strategy() + .clone() + .build_stream(&PollContext::new(this.wrk.clone(), this.prev_count.clone())); + this.delay_stream = Some(Box::pin(strategy)); + } + + loop { + match this.state { + StreamState::Ready => { + let stream = fetch_next::( + this.pool.clone(), + this.config.clone(), + this.wrk.clone(), + ); + this.state = StreamState::Fetch(stream.boxed()); + } + StreamState::Delay => { + if let Some(delay_stream) = this.delay_stream.as_mut() { + match delay_stream.as_mut().poll_next(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Some(_)) => { + this.state = StreamState::Ready; + } + Poll::Ready(None) => { + this.state = StreamState::Empty; + return Poll::Ready(None); + } + } + } else { + this.state = StreamState::Empty; + return Poll::Ready(None); + } + } + + StreamState::Fetch(ref mut fut) => match fut.poll_unpin(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(item) => match item { + Ok(requests) => { + if requests.is_empty() { + this.state = StreamState::Delay; + } else { + let mut buffer = VecDeque::new(); + for request in requests { + buffer.push_back(request); + } + + this.state = StreamState::Buffered(buffer); + } + } + Err(e) => { + this.state = StreamState::Empty; + return Poll::Ready(Some(Err(e))); + } + }, + }, + + StreamState::Buffered(ref mut buffer) => { + if let Some(request) = buffer.pop_front() { + // Yield the next buffered item + if buffer.is_empty() { + // Buffer is now empty, transition to ready for next fetch + this.state = StreamState::Ready; + } + return Poll::Ready(Some(Ok(Some(request)))); + } else { + // Buffer is empty, transition to ready + this.state = StreamState::Ready; + } + } + + StreamState::Empty => return Poll::Ready(None), + } + } + } +} + +impl SqlitePollFetcher { + pub fn take_pending(&mut self) -> VecDeque> { + match &mut self.state { + StreamState::Buffered(tasks) => std::mem::take(tasks), + _ => VecDeque::new(), + } + } +} diff --git a/src/from_row.rs b/src/from_row.rs new file mode 100644 index 0000000..a08d351 --- /dev/null +++ b/src/from_row.rs @@ -0,0 +1,115 @@ +use std::str::FromStr; + +use apalis_core::{ + backend::codec::Codec, + task::{attempt::Attempt, builder::TaskBuilder, status::Status, task_id::TaskId}, +}; + +use crate::{CompactType, SqliteTask, context::SqliteContext}; + +#[derive(Debug)] +pub(crate) struct TaskRow { + pub(crate) job: CompactType, + pub(crate) id: Option, + pub(crate) job_type: Option, + pub(crate) status: Option, + pub(crate) attempts: Option, + pub(crate) max_attempts: Option, + pub(crate) run_at: Option, + pub(crate) last_result: Option, + pub(crate) lock_at: Option, + pub(crate) lock_by: Option, + pub(crate) done_at: Option, + pub(crate) priority: Option, + // pub(crate) meta: Option>, +} + +impl TaskRow { + pub fn try_into_task, Args>( + self, + ) -> Result, sqlx::Error> + where + D::Error: std::error::Error + Send + Sync + 'static, + { + let ctx = SqliteContext::default() + .with_done_at(self.done_at) + .with_lock_by(self.lock_by) + .with_max_attempts(self.max_attempts.unwrap_or(25) as i32) + .with_last_result(self.last_result) + .with_priority(self.priority.unwrap_or(0) as i32) + .with_queue( + self.job_type + .ok_or(sqlx::Error::ColumnNotFound("job_type".to_owned()))?, + ) + .with_lock_at(self.lock_at); + let args = D::decode(&self.job).map_err(|e| sqlx::Error::Decode(e.into()))?; + let task = TaskBuilder::new(args) + .with_ctx(ctx) + .with_attempt(Attempt::new_with_value( + self.attempts + .ok_or(sqlx::Error::ColumnNotFound("attempts".to_owned()))? + as usize, + )) + .with_status( + self.status + .ok_or(sqlx::Error::ColumnNotFound("status".to_owned())) + .and_then(|s| { + Status::from_str(&s).map_err(|e| sqlx::Error::Decode(e.into())) + })?, + ) + .with_task_id( + self.id + .ok_or(sqlx::Error::ColumnNotFound("task_id".to_owned())) + .and_then(|s| { + TaskId::from_str(&s).map_err(|e| sqlx::Error::Decode(e.into())) + })?, + ) + .run_at_timestamp( + self.run_at + .ok_or(sqlx::Error::ColumnNotFound("run_at".to_owned()))? + as u64, + ); + Ok(task.build()) + } + pub fn try_into_task_compact(self) -> Result, sqlx::Error> { + let ctx = SqliteContext::default() + .with_done_at(self.done_at) + .with_lock_by(self.lock_by) + .with_max_attempts(self.max_attempts.unwrap_or(25) as i32) + .with_last_result(self.last_result) + .with_priority(self.priority.unwrap_or(0) as i32) + .with_queue( + self.job_type + .ok_or(sqlx::Error::ColumnNotFound("job_type".to_owned()))?, + ) + .with_lock_at(self.lock_at); + + let task = TaskBuilder::new(self.job) + .with_ctx(ctx) + .with_attempt(Attempt::new_with_value( + self.attempts + .ok_or(sqlx::Error::ColumnNotFound("attempts".to_owned()))? + as usize, + )) + .with_status( + self.status + .ok_or(sqlx::Error::ColumnNotFound("status".to_owned())) + .and_then(|s| { + Status::from_str(&s).map_err(|e| sqlx::Error::Decode(e.into())) + })?, + ) + .with_task_id( + self.id + .ok_or(sqlx::Error::ColumnNotFound("task_id".to_owned())) + .and_then(|s| { + TaskId::from_str(&s).map_err(|e| sqlx::Error::Decode(e.into())) + })?, + ) + .run_at_timestamp( + self.run_at + .ok_or(sqlx::Error::ColumnNotFound("run_at".to_owned()))? + as u64, + ); + Ok(task.build()) + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..1b2be94 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,626 @@ +//! # apalis-sqlite +//! +//! Background task processing for rust using apalis and sqlite. +//! +//! ## Features +//! +//! - **Reliable job queue** using SQLite as the backend. +//! - **Multiple storage types**: standard polling and event-driven (hooked) storage. +//! - **Custom codecs** for serializing/deserializing job arguments. +//! - **Heartbeat and orphaned job re-enqueueing** for robust job processing. +//! - **Integration with apalis workers and middleware.** +//! +//! ## Storage Types +//! +//! - [`SqliteStorage`]: Standard polling-based storage. +//! - [`SqliteStorageWithHook`]: Event-driven storage using SQLite update hooks for low-latency job fetching. +//! - [`SharedSqliteStorage`]: Shared storage for multiple job types. +//! +//! The naming is designed to clearly indicate the storage mechanism and its capabilities, but under the hood its the result is the `SqliteStorage` struct with different configurations. +//! +//! ## Examples +//! +//! ### Basic Worker Example +//! +//! ```rust +//! # use apalis_sqlite::{SqliteStorage, SqliteContext}; +//! # use apalis_core::task::Task; +//! # use apalis_core::worker::context::WorkerContext; +//! # use sqlx::SqlitePool; +//! # use futures::stream; +//! # use std::time::Duration; +//! # use apalis_core::error::BoxDynError; +//! # use futures::StreamExt; +//! # use futures::SinkExt; +//! # use apalis_core::worker::builder::WorkerBuilder; +//! #[tokio::main] +//! async fn main() { +//! let pool = SqlitePool::connect(":memory:").await.unwrap(); +//! SqliteStorage::setup(&pool).await.unwrap(); +//! let mut backend = SqliteStorage::new(&pool); +//! +//! let mut start = 0; +//! let mut items = stream::repeat_with(move || { +//! start += 1; +//! let task = Task::builder(start) +//! .run_after(Duration::from_secs(1)) +//! .with_ctx(SqliteContext::new().with_priority(1)) +//! .build(); +//! Ok(task) +//! }) +//! .take(10); +//! backend.send_all(&mut items).await.unwrap(); +//! +//! async fn send_reminder(item: usize, wrk: WorkerContext) -> Result<(), BoxDynError> { +//! # if item == 10 { +//! # wrk.stop().unwrap(); +//! # } +//! Ok(()) +//! } +//! +//! let worker = WorkerBuilder::new("worker-1") +//! .backend(backend) +//! .build(send_reminder); +//! worker.run().await.unwrap(); +//! } +//! ``` +//! +//! ### Hooked Worker Example (Event-driven) +//! +//! ```rust,no_run +//! # use apalis_sqlite::{SqliteStorage, SqliteContext, Config}; +//! # use apalis_core::task::Task; +//! # use apalis_core::worker::context::WorkerContext; +//! # use apalis_core::backend::poll_strategy::{IntervalStrategy, StrategyBuilder}; +//! # use sqlx::SqlitePool; +//! # use futures::stream; +//! # use std::time::Duration; +//! # use apalis_core::error::BoxDynError; +//! # use futures::StreamExt; +//! # use futures::SinkExt; +//! # use apalis_core::worker::builder::WorkerBuilder; +//! +//! #[tokio::main] +//! async fn main() { +//! let pool = SqlitePool::connect(":memory:").await.unwrap(); +//! SqliteStorage::setup(&pool).await.unwrap(); +//! +//! let lazy_strategy = StrategyBuilder::new() +//! .apply(IntervalStrategy::new(Duration::from_secs(5))) +//! .build(); +//! let config = Config::new("queue") +//! .with_poll_interval(lazy_strategy) +//! .set_buffer_size(5); +//! let backend = SqliteStorage::new_with_callback(&pool, &config); +//! +//! tokio::spawn({ +//! let pool = pool.clone(); +//! let config = config.clone(); +//! async move { +//! tokio::time::sleep(Duration::from_secs(2)).await; +//! let mut start = 0; +//! let items = stream::repeat_with(move || { +//! start += 1; +//! Task::builder(serde_json::to_string(&start).unwrap()) +//! .run_after(Duration::from_secs(1)) +//! .with_ctx(SqliteContext::new().with_priority(start)) +//! .build() +//! }) +//! .take(20) +//! .collect::>() +//! .await; +//! // push encoded tasks +//! apalis_sqlite::sink::push_tasks(pool, config, items).await.unwrap(); +//! } +//! }); +//! +//! async fn send_reminder(item: usize, wrk: WorkerContext) -> Result<(), BoxDynError> { +//! if item == 1 { +//! apalis_core::timer::sleep(Duration::from_secs(1)).await; +//! wrk.stop().unwrap(); +//! } +//! Ok(()) +//! } +//! +//! let worker = WorkerBuilder::new("worker-2") +//! .backend(backend) +//! .build(send_reminder); +//! worker.run().await.unwrap(); +//! } +//! ``` +//! +//! ## Migrations +//! +//! If the `migrate` feature is enabled, you can run built-in migrations with: +//! +//! ```rust,no_run +//! # use sqlx::SqlitePool; +//! # use apalis_sqlite::SqliteStorage; +//! # #[tokio::main] async fn main() { +//! let pool = SqlitePool::connect(":memory:").await.unwrap(); +//! SqliteStorage::setup(&pool).await.unwrap(); +//! # } +//! ``` +//! +//! ## License +//! +//! Licensed under either of Apache License, Version 2.0 or MIT license at your option. +use std::{fmt, marker::PhantomData}; + +use apalis_core::{ + backend::{ + Backend, TaskStream, + codec::{Codec, json::JsonCodec}, + }, + task::Task, + worker::{context::WorkerContext, ext::ack::AcknowledgeLayer}, +}; +use futures::{ + FutureExt, StreamExt, TryFutureExt, TryStreamExt, + channel::mpsc, + future::ready, + stream::{self, BoxStream, select}, +}; +use libsqlite3_sys::{sqlite3, sqlite3_update_hook}; +use sqlx::{Pool, Sqlite}; +use std::ffi::c_void; +use tower_layer::Stack; +use ulid::Ulid; + +use crate::{ + ack::{LockTaskLayer, SqliteAck}, + callback::{HookCallbackListener, update_hook_callback}, + fetcher::{SqliteFetcher, SqlitePollFetcher, fetch_next}, + queries::{ + keep_alive::{initial_heartbeat, keep_alive, keep_alive_stream}, + reenqueue_orphaned::reenqueue_orphaned_stream, + }, + sink::SqliteSink, +}; + +mod ack; +mod callback; +mod config; +mod context; +pub mod fetcher; +pub mod from_row; +pub mod queries; +mod shared; +pub mod sink; + +pub type SqliteTask = Task; +pub use callback::{CallbackListener, DbEvent}; +pub use config::Config; +pub use context::SqliteContext; +pub use shared::{SharedPostgresError, SharedSqliteStorage}; +pub use sqlx::SqlitePool; + +#[cfg(feature = "json")] +pub type CompactType = String; + +#[cfg(feature = "bytes")] +pub type CompactType = Vec; + +const INSERT_OPERATION: &str = "INSERT"; +const JOBS_TABLE: &str = "Jobs"; + +#[pin_project::pin_project] +pub struct SqliteStorage { + pool: Pool, + job_type: PhantomData, + config: Config, + codec: PhantomData, + #[pin] + sink: SqliteSink, + #[pin] + fetcher: Fetcher, +} + +impl fmt::Debug for SqliteStorage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SqliteStorage") + .field("pool", &self.pool) + .field("job_type", &"PhantomData") + .field("config", &self.config) + .field("codec", &std::any::type_name::()) + .finish() + } +} + +impl Clone for SqliteStorage { + fn clone(&self) -> Self { + SqliteStorage { + sink: self.sink.clone(), + pool: self.pool.clone(), + job_type: PhantomData, + config: self.config.clone(), + codec: self.codec, + fetcher: self.fetcher.clone(), + } + } +} + +impl SqliteStorage<(), (), ()> { + /// Perform migrations for storage + #[cfg(feature = "migrate")] + pub async fn setup(pool: &Pool) -> Result<(), sqlx::Error> { + sqlx::query("PRAGMA journal_mode = 'WAL';") + .execute(pool) + .await?; + sqlx::query("PRAGMA temp_store = 2;").execute(pool).await?; + sqlx::query("PRAGMA synchronous = NORMAL;") + .execute(pool) + .await?; + sqlx::query("PRAGMA cache_size = 64000;") + .execute(pool) + .await?; + Self::migrations().run(pool).await?; + Ok(()) + } + + /// Get sqlite migrations without running them + #[cfg(feature = "migrate")] + pub fn migrations() -> sqlx::migrate::Migrator { + if cfg!(feature = "bytes") && cfg!(not(feature = "json")) { + sqlx::migrate!("./migrations/bytes") + } else if cfg!(feature = "json") && cfg!(not(feature = "bytes")) { + sqlx::migrate!("./migrations/json") + } else { + panic!( + "One of either the 'json' or 'bytes' feature must be enabled for migrations to work." + ); + } + } +} + +impl SqliteStorage { + /// Create a new SqliteStorage + pub fn new( + pool: &Pool, + ) -> SqliteStorage< + T, + JsonCodec, + fetcher::SqliteFetcher>, + > { + let config = Config::new(std::any::type_name::()); + SqliteStorage { + pool: pool.clone(), + job_type: PhantomData, + sink: SqliteSink::new(pool, &config), + config, + codec: PhantomData, + fetcher: fetcher::SqliteFetcher { + _marker: PhantomData, + }, + } + } + + pub fn new_with_codec( + pool: &Pool, + config: &Config, + ) -> SqliteStorage> { + SqliteStorage { + pool: pool.clone(), + job_type: PhantomData, + config: config.clone(), + codec: PhantomData, + sink: SqliteSink::new(pool, config), + fetcher: fetcher::SqliteFetcher { + _marker: PhantomData, + }, + } + } + + pub fn new_with_config( + pool: &Pool, + config: &Config, + ) -> SqliteStorage< + T, + JsonCodec, + fetcher::SqliteFetcher>, + > { + SqliteStorage { + pool: pool.clone(), + job_type: PhantomData, + config: config.clone(), + codec: PhantomData, + sink: SqliteSink::new(pool, config), + fetcher: fetcher::SqliteFetcher { + _marker: PhantomData, + }, + } + } + + pub fn new_with_callback( + pool: &Pool, + config: &Config, + ) -> SqliteStorage, HookCallbackListener> { + SqliteStorage { + pool: pool.clone(), + job_type: PhantomData, + config: config.clone(), + codec: PhantomData, + sink: SqliteSink::new(&pool, config), + fetcher: HookCallbackListener, + } + } + + pub fn new_with_codec_callback( + pool: &Pool, + config: &Config, + ) -> SqliteStorage { + SqliteStorage { + pool: pool.clone(), + job_type: PhantomData, + config: config.clone(), + codec: PhantomData, + sink: SqliteSink::new(&pool, config), + fetcher: HookCallbackListener, + } + } +} + +impl SqliteStorage { + pub fn config(&self) -> &Config { + &self.config + } +} + +impl Backend for SqliteStorage> +where + Args: Send + 'static + Unpin, + Decode: Codec + 'static + Send, + Decode::Error: std::error::Error + Send + Sync + 'static, +{ + type Args = Args; + type IdType = Ulid; + + type Context = SqliteContext; + + type Codec = Decode; + + type Compact = CompactType; + + type Error = sqlx::Error; + + type Stream = TaskStream, sqlx::Error>; + + type Beat = BoxStream<'static, Result<(), sqlx::Error>>; + + type Layer = Stack>; + + fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { + let pool = self.pool.clone(); + let config = self.config.clone(); + let worker = worker.clone(); + let keep_alive = keep_alive_stream(pool, config, worker); + let reenqueue = reenqueue_orphaned_stream( + self.pool.clone(), + self.config.clone(), + *self.config.keep_alive(), + ) + .map_ok(|_| ()); + futures::stream::select(keep_alive, reenqueue).boxed() + } + + fn middleware(&self) -> Self::Layer { + let lock = LockTaskLayer::new(self.pool.clone()); + let ack = AcknowledgeLayer::new(SqliteAck::new(self.pool.clone())); + Stack::new(lock, ack) + } + + fn poll(self, worker: &WorkerContext) -> Self::Stream { + let fut = initial_heartbeat( + self.pool.clone(), + self.config().clone(), + worker.clone(), + "SqliteStorage", + ); + let register = stream::once(fut.map(|_| Ok(None))); + register + .chain(SqlitePollFetcher::::new( + &self.pool, + &self.config, + worker, + )) + .boxed() + } +} + +impl Backend for SqliteStorage +where + Args: Send + 'static + Unpin, + Decode: Codec + Send + 'static, + Decode::Error: std::error::Error + Send + Sync + 'static, +{ + type Args = Args; + type IdType = Ulid; + + type Context = SqliteContext; + + type Codec = Decode; + + type Compact = CompactType; + + type Error = sqlx::Error; + + type Stream = TaskStream, sqlx::Error>; + + type Beat = BoxStream<'static, Result<(), sqlx::Error>>; + + type Layer = Stack>; + + fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { + let pool = self.pool.clone(); + let config = self.config.clone(); + let worker = worker.clone(); + let keep_alive = keep_alive_stream(pool, config, worker); + let reenqueue = reenqueue_orphaned_stream( + self.pool.clone(), + self.config.clone(), + *self.config.keep_alive(), + ) + .map_ok(|_| ()); + futures::stream::select(keep_alive, reenqueue).boxed() + } + + fn middleware(&self) -> Self::Layer { + let lock = LockTaskLayer::new(self.pool.clone()); + let ack = AcknowledgeLayer::new(SqliteAck::new(self.pool.clone())); + Stack::new(lock, ack) + } + + fn poll(self, worker: &WorkerContext) -> Self::Stream { + let (tx, rx) = mpsc::unbounded::(); + + let listener = CallbackListener::new(rx); + + let pool = self.pool.clone(); + let config = self.config.clone(); + let worker = worker.clone(); + let register_worker = initial_heartbeat( + self.pool.clone(), + self.config.clone(), + worker.clone(), + "SqliteStorageWithHook", + ); + let p = pool.clone(); + let register_worker = stream::once( + register_worker + .and_then(|_| async move { + // This is still a little tbd, but the idea is to test the update hook + let mut conn = p.acquire().await?; + // Get raw sqlite3* handle + let handle: *mut sqlite3 = + conn.lock_handle().await.unwrap().as_raw_handle().as_ptr(); + + // Put sender in a Box so it has a stable memory address + let tx_box = Box::new(tx); + let tx_ptr = Box::into_raw(tx_box) as *mut c_void; + + unsafe { + sqlite3_update_hook(handle, Some(update_hook_callback), tx_ptr); + } + Ok(()) + }) + .map(|_| Ok(None)), + ); + let eager_fetcher: SqlitePollFetcher = + SqlitePollFetcher::new(&self.pool, &self.config, &worker); + let lazy_fetcher = listener + .filter(|a| ready(a.operation() == INSERT_OPERATION && a.table_name() == JOBS_TABLE)) + .ready_chunks(self.config.buffer_size()) + .then(move |_| fetch_next::(pool.clone(), config.clone(), worker.clone())) + .flat_map(|res| match res { + Ok(tasks) => stream::iter(tasks).map(Ok).boxed(), + Err(e) => stream::iter(vec![Err(e)]).boxed(), + }) + .map(|res| match res { + Ok(task) => Ok(Some(task)), + Err(e) => Err(e), + }); + + register_worker + .chain(select(lazy_fetcher, eager_fetcher)) + .boxed() + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use chrono::Local; + + use apalis_core::{ + backend::poll_strategy::{IntervalStrategy, StrategyBuilder}, + error::BoxDynError, + worker::builder::WorkerBuilder, + }; + use futures::SinkExt; + + use super::*; + + #[tokio::test] + async fn basic_worker() { + const ITEMS: usize = 10; + let pool = SqlitePool::connect(":memory:").await.unwrap(); + SqliteStorage::setup(&pool).await.unwrap(); + + let mut backend = SqliteStorage::new(&pool); + + let mut start = 0; + + let mut items = stream::repeat_with(move || { + start += 1; + let task = Task::builder(start) + .run_after(Duration::from_secs(1)) + .with_ctx(SqliteContext::new().with_priority(1)) + .build(); + Ok(task) + }) + .take(ITEMS); + backend.send_all(&mut items).await.unwrap(); + + println!("Starting worker at {}", Local::now()); + + async fn send_reminder(item: usize, wrk: WorkerContext) -> Result<(), BoxDynError> { + if ITEMS == item { + wrk.stop().unwrap(); + } + Ok(()) + } + + let worker = WorkerBuilder::new("rango-tango-1") + .backend(backend) + .build(send_reminder); + worker.run().await.unwrap(); + } + + #[tokio::test] + async fn hooked_worker() { + const ITEMS: usize = 20; + let pool = SqlitePool::connect(":memory:").await.unwrap(); + SqliteStorage::setup(&pool).await.unwrap(); + + let lazy_strategy = StrategyBuilder::new() + .apply(IntervalStrategy::new(Duration::from_secs(5))) + .build(); + let config = Config::new("rango-tango-queue") + .with_poll_interval(lazy_strategy) + .set_buffer_size(5); + let backend = SqliteStorage::new_with_callback(&pool, &config); + + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(2)).await; + let mut start = 0; + + let items = stream::repeat_with(move || { + start += 1; + + Task::builder(serde_json::to_string(&start).unwrap()) + .run_after(Duration::from_secs(1)) + .with_ctx(SqliteContext::new().with_priority(start)) + .build() + }) + .take(ITEMS) + .collect::>() + .await; + sink::push_tasks(pool, config, items).await.unwrap(); + }); + + async fn send_reminder(item: usize, wrk: WorkerContext) -> Result<(), BoxDynError> { + // Priority is in reverse order + if item == 1 { + apalis_core::timer::sleep(Duration::from_secs(1)).await; + wrk.stop().unwrap(); + } + Ok(()) + } + + let worker = WorkerBuilder::new("rango-tango-1") + .backend(backend) + .build(send_reminder); + worker.run().await.unwrap(); + } +} diff --git a/src/queries/fetch_by_id.rs b/src/queries/fetch_by_id.rs new file mode 100644 index 0000000..04d5658 --- /dev/null +++ b/src/queries/fetch_by_id.rs @@ -0,0 +1,31 @@ +use apalis_core::{ + backend::{Backend, FetchById, codec::Codec}, + task::task_id::TaskId, +}; +use ulid::Ulid; + +use crate::{CompactType, SqliteContext, SqliteStorage, SqliteTask, from_row::TaskRow}; + +impl FetchById for SqliteStorage +where + SqliteStorage: + Backend, + D: Codec, + D::Error: std::error::Error + Send + Sync + 'static, +{ + fn fetch_by_id( + &mut self, + id: &TaskId, + ) -> impl Future>, Self::Error>> + Send { + let pool = self.pool.clone(); + let id = id.to_string(); + async move { + let task = sqlx::query_file_as!(TaskRow, "queries/task/find_by_id.sql", id) + .fetch_optional(&pool) + .await? + .map(|r| r.try_into_task::()) + .transpose()?; + Ok(task) + } + } +} diff --git a/src/queries/keep_alive.rs b/src/queries/keep_alive.rs new file mode 100644 index 0000000..9c7218b --- /dev/null +++ b/src/queries/keep_alive.rs @@ -0,0 +1,50 @@ +use apalis_core::worker::context::WorkerContext; +use futures::{FutureExt, Stream, stream}; +use sqlx::SqlitePool; + +use crate::{ + Config, + queries::{reenqueue_orphaned::reenqueue_orphaned, register_worker::register_worker}, +}; + +pub async fn keep_alive( + pool: SqlitePool, + config: Config, + worker: WorkerContext, +) -> Result<(), sqlx::Error> { + let worker = worker.name().to_owned(); + let queue = config.queue().to_string(); + let res = sqlx::query_file!("queries/backend/keep_alive.sql", worker, queue) + .execute(&pool) + .await?; + if res.rows_affected() == 0 { + return Err(sqlx::Error::Io(std::io::Error::new( + std::io::ErrorKind::NotFound, + "WORKER_DOES_NOT_EXIST", + ))); + } + Ok(()) +} + +pub async fn initial_heartbeat( + pool: SqlitePool, + config: Config, + worker: WorkerContext, + storage_type: &str, +) -> Result<(), sqlx::Error> { + reenqueue_orphaned(pool.clone(), config.clone()).await?; + register_worker(pool, config, worker, storage_type).await?; + Ok(()) +} + +pub fn keep_alive_stream( + pool: SqlitePool, + config: Config, + worker: WorkerContext, +) -> impl Stream> + Send { + stream::unfold((), move |_| { + let register = keep_alive(pool.clone(), config.clone(), worker.clone()); + let interval = apalis_core::timer::Delay::new(*config.keep_alive()); + interval.then(move |_| register.map(|res| Some((res, ())))) + }) +} diff --git a/src/queries/list_queues.rs b/src/queries/list_queues.rs new file mode 100644 index 0000000..2be5499 --- /dev/null +++ b/src/queries/list_queues.rs @@ -0,0 +1,42 @@ +use apalis_core::backend::{Backend, ListQueues, QueueInfo}; +use ulid::Ulid; + +use crate::{CompactType, SqliteContext, SqliteStorage}; + +struct QueueInfoRow { + name: String, + stats: String, // JSON string + workers: String, // JSON string + activity: String, // JSON string +} + +impl From for QueueInfo { + fn from(row: QueueInfoRow) -> Self { + QueueInfo { + name: row.name, + stats: serde_json::from_str(&row.stats).unwrap(), + workers: serde_json::from_str(&row.workers).unwrap(), + activity: serde_json::from_str(&row.activity).unwrap(), + } + } +} + +impl ListQueues for SqliteStorage +where + SqliteStorage: + Backend, +{ + fn list_queues(&self) -> impl Future, Self::Error>> + Send { + let pool = self.pool.clone(); + + async move { + let queues = sqlx::query_file_as!(QueueInfoRow, "queries/backend/list_queues.sql") + .fetch_all(&pool) + .await? + .into_iter() + .map(QueueInfo::from) + .collect(); + Ok(queues) + } + } +} diff --git a/src/queries/list_tasks.rs b/src/queries/list_tasks.rs new file mode 100644 index 0000000..d86843b --- /dev/null +++ b/src/queries/list_tasks.rs @@ -0,0 +1,84 @@ +use apalis_core::{ + backend::{Backend, Filter, ListAllTasks, ListTasks, codec::Codec}, + task::{Task, status::Status}, +}; +use ulid::Ulid; + +use crate::{CompactType, SqliteContext, SqliteStorage, SqliteTask, from_row::TaskRow}; + +impl ListTasks for SqliteStorage +where + SqliteStorage: + Backend, + D: Codec, + D::Error: std::error::Error + Send + Sync + 'static, +{ + fn list_tasks( + &self, + queue: &str, + filter: &Filter, + ) -> impl Future>, Self::Error>> + Send { + let queue = queue.to_string(); + let pool = self.pool.clone(); + let limit = filter.limit() as i32; + let offset = filter.offset() as i32; + let status = filter + .status + .as_ref() + .unwrap_or(&Status::Pending) + .to_string(); + async move { + let tasks = sqlx::query_file_as!( + TaskRow, + "queries/backend/list_jobs.sql", + status, + queue, + limit, + offset + ) + .fetch_all(&pool) + .await? + .into_iter() + .map(|r| r.try_into_task::()) + .collect::, _>>()?; + Ok(tasks) + } + } +} + +impl ListAllTasks for SqliteStorage +where + SqliteStorage: + Backend, +{ + fn list_all_tasks( + &self, + filter: &Filter, + ) -> impl Future< + Output = Result>, Self::Error>, + > + Send { + let status = filter + .status + .as_ref() + .map(|s| s.to_string()) + .unwrap_or(Status::Pending.to_string()); + let pool = self.pool.clone(); + let limit = filter.limit() as i32; + let offset = filter.offset() as i32; + async move { + let tasks = sqlx::query_file_as!( + TaskRow, + "queries/backend/list_all_jobs.sql", + status, + limit, + offset + ) + .fetch_all(&pool) + .await? + .into_iter() + .map(|r| r.try_into_task_compact()) + .collect::, _>>()?; + Ok(tasks) + } + } +} diff --git a/src/queries/list_workers.rs b/src/queries/list_workers.rs new file mode 100644 index 0000000..9dcedec --- /dev/null +++ b/src/queries/list_workers.rs @@ -0,0 +1,85 @@ +use apalis_core::backend::{Backend, ListWorkers, RunningWorker}; +use futures::TryFutureExt; +use ulid::Ulid; + +use crate::{CompactType, SqliteContext, SqliteStorage}; + +struct Worker { + id: String, + worker_type: String, + storage_name: String, + layers: Option, + last_seen: i64, + started_at: Option, +} + +impl ListWorkers for SqliteStorage +where + SqliteStorage: + Backend, +{ + fn list_workers( + &self, + queue: &str, + ) -> impl Future, Self::Error>> + Send { + let queue = queue.to_string(); + let pool = self.pool.clone(); + let limit = 100; + let offset = 0; + async move { + let workers = sqlx::query_file_as!( + Worker, + "queries/backend/list_workers.sql", + queue, + limit, + offset + ) + .fetch_all(&pool) + .map_ok(|w| { + w.into_iter() + .map(|w| RunningWorker { + id: w.id, + backend: w.storage_name, + started_at: w.started_at.unwrap_or_default() as u64, + last_heartbeat: w.last_seen as u64, + layers: w.layers.unwrap_or_default(), + queue: w.worker_type, + }) + .collect() + }) + .await?; + Ok(workers) + } + } + + fn list_all_workers( + &self, + ) -> impl Future, Self::Error>> + Send { + let pool = self.pool.clone(); + let limit = 100; + let offset = 0; + async move { + let workers = sqlx::query_file_as!( + Worker, + "queries/backend/list_all_workers.sql", + limit, + offset + ) + .fetch_all(&pool) + .map_ok(|w| { + w.into_iter() + .map(|w| RunningWorker { + id: w.id, + backend: w.storage_name, + started_at: w.started_at.unwrap_or_default() as u64, + last_heartbeat: w.last_seen as u64, + layers: w.layers.unwrap_or_default(), + queue: w.worker_type, + }) + .collect() + }) + .await?; + Ok(workers) + } + } +} diff --git a/src/queries/metrics.rs b/src/queries/metrics.rs new file mode 100644 index 0000000..ec31493 --- /dev/null +++ b/src/queries/metrics.rs @@ -0,0 +1,64 @@ +use apalis_core::backend::{Backend, Metrics, Statistic}; +use ulid::Ulid; + +use crate::{CompactType, SqliteContext, SqliteStorage}; + +struct StatisticRow { + /// The priority of the statistic (lower number means higher priority) + pub priority: i64, + /// The statistics type + pub r#type: String, + /// Overall statistics of the backend + pub statistic: String, + /// The value of the statistic + pub value: Option, +} + +impl Metrics for SqliteStorage +where + SqliteStorage: + Backend, +{ + fn global(&self) -> impl Future, Self::Error>> + Send { + let pool = self.pool.clone(); + async move { + let rec = sqlx::query_file_as!(StatisticRow, "queries/backend/overview.sql") + .fetch_all(&pool) + .await? + .into_iter() + .map(|r| Statistic { + priority: Some(r.priority as u64), + stat_type: super::stat_type_from_string(&r.r#type), + title: r.statistic, + value: r.value.unwrap_or_default().to_string(), + }) + .collect(); + Ok(rec) + } + } + fn fetch_by_queue( + &self, + queue_id: &str, + ) -> impl Future, Self::Error>> + Send { + let pool = self.pool.clone(); + let queue_id = queue_id.to_string(); + async move { + let rec = sqlx::query_file_as!( + StatisticRow, + "queries/backend/overview_by_queue.sql", + queue_id + ) + .fetch_all(&pool) + .await? + .into_iter() + .map(|r| Statistic { + priority: Some(r.priority as u64), + stat_type: super::stat_type_from_string(&r.r#type), + title: r.statistic, + value: r.value.unwrap_or_default().to_string(), + }) + .collect(); + Ok(rec) + } + } +} diff --git a/src/queries/mod.rs b/src/queries/mod.rs new file mode 100644 index 0000000..52ef448 --- /dev/null +++ b/src/queries/mod.rs @@ -0,0 +1,20 @@ +use apalis_core::backend::StatType; + +pub mod fetch_by_id; +pub mod list_queues; +pub mod list_tasks; +pub mod list_workers; +pub mod metrics; +pub mod reenqueue_orphaned; +pub mod register_worker; +pub mod keep_alive; + +fn stat_type_from_string(s: &str) -> StatType { + match s { + "Number" => StatType::Number, + "Decimal" => StatType::Decimal, + "Percentage" => StatType::Percentage, + "Timestamp" => StatType::Timestamp, + _ => StatType::Number, // default fallback + } +} diff --git a/src/queries/reenqueue_orphaned.rs b/src/queries/reenqueue_orphaned.rs new file mode 100644 index 0000000..93788da --- /dev/null +++ b/src/queries/reenqueue_orphaned.rs @@ -0,0 +1,52 @@ +use std::time::Duration; + +use futures::{FutureExt, Stream, stream}; +use sqlx::SqlitePool; + +use crate::Config; + +pub fn reenqueue_orphaned( + pool: SqlitePool, + config: Config, +) -> impl Future> + Send { + let dead_for = config.reenqueue_orphaned_after().as_secs() as i64; + let queue = config.queue().to_string(); + async move { + match sqlx::query_file!("queries/backend/reenqueue_orphaned.sql", dead_for, queue,) + .execute(&pool) + .await + { + Ok(res) => { + if res.rows_affected() > 0 { + log::info!( + "Re-enqueued {} orphaned tasks that were being processed by dead workers", + res.rows_affected() + ); + } + return Ok(res.rows_affected()); + } + Err(e) => { + log::error!("Failed to re-enqueue orphaned tasks: {}", e); + return Err(e); + } + } + } +} + +pub fn reenqueue_orphaned_stream( + pool: SqlitePool, + config: Config, + interval: Duration, +) -> impl Stream> + Send { + let config = config.clone(); + stream::unfold((), move |_| { + let pool = pool.clone(); + let config = config.clone(); + let interval = apalis_core::timer::Delay::new(interval); + let fut = async move { + interval.await; + reenqueue_orphaned(pool, config).await + }; + fut.map(|res| Some((res, ()))) + }) +} diff --git a/src/queries/register_worker.rs b/src/queries/register_worker.rs new file mode 100644 index 0000000..59d1e79 --- /dev/null +++ b/src/queries/register_worker.rs @@ -0,0 +1,33 @@ +use apalis_core::worker::context::WorkerContext; +use sqlx::SqlitePool; + +use crate::Config; + +pub async fn register_worker( + pool: SqlitePool, + config: Config, + worker: WorkerContext, + storage_type: &str, +) -> Result<(), sqlx::Error> { + let worker_id = worker.name().to_owned(); + let queue = config.queue().to_string(); + let layers = worker.get_service().to_owned(); + let keep_alive = config.keep_alive().as_secs() as i64; + let res = sqlx::query_file!( + "queries/backend/register_worker.sql", + worker_id, + queue, + storage_type, + layers, + keep_alive, + ) + .execute(&pool) + .await?; + if res.rows_affected() == 0 { + return Err(sqlx::Error::Io(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "WORKER_ALREADY_EXISTS", + ))); + } + Ok(()) +} diff --git a/src/shared.rs b/src/shared.rs new file mode 100644 index 0000000..4ca3844 --- /dev/null +++ b/src/shared.rs @@ -0,0 +1,341 @@ +use std::{ + cmp::max, + collections::{HashMap, HashSet}, + ffi::c_void, + future::ready, + marker::PhantomData, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use crate::{ + CompactType, Config, INSERT_OPERATION, JOBS_TABLE, SqliteStorage, SqliteTask, + ack::{LockTaskLayer, SqliteAck}, + callback::{DbEvent, update_hook_callback}, + context::SqliteContext, + fetcher::SqlitePollFetcher, + initial_heartbeat, keep_alive, +}; +use crate::{from_row::TaskRow, sink::SqliteSink}; +use apalis_core::{ + backend::{ + Backend, TaskStream, + codec::{Codec, json::JsonCodec}, + shared::MakeShared, + }, + worker::{context::WorkerContext, ext::ack::AcknowledgeLayer}, +}; +use futures::{ + FutureExt, SinkExt, Stream, StreamExt, TryStreamExt, + channel::mpsc::{self, Receiver, Sender}, + future::{self, BoxFuture, Shared}, + lock::Mutex, + stream::{self, BoxStream, select}, +}; +use libsqlite3_sys::{sqlite3, sqlite3_update_hook}; +use sqlx::SqlitePool; +use tower_layer::Stack; +use ulid::Ulid; + +pub struct SharedSqliteStorage { + pool: SqlitePool, + registry: Arc>>>>, + drive: Shared>, + _marker: PhantomData, +} + +impl SharedSqliteStorage> { + pub fn new(pool: SqlitePool) -> Self { + SharedSqliteStorage::new_with_codec(pool) + } + pub fn new_with_codec(pool: SqlitePool) -> SharedSqliteStorage { + let registry: Arc>>>> = + Arc::new(Mutex::new(HashMap::default())); + let p = pool.clone(); + let instances = registry.clone(); + SharedSqliteStorage { + pool, + drive: async move { + let (tx, rx) = mpsc::unbounded::(); + + let mut conn = p.acquire().await.unwrap(); + // Get raw sqlite3* handle + let handle: *mut sqlite3 = + conn.lock_handle().await.unwrap().as_raw_handle().as_ptr(); + + // Put sender in a Box so it has a stable memory address + let tx_box = Box::new(tx); + let tx_ptr = Box::into_raw(tx_box) as *mut c_void; + + unsafe { + sqlite3_update_hook(handle, Some(update_hook_callback), tx_ptr); + } + + rx.filter(|a| { + ready(a.operation() == INSERT_OPERATION && a.table_name() == JOBS_TABLE) + }) + .ready_chunks(instances.try_lock().map(|r| r.len()).unwrap_or(10)) + .then(|events| { + let row_ids = events.iter().map(|e| e.rowid()).collect::>(); + let instances = instances.clone(); + let pool = p.clone(); + async move { + let instances = instances.lock().await; + let job_types = serde_json::to_string( + &instances.keys().cloned().collect::>(), + ) + .unwrap(); + let row_ids = serde_json::to_string(&row_ids).unwrap(); + let mut tx = pool.begin().await?; + let buffer_size = max(10, instances.len()) as i32; + let res: Vec<_> = sqlx::query_file_as!( + TaskRow, + "queries/backend/fetch_next_shared.sql", + job_types, + row_ids, + buffer_size, + ) + .fetch(&mut *tx) + .map(|r| r?.try_into_task_compact()) + .try_collect() + .await?; + tx.commit().await?; + Ok::<_, sqlx::Error>(res) + } + }) + .for_each(|r| async { + match r { + Ok(tasks) => { + let mut instances = instances.lock().await; + for task in tasks { + if let Some(tx) = instances.get_mut( + task.parts + .ctx + .queue() + .as_ref() + .expect("Namespace must be set"), + ) { + let _ = tx.send(task).await; + } + } + } + Err(e) => { + log::error!("Error fetching tasks: {e:?}"); + } + } + }) + .await; + } + .boxed() + .shared(), + registry, + _marker: PhantomData, + } + } +} +#[derive(Debug, thiserror::Error)] +pub enum SharedPostgresError { + #[error("Namespace {0} already exists")] + NamespaceExists(String), + #[error("Could not acquire registry lock")] + RegistryLocked, +} + +impl> MakeShared + for SharedSqliteStorage +{ + type Backend = SqliteStorage>; + type Config = Config; + type MakeError = SharedPostgresError; + fn make_shared(&mut self) -> Result + where + Self::Config: Default, + { + Self::make_shared_with_config(self, Config::new(std::any::type_name::())) + } + fn make_shared_with_config( + &mut self, + config: Self::Config, + ) -> Result { + let (tx, rx) = mpsc::channel(config.buffer_size()); + let mut r = self + .registry + .try_lock() + .ok_or(SharedPostgresError::RegistryLocked)?; + if r.insert(config.queue().to_string(), tx).is_some() { + return Err(SharedPostgresError::NamespaceExists( + config.queue().to_string(), + )); + } + let sink = SqliteSink::new(&self.pool, &config); + Ok(SqliteStorage { + config, + fetcher: SharedFetcher { + poller: self.drive.clone(), + receiver: rx, + }, + pool: self.pool.clone(), + sink, + job_type: PhantomData, + codec: PhantomData, + }) + } +} + +pub struct SharedFetcher { + poller: Shared>, + receiver: Receiver>, +} + +impl Stream for SharedFetcher { + type Item = SqliteTask; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + // Keep the poller alive by polling it, but ignoring the output + let _ = this.poller.poll_unpin(cx); + + // Delegate actual items to receiver + this.receiver.poll_next_unpin(cx) + } +} + +impl Backend for SqliteStorage> +where + Args: Send + 'static + Unpin + Sync, + Decode: Codec + 'static + Unpin + Send + Sync, + Decode::Error: std::error::Error + Send + Sync + 'static, +{ + type Args = Args; + + type IdType = Ulid; + + type Error = sqlx::Error; + + type Stream = TaskStream, sqlx::Error>; + + type Beat = BoxStream<'static, Result<(), sqlx::Error>>; + + type Codec = Decode; + + type Compact = CompactType; + + type Context = SqliteContext; + + type Layer = Stack>; + + fn heartbeat(&self, worker: &WorkerContext) -> Self::Beat { + let keep_alive_interval = *self.config.keep_alive(); + let pool = self.pool.clone(); + let worker = worker.clone(); + let config = self.config.clone(); + + stream::unfold((), move |()| async move { + apalis_core::timer::sleep(keep_alive_interval).await; + Some(((), ())) + }) + .then(move |_| keep_alive(pool.clone(), config.clone(), worker.clone())) + .boxed() + } + + fn middleware(&self) -> Self::Layer { + let lock = LockTaskLayer::new(self.pool.clone()); + let ack = AcknowledgeLayer::new(SqliteAck::new(self.pool.clone())); + Stack::new(lock, ack) + } + + fn poll(self, worker: &WorkerContext) -> Self::Stream { + let pool = self.pool.clone(); + let worker = worker.clone(); + // Initial registration heartbeat + // This ensures that the worker is registered before fetching any tasks + // This also ensures that the worker is marked as alive in case it crashes + // before fetching any tasks + // Subsequent heartbeats are handled in the heartbeat stream + let init = initial_heartbeat( + pool.clone(), + self.config.clone(), + worker.clone(), + "SharedSqliteStorage", + ); + let starter = stream::once(init) + .filter_map(|s| future::ready(s.ok().map(|_| Ok(None::>)))) + .boxed(); + let lazy_fetcher = self + .fetcher + .map(|t| { + t.try_map(|args| Decode::decode(&args).map_err(|e| sqlx::Error::Decode(e.into()))) + }) + .flat_map(|vec| match vec { + Ok(task) => stream::iter(vec![Ok(Some(task))]).boxed(), + Err(e) => stream::once(ready(Err(e))).boxed(), + }) + .boxed(); + + let eager_fetcher = StreamExt::boxed(SqlitePollFetcher::::new( + &self.pool, + &self.config, + &worker, + )); + starter.chain(select(lazy_fetcher, eager_fetcher)).boxed() + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use apalis_core::{ + backend::TaskSink, + error::BoxDynError, + task::{Task, task_id::TaskId}, + worker::builder::WorkerBuilder, + }; + + use crate::context::SqliteContext; + + use super::*; + + #[tokio::test] + async fn basic_worker() { + let pool = SqlitePool::connect(":memory:").await.unwrap(); + + SqliteStorage::setup(&pool).await.unwrap(); + + let mut store = SharedSqliteStorage::new(pool); + + let mut map_store = store.make_shared().unwrap(); + + let mut int_store = store.make_shared().unwrap(); + + let task = Task::builder(99u32) + .run_after(Duration::from_secs(2)) + .with_ctx(SqliteContext::new().with_priority(1)) + .build(); + + map_store + .send_all(&mut stream::iter(vec![task].into_iter().map(Ok))) + .await + .unwrap(); + int_store.push(99).await.unwrap(); + + async fn send_reminder( + _: T, + _task_id: TaskId, + wrk: WorkerContext, + ) -> Result<(), BoxDynError> { + tokio::time::sleep(Duration::from_secs(2)).await; + wrk.stop().unwrap(); + Ok(()) + } + + let int_worker = WorkerBuilder::new("rango-tango-2") + .backend(int_store) + .build(send_reminder); + let map_worker = WorkerBuilder::new("rango-tango-1") + .backend(map_store) + .build(send_reminder); + tokio::try_join!(int_worker.run(), map_worker.run()).unwrap(); + } +} diff --git a/src/sink.rs b/src/sink.rs new file mode 100644 index 0000000..95453ff --- /dev/null +++ b/src/sink.rs @@ -0,0 +1,155 @@ +use std::{ + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use apalis_core::backend::codec::Codec; +use futures::{ + FutureExt, Sink, + future::{BoxFuture, Shared}, +}; +use sqlx::SqlitePool; +use ulid::Ulid; + +use crate::{CompactType, SqliteStorage, SqliteTask, config::Config}; + +type FlushFuture = BoxFuture<'static, Result<(), Arc>>; + +#[pin_project::pin_project] +pub struct SqliteSink { + pool: SqlitePool, + config: Config, + buffer: Vec>, + #[pin] + flush_future: Option>, + _marker: std::marker::PhantomData<(Args, Codec)>, +} + +impl Clone for SqliteSink { + fn clone(&self) -> Self { + Self { + pool: self.pool.clone(), + config: self.config.clone(), + buffer: Vec::new(), + flush_future: None, + _marker: std::marker::PhantomData, + } + } +} + +pub async fn push_tasks( + pool: SqlitePool, + cfg: Config, + buffer: Vec>, +) -> Result<(), Arc> { + let mut tx = pool.begin().await?; + for task in buffer { + let id = task + .parts + .task_id + .map(|id| id.to_string()) + .unwrap_or(Ulid::new().to_string()); + let run_at = task.parts.run_at as i64; + let max_attempts = task.parts.ctx.max_attempts(); + let priority = task.parts.ctx.priority(); + let args = task.args; + // Use specified queue if specified, otherwise use default + let job_type = match task.parts.queue { + Some(ref queue) => queue.to_string(), + None => cfg.queue().to_string(), + }; + sqlx::query_file!( + "queries/task/sink.sql", + args, + id, + job_type, + max_attempts, + run_at, + priority, + ) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + + Ok(()) +} + +impl SqliteSink { + pub fn new(pool: &SqlitePool, config: &Config) -> Self { + Self { + pool: pool.clone(), + config: config.clone(), + buffer: Vec::new(), + _marker: std::marker::PhantomData, + flush_future: None, + } + } +} + +impl Sink> for SqliteStorage +where + Args: Send + Sync + 'static, + Encode: Codec, + Encode::Error: std::error::Error + Send + Sync + 'static, +{ + type Error = sqlx::Error; + + fn poll_ready(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn start_send(self: Pin<&mut Self>, item: SqliteTask) -> Result<(), Self::Error> { + // Add the item to the buffer + self.project() + .sink + .buffer + .push(item.try_map(|s| Encode::encode(&s).map_err(|e| sqlx::Error::Encode(e.into())))?); + Ok(()) + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let mut this = self.project(); + + // If there's no existing future and buffer is empty, we're done + if this.sink.flush_future.is_none() && this.sink.buffer.is_empty() { + return Poll::Ready(Ok(())); + } + + // Create the future only if we don't have one and there's work to do + if this.sink.flush_future.is_none() && !this.sink.buffer.is_empty() { + let pool = this.pool.clone(); + let config = this.config.clone(); + let buffer = std::mem::take(&mut this.sink.buffer); + let sink_fut = push_tasks(pool, config, buffer); + this.sink.flush_future = Some((Box::pin(sink_fut) as FlushFuture).shared()); + } + + // Poll the existing future + if let Some(mut fut) = this.sink.flush_future.take() { + match fut.poll_unpin(cx) { + Poll::Ready(Ok(())) => { + // Future completed successfully, don't put it back + Poll::Ready(Ok(())) + } + Poll::Ready(Err(e)) => { + // Future completed with error, don't put it back + Poll::Ready(Err(Arc::into_inner(e).unwrap())) + } + Poll::Pending => { + // Future is still pending, put it back and return Pending + this.sink.flush_future = Some(fut); + Poll::Pending + } + } + } else { + // No future and no work to do + Poll::Ready(Ok(())) + } + } + + fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.poll_flush(cx) + } +}