Merge pull request #21 from COMBINE-lab/dev

Merge dev into main

.github/workflows/release.yml

@@ -1,20 +1,19 @@
-# Copyright 2022-2023, axodotdev
+# Copyright 2022-2024, axodotdev
 # SPDX-License-Identifier: MIT or Apache-2.0
 #
 # CI that:
 #
 # * checks for a Git Tag that looks like a release
 # * builds artifacts with cargo-dist (archives, installers, hashes)
 # * uploads those artifacts to temporary workflow zip
-# * on success, uploads the artifacts to a Github Release
+# * on success, uploads the artifacts to a GitHub Release
 #
-# Note that the Github Release will be created with a generated
+# Note that the GitHub Release will be created with a generated
 # title/body based on your changelogs.
 
 name: Release
-
 permissions:
-  contents: write
+  "contents": "write"
 
 # This task will run whenever you push a git tag that looks like a version
 # like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc.
@@ -31,22 +30,22 @@ permissions:
 # packages versioned/released in lockstep).
 #
 # If you push multiple tags at once, separate instances of this workflow will
-# spin up, creating an independent announcement for each one. However Github
+# spin up, creating an independent announcement for each one. However, GitHub
 # will hard limit this to 3 tags per commit, as it will assume more tags is a
 # mistake.
 #
 # If there's a prerelease-style suffix to the version, then the release(s)
 # will be marked as a prerelease.
 on:
+  pull_request:
   push:
     tags:
       - '**[0-9]+.[0-9]+.[0-9]+*'
-  pull_request:
 
 jobs:
   # Run 'cargo dist plan' (or host) to determine what tasks we need to do
   plan:
-    runs-on: ubuntu-latest
+    runs-on: "ubuntu-20.04"
     outputs:
       val: ${{ steps.plan.outputs.manifest }}
       tag: ${{ !github.event.pull_request && github.ref_name || '' }}
@@ -62,7 +61,12 @@ jobs:
         # we specify bash to get pipefail; it guards against the `curl` command
         # failing. otherwise `sh` won't catch that `curl` returned non-0
         shell: bash
-        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.12.0/cargo-dist-installer.sh | sh"
+        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.19.1/cargo-dist-installer.sh | sh"
+      - name: Cache cargo-dist
+        uses: actions/upload-artifact@v4
+        with:
+          name: cargo-dist-cache
+          path: ~/.cargo/bin/cargo-dist
       # sure would be cool if github gave us proper conditionals...
       # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible
       # functionality based on whether this is a pull_request, and whether it's from a fork.
@@ -105,10 +109,12 @@ jobs:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json
     steps:
+      - name: enable windows longpaths
+        run: |
+          git config --global core.longpaths true
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - uses: swatinem/rust-cache@v2
       - name: Install cargo-dist
         run: ${{ matrix.install_dist }}
       # Get the dist-manifest
@@ -135,7 +141,7 @@ jobs:
         run: |
           # Parse out what we just built and upload it to scratch storage
           echo "paths<<EOF" >> "$GITHUB_OUTPUT"
-          jq --raw-output ".artifacts[]?.path | select( . != null )" dist-manifest.json >> "$GITHUB_OUTPUT"
+          jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT"
           echo "EOF" >> "$GITHUB_OUTPUT"
 
           cp dist-manifest.json "$BUILD_MANIFEST_NAME"
@@ -160,9 +166,12 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - name: Install cargo-dist
-        shell: bash
-        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.12.0/cargo-dist-installer.sh | sh"
+      - name: Install cached cargo-dist
+        uses: actions/download-artifact@v4
+        with:
+          name: cargo-dist-cache
+          path: ~/.cargo/bin/
+      - run: chmod +x ~/.cargo/bin/cargo-dist
       # Get all the local artifacts for the global tasks to use (for e.g. checksums)
       - name: Fetch local artifacts
         uses: actions/download-artifact@v4
@@ -178,7 +187,7 @@ jobs:
 
           # Parse out what we just built and upload it to scratch storage
           echo "paths<<EOF" >> "$GITHUB_OUTPUT"
-          jq --raw-output ".artifacts[]?.path | select( . != null )" dist-manifest.json >> "$GITHUB_OUTPUT"
+          jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT"
           echo "EOF" >> "$GITHUB_OUTPUT"
 
           cp dist-manifest.json "$BUILD_MANIFEST_NAME"
@@ -206,16 +215,19 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - name: Install cargo-dist
-        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.12.0/cargo-dist-installer.sh | sh"
+      - name: Install cached cargo-dist
+        uses: actions/download-artifact@v4
+        with:
+          name: cargo-dist-cache
+          path: ~/.cargo/bin/
+      - run: chmod +x ~/.cargo/bin/cargo-dist
       # Fetch artifacts from scratch-storage
       - name: Fetch artifacts
         uses: actions/download-artifact@v4
         with:
           pattern: artifacts-*
           path: target/distrib/
           merge-multiple: true
-      # This is a harmless no-op for Github Releases, hosting for that happens in "announce"
       - id: host
         shell: bash
         run: |
@@ -229,8 +241,29 @@ jobs:
           # Overwrite the previous copy
           name: artifacts-dist-manifest
           path: dist-manifest.json
+      # Create a GitHub Release while uploading all files to it
+      - name: "Download GitHub Artifacts"
+        uses: actions/download-artifact@v4
+        with:
+          pattern: artifacts-*
+          path: artifacts
+          merge-multiple: true
+      - name: Cleanup
+        run: |
+          # Remove the granular manifests
+          rm -f artifacts/*-dist-manifest.json
+      - name: Create GitHub Release
+        env:
+          PRERELEASE_FLAG: "${{ fromJson(steps.host.outputs.manifest).announcement_is_prerelease && '--prerelease' || '' }}"
+          ANNOUNCEMENT_TITLE: "${{ fromJson(steps.host.outputs.manifest).announcement_title }}"
+          ANNOUNCEMENT_BODY: "${{ fromJson(steps.host.outputs.manifest).announcement_github_body }}"
+          RELEASE_COMMIT: "${{ github.sha }}"
+        run: |
+          # Write and read notes from a file to avoid quoting breaking things
+          echo "$ANNOUNCEMENT_BODY" > $RUNNER_TEMP/notes.txt
+
+          gh release create "${{ needs.plan.outputs.tag }}" --target "$RELEASE_COMMIT" $PRERELEASE_FLAG --title "$ANNOUNCEMENT_TITLE" --notes-file "$RUNNER_TEMP/notes.txt" artifacts/*
 
-  # Create a Github Release while uploading all files to it
   announce:
     needs:
       - plan
@@ -246,21 +279,3 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - name: "Download Github Artifacts"
-        uses: actions/download-artifact@v4
-        with:
-          pattern: artifacts-*
-          path: artifacts
-          merge-multiple: true
-      - name: Cleanup
-        run: |
-          # Remove the granular manifests
-          rm -f artifacts/*-dist-manifest.json
-      - name: Create Github Release
-        uses: ncipollo/release-action@v1
-        with:
-          tag: ${{ needs.plan.outputs.tag }}
-          name: ${{ fromJson(needs.host.outputs.val).announcement_title }}
-          body: ${{ fromJson(needs.host.outputs.val).announcement_github_body }}
-          prerelease: ${{ fromJson(needs.host.outputs.val).announcement_is_prerelease }}
-          artifacts: "artifacts/*"

.github/workflows/sanitize-cargo.yml

@@ -0,0 +1,38 @@
+name: Sanitize Cargo
+on: 
+  push:
+    branches:
+      - main
+
+permissions:
+  "contents": "write"
+
+jobs:
+  sanitize_cargo_file:
+    if: "contains(github.event.head_commit.message, '[do_tag]')"
+    runs-on: ubuntu-latest
+    # This is optional; it exposes the plan to your job as an environment variable
+    #env:
+    #  PLAN: ${{ inputs.plan }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.PAT_TOK }}
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Install cargo-sanitize
+        run: |
+          cargo install cargo-sanitize
+      - name: Sanitize Cargo.toml file
+        run: |
+          cp Cargo.toml Cargo.toml.cs_orig
+          cargo-sanitize -i Cargo.toml.cs_orig -o Cargo.toml
+      - name: Tag and push new commit
+        run: |
+          export VERSION_TAG=`cargo read-manifest | jq ".version" | tr -d '"'`
+          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add Cargo.toml.cs_orig
+          git add Cargo.toml
+          git commit -m "create sanitized release"
+          git tag --force -a v${VERSION_TAG} -m "version ${VERSION_TAG}"
+          git push --force origin v${VERSION_TAG}

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "oarfish"
-version = "0.4.0"
+version = "0.5.0"
 edition = "2021"
 authors = [
   "Zahra Zare Jousheghani <[email protected]>",
@@ -31,29 +31,30 @@ categories = ["command-line-utilities", "science"]
 
 [dependencies]
 seqcol_rs = { git = "https://github.com/COMBINE-lab/seqcol_rs", branch = "dev", version = "0.1.0" }
-anyhow = "1.0.81"
-bio-types = "1.0.1"
-clap = { version = "4.5.4", features = ["derive"] }
+anyhow = "1.0.86"
+bio-types = { version = "1.0.3", features = ["serde"] }
+clap = { version = "4.5.11", features = ["derive"] }
 coitrees = "0.4.0"
-noodles-bam = "0.56.0"
-noodles-gtf = "0.23.0"
-noodles-sam = "0.53.0"
+noodles-bam = "0.65.0"
+noodles-gtf = "0.30.0"
+noodles-sam = "0.62.0"
 num-format = "0.4.4"
 tabled = "0.15.0"
 tracing = "0.1.40"
 tracing-subscriber = {version="0.3.18", features=["env-filter"]}
-typed-builder = "0.18.1"
+typed-builder = "0.19.1"
 rayon = "1.10"
-statrs = "0.16"
+statrs = "0.17"
 csv = "1.3"
 ndarray = "0.15"
 serde = { version = "1", features = ["derive"] }
-itertools = "0.12.1"
-serde_json = "1.0.115"
+itertools = "0.13.0"
+serde_json = "1.0.121"
 path-tools = "0.1.0"
-atomic_float = "0.1.0"
+atomic_float = "1.0.0"
 rand = "0.8.5"
 arrow2 = { version = "0.18.0", features = ["io_parquet", "io_parquet_gzip", "io_parquet_zstd", "io_parquet_snappy"] }
+kders = { git = "https://github.com/COMBINE-lab/kde-rs.git", branch = "dev", version = "0.1.0" }
 
 [[bin]]
 name = "oarfish"
@@ -72,9 +73,9 @@ lto = "thin"
 # Config for 'cargo dist'
 [workspace.metadata.dist]
 # The preferred cargo-dist version to use in CI (Cargo.toml SemVer syntax)
-cargo-dist-version = "0.12.0"
+cargo-dist-version = "0.19.1"
 # CI backends to support
-ci = ["github"]
+ci = "github"
 # The installers to generate for each app
 installers = ["shell"]
 # Target platforms to build apps for (Rust target-triple syntax)
@@ -83,6 +84,8 @@ targets = ["aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-unknown-linux-
 pr-run-mode = "plan"
 # Whether to install an updater program
 install-updater = false
+# Path that installers should place binaries in
+install-path = "CARGO_HOME"
 
 [workspace.metadata.dist.github-custom-runners]
 aarch64-apple-darwin = "macos-14"

scripts/eval_sim.py

@@ -5,15 +5,20 @@
 
 
 def eval_sim(args):
-    x = pd.read_csv(args.true_abundances, names=["txp_name", "num_reads"], sep="\t")
+    x = pd.read_csv(args.true_abundances, names=["txp_name", "true_reads"], sep="\s+")
     y = pd.read_csv(args.predicted_abundances, sep="\s+")
-    print(y)
     y["txp_name"] = y.tname.str.split(".").str.get(0)
 
     m = pd.merge(x, y, on="txp_name", how="outer")
     m = m.fillna(0.0)
+    print(m.loc[:, ['txp_name', 'true_reads', 'num_reads']])
+    print(m.loc[:, ['true_reads', 'num_reads']].corr(method="spearman"))
+
+    ards = (m.loc[:, 'true_reads'] - m.loc[:, 'num_reads']).abs() / (m.loc[:, 'true_reads'] + m.loc[:, 'num_reads']).abs()
+    ards.fillna(0.0)
+    print(ards.mean())
+
 
-    print(m.corr(method="spearman"))
 
 
 if __name__ == "__main__":

src/alignment_parser.rs

@@ -25,7 +25,7 @@ pub fn read_and_verify_header<R: io::BufRead>(
     let mut progs = vec![];
     // explicitly check that alignment was done with a supported
     // aligner (right now, just minimap2).
-    for (prog, _pmap) in header.programs().iter() {
+    for (prog, _pmap) in header.programs().roots() {
         if prog == "minimap2" {
             saw_minimap2 = true;
             break;
@@ -88,6 +88,9 @@ pub fn parse_alignments<R: io::BufRead>(
                 // previous read record.
                 if !prev_read.is_empty() {
                     store.add_group(txps, &mut records_for_read);
+                    if records_for_read.len() == 1 {
+                        store.inc_unique_alignments();
+                    }
                     records_for_read.clear();
                 }
                 // the new "prev_read" name is the current read name
@@ -104,6 +107,9 @@ pub fn parse_alignments<R: io::BufRead>(
     // add that group.
     if !records_for_read.is_empty() {
         store.add_group(txps, &mut records_for_read);
+        if records_for_read.len() == 1 {
+            store.inc_unique_alignments();
+        }
         records_for_read.clear();
     }