From MuData to SpatialData

.github/workflows/spatial_deconvolution-ci.yml → .github/workflows/spatial_deconvolution_cell2location-ci.yml

@@ -1,4 +1,4 @@
-name: Run tutorials (spatial deconvolution)
+name: Run tutorials (spatial deconvolution cell2location)
 
 on:
   push:
@@ -12,7 +12,7 @@ env:
   debug: 'true'
 
 jobs:
-  spatial_deconvolution:
+  spatial_deconvolution_cell2location:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -60,7 +60,9 @@ jobs:
           cd deconvolution/data 
           curl -L -o Human_Heart_reference.h5mu https://figshare.com/ndownloader/files/44969677
           cd spatial_data
-          curl -L -o Human_Heart.h5mu https://figshare.com/ndownloader/files/44969488
+          curl -L -o Human_Heart.zarr.zip https://figshare.com/ndownloader/files/51667673
+          unzip Human_Heart.zarr.zip
+          rm Human_Heart.zarr.zip
 
 
       # Note: we run the following to test that the commands works

.github/workflows/spatial_deconvolution_tangram-ci.yml

@@ -0,0 +1,98 @@
+name: Run tutorials (spatial deconvolution tangram)
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env:
+  debug: 'true'
+
+jobs:
+  spatial_deconvolution_tangram:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"] # , "macos-latest", "windows-latest"
+        python-version: ["3.10"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
+          auto-activate-base: true
+          auto-update-conda: true
+          channels: conda-forge
+          channel-priority: strict
+          activate-environment: pipeline_env
+          environment-file: pipeline_env.yaml
+
+      - name: Install Panpipes
+        shell: bash -el {0}
+        run: |
+          pip install .[spatial]
+          conda list
+
+      - name: Conda info
+        if: env.debug == 'true'
+        shell: bash -el {0}
+        run: conda info
+
+      - name: Conda list
+        if: env.debug == 'true'
+        shell: pwsh
+        run: conda list
+
+
+      - name: Preparing the data
+        run: |
+          mkdir deconvolution_tangram deconvolution_tangram/data deconvolution_tangram/data/spatial_data
+          cd deconvolution_tangram/data 
+          curl -L -o Human_Heart_reference.h5mu https://figshare.com/ndownloader/files/44969677
+          cd spatial_data
+          curl -L -o Human_Heart.zarr.zip https://figshare.com/ndownloader/files/51667673
+          unzip Human_Heart.zarr.zip
+          rm Human_Heart.zarr.zip
+
+
+      # Note: we run the following to test that the commands works
+      - name: Preparing the configuration file
+        shell: bash -el {0}
+        run: |
+          cd deconvolution_tangram
+          panpipes deconvolution_spatial config
+
+      - name: Edit the submission file
+        run: |
+          cd deconvolution_tangram
+          curl -o pipeline.yml https://raw.githubusercontent.com/DendrouLab/panpipes-tutorials/sarah_spatialData/docs/deconvolution_tangram/pipeline.yml
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree deconvolution_tangram
+
+      - name: Review pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd deconvolution_tangram
+          panpipes deconvolution_spatial show full --local
+
+      - name: Run pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd deconvolution_tangram
+          panpipes deconvolution_spatial make full --local
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree deconvolution_tangram

.github/workflows/spatial_ingestion_merfish-ci.yml

@@ -58,11 +58,11 @@ jobs:
         run: |
           mkdir spatial spatial/ingestion_merfish spatial/ingestion_merfish/data
           cd spatial/ingestion_merfish/data 
-          curl -L -o datasets_mouse_brain_map_BrainReceptorShowcase_Slice1_Replicate1_cell_by_gene_S1R1.csv https://figshare.com/ndownloader/files/45028624
-          curl -L -o datasets_mouse_brain_map_BrainReceptorShowcase_Slice1_Replicate1_cell_metadata_S1R1.csv https://figshare.com/ndownloader/files/45028621
+          curl -L -o cell_by_gene.csv https://figshare.com/ndownloader/files/45028624
+          curl -L -o cell_metadata.csv https://figshare.com/ndownloader/files/45028621
           mkdir images
           cd images 
-          curl -L -o datasets_mouse_brain_map_BrainReceptorShowcase_Slice1_Replicate1_images_micron_to_mosaic_pixel_transform.csv https://figshare.com/ndownloader/files/45028645
+          curl -L -o micron_to_mosaic_pixel_transform.csv https://figshare.com/ndownloader/files/45028645
 
 
       # Note: we run the following to test that the commands works
@@ -75,12 +75,12 @@ jobs:
       - name: Preparing the submission file
         run: |
           cd spatial/ingestion_merfish
-          curl -o sample_file_qc_spatial.txt https://raw.githubusercontent.com/DendrouLab/panpipes-tutorials/main/docs/ingesting_merfish_data/sample_file_qc_spatial.txt
+          curl -o sample_file_qc_merfish.txt https://raw.githubusercontent.com/DendrouLab/panpipes-tutorials/sarah_spatialData/docs/ingesting_merfish_data/sample_file_qc_merfish.txt
 
       - name: Preparing the yaml file
         run: |
           cd spatial/ingestion_merfish
-          curl -o pipeline.yml https://raw.githubusercontent.com/DendrouLab/panpipes-tutorials/main/docs/ingesting_merfish_data/pipeline.yml
+          curl -o pipeline.yml https://raw.githubusercontent.com/DendrouLab/panpipes-tutorials/sarah_spatialData/docs/ingesting_merfish_data/pipeline.yml
 
       - name: File tree
         if: env.debug == 'true'

.github/workflows/spatial_ingestion_merscope-ci.yml

@@ -0,0 +1,105 @@
+name: Run tutorials (spatial ingest merscope)
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env:
+  debug: 'true'
+
+jobs:
+  spatial_ingest_merscope:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"] # , "macos-latest", "windows-latest"
+        python-version: ["3.10"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
+          auto-activate-base: true
+          auto-update-conda: true
+          channels: conda-forge
+          channel-priority: strict
+          activate-environment: pipeline_env
+          environment-file: pipeline_env.yaml
+
+      - name: Install Panpipes
+        shell: bash -el {0}
+        run: |
+          pip install .[spatial]
+          conda list
+
+      - name: Conda info
+        if: env.debug == 'true'
+        shell: bash -el {0}
+        run: conda info
+
+      - name: Conda list
+        if: env.debug == 'true'
+        shell: pwsh
+        run: conda list
+
+
+      - name: Preparing the data
+        run: |
+          mkdir spatial spatial/ingestion_merscope spatial/ingestion_merscope/data
+          cd spatial/ingestion_merscope/data 
+          curl -L -o cell_by_gene.csv https://figshare.com/ndownloader/files/50899455
+          curl -L -o cell_metadata.csv https://figshare.com/ndownloader/files/50899452
+          curl -L -o cellpose_micron_space.parquet https://figshare.com/ndownloader/files/50899458
+          # curl -L -o detected_transcripts.csv https://figshare.com/ndownloader/files/50899476
+          mkdir images
+          cd images 
+          curl -L -o micron_to_mosaic_pixel_transform.csv https://figshare.com/ndownloader/files/50899449
+
+
+      # Note: we run the following to test that the commands works
+      - name: Preparing the configuration file
+        shell: bash -el {0}
+        run: |
+          cd spatial/ingestion_merscope
+          panpipes qc_spatial config
+
+      - name: Preparing the submission file
+        run: |
+          cd spatial/ingestion_merscope
+          curl -o sample_file_qc_merscope.txt https://raw.githubusercontent.com/DendrouLab/panpipes-tutorials/sarah_spatialData/docs/ingesting_merscope_data/sample_file_qc_merscope.txt
+
+      - name: Preparing the yaml file
+        run: |
+          cd spatial/ingestion_merscope
+          curl -o pipeline.yml https://raw.githubusercontent.com/DendrouLab/panpipes-tutorials/sarah_spatialData/docs/ingesting_merscope_data/pipeline.yml
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree spatial/ingestion_merscope
+
+      - name: Review pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd spatial/ingestion_merscope
+          panpipes qc_spatial show full --local
+
+      - name: Run pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd spatial/ingestion_merscope
+          panpipes qc_spatial make full --local
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree spatial/ingestion_merscope

.github/workflows/spatial_ingestion_visium-ci.yml

@@ -78,12 +78,11 @@ jobs:
       - name: Preparing the submission file
         run: |
           cd spatial/ingestion
-          curl -o sample_file_qc_spatial.txt https://raw.githubusercontent.com/DendrouLab/panpipes-tutorials/main/docs/ingesting_visium_data/sample_file_qc_spatial.txt
-
+          curl -o sample_file_qc_visium.txt https://raw.githubusercontent.com/DendrouLab/panpipes-tutorials/sarah_spatialData/docs/ingesting_visium_data/sample_file_qc_visium.txt
       - name: Preparing the yaml file
         run: |
           cd spatial/ingestion
-          curl -o pipeline.yml https://raw.githubusercontent.com/DendrouLab/panpipes-tutorials/main/docs/ingesting_visium_data/pipeline.yml
+          curl -o pipeline.yml https://raw.githubusercontent.com/DendrouLab/panpipes-tutorials/sarah_spatialData/docs/ingesting_visium_data/pipeline.yml
 
       - name: File tree
         if: env.debug == 'true'

.github/workflows/spatial_ingestion_xenium.yml

@@ -0,0 +1,109 @@
+name: Run tutorials (spatial ingest xenium)
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+env:
+  debug: 'true'
+
+jobs:
+  spatial_ingest_xenium:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"] # , "macos-latest", "windows-latest"
+        python-version: ["3.10"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniforge-version: latest
+          auto-activate-base: true
+          auto-update-conda: true
+          channels: conda-forge
+          channel-priority: strict
+          activate-environment: pipeline_env
+          environment-file: pipeline_env.yaml
+
+      - name: Install Panpipes
+        shell: bash -el {0}
+        run: |
+          pip install .[spatial]
+          conda list
+
+      - name: Conda info
+        if: env.debug == 'true'
+        shell: bash -el {0}
+        run: conda info
+
+      - name: Conda list
+        if: env.debug == 'true'
+        shell: pwsh
+        run: conda list
+
+
+      - name: Preparing the data
+        run: |
+          mkdir spatial spatial/ingestion_xenium spatial/ingestion_xenium/data
+          cd spatial/ingestion_xenium/data 
+          curl -L -o experiment.xenium https://figshare.com/ndownloader/files/51244265
+          curl -L -o nucleus_boundaries.parquet https://figshare.com/ndownloader/files/51244286
+          curl -L -o cell_boundaries.parquet https://figshare.com/ndownloader/files/51244244
+          curl -L -o transcripts.parquet https://figshare.com/ndownloader/files/51244283
+          curl -L -o cell_feature_matrix.h5 https://figshare.com/ndownloader/files/51244247
+          curl -L -o cells.parquet https://figshare.com/ndownloader/files/51244259
+          curl -L -o morphology_mip.ome.tif https://figshare.com/ndownloader/files/51244415
+          curl -L -o cells.zarr.zip https://figshare.com/ndownloader/files/51244262
+          mkdir morphology_focus
+          cd morphology_focus
+          curl -L -o morphology_focus_0000.ome.tif https://figshare.com/ndownloader/files/51244277
+
+
+      # Note: we run the following to test that the commands works
+      - name: Preparing the configuration file
+        shell: bash -el {0}
+        run: |
+          cd spatial/ingestion_xenium
+          panpipes qc_spatial config
+
+      - name: Preparing the submission file
+        run: |
+          cd spatial/ingestion_xenium
+          curl -o sample_file_qc_xenium.txt https://raw.githubusercontent.com/DendrouLab/panpipes-tutorials/sarah_spatialData/docs/ingesting_xenium_data/sample_file_qc_xenium.txt
+
+      - name: Preparing the yaml file
+        run: |
+          cd spatial/ingestion_xenium
+          curl -o pipeline.yml https://raw.githubusercontent.com/DendrouLab/panpipes-tutorials/sarah_spatialData/docs/ingesting_xenium_data/pipeline.yml
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree spatial/ingestion_xenium
+
+      - name: Review pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd spatial/ingestion_xenium
+          panpipes qc_spatial show full --local
+
+      - name: Run pipeline tasks
+        shell: bash -el {0}
+        run: |
+          cd spatial/ingestion_xenium
+          panpipes qc_spatial make full --local
+
+      - name: File tree
+        if: env.debug == 'true'
+        run: tree spatial/ingestion_xenium