TSPS-183 Beagle imputation hg38 wdl and associated support wdls

.dockstore.yml

@@ -83,6 +83,14 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /pipelines/broad/arrays/imputation/Imputation.wdl
 
+  - name: ImputationBeagle
+    subclass: WDL
+    primaryDescriptorPath: /pipelines/broad/arrays/imputation_beagle/ImputationBeagle.wdl
+
+  - name: ArrayImputationQuotaConsumed
+    subclass: WDL
+    primaryDescriptorPath: /pipelines/broad/arrays/imputation_beagle/ArrayImputationQuotaConsumed.wdl
+
   - name: RNAWithUMIsPipeline
     subclass: WDL
     primaryDescriptorPath: /pipelines/broad/rna_seq/RNAWithUMIsPipeline.wdl
@@ -155,6 +163,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestImputation.wdl
 
+  - name: TestImputationBeagle
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestImputationBeagle.wdl
+
   - name: TestJointGenotyping
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestJointGenotyping.wdl

.github/workflows/test_imputation_beagle.yml

@@ -0,0 +1,75 @@
+name: Test ImputationBeagle
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/broad/arrays/imputation_beagle/**'
+      - 'structs/imputation/ImputationBeagleStructs.wdl'
+      - 'tasks/broad/ImputationTasks.wdl'
+      - 'tasks/broad/ImputationBeagleTasks.wdl'
+      - 'verification/VerifyImputationBeagle.wdl'
+      - 'verification/test-wdls/TestImputationBeagle.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_imputation_beagle.yml'
+      - '.github/workflows/warp_test_workflow.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+env:
+  # pipeline configuration
+  PIPELINE_NAME: TestImputationBeagle
+  DOCKSTORE_PIPELINE_NAME: ImputationBeagle
+  PIPELINE_DIR: "pipelines/broad/arrays/imputation_beagle"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: [email protected]
+
+
+jobs:
+  TestImputationBeagle:
+    uses: ./.github/workflows/warp_test_workflow.yml
+    with:
+      pipeline_name: TestImputationBeagle
+      dockstore_pipeline_name: ImputationBeagle
+      pipeline_dir: pipelines/broad/arrays/imputation_beagle
+      use_call_cache: ${{ github.event.inputs.useCallCache || 'true' }}
+      update_truth: ${{ github.event.inputs.updateTruth || 'false' }}
+      test_type: ${{ github.event.inputs.testType }}
+      truth_branch: ${{ github.event.inputs.truthBranch || 'master' }}
+    secrets:
+      PDT_TESTER_SA_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+      DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}

.github/workflows/warp_test_workflow.yml

@@ -188,6 +188,7 @@ jobs:
 
           echo "Starting hash comparison with retry mechanism..."
 
+
           while [ $TOTAL_WAITED -lt $MAX_WAIT_TIME ]; do
             echo "Fetching Dockstore Commit Hash..."
             DOCKSTORE_COMMIT_HASH=$(python scripts/dockstore_api/fetch_dockstore_commit.py \

pipeline_versions.txt

@@ -1,40 +1,42 @@
 Pipeline Name	Version	Date of Last Commit
-snm3C	 4.0.4	2024-08-06 
-BuildIndices	 4.0.0	2025-01-17 
-scATAC	 1.3.2	2023-08-03 
-MultiSampleSmartSeq2SingleNucleus	 2.0.8	2025-02-25 
-atac	 2.7.1	2025-02-25 
-Optimus	 7.9.2	2025-02-25 
-SmartSeq2SingleSample	 5.1.21	2024-09-11 
-Multiome	 5.11.0	2025-02-25 
-PairedTag	 1.10.2 	2025-02-25 
-SlideSeq	 3.4.9	2025-02-25 
-MultiSampleSmartSeq2	 2.2.22	2024-09-11 
-AnnotationFiltration	 1.2.7	2024-11-04 
-RNAWithUMIsPipeline	 1.0.18	2024-11-04 
-Imputation	 1.1.15	2024-11-04 
 Arrays	 2.6.30	2024-11-04 
-MultiSampleArrays	 1.6.2	2024-08-02 
 ValidateChip	 1.16.7	2024-11-04 
-JointGenotyping	 1.7.2	2024-11-04 
+ArrayImputationQuotaConsumed	 1.0.0	2025-02-24 
+ImputationBeagle	 1.0.0	2025-02-24 
+Imputation	 1.1.16	2025-02-24 
+MultiSampleArrays	 1.6.2	2024-08-02 
+WholeGenomeReprocessing	 3.3.3	2024-11-04 
+ExomeReprocessing	 3.3.3	2024-11-04 
+CramToUnmappedBams	 1.1.3	2024-08-02 
+ExternalWholeGenomeReprocessing	 2.3.3	2024-11-04 
+ExternalExomeReprocessing	 3.3.3	2024-11-04 
+BroadInternalArrays	 1.1.14	2024-11-04 
+BroadInternalImputation	 1.1.15	2025-02-24 
+BroadInternalRNAWithUMIs	 1.0.36	2024-11-04 
+BroadInternalUltimaGenomics	 1.1.3	2024-12-05 
+RNAWithUMIsPipeline	 1.0.18	2024-11-04 
+IlluminaGenotypingArray	 1.12.24	2024-11-04 
+AnnotationFiltration	 1.2.7	2024-11-04 
+UltimaGenomicsWholeGenomeCramOnly	 1.0.23	2024-11-04 
+GDCWholeGenomeSomaticSingleSample	 1.3.4	2024-11-04 
+UltimaGenomicsWholeGenomeGermline	 1.1.3	2024-12-05 
+WholeGenomeGermlineSingleSample	 3.3.3	2024-11-04 
+ExomeGermlineSingleSample	 3.2.3	2024-11-04 
+VariantCalling	 2.2.4	2024-11-04 
 ReblockGVCF	 2.4.0	2024-12-05 
 UltimaGenomicsJointGenotyping	 1.2.2	2024-11-04 
-JointGenotypingByChromosomePartTwo	 1.5.2	2024-11-04 
 JointGenotypingByChromosomePartOne	 1.5.2	2024-11-04 
-ExomeGermlineSingleSample	 3.2.3	2024-11-04 
-WholeGenomeGermlineSingleSample	 3.3.3	2024-11-04 
-UltimaGenomicsWholeGenomeGermline	 1.1.3	2024-12-05 
-VariantCalling	 2.2.4	2024-11-04 
-GDCWholeGenomeSomaticSingleSample	 1.3.4	2024-11-04 
-UltimaGenomicsWholeGenomeCramOnly	 1.0.23	2024-11-04 
+JointGenotypingByChromosomePartTwo	 1.5.2	2024-11-04 
+JointGenotyping	 1.7.2	2024-11-04 
 CheckFingerprint	 1.0.22	2024-10-28 
-BroadInternalRNAWithUMIs	 1.0.36	2024-11-04 
-BroadInternalImputation	 1.1.14	2024-11-04 
-BroadInternalArrays	 1.1.14	2024-11-04 
-BroadInternalUltimaGenomics	 1.1.3	2024-12-05 
-IlluminaGenotypingArray	 1.12.24	2024-11-04 
-ExternalExomeReprocessing	 3.3.3	2024-11-04 
-ExternalWholeGenomeReprocessing	 2.3.3	2024-11-04 
-ExomeReprocessing	 3.3.3	2024-11-04 
-CramToUnmappedBams	 1.1.3	2024-08-02 
-WholeGenomeReprocessing	 3.3.3	2024-11-04 
+scATAC	 1.3.2	2023-08-03 
+MultiSampleSmartSeq2	 2.2.22	2024-09-11 
+BuildIndices	 4.0.0	2025-01-17 
+SlideSeq	 3.4.9	2025-02-25 
+PairedTag	 1.10.2 	2025-02-25 
+MultiSampleSmartSeq2SingleNucleus	 2.0.8	2025-02-25 
+atac	 2.7.1	2025-02-25 
+snm3C	 4.0.4	2024-08-06 
+SmartSeq2SingleSample	 5.1.21	2024-09-11 
+Optimus	 7.9.2	2025-02-25 
+Multiome	 5.11.0	2025-02-25 

pipelines/broad/arrays/imputation/Imputation.changelog.md

@@ -1,3 +1,8 @@
+# 1.1.16
+2025-02-24 (Date of Last Commit)
+
+* Updated runtime parameters in some ImputationTasks, and added an explicit definition of a vcf_index.
+
 # 1.1.15
 2024-11-04 (Date of Last Commit)
 

pipelines/broad/arrays/imputation/Imputation.wdl

@@ -6,7 +6,7 @@ import "../../../../tasks/broad/Utilities.wdl" as utils
 
 workflow Imputation {
 
-  String pipeline_version = "1.1.15"
+  String pipeline_version = "1.1.16"
 
   input {
     Int chunkLength = 25000000
@@ -242,6 +242,7 @@ workflow Imputation {
       call tasks.SelectVariantsByIds {
         input:
           vcf = SetIdsVcfToImpute.output_vcf,
+          vcf_index = SetIdsVcfToImpute.output_vcf_index,
           ids = FindSitesUniqueToFileTwoOnly.missing_sites,
           basename = "imputed_sites_to_recover"
       }

pipelines/broad/arrays/imputation_beagle/ArrayImputationQuotaConsumed.changelog.md

@@ -0,0 +1,4 @@
+# 1.0.0
+2025-02-24 (Date of Last Commit)
+
+* Initial release of pipeline to calculate the number of samples, i.e. quota used by an imputation service that uses ImputationBeagle.wdl.

pipelines/broad/arrays/imputation_beagle/ArrayImputationQuotaConsumed.wdl

@@ -0,0 +1,29 @@
+version 1.0
+
+import "../../../../tasks/broad/ImputationTasks.wdl" as tasks
+
+workflow QuotaConsumed {
+    String pipeline_version = "1.0.0"
+
+    input {
+        Int chunkLength = 25000000
+        Int chunkOverlaps = 5000000
+
+        File multi_sample_vcf
+
+        File ref_dict
+        Array[String] contigs
+        String reference_panel_path_prefix
+        String genetic_maps_path
+        String output_basename
+    }
+
+    call tasks.CountSamples {
+        input:
+            vcf = multi_sample_vcf
+    }
+
+    output {
+        Int quota_consumed = CountSamples.nSamples
+    }
+}

pipelines/broad/arrays/imputation_beagle/ImputationBeagle.changelog.md

@@ -0,0 +1,6 @@
+# 1.0.0
+2025-02-24 (Date of Last Commit)
+
+* * Initial public release of the ImputationBeagle pipeline.
+
+  * The ImputationBeagle pipeline imputes missing genotypes from a multi-sample VCF using a large genomic reference panel. It is based on the Michigan Imputation Server pipeline but uses the Beagle imputation tool instead of minimac. Overall, the pipeline filters, phases, and performs imputation on a multi-sample VCF. It outputs the imputed VCF.