Merge pull request #2083 from broadinstitute/development

Release 1.76.0

app/controllers/api/v1/visualization/annotations_controller.rb

@@ -188,6 +188,21 @@ def cell_values
               key :type, :string
               key :required, true
             end
+            parameter do
+              key :name, :loaded_annotation
+              key :in, :query
+              key :description, 'Name of currently loaded annotation (for sourcing cluster cells)'
+              key :type, :string
+              key :required, false
+            end
+            parameter do
+              key :name, :subsample
+              key :in, :query
+              key :description, 'Subsampling threshold'
+              key :type, :string
+              key :required, false
+              key :enum, [nil, 'all', 100000]
+            end
             response 200 do
               key :description, 'Array of integer-based annotation assignments for all cells in requested cluster'
               schema do
@@ -260,23 +275,29 @@ def facets
             render json: { error: "Cannot find annotations: #{missing.join(', ')}" }, status: :not_found and return
           end
 
+          if params[:subsample] == 'all' || params[:subsample] == 'null' || params[:subsample].nil?
+            subsample_threshold = nil
+          else
+            subsample_threshold = params[:subsample].to_i
+          end
+          subsample_annotation = params[:loaded_annotation]
           # use new cell index arrays to load data much faster
-          indexed_cluster_cells = cluster.cell_index_array
+          indexed_cluster_cells = cluster.cell_index_array(subsample_annotation: , subsample_threshold:)
           annotation_arrays = {}
           facets = []
           # build arrays of annotation values, and populate facets response array
           annotations.each do |annotation|
             annot_scope = annotation[:scope]
             annot_type = annotation[:type]
             identifier = annotation[:identifier]
-
             data_obj = annot_scope == 'study' ? @study.cell_metadata.by_name_and_type(annotation[:name], annot_type) : cluster
             study_file_id = annot_scope == 'study' ? @study.metadata_file.id : cluster.study_file_id
             array_query = {
               name: annotation[:name], array_type: 'annotations', linear_data_type: data_obj.class.name,
               linear_data_id: data_obj.id, study_id: @study.id, study_file_id:, subsample_annotation: nil,
               subsample_threshold: nil
             }
+
             annotation_arrays[identifier] = DataArray.concatenate_arrays(array_query)
             facets << { annotation: identifier, groups: annotation[:values] }
           end
@@ -288,7 +309,7 @@ def facets
             facets.map do |facet|
               annotation = facet[:annotation]
               _, annotation_type, annotation_scope = annotation.split('--')
-              if annotation_scope == 'study'
+              if annotation_scope == 'study' || subsample_threshold.present?
                 label = annotation_arrays[annotation][value] || '--Unspecified--'
               else
                 label = annotation_arrays[annotation][index] || '--Unspecified--'

app/javascript/components/explore/DifferentialExpressionPanel.jsx

@@ -224,13 +224,8 @@ function DifferentialExpressionTable({
     pageSize: numRows
   }
 
-  const defaultSorting = [
-    { id: 'significance', desc: false },
-    { id: 'size', desc: true }
-  ]
-
   const [rowSelection, setRowSelection] = useState({})
-  const [sorting, setSorting] = React.useState(defaultSorting)
+  const [sorting, setSorting] = React.useState([])
   const [pagination, setPagination] = React.useState(defaultPagination)
 
   const logProps = {
@@ -350,7 +345,7 @@ function DifferentialExpressionTable({
   /** Put DE table back to its original state */
   function resetDifferentialExpression() {
     setRowSelection({})
-    setSorting(defaultSorting)
+    setSorting([])
     setPagination(defaultPagination)
     handleClear()
   }

app/javascript/components/explore/ExploreDisplayPanelManager.jsx

@@ -253,11 +253,8 @@ export default function ExploreDisplayPanelManager({
     }
   }
 
-  /** Toggle cell filtering panel, and remove subsampling if needed */
+  /** Toggle cell filtering panel */
   function toggleCellFilterPanel() {
-    if (isSubsampled) {
-      updateClusterParams({ subsample: 'All Cells' })
-    }
     togglePanel('cell-filtering')
   }
 

app/javascript/components/explore/ExploreDisplayTabs.jsx

@@ -122,8 +122,9 @@ function getCellFacetingData(cluster, annotation, setterFunctions, context, prev
     const allAnnots = exploreInfo?.annotationList.annotations
     if (allAnnots && allAnnots.length > 0) {
       if (!prevCellFaceting?.isFullyLoaded) {
+        const subsample = exploreParams?.subsample || (exploreInfo?.cluster?.numPoints > 100_000 ? 100_000 : null)
         initCellFaceting(
-          cluster, annotation, studyAccession, allAnnots, prevCellFaceting
+          cluster, annotation, studyAccession, allAnnots, prevCellFaceting, subsample
         ).then(newCellFaceting => {
           const initSelection = {}
           if (!cellFilteringSelection) {
@@ -249,11 +250,6 @@ export default function ExploreDisplayTabs({
   const [clusterCanFilter, setClusterCanFilter] = useState(true)
   const [filterErrorText, setFilterErrorText] = useState(null)
 
-  // map of colors for maintaining associations in spatial UX
-  const [refColorMap, setRefColorMap] = useState({})
-  // state tracker to ensure that spatial/secondary plots update with correct color map after main plot finishes
-  const [refClusterRendered, setRefClusterRendered] = useState(false)
-
   const {
     enabledTabs, disabledTabs, isGeneList, isGene, isMultiGene, hasIdeogramOutputs
   } = getEnabledTabs(exploreInfo, exploreParamsWithDefaults, cellFaceting)
@@ -342,7 +338,7 @@ export default function ExploreDisplayTabs({
       cellFilteringSelection
     }
     getCellFacetingData(newCluster, newAnnot, setterFunctions, context)
-  }, [exploreParams?.cluster, exploreParams?.annotation])
+  }, [exploreParams?.cluster, exploreParams?.annotation, exploreParams?.subsample])
 
 
   /** Update filtered cells to only those that match filter selections */
@@ -561,11 +557,7 @@ export default function ExploreDisplayTabs({
                     setCountsByLabelForDe,
                     dataCache,
                     filteredCells,
-                    cellFilteringSelection,
-                    refColorMap,
-                    setRefColorMap,
-                    refClusterRendered,
-                    setRefClusterRendered
+                    cellFilteringSelection
                   }}/>
               </div>
             }
@@ -623,6 +615,7 @@ export default function ExploreDisplayTabs({
                   trackFileName={exploreParams.trackFileName}
                   uniqueGenes={exploreInfo.uniqueGenes}
                   isVisible={shownTab === 'genome'}
+                  cellFilteringSelection={cellFilteringSelection}
                   queriedGenes={exploreParams.genes}
                   updateExploreParams={updateExploreParams}
                 />
@@ -764,6 +757,10 @@ export function getEnabledTabs(exploreInfo, exploreParams, cellFaceting) {
     )
   })
 
+  if (hasGenomeFiles && !enabledTabs.includes('genome')) {
+    disabledTabs.push('genome')
+  }
+
   if (
     !exploreInfo ||
     (exploreParams.facets !== '' && !cellFaceting?.isFullyLoaded)

app/javascript/components/explore/GenomeView.jsx

@@ -12,9 +12,16 @@ import { profileWarning } from '~/lib/study-overview/terra-profile-warning'
 
 /** Component for displaying IGV for any BAM/BAI files provided with the study */
 function GenomeView({
-  studyAccession, trackFileName, uniqueGenes, isVisible, queriedGenes, updateExploreParams
+  studyAccession, trackFileName, uniqueGenes, isVisible, cellFilteringSelection,
+  queriedGenes, updateExploreParams
 }) {
   const [isLoading, setIsLoading] = useState(false)
+
+  let numFacets
+  if (cellFilteringSelection) {
+    numFacets = Object.keys(cellFilteringSelection).length
+  }
+  const [hasAppliedInitFilters, setHasAppliedInitFilters] = useState(cellFilteringSelection && numFacets > 0)
   const [trackFileList, setTrackFileList] = useState(null)
   const [igvInitializedFiles, setIgvInitializedFiles] = useState('')
   const [igvContainerId] = useState(_uniqueId('study-igv-'))
@@ -55,19 +62,41 @@ function GenomeView({
       // So we track what the last files are that we initialized
       // IGV with, and only rerender if they are different.
       if (igvInitializedFiles !== fileNamesToShow) {
-        initializeIgv(igvContainerId, listToShow, trackFileList.gtfFiles, uniqueGenes, queriedGenes)
+        const igvCellFilteringSelection = hasAppliedInitFilters ? cellFilteringSelection : null
+        initializeIgv(
+          igvContainerId, listToShow, trackFileList.gtfFiles, uniqueGenes,
+          queriedGenes, igvCellFilteringSelection, setHasAppliedInitFilters
+        )
       }
       setIgvInitializedFiles(fileNamesToShow)
     }
   }, [fileListString, trackFileName, isVisible])
 
-  // Search gene in IGV upon searching gene in Explore
-  useEffect(() => {
+  /** Wrap igvBrowser.search, retryable */
+  function igvSearch(queriedGenes, retryAttempt=0) {
     if (window.igvBrowser) {
-      const genomeId = trackFileList.tracks[0].genomeAssembly
-      getDefaultLocus(queriedGenes, uniqueGenes, genomeId)
-      window.igvBrowser.search(queriedGenes[0])
+
+      // Retry search every .25 s, up to ~5 s, if needed track absent
+      if (!trackFileList && retryAttempt < 20) {
+        setTimeout(() => {
+          igvSearch(queriedGenes, retryAttempt++)
+        }, 250)
+      } else {
+        const genomeId = trackFileList.tracks[0].genomeAssembly
+        getDefaultLocus(queriedGenes, uniqueGenes, genomeId)
+        window.igvBrowser.search(queriedGenes[0])
+
+        const filteredCellNames = window.SCP.filteredCellNames
+        if (filteredCellNames) {
+          filterIgvFeatures(filteredCellNames)
+        }
+      }
     }
+  }
+
+  // Search gene in IGV upon searching gene in Explore
+  useEffect(() => {
+    igvSearch(queriedGenes)
   }, [queriedGenes.join(',')])
 
   /** handle clicks on the download 'browse in genome' buttons
@@ -118,13 +147,17 @@ export default SafeGenomeView
 
 /** Get unfiltered genomic features on current chromosome */
 function getOriginalChrFeatures(trackIndex, igvBrowser) {
-  const chr = igvBrowser.tracks[0].trackView.viewports[0].featureCache.chr
+  const chr = igvBrowser.referenceFrameList[0].chr
 
   if (
     typeof window.originalFeatures === 'undefined' ||
     chr in window.originalFeatures === false
   ) {
-    window.originalFeatures = igvBrowser.trackViews[trackIndex].track.featureSource.featureCache.allFeatures
+    if (igvBrowser.trackViews[trackIndex].track.featureSource.featureCache) {
+      window.originalFeatures = igvBrowser.trackViews[trackIndex].track.featureSource.featureCache.allFeatures
+    } else {
+      return
+    }
   }
 
   const originalChrFeatures = window.originalFeatures[chr]
@@ -191,11 +224,23 @@ function getIsFeatureInFrame(feature, igvBrowser) {
 // }
 
 /** Filter genomic features */
-export function filterIgvFeatures(filteredCellNames) {
-  const trackIndex = 4 // TODO (SCP-5662): Robustify this
+export function filterIgvFeatures(filteredCellNames, retryAttempt=0) {
   const igvBrowser = window.igvBrowser
+  if (!igvBrowser?.tracks) {return}
+  const trackIndex = igvBrowser.tracks.findIndex(
+    track => track.config?.dataType === 'atac-fragment'
+  )
 
   const originalChrFeatures = getOriginalChrFeatures(trackIndex, igvBrowser)
+
+  if (typeof originalChrFeatures === 'undefined') {
+    if (retryAttempt < 20) { // Poll RAM every 250 ms, up to 20 times (~5 s)
+      setTimeout(() => {
+        filterIgvFeatures(filteredCellNames, retryAttempt++)
+      }, 250)
+    }
+    return
+  }
   const filteredFeatures = originalChrFeatures.filter(
     feature => filteredCellNames.has(feature.name) && getIsFeatureInFrame(feature, igvBrowser)
   )
@@ -220,6 +265,7 @@ function getTracks(tsvAndIndexFiles, dataType) {
     tsvTrack.label = tsvTrack.name
     tsvTrack.indexURL = decodeURIComponent(tsvTrack.indexUrl)
     tsvTrack.url = decodeURIComponent(tsvTrack.url)
+    tsvTrack.visibilityWindow = 1_000_000 // 1 Mbp
     if (dataType && dataType === 'atac-fragment') {
       const atacProps = {
         displayMode: 'SQUISHED',
@@ -246,7 +292,11 @@ function getTracks(tsvAndIndexFiles, dataType) {
           '3': '#C66',
           '4': '#E44',
           '5': '#E44',
-          '6': '#E44'
+          '6': '#E44',
+          '7': '#E44',
+          '8': '#F00',
+          '9': '#F00',
+          '10': '#F00'
         },
 
         // "dataType" is an SCP-custom IGV track attribute, which lets us
@@ -394,11 +444,28 @@ export function getIgvOptions(tracks, gtfFiles, uniqueGenes, queriedGenes) {
   return igvOptions
 }
 
+/** Apply cell filtering to IGV, retryable */
+function applyIgvFilters(retryAttempt=0, setHasAppliedInitFilters) {
+  const filteredCellNames = window.SCP.filteredCellNames
+  if (filteredCellNames) {
+    filterIgvFeatures(filteredCellNames)
+    setHasAppliedInitFilters(true)
+  } else {
+    if (retryAttempt < 20) {
+      setTimeout(() => {
+        applyIgvFilters(retryAttempt++)
+      }, 250)
+    }
+  }
+}
 
 /**
  * Instantiates and renders igv.js widget on the page
  */
-async function initializeIgv(containerId, tracks, gtfFiles, uniqueGenes, queriedGenes) {
+async function initializeIgv(
+  containerId, tracks, gtfFiles, uniqueGenes, queriedGenes,
+  igvCellFilteringSelection, setHasAppliedInitFilters
+) {
   // Bail if already displayed
   delete igv.browser
 
@@ -415,9 +482,14 @@ async function initializeIgv(containerId, tracks, gtfFiles, uniqueGenes, queried
   }
 
   window.igv = igv
-  window.igvBrowser = await igv.createBrowser(igvContainer, igvOptions)
+  const igvBrowser = await igv.createBrowser(igvContainer, igvOptions)
+  window.igvBrowser = igvBrowser
+
+  if (igvCellFilteringSelection) {
+    applyIgvFilters(0, setHasAppliedInitFilters)
+  }
 
-  window.igvBrowser.on('trackclick', (track, popoverData) => {
+  igvBrowser.on('trackclick', (track, popoverData) => {
     // Don't show popover when there's no data.
     if (!popoverData || !popoverData.length) {
       return false
@@ -453,7 +525,13 @@ async function initializeIgv(containerId, tracks, gtfFiles, uniqueGenes, queried
     return markup
   })
 
-  // Log igv.js initialization
+  igvBrowser.on('locuschange', () => {
+    const filteredCellNames = window.SCP.filteredCellNames
+    if (filteredCellNames) {
+      filterIgvFeatures(filteredCellNames)
+    }
+  })
+
   log('igv:initialize')
 }