Merge pull request #2117 from broadinstitute/development

Release 1.78.0

Gemfile.lock

@@ -426,7 +426,7 @@ GEM
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     retriable (3.1.2)
-    rexml (3.3.3)
+    rexml (3.3.6)
       strscan
     rubocop (1.36.0)
       json (~> 2.3)

app/javascript/components/explore/ExploreDisplayTabs.jsx

@@ -266,6 +266,11 @@ export default function ExploreDisplayTabs({
   const referencePlotDataParams = _clone(exploreParams)
   referencePlotDataParams.genes = []
 
+  // TODO (SCP-5760): Refactor pathway diagrams into independent component where
+  // React state can be propagated conventionally, then remove this
+  window.SCP.exploreParamsWithDefaults = exploreParamsWithDefaults
+  window.SCP.exploreInfo = exploreInfo
+
   /** helper function so that StudyGeneField doesn't have to see the full exploreParams object */
   function searchGenes(genes) {
     // also unset any selected gene lists or ideogram files
@@ -506,6 +511,9 @@ export default function ExploreDisplayTabs({
                 genesInScope={exploreInfo.uniqueGenes}
                 searchGenes={searchGenes}
                 speciesList={exploreInfo.taxonNames}
+
+                studyAccession={studyAccession}
+                {... exploreParamsWithDefaults}
               />
             }
             { enabledTabs.includes('annotatedScatter') &&
@@ -588,8 +596,8 @@ export default function ExploreDisplayTabs({
                   studyAccession={studyAccession}
                   {... exploreParamsWithDefaults}
                   annotationValues={getAnnotationValues(
-                     exploreParamsWithDefaults?.annotation,
-                     exploreParamsWithDefaults?.annotationList?.annotations
+                    exploreParamsWithDefaults?.annotation,
+                    exploreParamsWithDefaults?.annotationList?.annotations
                   )}
                   setMorpheusData={setMorpheusData}
                   dimensions={getPlotDimensions({ showViewOptionsControls, showDifferentialExpressionTable })}

app/javascript/components/upload/AnnDataExpressionStep.jsx

@@ -9,21 +9,21 @@ const DEFAULT_NEW_PROCESSED_FILE = {
   expression_file_info: {
     is_raw_counts: false,
     biosample_input_type: 'Whole cell',
-    modality: 'Transcriptomic: unbiased'
+    modality: 'Transcriptomic: unbiased',
+    raw_counts_associations: []
   },
   file_type: 'Expression Matrix'
 }
 
 export const fileTypes = ['Expression Matrix', 'MM Coordinate Matrix']
-export const processedFileFilter = file => fileTypes.includes(file.file_type) &&
-  !file.expression_file_info?.is_raw_counts
+export const annDataExpFilter = file => fileTypes.includes(file.file_type)
 
 export default {
   title: 'Expression matrices',
   header: 'Expression matrices',
   name: 'combined expression matrices',
   component: ExpressionUploadForm,
-  fileFilter: processedFileFilter
+  fileFilter: annDataExpFilter
 }
 
 /** form for uploading a parent expression file and any children */
@@ -37,24 +37,24 @@ function ExpressionUploadForm({
   isAnnDataExperience
 }) {
   const fragmentType = isAnnDataExperience ? 'expression' : null
-  const processedParentFiles = matchingFormFiles(
-    formState.files, processedFileFilter, isAnnDataExperience, fragmentType
+  const annDataExpFiles = matchingFormFiles(
+    formState.files, annDataExpFilter, isAnnDataExperience, fragmentType
   )
   const fileMenuOptions = serverState.menu_options
 
   const featureFlagState = serverState.feature_flags
 
   useEffect(() => {
-    if (processedParentFiles.length === 0) {
+    if (annDataExpFiles.length === 0) {
       addNewFile(DEFAULT_NEW_PROCESSED_FILE)
     }
-  }, [processedParentFiles.length])
+  }, [annDataExpFiles.length])
 
   return <div>
     <div className="row">
       <div className="col-md-12">
         {getExpressionFileInfoMessage(isAnnDataExperience, 'Processed')}
-        { processedParentFiles.map(file => {
+        { annDataExpFiles.map(file => {
           return <ExpressionFileForm
             key={file.oldId ? file.oldId : file._id}
             file={file}

app/javascript/components/upload/ExpressionFileForm.jsx

@@ -1,4 +1,4 @@
-import React from 'react'
+import React, { useState } from 'react'
 import _kebabCase from 'lodash/kebabCase'
 
 import Select from '~/lib/InstrumentedSelect'
@@ -39,8 +39,9 @@ export default function ExpressionFileForm({
   const speciesOptions = fileMenuOptions.species.map(spec => ({ label: spec.common_name, value: spec.id }))
   const selectedSpecies = speciesOptions.find(opt => opt.value === file.taxon_id)
   const isMtxFile = file.file_type === 'MM Coordinate Matrix'
-  const isRawCountsFile = file.expression_file_info.is_raw_counts
-  const showRawCountsUnits = isRawCountsFile
+const rawCountsInfo = file.expression_file_info.is_raw_counts
+const isRawCountsFile = rawCountsInfo === 'true' || rawCountsInfo
+  const [showRawCountsUnits, setShowRawCountsUnits] = useState(isRawCountsFile)
 
   const allowedFileExts = isMtxFile ? FileTypeExtensions.mtx : FileTypeExtensions.plainText
   let requiredFields = showRawCountsUnits ? RAW_COUNTS_REQUIRED_FIELDS : REQUIRED_FIELDS
@@ -55,6 +56,11 @@ export default function ExpressionFileForm({
     value: id
   }))
 
+  function toggleIsRawCounts(rawCountsVal) {
+    updateFile(file._id, { expression_file_info: {is_raw_counts: rawCountsVal} })
+    setShowRawCountsUnits(rawCountsVal)
+  }
+
   return <ExpandableFileForm {...{
     file, allFiles, updateFile, saveFile,
     allowedFileExts, deleteFile, validationMessages, bucketName, isInitiallyExpanded, isAnnDataExperience
@@ -106,14 +112,46 @@ export default function ExpressionFileForm({
       </label>
     </div>
 
-    { showRawCountsUnits &&
+    { showRawCountsUnits && !isAnnDataExperience &&
       <ExpressionFileInfoSelect label="Units *"
         propertyName="units"
         rawOptions={fileMenuOptions.units}
         file={file}
         updateFile={updateFile}/>
     }
 
+    { isAnnDataExperience &&
+      <div className="row">
+        <div className="form-radio col-sm-4">
+          <label className="labeled-select">I have raw count data in the <strong>adata.raw</strong> slot</label>
+          <label className="sublabel">
+            <input type="radio"
+                   name={`anndata-raw-counts-${file._id}`}
+                   value="true"
+                   checked={isRawCountsFile}
+                   onChange={e => toggleIsRawCounts(true) } />
+            &nbsp;Yes
+          </label>
+          <label className="sublabel">
+            <input type="radio"
+                   name={`anndata-raw-counts-${file._id}`}
+                   value="false"
+                   checked={!isRawCountsFile}
+                   onChange={e => toggleIsRawCounts(false) }/>
+            &nbsp;No
+          </label>
+        </div>
+        {showRawCountsUnits && <div className="col-sm-8">
+          <ExpressionFileInfoSelect label="Units *"
+                                    propertyName="units"
+                                    rawOptions={fileMenuOptions.units}
+                                    file={file}
+                                    updateFile={updateFile}/>
+        </div>
+        }
+      </div>
+    }
+
     <ExpressionFileInfoSelect label="Biosample input type *"
       propertyName="biosample_input_type"
       rawOptions={fileMenuOptions.biosample_input_type}
@@ -167,4 +205,3 @@ function ExpressionFileInfoSelect({ label, propertyName, rawOptions, file, updat
     </label>
   </div>
 }
-

app/javascript/components/upload/FileUploadControl.jsx

@@ -19,7 +19,7 @@ export default function FileUploadControl({
   file, allFiles, updateFile,
   allowedFileExts=['*'],
   validationIssues={},
-  bucketName
+  bucketName, isAnnDataExperience
 }) {
   const [fileValidation, setFileValidation] = useState({
     validating: false, issues: {}, fileName: null
@@ -151,9 +151,10 @@ export default function FileUploadControl({
     </div>
   }
 
+  const displayName = isAnnDataExperience && file?.data_type === 'cluster' ? file?.name : file?.upload_file_name
   return <div className="form-inline">
     <label>
-      { !file.uploadSelection && <h5 data-testid="file-uploaded-name">{file.upload_file_name}</h5> }
+      { !file.uploadSelection && <h5 data-testid="file-uploaded-name">{displayName}</h5> }
       { file.uploadSelection && <h5 data-testid="file-selection-name">
         {file.uploadSelection.name} ({bytesToSize(file.uploadSelection.size)})
       </h5> }

app/javascript/components/upload/MetadataStep.jsx

@@ -1,5 +1,6 @@
 import React, { useEffect, useContext } from 'react'
 import metadataExplainerImage from '~/../assets/images/metadata-convention-explainer.jpg'
+import annDataMetadataExplainerImage from '~/../assets/images/metadata-convention-explainer-anndata.png'
 import { faInfoCircle } from '@fortawesome/free-solid-svg-icons'
 import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'
 import { Popover, OverlayTrigger } from 'react-bootstrap'
@@ -39,6 +40,10 @@ function MetadataForm({
   const userState = useContext(UserContext)
   const featureFlagState = serverState.feature_flags
   const conventionRequired = featureFlagState && featureFlagState.convention_required
+  const explainerImg = isAnnDataExperience ? annDataMetadataExplainerImage : metadataExplainerImage
+  const infoText = isAnnDataExperience ?
+    <>Uploaded data must conform to the SCP metadata convention by including the following required metadata in <strong>adata.obs</strong>.</> :
+    <>A <b>metadata file</b> lists all cells in the study.</>
 
   const file = formState.files.find(metadataFileFilter)
   const bucketName = formState.study.bucket_id
@@ -61,8 +66,8 @@ function MetadataForm({
         <div className="form-terra">
           <div className="row">
             <div className="col-md-12" id="overflow-x-scroll">
-              A <b>metadata file</b> lists all cells in the study
-              <img src={metadataExplainerImage} alt={'Diagram of data required for metadata file'}/>
+              {infoText}
+              <img src={explainerImg} alt={'Diagram of data required for metadata file'}/>
             </div>
           </div>
           <div className="row">

app/javascript/components/upload/RawCountsStep.jsx

@@ -92,7 +92,7 @@ export function getExpressionFileInfoMessage(isAnnDataExperience, expressionType
         </div>
       </div>
     </div>
-  } else if (isAnnDataExperience) {return <AnnDataPreUploadDirections/>}
+  } else if (isAnnDataExperience) {return <AnnDataPreUploadDirections extraInfoType={expressionType} />}
 }
 
 const expressionFileStructureHelp = <>

app/javascript/components/upload/form-components.jsx

@@ -123,13 +123,28 @@ export function SaveDeleteButtons({ file, updateFile, saveFile, deleteFile, vali
   </div>
 }
 
+function getExtraInfo(extraInfoType) {
+  switch (extraInfoType) {
+    case 'Processed':
+      return <><br /><br />
+        To visualize processed expression data, <strong>adata.X</strong> should have processed expression data. Raw
+        count data can be included in <strong>adata.raw</strong> (not used for visualization). SCP plans to support
+        “exploratory differential expression” analysis for raw count data in <strong>adata.raw</strong>. If you have
+        raw count data but cannot include it in the <strong>adata.raw</strong> slot, please contact&nbsp;
+        <a href="mailto:[email protected]">[email protected]</a> for
+        further assistance.
+      </>
+  }
+}
+
 /** renders the note that AnnData upload will occur later for preceeding upload steps */
-export function AnnDataPreUploadDirections() {
+export function AnnDataPreUploadDirections({extraInfoType=null}) {
   return <>
     <div className="row">
       <div className="col-md-12">
         <p className="form-terra">
-        Fill in data here, the file upload will occur in the AnnData tab.
+        Fill in form below, the file upload will occur in the AnnData tab.
+          { getExtraInfo(extraInfoType) }
         </p>
       </div>
     </div></>

app/javascript/components/upload/upload-utils.js

@@ -5,7 +5,7 @@ export const PARSEABLE_TYPES = ['Cluster', 'Coordinate Labels', 'Expression Matr
   '10X Genes File', '10X Barcodes File', 'Gene List', 'Metadata', 'Analysis Output', 'AnnData',
   'Differential Expression']
 // file types to ignore in CSFV context (still validated server-side)
-export const UNVALIDATED_TYPES = ['AnnData', 'Documentation', 'Other']
+export const UNVALIDATED_TYPES = ['Documentation', 'Other']
 export const CSFV_VALIDATED_TYPES = PARSEABLE_TYPES.filter(ft => !UNVALIDATED_TYPES.includes(ft))
 
 const EXPRESSION_INFO_TYPES = ['Expression Matrix', 'MM Coordinate Matrix']

app/javascript/components/validation/ValidationMessage.jsx

@@ -72,11 +72,7 @@ export default function ValidationMessage({
       { suggestSync &&
       <div className="validation-info" data-testid="validation-info">
         <>
-      Your file is large.  If it is already in a Google bucket,{' '}
-          <a href="sync" target="_blank" data-analytics-name="sync-suggestion">
-          sync your file
-          </a>{' '}
-      to add it faster.
+      Your file is large.  If it is already in a Google bucket, click "Use bucket path" to add it faster.
         </>
       </div>
       }

app/javascript/components/visualization/DotPlot.jsx

@@ -180,7 +180,7 @@ function patchServiceWorkerCache() {
   }
 }
 
-/** renders a morpheus powered dotPlot for the given URL paths and annotation
+/** Renders a Morpheus-powered dot plot for the given URL paths and annotation
   * Note that this has a lot in common with Heatmap.js.  they are separate for now
   * as their display capabilities may diverge (esp. since DotPlot is used in global gene search)
   * @param cluster {string} the name of the cluster, or blank/null for the study's default
@@ -257,9 +257,9 @@ const DotPlot = withErrorBoundary(RawDotPlot)
 export default DotPlot
 
 /** Render Morpheus dot plot */
-function renderDotPlot({
+export function renderDotPlot({
   target, dataset, annotationName, annotationValues,
-  setShowError, setErrorContent, genes
+  setShowError, setErrorContent, genes, drawCallback
 }) {
   const $target = $(target)
   $target.empty()
@@ -322,8 +322,14 @@ function renderDotPlot({
 
   patchServiceWorkerCache()
 
+  config.drawCallback = function() {
+    const dotPlot = this
+    if (drawCallback) {drawCallback(dotPlot)}
+  }
+
   // Instantiate dot plot and embed in DOM element
-  new window.morpheus.HeatMap(config)
+  delete window.dotPlot
+  window.dotPlot = new window.morpheus.HeatMap(config)
 }
 
 /** return a trivial tab manager that handles focus and sizing

app/javascript/components/visualization/RelatedGenesIdeogram.jsx

@@ -18,6 +18,7 @@ import PlotUtils from '~/lib/plot'
 const ideogramHeight = PlotUtils.ideogramHeight
 import { log } from '~/lib/metrics-api'
 import { logStudyGeneSearch } from '~/lib/search-metrics'
+import { manageDrawPathway } from '~/lib/pathway-expression'
 
 /** Handle clicks on Ideogram annotations */
 function onClickAnnot(annot) {
@@ -116,6 +117,7 @@ function onWillShowAnnotTooltip(annot) {
 /** Persist click handling for tissue toggle click */
 function addTissueToggleClickHandler(newTitle) {
   const ideoTissueToggle = document.querySelector('._ideoMoreOrLessTissue')
+  if (!ideoTissueToggle) {return} // Some genes (e.g. CSN2) have <= 3 tissue entries
   ideoTissueToggle.addEventListener('click', () => {
     const ideoTissuePlotTitle = document.querySelector('._ideoTissuePlotTitle')
     ideoTissuePlotTitle.innerHTML = newTitle
@@ -161,7 +163,8 @@ function onPlotRelatedGenes() {
  * This is only done in the context of single-gene search in Study Overview
  */
 export default function RelatedGenesIdeogram({
-  gene, taxon, target, genesInScope, searchGenes, speciesList
+  gene, taxon, target, genesInScope, searchGenes, speciesList,
+  studyAccession, cluster, annotation
 }) {
   if (taxon === null) {
     // Quick fix to decrease Sentry error log rate
@@ -196,13 +199,19 @@ export default function RelatedGenesIdeogram({
         showRelatedGenesIdeogram(target)
       }
     }
-    window.ideogram =
-      Ideogram.initRelatedGenes(ideoConfig, genesInScope)
+    const ideogram = Ideogram.initRelatedGenes(ideoConfig, genesInScope)
+    window.ideogram = ideogram
+
+    manageDrawPathway(studyAccession, cluster, annotation, ideogram)
 
     // Extend ideogram with custom SCP function to search genes
     window.ideogram.SCP = { searchGenes, speciesList }
   }, [gene])
 
+  useEffect(() => {
+    manageDrawPathway(studyAccession, cluster, annotation, window.ideogram)
+  }, [cluster, annotation])
+
   return (
     <div
       id="related-genes-ideogram-container"

app/javascript/components/visualization/ScatterPlot.jsx

@@ -747,6 +747,12 @@ function getPlotlyTraces({
     expressionFilter, expressionData: data.expression, isSplitLabelArrays
   })
 
+  if (Object.keys(countsByLabel).length > 1) {
+    // TODO (SCP-5760): Refactor pathway diagrams into independent component where
+    // React state can be propagated conventionally, then remove this
+    window.SCP.countsByLabel = countsByLabel
+  }
+
   if (isRefGroup) {
     const labels = getLegendSortedLabels(countsByLabel)
     traces.forEach(groupTrace => {