Skip to content

Commit

Permalink
Merge pull request #1883 from broadinstitute/development
Browse files Browse the repository at this point in the history 
Release 1.55.0
  • Loading branch information
@jlchang
jlchang authored Sep 20, 2023
2 parents 26d33dd + cd8876b commit 14b0e8b
Show file tree
Hide file tree
Showing 27 changed files with 849 additions and 220 deletions.
170 changes: 149 additions & 21 deletions app/controllers/api/v1/studies_controller.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -518,6 +518,19 @@ def destroy
end
end
end
key :title, 'JSON object of StudyShares, StudyFiles, and DirectoryListings'
property :page_token do
key :type, :string
key :description, 'Token to next batch of files (if present)'
end
property :remaining_files do
key :type, :integer
key :description, 'Number of remaining files yet to process for sync'
end
property :next_page do
key :type, :string
key :description, 'URL for next page of results, with token (if present)'
end
end
end
response 403 do
Expand Down Expand Up @@ -545,29 +558,114 @@ def sync_study
alert: "We were unable to sync with your workspace bucket due to an error: #{view_context.simple_format(e.message)}. #{SCP_SUPPORT_EMAIL}" and return
end

# begin determining sync status with study_files and primary or other data
begin
@unsynced_files = StudySyncService.process_all_remotes(@study)
rescue => e
ErrorTracker.report_exception(e, current_user, @study, params)
MetricsService.report_error(e, request, current_user, @study)
logger.error "#{Time.zone.now}: error syncing files in workspace bucket #{@study.firecloud_workspace} due to error: #{e.message}"
redirect_to merge_default_redirect_params(studies_path, scpbr: params[:scpbr]),
alert: "We were unable to sync with your workspace bucket due to an error: #{view_context.simple_format(e.message)}. #{SCP_SUPPORT_EMAIL}" and return
end

# refresh study state before continuing as new records may have been inserted
@study.reload
@synced_directories = @study.directory_listings.are_synced
@unsynced_directories = @study.directory_listings.unsynced
sync_file_batch
end

# split directories into primary data types and 'others'
@unsynced_primary_data_dirs, @unsynced_other_dirs = StudySyncService.load_unsynced_directories(@study)
swagger_path '/studies/{id}/sync_batch' do
operation :post do
key :tags, [
'Studies'
]
key :summary, 'Sync a batch of files in a Study'
key :description, 'Synchronize a batch of remote files from GCS bucket for a study'
key :operationId, 'sync_batch_study_path'
parameter do
key :name, :id
key :in, :path
key :description, 'ID of Study to sync'
key :required, true
key :type, :string
end
parameter do
key :name, :page_token
key :in, :query
key :description, 'token for next batch of files'
key :required, true
key :type, :string
end
response 200 do
key :description, 'StudyShares, StudyFiles, and DirectoryListings'
schema do
key :title, 'JSON object of StudyShares, StudyFiles, and DirectoryListings'
property :study_shares do
key :type, :array
key :title, 'StudyShare array'
items do
key :title, 'StudyShare'
key :'$ref', :StudyShare
end
end
property :study_files do
key :type, :object
key :title, 'JSON object of synced, unsynced, and orphaned StudyFiles'
property :unsynced do
key :title, 'Unsynced StudyFile array'
key :type, :array
items do
key :title, 'StudyFile'
key :'$ref', :StudyFile
end
end
property :synced do
key :title, 'Synced StudyFile array'
key :type, :array
items do
key :title, 'StudyFile'
key :'$ref', :StudyFile
end
end
property :orphaned do
key :title, 'Orphaned StudyFile array'
key :type, :array
items do
key :title, 'StudyFile'
key :'$ref', :StudyFile
end
end
end
property :directory_listings do
key :type, :object
key :title, 'JSON object of synced and unsynced DirectoryListings'
property :unsynced do
key :type, :array
key :title, 'Unsynced DirectoryListing array'
items do
key :title, 'DirectoryListing'
key :'$ref', :DirectoryListing
end
end
property :synced do
key :type, :array
key :title, 'Synced DirectoryListing array'
items do
key :title, 'DirectoryListing'
key :'$ref', :DirectoryListing
end
end
end
property :page_token do
key :type, :string
key :description, 'Token to next batch of files (if present)'
end
property :remaining_files do
key :type, :integer
key :description, 'Number of remaining files yet to process for sync'
end
end
end
response 403 do
key :description, ApiBaseController.forbidden('edit Study')
end
extend SwaggerResponses::StudyControllerResponses
response 500 do
key :description, 'Server error when attempting to synchronize FireCloud workspace or access GCS objects'
end
end
end

# now determine if we have study_files that have been 'orphaned' (cannot find a corresponding bucket file)
@orphaned_study_files = StudySyncService.find_orphaned_files(@study)
@available_files = StudySyncService.set_available_files(@unsynced_files)
@synced_study_files = StudySyncService.set_synced_files(@study, @orphaned_study_files)
def sync_next_file_batch
sync_file_batch
render action: :sync_study
end

swagger_path '/studies/{id}/manifest' do
Expand Down Expand Up @@ -627,6 +725,36 @@ def generate_manifest
render plain: manifest_obj
end

# sync a batch of up to 1000 remote files
def sync_file_batch
# begin determining sync status with study_files and primary or other data
begin
remote_info = StudySyncService.process_remotes(@study, token: params[:page_token])
@unsynced_files = remote_info[:unsynced_study_files]
@next_page = remote_info[:page_token]
@remaining_files = remote_info[:remaining_files]
rescue => e
ErrorTracker.report_exception(e, current_user, @study, params)
MetricsService.report_error(e, request, current_user, @study)
logger.error "#{Time.zone.now}: error syncing files in workspace bucket #{@study.firecloud_workspace} due to error: #{e.message}"
redirect_to merge_default_redirect_params(studies_path, scpbr: params[:scpbr]),
alert: "We were unable to sync with your workspace bucket due to an error: #{view_context.simple_format(e.message)}. #{SCP_SUPPORT_EMAIL}" and return
end

# refresh study state before continuing as new records may have been inserted
@study.reload
@synced_directories = @study.directory_listings.are_synced
@unsynced_directories = @study.directory_listings.unsynced

# split directories into primary data types and 'others'
@unsynced_primary_data_dirs, @unsynced_other_dirs = StudySyncService.load_unsynced_directories(@study)

# now determine if we have study_files that have been 'orphaned' (cannot find a corresponding bucket file)
@orphaned_study_files = StudySyncService.find_orphaned_files(@study)
@available_files = StudySyncService.set_available_files(@unsynced_files)
@synced_study_files = StudySyncService.set_synced_files(@study, @orphaned_study_files)
end

private

# copy the 'id' param to 'study_id' so that it can be compatible with StudyAware concern
Expand Down
1 change: 1 addition & 0 deletions app/controllers/site_controller.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ def study
# normally we would do this in React but the tab display is in the Rails HTML view
# we need to check server-side since we have to account for @study.can_visualize? as well
@explore_tab_default = @study.can_visualize?
@show_appcue_pin = FeatureFlaggable.feature_flags_for_instances(current_user).[](:show_appcue_viz_tour)
end

def record_download_acceptance
Expand Down
59 changes: 36 additions & 23 deletions app/controllers/studies_controller.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,30 +111,13 @@ def sync_study
redirect_to merge_default_redirect_params(studies_path, scpbr: params[:scpbr]),
alert: "We were unable to sync with your workspace bucket due to an error: #{view_context.simple_format(e.message)}. #{SCP_SUPPORT_EMAIL}" and return
end
sync_file_batch
end

# begin determining sync status with study_files and primary or other data
begin
@unsynced_files = StudySyncService.process_all_remotes(@study)
rescue => e
ErrorTracker.report_exception(e, current_user, @study, params)
MetricsService.report_error(e, request, current_user, @study)
logger.error "#{Time.zone.now}: error syncing files in workspace bucket #{@study.firecloud_workspace} due to error: #{e.message}"
redirect_to merge_default_redirect_params(studies_path, scpbr: params[:scpbr]),
alert: "We were unable to sync with your workspace bucket due to an error: #{view_context.simple_format(e.message)}. #{SCP_SUPPORT_EMAIL}" and return
end

# refresh study state before continuing as new records may have been inserted
@study.reload
@synced_directories = @study.directory_listings.are_synced
@unsynced_directories = @study.directory_listings.unsynced

# split directories into primary data types and 'others'
@unsynced_primary_data_dirs, @unsynced_other_dirs = StudySyncService.load_unsynced_directories(@study)

# now determine if we have study_files that have been 'orphaned' (cannot find a corresponding bucket file)
@orphaned_study_files = StudySyncService.find_orphaned_files(@study)
@available_files = StudySyncService.set_available_files(@unsynced_files)
@synced_study_files = StudySyncService.set_synced_files(@study, @orphaned_study_files)
# paginate batch of files to sync (same as sync_study but does not alter workspace permissions)
def sync_next_file_batch
sync_file_batch
render action: :sync_study
end

# sync outputs from a specific submission
Expand Down Expand Up @@ -949,6 +932,36 @@ def load_annotation_options
end
end

# sync a batch of up to 1000 remote files
def sync_file_batch
# begin determining sync status with study_files and primary or other data
begin
remote_info = StudySyncService.process_remotes(@study, token: params[:page_token])
@unsynced_files = remote_info[:unsynced_study_files]
@next_page = remote_info[:page_token]
@remaining_files = remote_info[:remaining_files]
rescue => e
ErrorTracker.report_exception(e, current_user, @study, params)
MetricsService.report_error(e, request, current_user, @study)
logger.error "#{Time.zone.now}: error syncing files in workspace bucket #{@study.firecloud_workspace} due to error: #{e.message}"
redirect_to merge_default_redirect_params(studies_path, scpbr: params[:scpbr]),
alert: "We were unable to sync with your workspace bucket due to an error: #{view_context.simple_format(e.message)}. #{SCP_SUPPORT_EMAIL}" and return
end

# refresh study state before continuing as new records may have been inserted
@study.reload
@synced_directories = @study.directory_listings.are_synced
@unsynced_directories = @study.directory_listings.unsynced

# split directories into primary data types and 'others'
@unsynced_primary_data_dirs, @unsynced_other_dirs = StudySyncService.load_unsynced_directories(@study)

# now determine if we have study_files that have been 'orphaned' (cannot find a corresponding bucket file)
@orphaned_study_files = StudySyncService.find_orphaned_files(@study)
@available_files = StudySyncService.set_available_files(@unsynced_files)
@synced_study_files = StudySyncService.set_synced_files(@study, @orphaned_study_files)
end

private

###
Expand Down
14 changes: 13 additions & 1 deletion app/javascript/components/HomePageContent.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import ResultsPanel from '~/components/search/results/ResultsPanel'
import StudyDetails from '~/components/search/results/StudySearchResult'
import StudySearchProvider, { StudySearchContext } from '~/providers/StudySearchProvider'
import SearchFacetProvider from '~/providers/SearchFacetProvider'
import UserProvider from '~/providers/UserProvider'
import UserProvider, { getFeatureFlagsWithDefaults } from '~/providers/UserProvider'
import ErrorBoundary from '~/lib/ErrorBoundary'

/** include search controls and results */
Expand All @@ -22,12 +22,17 @@ export function StudySearchView() {
</>
}



const LinkableSearchTabs = function(props) {
// we can't use the regular ReachRouter methods for link highlighting
// since the Reach router doesn't own the home path
const location = useLocation()
const basePath = location.pathname.includes('covid19') ? '/single_cell/covid19' : '/single_cell'
const showGenesTab = location.pathname.includes('/app/genes')
const featureFlags = getFeatureFlagsWithDefaults()
const showVizAppcue = featureFlags && featureFlags?.show_appcue_viz_tour

// the queryParams object does not support the more typical hasOwnProperty test
return (
<div>
Expand All @@ -40,6 +45,13 @@ const LinkableSearchTabs = function(props) {
className={showGenesTab ? 'active' : ''}>
<span className="fas fa-dna"></span> Search genes
</Link>
{ showVizAppcue &&
<a className='btn btn-primary appcue-pin'
href='https://singlecell.broadinstitute.org/single_cell/study/SCP2271'
data-analytics-name='appcue-viz-demo'>
Demo of SCP visualization tools
</a>
}
</nav>
<div className="tab-content top-pad">
<Router basepath={basePath}>
Expand Down
4 changes: 3 additions & 1 deletion app/javascript/components/explore/DifferentialExpressionPanel.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,19 +54,21 @@ function BroadenSearchMessage() {
/** Top matter for differential expression panel shown at right in Explore tab */
export function DifferentialExpressionPanelHeader({
setDeGenes, setDeGroup, setShowDifferentialExpressionPanel, setShowUpstreamDifferentialExpressionPanel, isUpstream,
cluster, annotation, setDeGroupB, isAuthorDe, deGenes
cluster, annotation, setDeGroupB, isAuthorDe, deGenes, togglePanel
}) {
const deSource = `${isAuthorDe ? 'Author' : 'SCP'}-computed`
return (
<>
<span>Differential expression {deGenes && <span className="margin-left de-source badge badge-inverse">{deSource}</span>}</span>
<button className="action fa-lg de-exit-panel"
onClick={() => {
togglePanel('default')
setDeGenes(null)
setDeGroup(null)
setDeGroupB(null)
setShowDifferentialExpressionPanel(false)
setShowUpstreamDifferentialExpressionPanel(false)

}}
title="Exit differential expression panel"
data-analytics-name="differential-expression-panel-exit">
Expand Down
Loading

0 comments on commit 14b0e8b

Please sign in to comment.