HP-1484 HP-1617 Feat/cde submission (#1547)

* feat: skeleton of tab panels * component actions * cde sub * existing cde wip and change label * setup tags and filters * style update * update submit btn status * cde dd vlmd update * refactor vlmd dl for metadata update * presist CDE selection * fix vlmd download * update doc * remove console * fix using the correct parse() * Consider '*' as a wildcard (#1566) * Consider '*' as a wildcard * upgrade fuse to latest version (#1559) * upgrade fuse to allow search with longer 32 characters * use grid for checkbox group * disable cde form for redcap * fix zendesk function calls * adjust on pr feedback * use enum * add pr feedbacks * update place holder text --------- Co-authored-by: Sai Shanmukha Narumanchi <[email protected]> Co-authored-by: Thanh Dang Nguyen <[email protected]>
uc-cdis · Sep 30, 2024 · 079b69d · 079b69d
1 parent 0ef57ff
commit 079b69d
Show file tree

Hide file tree

Showing 25 changed files with 710 additions and 179 deletions.
diff --git a/docs/portal_config.md b/docs/portal_config.md
@@ -463,12 +463,19 @@ Below is an example, with inline comments describing what each JSON block config
       "exportToWorkspace": { // configures the export to workspace feature. If enabled, the Discovery page data must contain a field which is a list of GUIDs for each study. See `manifestFieldName`
           "enable": true,
           "enableDownloadManifest": true, // enables a button which allows user to download a manifest file for gen3 client
-          "downloadManifestButtonText": true, // text to be displayed on the download manifest button
+          "downloadManifestButtonText": "Download Manifest", // text to be displayed on the download manifest button
           "manifestFieldName": "__manifest", // the field in the Discovery page data that contains the list of GUIDs that link to each study's data files.
+          "enableDownloadZip": true,  // enables a button which allows user to download all the files as a zip file (with pre-set size limits)
+          "downloadZipButtonText": "Download Zip", // text to be displayed on the download zip file button
+          "enableDownloadVariableMetadata": true,  // enables a button (on discovery details page) which allows user to download variable-level metadata
+          "variableMetadataFieldName": "variable_level_metadata", // field name in metadata record to reference variable-level metadata
+          "enableDownloadStudyMetadata": true, // enables a button (on discovery details page) which allows user to download study-level metadata
+          "studyMetadataFieldName": "study_metadata", // field name in metadata record to reference study-level metadata
           "documentationLinks": {
               "gen3Client": "https://gen3-client", // link to documentation about the gen3 client. Used for button tooltips
               "gen3Workspaces": "https://gen3-workspace-docs", // link to documentation about gen3 workspaces. Used for button tooltips.
-          }
+          },
+          "verifyExternalLogins": true // enables verification if the user has access to all the data files selected, by using WTS (a Gen3 ecosystem feature)
       },
       "pageTitle": {
         "enabled": true,
@@ -758,9 +765,10 @@ Below is an example, with inline comments describing what each JSON block config
           "studyRegistrationUIDField": "_hdp_uid", // optional, if omitted, value defaults the same as the "minimalFieldMapping.uid" value. In metadata, values from this field MUST be the same as their GUIDs for each metadata record
           "studyRegistrationFormDisclaimerField": "This is a disclaimer", //optional, the disclaimer text that appears under the submit button on the study registration request access form. Defaults to undefined
           "clinicalTrialFields": [], // optional, list of fields to fetch from ClinicalTrials.gov
-          "dataDictionaryField": "data_dictionaries", // optional, specify the field name in metadata for variable-level metadata, default to ""
-          "dataDictionarySubmissionBucket": "bucket-1", // optional, customize the S3 bucket that will be used for VLMD submission. Default to the data upload bucket from fence config if omitted
-          "dataDictionarySubmissionDisclaimerField": "some disclaimer text" // optional, the disclaimer text that appears under the submit button on the VLMD submission page. Defaults to undefined
+          "variableMetadataField": "variable_level_metadata", // optional, specify the field name in metadata for variable-level metadata, default to ""
+          "dataDictionarySubmissionBucket": "bucket-1", // optional, customize the S3 bucket that will be used for data dictionary submission. Default to the data upload bucket from fence config if omitted
+          "dataDictionarySubmissionDisclaimerField": "some disclaimer text", // optional, the disclaimer text that appears under the submit button on the data dictionary submission page. Defaults to undefined
+          "cdeSubmissionDisclaimerField": "some disclaimer text"  // optional, the disclaimer text that appears under the submit button on the CDE submission page. Defaults to undefined
         },
         "workspaceRegistrationConfig" : { // optional, config for Workspace Registration Request Access page.
         "workspacePolicyId": "workspace", // optional, name of the policy that is needed to provide workspace access; if missing, defaults to 'workspace'

diff --git a/...etails/Components/FieldGrouping/TabField/DataDownloadList/ActionButtons/ActionButtons.tsx b/...etails/Components/FieldGrouping/TabField/DataDownloadList/ActionButtons/ActionButtons.tsx
@@ -11,8 +11,8 @@ import DownloadDataFiles from './DownloadUtils/DownloadDataFiles/DownloadDataFil
 import DownloadJsonFile from './DownloadUtils/DownloadJsonFile';
 import DownloadVariableMetadata from './DownloadUtils/DownloadVariableMetadata/DownloadVariableMetadata';
 import './ActionButtons.css';
-import DownloadDataDictionaryInfo from './DownloadUtils/DownloadDataDictionaryInfo';
-import DataDictionaries from '../Interfaces/DataDictionaries';
+import DownloadVariableMetadataInfo from './DownloadUtils/DownloadVariableMetadataInfo';
+import VariableLevelMetadata from '../Interfaces/VariableLevelMetadata';
 import DownloadStatus from '../Interfaces/DownloadStatus';
 
 interface ActionButtonsProps {
@@ -49,7 +49,7 @@ const ActionButtons = ({
     fileManifest = resourceInfo?.[manifestFieldName] || [];
   }
 
-  // Study level metabutton should show only if the downloading study level metadata value is enabled
+  // Study level meta button should show only if the downloading study level metadata value is enabled
   // and resourceInfo includes the study metadata field name reference from the discovery config
   const showDownloadStudyLevelMetadataButton = Boolean(
     discoveryConfig?.features.exportToWorkspace.enableDownloadStudyMetadata
@@ -70,9 +70,9 @@ const ActionButtons = ({
       && discoveryConfig.features.exportToWorkspace.enableDownloadVariableMetadata,
   );
 
-  const [dataDictionaryInfo, setDataDictionaryInfo] = useState({
+  const [variableMetadataInfo, setVariableMetadataInfo] = useState({
     noVariableLevelMetadata: true,
-    dataDictionaries: {} as DataDictionaries,
+    variableLevelMetadataRecords: {} as VariableLevelMetadata,
   });
 
   let uid = '';
@@ -81,11 +81,11 @@ const ActionButtons = ({
   }
 
   useEffect(() => {
-    DownloadDataDictionaryInfo(
+    DownloadVariableMetadataInfo(
       discoveryConfig,
       resourceInfo,
       showDownloadVariableMetadataButton,
-      setDataDictionaryInfo,
+      setVariableMetadataInfo,
     );
   }, [resourceInfo]);
 
@@ -141,12 +141,12 @@ const ActionButtons = ({
               data-testid='download-variable-level-metadata'
               disabled={Boolean(
                 downloadStatus.inProgress
-                  || dataDictionaryInfo.noVariableLevelMetadata,
+                  || variableMetadataInfo.noVariableLevelMetadata,
               )}
               loading={downloadStatus.inProgress === 'DownloadVariableMetadata'}
               onClick={() => {
                 DownloadVariableMetadata(
-                  dataDictionaryInfo.dataDictionaries,
+                  variableMetadataInfo.variableLevelMetadataRecords,
                   resourceInfo,
                   setDownloadStatus,
                 );

diff --git a/...uping/TabField/DataDownloadList/ActionButtons/DownloadUtils/DownloadDataDictionaryInfo.ts b/...uping/TabField/DataDownloadList/ActionButtons/DownloadUtils/DownloadDataDictionaryInfo.ts
diff --git a/...st/ActionButtons/DownloadUtils/DownloadVariableMetadata/DownloadVariableMetadata.test.tsx b/...st/ActionButtons/DownloadUtils/DownloadVariableMetadata/DownloadVariableMetadata.test.tsx
@@ -17,8 +17,8 @@ const mockResourceInfo = {
   project_title: 'Sample Project',
 };
 
-const mockDataDictionaries = {
-  'anotherDataDictionary.json': 'string-associated-with-file-name',
+const mockVariableLevelMetadata = {
+  dataDictionaries: { 'anotherDataDictionary.json': 'string-associated-with-file-name' },
 };
 
 const mockSetDownloadStatus = jest.fn();
@@ -41,7 +41,7 @@ describe('DownloadVariableMetadata', () => {
     (fetchWithCreds as jest.Mock).mockResolvedValue(mockStatusResponse);
     await act(async () => {
       await DownloadVariableMetadata(
-        mockDataDictionaries,
+        mockVariableLevelMetadata,
         mockResourceInfo as unknown as DiscoveryResource,
         mockSetDownloadStatus,
       );
@@ -66,7 +66,7 @@ describe('DownloadVariableMetadata', () => {
     (fetchWithCreds as jest.Mock).mockResolvedValue(mockStatusResponse);
     await act(async () => {
       await DownloadVariableMetadata(
-        mockDataDictionaries,
+        mockVariableLevelMetadata,
         mockResourceInfo as unknown as DiscoveryResource,
         mockSetDownloadStatus,
       );

diff --git a/...oadList/ActionButtons/DownloadUtils/DownloadVariableMetadata/DownloadVariableMetadata.tsx b/...oadList/ActionButtons/DownloadUtils/DownloadVariableMetadata/DownloadVariableMetadata.tsx
@@ -6,11 +6,16 @@ import { mdsURL } from '../../../../../../../../../localconf';
 import { INITIAL_DOWNLOAD_STATUS } from '../Constants';
 import DownloadStatus from '../../../Interfaces/DownloadStatus';
 import { DiscoveryResource } from '../../../../../../../../Discovery';
-import DataDictionaries from '../../../Interfaces/DataDictionaries';
+import VariableLevelMetadata from '../../../Interfaces/VariableLevelMetadata';
 import SanitizeFileName from './SanitizeFileName';
 
+enum fetchType {
+  CDE = 'cde',
+  DD = 'dd'
+}
+
 const DownloadVariableMetadata = async (
-  dataDictionaries: DataDictionaries,
+  variableLevelMetadataRecords: VariableLevelMetadata,
   resourceInfo: DiscoveryResource,
   setDownloadStatus: Function,
 ) => {
@@ -33,16 +38,24 @@ const DownloadVariableMetadata = async (
     },
   } as DownloadStatus);
 
-  const fetchData = async (key: string, value: string) => new Promise((resolve, reject) => {
+  const fetchData = async (key: string, value: string, type: fetchType) => new Promise((resolve, reject) => {
     fetchWithCreds({ path: `${mdsURL}/${value}` }).then((statusResponse) => {
-      const { data } = statusResponse;
+      let { data } = statusResponse;
       if (statusResponse.status !== 200 || !data) {
         setDownloadStatus(createUniqueDownloadErrorMsg(key));
         reject(new Error(`Issue with ${key}: ${value}`));
       } else {
         const sanitizedFileName = SanitizeFileName(key);
-        zip.file(sanitizedFileName, JSON.stringify(data));
-        resolve(`Data resolved for ${key}: ${value}`);
+        let subDirectoryName = '';
+        if (type === fetchType.CDE) {
+          subDirectoryName = 'common_data_elements/';
+          data = data.cde_metadata;
+        } else if (type === fetchType.DD) {
+          subDirectoryName = 'data_dictionaries/';
+          data = data.data_dictionary;
+        }
+        zip.file(`${subDirectoryName}${sanitizedFileName}`, JSON.stringify(data));
+        resolve(`Data resolved for ${key}: ${value}, with type ${type}`);
       }
     });
   });
@@ -54,11 +67,13 @@ const DownloadVariableMetadata = async (
         inProgress: 'DownloadVariableMetadata',
       });
       await Promise.all(
-        Object.entries(dataDictionaries).map(([key, value]) => fetchData(key, value),
+        [...Object.entries(variableLevelMetadataRecords.dataDictionaries || []).map(([key, value]) => fetchData(key, value, fetchType.DD),
         ),
+        ...Object.entries(variableLevelMetadataRecords.cdeMetadata || []).map(([key, value]) => fetchData(key, value, fetchType.CDE),
+        )],
       ).then(() => {
         zip.generateAsync({ type: 'blob' }).then((content) => {
-          FileSaver.saveAs(content, 'variable-metadata.zip');
+          FileSaver.saveAs(content, 'variable-level-metadata.zip');
         });
         setDownloadStatus(INITIAL_DOWNLOAD_STATUS);
       });

diff --git a/...dUtils/DownloadDataDictionaryInfo.test.ts → ...tils/DownloadVariableMetadataInfo.test.ts b/...dUtils/DownloadDataDictionaryInfo.test.ts → ...tils/DownloadVariableMetadataInfo.test.ts
@@ -1,17 +1,23 @@
 import { act } from 'react-dom/test-utils';
-import DownloadDataDictionaryInfo from './DownloadDataDictionaryInfo';
+import DownloadVariableMetadataInfo from './DownloadVariableMetadataInfo';
 import { DiscoveryConfig } from '../../../../../../../DiscoveryConfig';
 import { DiscoveryResource } from '../../../../../../../Discovery';
 import { mdsURL } from '../../../../../../../../localconf';
 import { fetchWithCreds } from '../../../../../../../../actions';
 
-const mockDataDictionaries = {
-  'QA_minimal_json_20230817.json': 'f79114a6-93bd-4970-b096-7b47aa6c16fa',
+const mockVariableLevelMetadata = {
+  data_dictionaries: { 'QA_minimal_json_20230817.json': 'f79114a6-93bd-4970-b096-7b47aa6c16fa' },
+  common_data_elements: { '5131 Pediatric Demographics': 'HDPCDE5131' },
+};
+
+const mockVariableLevelMetadataRecords = {
+  dataDictionaries: { 'QA_minimal_json_20230817.json': 'f79114a6-93bd-4970-b096-7b47aa6c16fa' },
+  cdeMetadata: { '5131 Pediatric Demographics': 'HDPCDE5131' },
 };
 
 const discoveryConfig = {
   features: {
-    exportToWorkspace: { variableMetadataFieldName: 'data_dictionaries' },
+    exportToWorkspace: { variableMetadataFieldName: 'variable_level_metadata' },
   },
 } as DiscoveryConfig;
 
@@ -26,17 +32,17 @@ afterEach(() => {
   jest.clearAllMocks();
 });
 
-describe('DownloadDataDictionaryInfo', () => {
-  it('should not set download data dictionary info when called with invalid status number', async () => {
+describe('DownloadVariableMetadataInfo', () => {
+  it('should not set download variable-level metadata info when called with invalid status number', async () => {
     const mockStatusResponse = {
       status: 500,
       data: {
-        data_dictionaries: mockDataDictionaries,
+        variable_level_metadata: mockVariableLevelMetadata,
       },
     };
     (fetchWithCreds as jest.Mock).mockResolvedValue(mockStatusResponse);
     await act(async () => {
-      DownloadDataDictionaryInfo(
+      DownloadVariableMetadataInfo(
         discoveryConfig,
         resourceInfo as unknown as DiscoveryResource,
         showDownloadVariableMetadataButton,
@@ -49,16 +55,16 @@ describe('DownloadDataDictionaryInfo', () => {
     expect(mockSetDownloadStatus).not.toHaveBeenCalled();
   });
 
-  it('should not download data dictionary info when called with invalid data', async () => {
+  it('should not download variable-level metadata info when called with invalid data', async () => {
     const mockStatusResponse = {
       status: 200,
       data: {
-        data_dictionaries: {},
+        variable_level_metadata: {},
       },
     };
     (fetchWithCreds as jest.Mock).mockResolvedValue(mockStatusResponse);
     await act(async () => {
-      DownloadDataDictionaryInfo(
+      DownloadVariableMetadataInfo(
         discoveryConfig,
         resourceInfo as unknown as DiscoveryResource,
         showDownloadVariableMetadataButton,
@@ -71,17 +77,17 @@ describe('DownloadDataDictionaryInfo', () => {
     expect(mockSetDownloadStatus).not.toHaveBeenCalled();
   });
 
-  it(`should not call fetchWithCreds or download data dictionary info when
+  it(`should not call fetchWithCreds or variable-level metadata info when
     called with showDownloadVariableMetadataButton = false`, async () => {
     const mockStatusResponse = {
       status: 200,
       data: {
-        data_dictionaries: mockDataDictionaries,
+        variable_level_metadata: mockVariableLevelMetadata,
       },
     };
     (fetchWithCreds as jest.Mock).mockResolvedValue(mockStatusResponse);
     await act(async () => {
-      DownloadDataDictionaryInfo(
+      DownloadVariableMetadataInfo(
         discoveryConfig,
         resourceInfo as unknown as DiscoveryResource,
         false, // showDownloadVariableMetadataButton
@@ -92,16 +98,16 @@ describe('DownloadDataDictionaryInfo', () => {
     expect(mockSetDownloadStatus).not.toHaveBeenCalled();
   });
 
-  it('should set data dictionary info when called with valid params', async () => {
+  it('should set variable-level metadata info when called with valid params', async () => {
     const mockStatusResponse = {
       status: 200,
       data: {
-        data_dictionaries: mockDataDictionaries,
+        variable_level_metadata: mockVariableLevelMetadata,
       },
     };
     (fetchWithCreds as jest.Mock).mockResolvedValue(mockStatusResponse);
     await act(async () => {
-      DownloadDataDictionaryInfo(
+      DownloadVariableMetadataInfo(
         discoveryConfig,
         resourceInfo as unknown as DiscoveryResource,
         showDownloadVariableMetadataButton,
@@ -113,7 +119,7 @@ describe('DownloadDataDictionaryInfo', () => {
     });
     expect(mockSetDownloadStatus).toHaveBeenCalledWith({
       noVariableLevelMetadata: false,
-      dataDictionaries: mockDataDictionaries,
+      variableLevelMetadataRecords: mockVariableLevelMetadataRecords,
     });
   });
 });
diff --git a/...ing/TabField/DataDownloadList/ActionButtons/DownloadUtils/DownloadVariableMetadataInfo.ts b/...ing/TabField/DataDownloadList/ActionButtons/DownloadUtils/DownloadVariableMetadataInfo.ts
@@ -0,0 +1,42 @@
+import { DiscoveryConfig } from '../../../../../../../DiscoveryConfig';
+import { DiscoveryResource } from '../../../../../../../Discovery';
+import { mdsURL } from '../../../../../../../../localconf';
+import { fetchWithCreds } from '../../../../../../../../actions';
+import VariableLevelMetadata from '../../Interfaces/VariableLevelMetadata';
+
+const DownloadVariableMetadataInfo = (
+  discoveryConfig: DiscoveryConfig,
+  resourceInfo: DiscoveryResource,
+  showDownloadVariableMetadataButton: Boolean,
+  setVariableMetadataInfo: Function,
+) => {
+  const vlmdFieldReference = discoveryConfig.features.exportToWorkspace.variableMetadataFieldName;
+  if (showDownloadVariableMetadataButton) {
+    const studyID = resourceInfo._hdp_uid;
+    fetchWithCreds({ path: `${mdsURL}/${studyID}` }).then((statusResponse) => {
+      const { data } = statusResponse;
+      if (
+        statusResponse.status === 200
+        && data
+        && data[vlmdFieldReference as string]
+        && Object.keys(data[vlmdFieldReference as string]).length !== 0
+      ) {
+        const variableLevelMetadataRecords: VariableLevelMetadata = {};
+        if (data[vlmdFieldReference as string].data_dictionaries) {
+          variableLevelMetadataRecords.dataDictionaries = data[vlmdFieldReference as string].data_dictionaries;
+        }
+        if (data[vlmdFieldReference as string].common_data_elements) {
+          variableLevelMetadataRecords.cdeMetadata = data[vlmdFieldReference as string].common_data_elements;
+        }
+        if (Object.keys(variableLevelMetadataRecords).length) {
+          setVariableMetadataInfo({
+            noVariableLevelMetadata: false,
+            variableLevelMetadataRecords,
+          });
+        }
+      }
+    });
+  }
+};
+
+export default DownloadVariableMetadataInfo;
diff --git a/...Details/Components/FieldGrouping/TabField/DataDownloadList/Interfaces/DataDictionaries.ts b/...Details/Components/FieldGrouping/TabField/DataDownloadList/Interfaces/DataDictionaries.ts