Fix db csv export (#4515)

* Fix db csv export * Fix test
charmverse · Aug 29, 2024 · 4288a72 · 4288a72
1 parent b68ceca
commit 4288a72
Show file tree

Hide file tree

Showing 2 changed files with 70 additions and 54 deletions.
diff --git a/lib/databases/__tests__/generateCsv.spec.ts b/lib/databases/__tests__/generateCsv.spec.ts
@@ -6,10 +6,10 @@ import { v4 } from 'uuid';
 import { generateBoard } from 'testing/setupDatabase';
 import { generateProposalSourceDb } from 'testing/utils/proposals';
 
-import type { Board, IPropertyOption, IPropertyTemplate } from '../board';
+import type { IPropertyOption, IPropertyTemplate } from '../board';
 import type { BoardViewFields } from '../boardView';
 import { Constants } from '../constants';
-import { generateTableArray, loadAndGenerateCsv } from '../generateCsv';
+import { generateCsvContent, loadAndGenerateCsv } from '../generateCsv';
 
 import { generateTableArrayInput } from './cardStubs';
 
@@ -114,9 +114,9 @@ describe('loadAndGenerateCsv()', () => {
         .map((c) => c.trim())
     ).toStrictEqual([
       'Title\tText\tSelect',
-      '"Card 1"\t"Card 1 Text"\t"Blue"',
-      '"Card 2"\t"Card 2 Text"\t"Red"',
-      '"Card 3"\t"Card 3 Text"\t""'
+      'Card 1\tCard 1 Text\tBlue',
+      'Card 2\tCard 2 Text\tRed',
+      'Card 3\tCard 3 Text'
     ]);
   });
 
@@ -208,11 +208,11 @@ describe('loadAndGenerateCsv()', () => {
   });
 });
 
-describe('generateTableArray()', () => {
+describe('generateCsvContent()', () => {
   it('should return a table array with correct properties and filters applied', async () => {
     const { board, cardMap, cards, context, formatters, view } = generateTableArrayInput;
 
-    const results = generateTableArray(
+    const results = generateCsvContent(
       board as any,
       cards as any,
       view as any,
@@ -222,7 +222,12 @@ describe('generateTableArray()', () => {
     );
 
     // Tests that correct columns are provided and that all data was taken into account, even if Proposal Step is hidden as a column
-    expect(results.rows).toStrictEqual([
+    expect(
+      results.csvContent
+        .trim()
+        .split('\n')
+        .map((c) => c.trim())
+    ).toStrictEqual([
       [
         'Title',
         'Reviewer Notes',
@@ -231,16 +236,18 @@ describe('generateTableArray()', () => {
         'Publish Date',
         'Proposal Url',
         'Proposal Reviewers'
-      ],
+      ].join('\t'),
       [
-        '"Getting Started"',
-        '""',
+        'Getting Started',
+        '',
         'test',
-        '"In Progress"',
-        '"N/A"',
+        'In Progress',
+        'N/A',
         'http://127.0.0.1:3335/demo-space-domain/getting-started-8198984395372089',
         ''
       ]
+        .join('\t')
+        .trim()
     ]);
   });
 });
diff --git a/lib/databases/generateCsv.ts b/lib/databases/generateCsv.ts
@@ -8,6 +8,7 @@ import { getBlocks as getBlocksForProposalSource } from '@root/lib/databases/pro
 import { permissionsApiClient } from '@root/lib/permissions/api/client';
 import { formatDate, formatDateTime } from '@root/lib/utils/dates';
 import { isTruthy } from '@root/lib/utils/types';
+import { stringify } from 'csv-stringify/sync';
 import { sortBy } from 'lodash';
 
 import type { Formatters, PropertyContext } from 'components/common/DatabaseEditor/octoUtils';
@@ -137,13 +138,7 @@ function generateCSV(
   context: PropertyContext,
   cardMap: Record<string, { title: string }>
 ) {
-  const { rows, rowIds } = generateTableArray(board, cards, view, formatters, context, cardMap);
-  let csvContent = '';
-
-  rows.forEach((row) => {
-    const encodedRow = row.join('\t');
-    csvContent += `${encodedRow}\r\n`;
-  });
+  const { csvContent, rowIds } = generateCsvContent(board, cards, view, formatters, context, cardMap);
 
   let fileTitle = view.title;
   if (view.fields.sourceType === 'google_form') {
@@ -159,19 +154,14 @@ function generateCSV(
   };
 }
 
-function encodeText(text: string): string {
-  return text.replace(/"/g, '""');
-}
-
-export function generateTableArray(
+export function generateCsvContent(
   board: Pick<Board, 'fields'>,
   cards: Card[],
   viewToExport: BoardView,
   formatters: Formatters,
   context: PropertyContext,
   cardMap: Record<string, { title: string }>
-): { rows: string[][]; rowIds: string[] } {
-  const rows: string[][] = [];
+) {
   const visiblePropertyIds: string[] = viewToExport.fields.visiblePropertyIds;
 
   const allCardProperties = board.fields.cardProperties as IPropertyTemplate[];
@@ -198,50 +188,64 @@ export function generateTableArray(
     }
   }
 
+  const visibleProperties = [...cardProperties];
   const titleProperty = cardProperties.find((prop) => prop.id === Constants.titleColumnId);
+
   // Header row
-  const row: string[] = titleProperty ? [] : ['Title'];
-  cardProperties.forEach((template: IPropertyTemplate) => {
-    row.push(template.name);
-  });
-  rows.push(row);
+  const headers = visibleProperties.map((prop: IPropertyTemplate) => prop.name);
+
+  // In case we don't have a title prop we creata it on the fly
+  if (!titleProperty) {
+    headers.unshift('Title');
+    visibleProperties.unshift({
+      id: Constants.titleColumnId,
+      name: 'Title',
+      type: 'text',
+      options: [],
+      readOnly: true,
+      readOnlyValues: true
+    });
+  }
+
+  const csvData: Record<string, string>[] = [];
 
-  filteredCards.forEach((card) => {
-    const rowColumns = getCSVColumns({
+  for (const card of filteredCards) {
+    const csvCard = getCSVContent({
       card,
       context,
       formatters,
-      hasTitleProperty: !!titleProperty,
-      visibleProperties: cardProperties,
+      visibleProperties,
       cardMap
     });
-    rows.push(rowColumns);
+
+    csvData.push(csvCard);
+  }
+
+  const csvContent = stringify(csvData, {
+    delimiter: '\t',
+    header: true,
+    columns: headers
   });
 
-  return { rows, rowIds: filteredCards.map(({ id }) => id) };
+  const rowIds = filteredCards.map(({ id }) => id);
+
+  return { csvContent, rowIds };
 }
 
-function getCSVColumns({
+function getCSVContent({
   card,
   context,
   formatters,
-  hasTitleProperty,
   visibleProperties,
   cardMap
 }: {
   cardMap: Record<string, { title: string }>;
   card: Card;
   context: PropertyContext;
   formatters: Formatters;
-  hasTitleProperty: boolean;
   visibleProperties: IPropertyTemplate<PropertyType>[];
 }) {
-  const columns: string[] = [];
-
-  if (!hasTitleProperty) {
-    columns.push(`"${encodeText(card.title)}"`);
-  }
-  visibleProperties.forEach((propertyTemplate: IPropertyTemplate) => {
+  const csvCard = visibleProperties.reduce<Record<string, string>>((acc, propertyTemplate) => {
     const propertyValue = card.fields.properties[propertyTemplate.id];
     const displayValue =
       OctoUtils.propertyDisplayValue({
@@ -253,7 +257,7 @@ function getCSVColumns({
         cardMap
       }) || '';
     if (propertyTemplate.id === Constants.titleColumnId) {
-      columns.push(`"${encodeText(card.title)}"`);
+      acc[propertyTemplate.name] = card.title;
     } else if (
       propertyTemplate.type === 'number' ||
       propertyTemplate.type === 'proposalEvaluationAverage' ||
@@ -263,7 +267,7 @@ function getCSVColumns({
       propertyTemplate.type === 'proposalRubricCriteriaAverage'
     ) {
       const numericValue = propertyValue ? Number(propertyValue).toString() : '';
-      columns.push(numericValue);
+      acc[propertyTemplate.name] = numericValue;
     } else if (
       propertyTemplate.type === 'multiSelect' ||
       propertyTemplate.type === 'person' ||
@@ -273,14 +277,19 @@ function getCSVColumns({
       propertyTemplate.type === 'relation'
     ) {
       const multiSelectValue = (((displayValue as unknown) || []) as string[]).join(',');
-      columns.push(multiSelectValue);
+      acc[propertyTemplate.name] = multiSelectValue;
     } else if (propertyTemplate.type === 'proposalUrl') {
       // proposalUrl is an array on the Rewards database of [title, url], and only a string on database-as-a-source
-      columns.push(`${baseUrl}${encodeText(Array.isArray(displayValue) ? displayValue[0] : displayValue.toString())}`);
+      acc[propertyTemplate.name] = `${baseUrl}${
+        Array.isArray(displayValue) ? displayValue[0] : displayValue.toString()
+      }`;
     } else {
       // Export as string
-      columns.push(`"${encodeText(displayValue.toString())}"`);
+      acc[propertyTemplate.name] = displayValue.toString();
     }
-  });
-  return columns;
+
+    return acc;
+  }, {});
+
+  return csvCard;
 }