Batch synonyms from a csv file

frontend/package.json

@@ -27,6 +27,7 @@
     "angular2-multiselect-dropdown": "^4.6.6",
     "angular2-toaster": "^11.0.1",
     "bootstrap": "^4.5.0",
+    "papaparse": "^5.5.2",
     "rxjs": "~6.6.0",
     "tslib": "^2.0.0",
     "zone.js": "~0.10.2"

frontend/src/app/app.module.ts

@@ -100,7 +100,7 @@ import {
     SuggestedFieldsComponent,
     SuggestedFieldsCreateComponent,
     SuggestedFieldsListComponent,
-    PreviewLinkComponent
+    PreviewLinkComponent,
   ],
   providers: [
     CommonsService,

frontend/src/app/components/details/rule-management/rule-management.component.ts

@@ -27,6 +27,7 @@ import {
   UpDownRule,
   PreviewSection
 } from '../../../models';
+import {randomUUID} from '../../../lib/uuid';
 import {
   CommonsService,
   FeatureToggleService,
@@ -284,7 +285,7 @@ export class RuleManagementComponent implements OnChanges, OnInit, AfterContentC
     console.log('In SearchInputDetailComponent :: addNewSynonym');
 
     const emptySynonymRule: SynonymRule = {
-      id: this.randomUUID(),
+      id: randomUUID(),
       synonymType: 0,
       term: '',
       isActive: true
@@ -308,7 +309,7 @@ export class RuleManagementComponent implements OnChanges, OnInit, AfterContentC
     console.log('In SearchInputDetailComponent :: addNewUpDownRule');
 
     const emptyUpDownRule: UpDownRule = {
-      id: this.randomUUID(),
+      id: randomUUID(),
       term: '',
       isActive: true
     };
@@ -340,7 +341,7 @@ export class RuleManagementComponent implements OnChanges, OnInit, AfterContentC
     console.log('In SearchInputDetailComponent :: addNewFilterRule');
 
     const emptyFilterRule: FilterRule = {
-      id: this.randomUUID(),
+      id: randomUUID(),
       term: '',
       isActive: true
     };
@@ -369,7 +370,7 @@ export class RuleManagementComponent implements OnChanges, OnInit, AfterContentC
     console.log('In SearchInputDetailComponent :: addNewDeleteRule');
 
     const emptyDeleteRule: DeleteRule = {
-      id: this.randomUUID(),
+      id: randomUUID(),
       term: '',
       isActive: true
     };
@@ -393,7 +394,7 @@ export class RuleManagementComponent implements OnChanges, OnInit, AfterContentC
     console.log('In SearchInputDetailComponent :: addNewRedirectRule');
 
     const emptyRedirectRule: RedirectRule = {
-      id: this.randomUUID(),
+      id: randomUUID(),
       target: '',
       isActive: true
     };
@@ -560,17 +561,6 @@ export class RuleManagementComponent implements OnChanges, OnInit, AfterContentC
     return idxMinimumDistance;
   }
 
-  // taken from https://stackoverflow.com/questions/105034/create-guid-uuid-in-javascript
-  private randomUUID() {
-    /* eslint-disable */
-    return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
-      const r = (Math.random() * 16) | 0,
-        v = c == 'x' ? r : (r & 0x3) | 0x8
-      return v.toString(16)
-    })
-    /* eslint-enable */
-  }
-
   private updateSelectedTagsInModel() {
     if (this.detailSearchInput) {
       this.detailSearchInput.tags = this.selectedTags;

frontend/src/app/components/rules-panel/rules-search/rules-search.component.css

@@ -10,3 +10,7 @@
 .create-button {
   margin-left: 0.5rem;
 }
+
+.full-width {
+  width: 100%;
+}

frontend/src/app/components/rules-panel/rules-search/rules-search.component.html

@@ -45,6 +45,13 @@
       >
         <i class="fa fa-plus" aria-hidden="true"></i> New
       </button>
+      <button
+        class="btn btn-success create-button"
+        type="button"
+        (click)="openFileModal()"
+      >
+        <i class="fa fa-upload" aria-hidden="true"></i> Import
+      </button>
     </span>
   </div>
 </app-smui-card>
@@ -77,3 +84,50 @@
     </button>
   </div>
 </app-smui-modal>
+<app-smui-modal
+  id="file-import"
+  title="Import rules from CSV"
+>
+  <div content>
+    <div class="progress" [style.visibility]="progress ? 'visible' : 'hidden'">
+      <div class="progress-bar"
+        [style.width.%]="progress"
+        [attr.aria-valuenow]="progress"
+        role="progressbar"
+        aria-valuemin="0"
+        aria-valuemax="100"
+      ></div>
+    </div>
+    <p>
+      Your csv must be in the following format (do not include the header row)
+    </p>
+    <table class="table">
+      <thead class="thead-dark">
+        <tr>
+          <th>search term</th>
+          <th>synonym</th>
+          <th>comment</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+          <td>boot</td>
+          <td>shoe</td>
+          <td>Requested by footware</td>
+        </tr>
+      </tbody>
+    </table>
+  </div>
+  <div footer class="btn-toolbar full-width">
+    <div class="custom-file">
+      <input
+        type="file"
+        accept="text/csv"
+        class="custom-file-input"
+        id="customFile"
+        (change)="fileSelect($event)"
+      >
+      <label class="custom-file-label" for="customFile">Choose file</label>
+    </div>
+  </div>
+</app-smui-modal>

frontend/src/app/components/rules-panel/rules-search/rules-search.component.ts

@@ -4,17 +4,20 @@ import {
   Output,
   EventEmitter,
   OnChanges,
-  SimpleChanges
+  SimpleChanges,
+  ChangeDetectorRef
 } from '@angular/core';
 
 import {
   FeatureToggleService,
   ModalService,
   RuleManagementService,
   SpellingsService,
-  TagsService
+  TagsService,
+  CSVImportService
 } from '../../../services';
-import {InputTag, ListItem} from '../../../models';
+import {InputTag, ListItem, ApiResult} from '../../../models';
+const fileImportModal = 'file-import';
 
 @Component({
   selector: 'app-smui-rules-search',
@@ -35,6 +38,7 @@ export class RulesSearchComponent implements OnChanges {
   @Output() showErrorMsg: EventEmitter<string> = new EventEmitter();
 
   allTags: InputTag[] = [];
+  progress: number = 0;
   readonly isTaggingActive = this.featureToggleService.isRuleTaggingActive();
   private readonly isSpellingActive = this.featureToggleService.getSyncToggleActivateSpelling();
 
@@ -43,7 +47,9 @@ export class RulesSearchComponent implements OnChanges {
     private ruleManagementService: RuleManagementService,
     private spellingsService: SpellingsService,
     private tagsService: TagsService,
-    private modalService: ModalService
+    private modalService: ModalService,
+    private csvImportService: CSVImportService,
+    private cdr: ChangeDetectorRef
   ) {}
 
   ngOnChanges(changes: SimpleChanges): void {
@@ -65,6 +71,37 @@ export class RulesSearchComponent implements OnChanges {
     });
   }
 
+  openFileModal(): void {
+    this.modalService.open(fileImportModal);
+  }
+
+  fileSelect(event: Event): void {
+    const element = event.currentTarget as HTMLInputElement;
+    if (element?.files?.length && this.currentSolrIndexId) {
+      const files: FileList = element?.files;
+      const file = element?.files?.[0];
+      const ruleCreations = this.csvImportService.import(
+        file,
+        this.currentSolrIndexId,
+        (percentage: number) => {
+          this.progress = percentage;
+          this.cdr.detectChanges();
+        }
+      );
+      ruleCreations
+        .then(
+          () => {
+            this.progress = 0;
+            this.modalService.close(fileImportModal)
+            this.refreshAndSelectListItemById.emit();
+          }
+        ).catch(err => {
+          this.showErrorMsg.emit(err.message)
+          this.progress = 0;
+        });
+    }
+  }
+
   createNewSpellingItem() {
     if (this.currentSolrIndexId) {
       this.spellingsService

frontend/src/app/lib/csv.spec.ts

@@ -0,0 +1,50 @@
+import {rowsToSearchInputs} from './csv';
+
+describe('rowsToSearchInputs', () => {
+  it('parses no inputs when there are no rows', () => {
+    expect(rowsToSearchInputs([])).toHaveSize(0);
+  });
+
+  it('ignores rows with less than 3 columns', () => {
+    expect(rowsToSearchInputs([['abc', '123']])).toHaveSize(0);
+  });
+
+  it('ignores rows with more than 3 columns', () => {
+    expect(rowsToSearchInputs([['abc', '123', 'def', '456']])).toHaveSize(0);
+  });
+
+  it('parses rows with 3 columns', () => {
+    const inputs = rowsToSearchInputs([
+      ['term', 'synonym', 'comment']
+    ]);
+    expect(inputs).toHaveSize(1);
+    expect(inputs[0].id).toBeInstanceOf(String);
+    expect(inputs[0].term).toEqual('term');
+    expect(inputs[0].redirectRules).toEqual([]);
+    expect(inputs[0].filterRules).toEqual([]);
+    expect(inputs[0].tags).toEqual([]);
+    expect(inputs[0].upDownRules).toEqual([]);
+    expect(inputs[0].synonymRules).toHaveSize(1);
+    expect(inputs[0].synonymRules[0].term).toEqual('synonym');
+    expect(inputs[0].synonymRules[0].isActive).toEqual(true);
+    expect(inputs[0].synonymRules[0].synonymType).toEqual(0);
+    expect(inputs[0].synonymRules[0].id).toBeInstanceOf(String);
+  });
+
+  it('groups rows with the same search term', () => {
+    const inputs = rowsToSearchInputs([
+      ['term', 'synonym', 'comment'],
+      ['term', 'another synonym', 'comment'],
+    ]);
+    expect(inputs).toHaveSize(1);
+    expect(inputs[0].synonymRules).toHaveSize(2);
+  });
+
+  it('creates one SearchInput per term', () => {
+    const inputs = rowsToSearchInputs([
+      ['term', 'synonym', 'comment'],
+      ['term2', 'another synonym', 'comment'],
+    ]);
+    expect(inputs).toHaveSize(2);
+  });
+});

frontend/src/app/lib/csv.ts

@@ -0,0 +1,37 @@
+import {SynonymRule, SearchInput} from '../models';
+import {randomUUID} from './uuid';
+
+function makeActiveSynonymTerm(synonymTerm: string): SynonymRule {
+  return {
+    term: synonymTerm,
+    isActive: true,
+    synonymType: 0,
+    id: randomUUID()
+  };
+}
+
+export function rowsToSearchInputs(rows: string[][]): SearchInput[] {
+  return rows
+    .filter(row => row.length === 3)
+    .reduce((searchInputs, row) => {
+      const [term, synonymTerm, comment] = row;
+      const searchInput = searchInputs.find(searchInput => searchInput.term === term);
+      if (!searchInput) {
+        searchInputs = searchInputs.concat({
+          id: randomUUID(),
+          term,
+          synonymRules: [makeActiveSynonymTerm(synonymTerm)],
+          isActive: true,
+          redirectRules: [],
+          deleteRules: [],
+          filterRules: [],
+          tags: [],
+          upDownRules: [],
+          comment,
+        })
+      } else {
+        searchInput.synonymRules.push(makeActiveSynonymTerm(synonymTerm))
+      }
+      return searchInputs;
+     }, [] as SearchInput[]);
+}

frontend/src/app/lib/uuid.ts

@@ -0,0 +1,10 @@
+
+// taken from https://stackoverflow.com/questions/105034/create-guid-uuid-in-javascript
+export function randomUUID(): string {
+  /* eslint-disable */
+  return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
+    const r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8);
+    return v.toString(16);
+  });
+  /* eslint-enable */
+}

frontend/src/app/services/commons.service.ts

@@ -1,15 +1,11 @@
 import { Injectable, SimpleChanges } from '@angular/core';
+import {randomUUID} from '../lib/uuid';
 
 @Injectable()
 export class CommonsService {
 
   generateUUID(): string {
-    /* eslint-disable */
-    return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
-      const r = Math.random() * 16 | 0, v = c == 'x' ? r : (r & 0x3 | 0x8);
-      return v.toString(16);
-    });
-    /* eslint-enable */
+    return randomUUID();
   }
 
   isDirty(obj: any, origObj: string): boolean {