Skip to content

Vihko import fixes #887

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Blodir merged 4 commits into from
Jan 14, 2026
Merged

Vihko import fixes #887

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0773a7c
fix import error messages not showing
aorin Sep 30, 2025
41491f8
use new API's batch job endpoints for vihko spreadsheet
olzraiti Nov 6, 2025
e6c4376
merge vihko import branches
aorin Jan 12, 2026
70f3c52
fix import service checking import job phase
olzraiti Jan 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 18 additions & 14 deletions projects/laji/src/app/shared-modules/spreadsheet/importer/importer.component.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import {
} from '@angular/core';
import { TranslateService } from '@ngx-translate/core';
import { forkJoin, from as ObservableFrom, Observable, of } from 'rxjs';
import { Document } from '../../../shared/model/Document';
import { FormService } from '../../../shared/service/form.service';
import { IFormField, VALUE_IGNORE } from '../model/excel';
import { CombineToDocument, IDocumentData, ImportService } from '../service/import.service';
Expand All @@ -32,9 +31,14 @@ import { FileService, instanceOfFileLoad } from '../service/file.service';
import { IUserMappingFile, MappingFileService } from '../service/mapping-file.service';
import { Form } from '../../../shared/model/Form';
import { Logger } from '../../../shared/logger';
import { DocumentJobPayload } from '../../../shared/api/DocumentApi';
import { toHtmlSelectElement } from '../../../shared/service/html-element.service';
import {ModalRef, ModalService} from 'projects/laji-ui/src/lib/modal/modal.service';
import { ModalRef, ModalService } from 'projects/laji-ui/src/lib/modal/modal.service';

import type { components } from 'projects/laji-api-client-b/generated/api';

type Document = components['schemas']['document'];
type PublicityRestrictions = Document['publicityRestrictions'];
type BatchJob = components['schemas']['BatchJobValidationStatusResponse'];

@Component({
selector: 'laji-importer',
Expand Down Expand Up @@ -72,7 +76,7 @@ export class ImporterComponent implements OnInit, OnDestroy {
header?: {[key: string]: string};
fields?: {[key: string]: IFormField};
dataColumns?: ImportTableColumn[];
jobPayload?: DocumentJobPayload;
job?: BatchJob;
docCnt = 0;
origColMap?: {[key: string]: string};
colMap?: {[key: string]: string};
Expand All @@ -83,8 +87,8 @@ export class ImporterComponent implements OnInit, OnDestroy {
mimeType?: string;
errors: any;
valid = false;
priv = Document.PublicityRestrictionsEnum.publicityRestrictionsPrivate;
publ = Document.PublicityRestrictionsEnum.publicityRestrictionsPublic;
priv: PublicityRestrictions = 'MZ.publicityRestrictionsPrivate';
publ: PublicityRestrictions = 'MZ.publicityRestrictionsPublic';
excludedFromCopy: string[] = [];
userMappings: any;
separator = MappingService.valueSplitter;
Expand Down Expand Up @@ -456,9 +460,9 @@ export class ImporterComponent implements OnInit, OnDestroy {
ObservableFrom(rowData).pipe(
concatMap(data => this.augmentService.augmentDocument(data.document, this.excludedFromCopy)),
toArray(),
switchMap(documents => this.importService.validateData(documents)),
tap(job => this.jobPayload = job),
switchMap(job => this.importService.waitToComplete('validate', job, (status) => {
switchMap(documents => this.importService.startBatchJob(documents)),
tap(job => this.job = job),
switchMap(job => this.importService.waitToComplete(job, (status) => {
this.current = status.processed;
this.cdr.markForCheck();
})),
Expand Down Expand Up @@ -508,7 +512,7 @@ export class ImporterComponent implements OnInit, OnDestroy {
);
}

save(publicityRestrictions: Document.PublicityRestrictionsEnum) {
save(publicityRestrictions: PublicityRestrictions) {
this.spreadsheetFacade.goToStep(Step.importing);
this.showOnlyErroneous = false;
let success = true;
Expand All @@ -522,14 +526,14 @@ export class ImporterComponent implements OnInit, OnDestroy {
const rowData = this.parsedData!.filter(data => data.document !== null);

this.importService.sendData({
...this.jobPayload,
dataOrigin: [Document.DataOriginEnum.dataOriginSpreadsheetFile],
...this.job,
dataOrigin: 'MY.dataOriginSpreadsheetFile',
publicityRestrictions
} as any).pipe(
switchMap(() => this.importService.waitToComplete('create', this.jobPayload as any, (status) => {
switchMap(() => this.importService.waitToComplete(this.job!, (status) => {
ticker += add;
this.current = status.processed === this.total ?
status.processed :
status.processed:
Math.min(Math.max(this.total - 1, 0), ticker);
this.cdr.markForCheck();
})),
Expand Down
2 changes: 1 addition & 1 deletion .../app/shared-modules/spreadsheet/importer/status-cell/error-list/error-list.component.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<div *ngFor="let data of _errors">
<strong *ngIf="fields[data.field]">{{ fields[data.field].fullLabel }}</strong>
<strong *ngIf="data.title">{{ data.title }}</strong>
<ul class="errors">
<li *ngFor="let error of data.errors">{{ error | translate }}</li>
</ul>
Expand Down
82 changes: 33 additions & 49 deletions ...rc/app/shared-modules/spreadsheet/importer/status-cell/error-list/error-list.component.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,97 +1,81 @@
import { ChangeDetectionStrategy, Component, Input } from '@angular/core';
import { ChangeDetectionStrategy, Component, Input, OnChanges } from '@angular/core';
import { IFormField } from '../../../model/excel';
import { TranslateService } from '@ngx-translate/core';
import { Util } from '../../../../../shared/service/util.service';

interface ErrorGroup {
title: string;
errors: string[];
}

@Component({
selector: 'laji-error-list',
templateUrl: './error-list.component.html',
styleUrls: ['./error-list.component.css'],
changeDetection: ChangeDetectionStrategy.OnPush
})
export class ErrorListComponent {
export class ErrorListComponent implements OnChanges {

@Input({ required: true }) fields!: {[key: string]: IFormField};
@Input() errors: unknown;

_errors: {field: string; errors: string[]}[] = [];
_errors: ErrorGroup[] = [];

constructor(private translateService: TranslateService) { }

@Input()
set errors(data: unknown) {
const errors = [];
ngOnChanges() {
this._errors = this.processErrors(this.errors, this.fields);
}

processErrors(data: unknown, fields: {[key: string]: IFormField}): ErrorGroup[] {
const errors: ErrorGroup[] = [];

if (Util.isObject(data)) {
if (Util.hasOwnProperty(data, 'status')) {
switch (data.status) {
case 403:
errors.push({
field: 'id',
title: this.translateService.instant('error'),
errors: [this.translateService.instant('form.permission.no-access')]
});
break;
case 422:
default:
errors.push({
field: 'id',
errors: [Util.hasOwnProperty(data, 'statusText') ? data.statusText : this.translateService.instant('haseka.form.genericError')]
title: this.translateService.instant('error'),
errors: [this.translateService.instant('haseka.form.genericError')]
});
}
} else {
Object.keys(data).forEach(field => {
errors.push({
field: this.pathToKey(field),
errors: Array.isArray(data[field]) ? data[field] : this.pickErrors(data[field])
title: this.getFieldName(field, fields),
errors: data[field] as string[]
});
});
}
} else if (Array.isArray(data)) {
data.forEach(err => {
if (typeof err !== 'object' || !Util.hasOwnProperty(err, 'dataPath')) {
return;
}
errors.push({
field: err.dataPath
.substring(err.dataPath.substring(0, 1) === '/' ? 1 : 0)
.replace(/\/[0-9]+/g, '[*]')
.replace(/\//g, '.'),
errors: [this.getMessage(err)]
});
});
}
this._errors = errors;

return errors;
}

private pathToKey(path: string) {
private getFieldName(path: string, fields: {[key: string]: IFormField}): string {
if (path.substring(0, 1) === '.') {
path = path.substring(1);
}
return path.replace(/\[[0-9]+]/g, '[*]');
}

private pickErrors(value: any): any[] {
if (typeof value === 'string') {
return [value];
} else if (Array.isArray(value)) {
return value;
} else if (typeof value === 'object') {
return Object.keys((value)).reduce((prev, current) => [...prev, ...this.pickErrors(current)] as any, []);
}
return [value];
}
const key = path.replace(/\[[0-9]+]/g, '[*]');

private getMessage(err: unknown): string {
if (!Util.isObject(err) || !Util.hasOwnProperty(err, 'message') || typeof err.message !== 'string') {
return this.translateService.instant('haseka.form.genericError');
if (fields[key]) {
return fields[key].fullLabel;
}
let base = err.message;
if (Util.hasOwnProperty(err, 'params') && typeof err.params === 'object' && err.params && !Array.isArray(err.params)) {
const info = Object.values(err.params);
if (info.length) {
base += ` '${info.join('\', \'')}'`;
}

const arrayKey = key + '[*]';

if (fields[arrayKey]) {
return fields[arrayKey].fullLabel;
}

return base;
return '';
}

}
4 changes: 3 additions & 1 deletion projects/laji/src/app/shared-modules/spreadsheet/service/augment.service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@ import { from as ObservableFrom, Observable, of as ObservableOf } from 'rxjs';
import { NamedPlace } from '../../../shared/model/NamedPlace';
import { NamedPlaceApi } from '../../../shared/api/NamedPlaceApi';
import { UserService } from '../../../shared/service/user.service';
import { Document } from '../../../shared/model/Document';
import { DocumentService } from '../../own-submissions/service/document.service';
import { MappingService } from './mapping.service';
import type { components } from 'projects/laji-api-client-b/generated/api';

type Document = components['schemas']['document'];

@Injectable()
export class AugmentService {
Expand Down
44 changes: 16 additions & 28 deletions projects/laji/src/app/shared-modules/spreadsheet/service/import.service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
import { Injectable } from '@angular/core';
import { TranslateService } from '@ngx-translate/core';
import { Observable, of } from 'rxjs';
import { DocumentApi, DocumentJobPayload } from '../../../shared/api/DocumentApi';
import { Document } from '../../../shared/model/Document';
import { UserService } from '../../../shared/service/user.service';
import {
IFormField,
LEVEL_DOCUMENT,
Expand All @@ -15,7 +11,11 @@ import {
import { MappingService } from './mapping.service';
import * as Hash from 'object-hash';
import { catchError, delay, switchMap } from 'rxjs/operators';
import { ArrayType } from '@angular/compiler';
import { LajiApiClientBService } from 'projects/laji-api-client-b/src/laji-api-client-b.service';
import type { components } from 'projects/laji-api-client-b/generated/api';

type Document = components['schemas']['document'];
type BatchJob = components['schemas']['BatchJobValidationStatusResponse'];

export interface IData {
rowIdx: number;
Expand Down Expand Up @@ -74,56 +74,44 @@ export class ImportService {

constructor(
private mappingService: MappingService,
private documentApi: DocumentApi,
private userService: UserService,
private translateService: TranslateService
private api: LajiApiClientBService
) { }

hasInvalidValue(value: unknown, field: IFormField) {
const mappedValue = this.mappingService.map(value, field);
return Array.isArray(mappedValue) ? mappedValue.indexOf(null) > -1 : mappedValue === null;
}

validateData(document: Document|Document[]): Observable<any> {
return this.documentApi.validate(document, {
personToken: this.userService.getToken(),
lang: this.translateService.currentLang,
validationErrorFormat: 'jsonPath'
});
startBatchJob(documents: Document[]) {
return this.api.post('/documents/batch', undefined, documents);
}

waitToComplete(type: keyof Pick<DocumentApi, 'validate'|'create'>, jobPayload: DocumentJobPayload, processCB: (status: JobStatus) => void): Observable<any> {
const personToken = this.userService.getToken();
const source$ = type === 'validate' ?
this.documentApi.validate(jobPayload, {personToken}) :
this.documentApi.create(jobPayload, personToken);
return source$.pipe(
waitToComplete(job: BatchJob, processCB: (status: BatchJob['status']) => void): Observable<BatchJob> {
return this.api.get('/documents/batch/{jobID}', { path: { jobID: job.id }, query: { validationErrorFormat: 'dotNotation' } }).pipe(
switchMap(response => {
processCB(response.status);
if (response.status.percentage === 100) {
if (!['VALIDATING', 'COMPLETING'].includes(response.phase)) {
return of(response);
}
return of(response).pipe(
delay(1000),
switchMap(() => this.waitToComplete(type, jobPayload, processCB))
switchMap(() => this.waitToComplete(job, processCB))
);
}),
catchError((e) => {
console.log('ERROR', e);
return of(e).pipe(
delay(1000),
switchMap(() => this.waitToComplete(type, jobPayload, processCB))
switchMap(() => this.waitToComplete(job, processCB))
);
})
);
}

sendData(
job: DocumentJobPayload
job: BatchJob
): Observable<any> {
return this.documentApi.create(job, this.userService.getToken(), {
lang: this.translateService.currentLang
});
return this.api.post('/documents/batch/{jobID}', { path: { jobID: job.id } });
}

flatFieldsToDocuments(
Expand Down Expand Up @@ -287,7 +275,7 @@ export class ImportService {
const docs: {[hash: string]: IDocumentData} = {};
Object.keys(documents).forEach(hash => {
if (!docs[hash]) {
docs[hash] = {document: {formID}, ref: {[hash]: {}}, rows: {}, skipped: []};
docs[hash] = {document: {formID} as Document, ref: {[hash]: {}}, rows: {}, skipped: []};
const docData = this.findDocumentData(documents[hash]);
docs[hash].rows[(docData as any).rowIdx] = true;
if (ignoreRowsWithNoCount && !this.hasCountValue((docData as any).data) && combineBy === CombineToDocument.none) {
Expand Down