Skip to content

Commit 13d4414

Browse files
PatowhizPatowhiz
authored
Merge pull request #150 from Patowhiz/pwa
Implements drill down in data availability and improved data import
2 parents 51662cb + 54a7c65 commit 13d4414

90 files changed

Lines changed: 1957 additions & 833 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎back-end/api/src/metadata/source-templates/dtos/create-import-source-tabular.dto.ts‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,23 @@ export class FlagDefinition {
146146
flagsToFetch?: { sourceId: string; databaseId: FlagEnum }[];
147147
}
148148

149+
export type DateTimeFormatTypes = '%Y-%m-%d %H:%M:%S' |
150+
'%Y-%m-%d %H:%M' |
151+
'%Y-%m-%d' |
152+
'%d-%m-%Y %H:%M:%S' |
153+
'%d-%m-%Y %H:%M' |
154+
'%d-%m-%Y' |
155+
'%Y/%m/%d %H:%M:%S' |
156+
'%Y/%m/%d %H:%M' |
157+
'%Y/%m/%d' |
158+
'%d/%m/%Y %H:%M:%S' |
159+
'%d/%m/%Y %H:%M' |
160+
'%d/%m/%Y';
161+
162+
export type DateFormatTypes = '%Y-%m-%d' | '%d-%m-%Y' | '%Y/%m/%d' | '%d/%m/%Y';
163+
164+
export type TimeFormatTypes = '%H:%M:%S' | '%H:%M';
165+
149166
/**
150167
* When dateTimeColumnPostion is not specified then dateInMultipleColumn should be specified, that is,
151168
* either dateTimeColumnPostion or dateInMultipleColumn must be provided, but not both.
@@ -156,16 +173,21 @@ export class DateTimeDefinition {
156173
* The date time column position
157174
* Expected format: 'yyyy-mm-dd hh:mm:ss'
158175
*/
159-
dateTimeColumnPostion?: number;
176+
dateTimeInSingleColumn?: {
177+
columnPosition: number;
178+
datetimeFormat: DateTimeFormatTypes;
179+
};
160180

161181
dateTimeInMultipleColumn?: {
162182

163183
dateInSingleColumn?: {
164-
dateColumnPosition: number;
165-
// yyyy-mm-dd, yyyy/mm/dd,
166-
// dd-mm-yyyy, dd/mm/yyyy,
167-
// iso dateformat that is, yyyy-mm-dd hh:mm:ss
168-
datetimeFormat: 'yyyy-mm-dd hh:mm:ss' | 'yyyy-mm-dd' | 'yyyy/mm/dd' | 'dd-mm-yyyy' | 'dd/mm/yyyy';
184+
columnPosition: number;
185+
dateFormat: DateFormatTypes;
186+
};
187+
188+
timeInSingleColumn?: {
189+
columnPosition: number;
190+
timeFormat: TimeFormatTypes;
169191
};
170192

171193
dateInMultipleColumn?: {
@@ -176,7 +198,6 @@ export class DateTimeDefinition {
176198

177199
hourDefinition: HourDefinition;
178200
};
179-
180201
}
181202

182203
/**
@@ -186,12 +207,10 @@ export class HourDefinition {
186207
/**
187208
* If provided, then defaultHour will not be used.
188209
*/
189-
@IsInt()
190210
columnPosition?: number;
191211

192212
/**
193213
* Should be provided when hourColumnPosition is not provided.
194214
*/
195-
@IsInt()
196215
defaultHour?: number;
197216
}

‎back-end/api/src/metadata/source-templates/dtos/create-import-source.dto.ts‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,6 @@ export class CreateImportSourceDTO implements SourceParametersValidity {
55
dataStructureType: DataStructureTypeEnum;
66
dataStructureParameters: DataStructureValidity;
77

8-
//TODO. Delete this and use the one that is in CreateUpdateSourceDto
9-
scaleValues: boolean;
10-
118
/**
129
* source values that represent missing.
1310
* Applicable only when import of missing values is allowed.
@@ -22,7 +19,7 @@ export class CreateImportSourceDTO implements SourceParametersValidity {
2219
export enum DataStructureTypeEnum {
2320
TABULAR = "tabular",
2421
KEY_VALUE = "key_value",
25-
BUFR = "bufr"
22+
BUFR = "bufr",
2623
}
2724

2825
export interface DataStructureValidity {

‎back-end/api/src/metadata/source-templates/dtos/create-update-source.dto.ts‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1-
import { IsBoolean, IsEnum, IsInt, IsOptional, IsString } from 'class-validator';
1+
import { Transform, Type } from 'class-transformer';
2+
import { IsEnum, IsInt, IsOptional, IsString } from 'class-validator';
23
import { SourceTypeEnum } from 'src/metadata/source-templates/enums/source-type.enum';
4+
import { StringUtils } from 'src/shared/utils/string.utils';
35

46
export class CreateUpdateSourceDto {
57
@IsString()
@@ -28,26 +30,33 @@ export class CreateUpdateSourceDto {
2830
* Determines whether to allow missing values or not.
2931
* If true, entry of missing values will be allowed.
3032
*/
31-
@IsBoolean()
32-
allowMissingValue: boolean;
33+
@IsOptional()
34+
@Type(() => String) // Required to stop transformer from converting the value type to boolean
35+
@Transform(({ value }) => value ? StringUtils.mapBooleanStringToBoolean(value.toString()) : false)
36+
allowMissingValue?: boolean;
3337

3438
/**
3539
* Determines whether to scale the values.
3640
* To be used when data being imported is not scaled
3741
*/
38-
@IsBoolean()
39-
scaleValues: boolean;
42+
@IsOptional()
43+
@Type(() => String) // Required to stop transformer from converting the value type to boolean
44+
@Transform(({ value }) => value ? StringUtils.mapBooleanStringToBoolean(value.toString()) : false)
45+
scaleValues?: boolean;
4046

4147
/** Sample paper image that resembles the source design */
48+
@IsOptional()
4249
@IsString()
43-
sampleImage: string;
50+
sampleImage?: string;
4451

45-
@IsBoolean()
46-
disabled: boolean;
52+
@IsOptional()
53+
@Type(() => String) // Required to stop transformer from converting the value type to boolean
54+
@Transform(({ value }) => value ? StringUtils.mapBooleanStringToBoolean(value.toString()) : false)
55+
disabled?: boolean;
4756

4857
@IsOptional()
4958
@IsString()
50-
comment: string | null;
59+
comment?: string | null;
5160
}
5261

5362
export interface SourceParametersValidity {

‎back-end/api/src/metadata/source-templates/entities/source-template.entity.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,14 @@ export class SourceTemplateEntity extends AppBaseEntity {
2222
@Column({ name: "utc_offset", type: "int" })
2323
utcOffset: number;
2424

25-
@Column({ name: "allow_missing_value", type: "boolean" })
25+
@Column({ name: "allow_missing_value", type: "boolean", default: false })
2626
allowMissingValue: boolean;
2727

28-
@Column({ name: "scale_values", type: "boolean" })
28+
@Column({ name: "scale_values", type: "boolean" , default: false})
2929
scaleValues: boolean;
3030

3131
@Column({ name: "sample_image", type: "varchar", nullable: true })
32-
sampleImage: string;
32+
sampleImage: string | null;
3333

3434
@Column({ name: "parameters", type: "jsonb" })
3535
parameters: SourceParametersValidity;

‎back-end/api/src/metadata/source-templates/services/source-templates.service.ts‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -82,11 +82,11 @@ export class SourceTemplatesService {
8282

8383
entity.description = dto.description;
8484
entity.sourceType = dto.sourceType;
85-
entity.utcOffset = dto.utcOffset;
86-
entity.allowMissingValue = dto.allowMissingValue;
87-
entity.scaleValues = dto.scaleValues;
88-
entity.sampleImage = dto.sampleImage;
8985
entity.parameters = dto.parameters;
86+
entity.utcOffset = dto.utcOffset;
87+
entity.allowMissingValue = dto.allowMissingValue ? true : false;
88+
entity.scaleValues = dto.scaleValues ? true : false;
89+
entity.sampleImage = dto.sampleImage ? dto.sampleImage : null;
9090
entity.disabled = dto.disabled ? true : false;
9191
entity.comment = dto.comment ? dto.comment : null;
9292
entity.entryUserId = userId;
@@ -106,10 +106,13 @@ export class SourceTemplatesService {
106106
entity.name = dto.name;
107107
entity.description = dto.description;
108108
entity.sourceType = dto.sourceType;
109-
entity.utcOffset = dto.utcOffset;
110-
entity.allowMissingValue = dto.allowMissingValue;
111-
entity.sampleImage = dto.sampleImage;
112109
entity.parameters = dto.parameters;
110+
entity.utcOffset = dto.utcOffset;
111+
entity.allowMissingValue = dto.allowMissingValue ? true : false;
112+
entity.scaleValues = dto.scaleValues ? true : false;
113+
entity.sampleImage = dto.sampleImage ? dto.sampleImage : null;
114+
entity.disabled = dto.disabled ? true : false;
115+
entity.comment = dto.comment ? dto.comment : null;
113116
entity.entryUserId = userId;
114117

115118
await this.sourceRepo.save(entity);
@@ -144,7 +147,7 @@ export class SourceTemplatesService {
144147
sourceType: entity.sourceType,
145148
utcOffset: entity.utcOffset,
146149
allowMissingValue: entity.allowMissingValue,
147-
sampleImage: entity.sampleImage,
150+
sampleImage: entity.sampleImage? entity.sampleImage: '' ,
148151
parameters: entity.parameters,
149152
scaleValues: entity.scaleValues,
150153
disabled: entity.disabled,

‎back-end/api/src/metadata/stations/services/stations-import-export.service.ts‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -68,14 +68,12 @@ export class StationsImportExportService {
6868
// Execute the duckdb DDL SQL commands
6969
await this.fileIOService.duckDb.exec(alterSQLs);
7070

71-
let duplicates: TableData | undefined;
72-
//check for duplicate ids
73-
duplicates = await DuckDBUtils.getDuplicateCount(this.fileIOService.duckDb, tmpTableName, this.ID_PROPERTY);
71+
//check for duplicate ids
72+
const duplicates: TableData | undefined = await DuckDBUtils.getDuplicateCount(this.fileIOService.duckDb, tmpTableName, this.ID_PROPERTY);
7473
if (duplicates.length > 0) throw new Error(`Error: ${JSON.stringify(duplicates)}`);
7574
//check for duplicate names
76-
duplicates = await DuckDBUtils.getDuplicateCount(this.fileIOService.duckDb, tmpTableName, this.NAME_PROPERTY);
77-
if (duplicates.length > 0) throw new Error(`Error: ${JSON.stringify(duplicates)}`);
78-
75+
// duplicates = await DuckDBUtils.getDuplicateCount(this.fileIOService.duckDb, tmpTableName, this.NAME_PROPERTY);
76+
// if (duplicates.length > 0) throw new Error(`Error: ${JSON.stringify(duplicates)}`);
7977

8078
// Get all the data imported
8179
const rows = await this.fileIOService.duckDb.all(`SELECT ${this.ID_PROPERTY}, ${this.NAME_PROPERTY}, ${this.DESCRIPTION_PROPERTY}, ${this.OBS_PROC_METHOD_PROPERTY}, ${this.LATITUDE_PROPERTY}, ${this.LONGITUDE_PROPERTY}, ${this.ELEVATION_PROPERTY}, ${this.OBS_ENVIRONMENT_ID_PROPERTY}, ${this.OBS_FOCUS_ID_PROPERTY}, ${this.WMO_ID_PROPERTY}, ${this.WIGOS_ID_PROPERTY}, ${this.ICAO_ID_PROPERTY}, ${this.STATUS_PROPERTY}, ${this.DATE_ESTABLISHED_PROPERTY}, ${this.DATE_CLOSED_PROPERTY}, ${this.COMMENT_PROPERTY} FROM ${tmpTableName};`);

‎back-end/api/src/observation/controllers/observations.controller.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,11 @@ import { AuthorisedExportsPipe } from 'src/user/pipes/authorised-exports.pipe';
1515
import { AuthorisedImportsPipe } from 'src/user/pipes/authorised-imports.pipe';
1616
import { StationStatusQueryDto } from '../dtos/station-status-query.dto';
1717
import { StationStatusDataQueryDto } from '../dtos/station-status-data-query.dto';
18-
import { DataAvailabilitySummaryQueryDto } from '../dtos/data-availability-summary-query.dto';
18+
import { DataAvailabilityQueryDto } from '../dtos/data-availability-query.dto';
1919
import { DataEntryCheckService } from '../services/data-entry-check.service';
2020
import { DataFlowQueryDto } from '../dtos/data-flow-query.dto';
2121
import { QCStatusEnum } from '../enums/qc-status.enum';
22+
import { DataAvailabilityDetailsQueryDto } from '../dtos/data-availability-details-query.dto';
2223

2324
@Controller('observations')
2425
export class ObservationsController {
@@ -63,10 +64,16 @@ export class ObservationsController {
6364

6465
@Get('data-availability-summary')
6566
getDataAvailabilitySummary(
66-
@Query(AuthorisedStationsPipe) query: DataAvailabilitySummaryQueryDto) {
67+
@Query(AuthorisedStationsPipe) query: DataAvailabilityQueryDto) {
6768
return this.observationsService.findDataAvailabilitySummary(query);
6869
}
6970

71+
@Get('data-availability-details')
72+
getDataAvailabilitydetails(
73+
@Query(AuthorisedStationsPipe) query: DataAvailabilityDetailsQueryDto) {
74+
return this.observationsService.findDataAvailabilityDetails(query);
75+
}
76+
7077
@Get('data-flow')
7178
getDataFlow(
7279
@Query(AuthorisedStationsPipe) query: DataFlowQueryDto) {

‎back-end/api/src/observation/controllers/quality-control.controller.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,12 @@ export class QualityControlController {
1111

1212
@Get('duplicates')
1313
findObservationsWithDuplicates(@Query(AuthorisedStationsPipe) queryDto: ViewObservationQueryDTO) {
14-
return this.sourceCheckService.findObservationsWithDuplicates(queryDto);
14+
return this.sourceCheckService.findSameObsWithDiffSources(queryDto);
1515
}
1616

1717
@Get('count-duplicates')
1818
countObservationsWithDuplicates(@Query(AuthorisedStationsPipe) queryDto: ViewObservationQueryDTO) {
19-
return this.sourceCheckService.countObservationsWithDuplicates(queryDto);
19+
return this.sourceCheckService.countSameObsWithDiffSources(queryDto);
2020
}
2121

2222
@Post('perform-qc')

‎back-end/api/src/observation/dtos/data-availability-details-query.dto.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import { Transform } from "class-transformer";
2+
import { IsDateString, IsInt, IsOptional, IsString } from "class-validator";
3+
import { StringUtils } from "src/shared/utils/string.utils";
4+
5+
export class DataAvailabilityDetailsQueryDto {
6+
@IsOptional()
7+
@Transform(({ value }) => value ? StringUtils.mapCommaSeparatedStringToStringArray(value.toString()) : [])
8+
@IsString({ each: true })
9+
stationIds?: string[];
10+
11+
@IsOptional()
12+
@Transform(({ value }) => value ? StringUtils.mapCommaSeparatedStringToIntArray(value.toString()) : [])
13+
@IsInt({ each: true })
14+
elementIds?: number[];
15+
16+
@IsOptional()
17+
@IsInt()
18+
interval?: number;
19+
20+
@IsOptional()
21+
@IsInt()
22+
level?: number;
23+
24+
@IsDateString()
25+
fromDate: string;
26+
27+
@IsDateString()
28+
toDate: string;
29+
}
30+

‎back-end/api/src/observation/dtos/data-availability-details.dto.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
export interface DataAvailaibilityDetailsDto {
2+
stationId: string;
3+
elementId: number;
4+
level: number;
5+
interval: number;
6+
fromDate: string;
7+
toDate: string;
8+
expected: number;
9+
nonMissing: number;
10+
confirmedMissing: number;
11+
gaps: number;
12+
gapsPlusMissing: number;
13+
qcNones: number;
14+
qcPasses: number;
15+
qcFails: number;
16+
}

0 commit comments

Comments
 (0)