feat(api): Add new dat model for surveys, enable new migration system

Vizzuality · Oct 18, 2024 · d3812f6 · d3812f6
1 parent 4ef9041
commit d3812f6
Show file tree

Hide file tree

Showing 17 changed files with 284 additions and 159 deletions.
diff --git a/api/README.md b/api/README.md
@@ -1,6 +1,5 @@
 # 4-GROWTH API
 
-
 This project is built with NestJS.
 
 ## NestJS Version
@@ -27,4 +26,5 @@ Here are some of the npm scripts that you can run:
 - `pnpm start`: Starts the application
 - `pnpm start:dev`: Starts the application in watch mode
 - `pnpm test`: Runs the tests
-- `pnpm test:e2e`: Runs the end-to-end tests
+- `pnpm typeorm`: Execute typeorm CLI
+- `pnpm typeorm migration:run -d src/infrastructure/data-sources/postgres-data-source.ts`: Run typeorm migrations
diff --git a/api/data/filters.sql b/api/data/filters.sql
@@ -1,16 +1,4 @@
-
--- TEMPORAL WORKAROUND FOR FILTERS
-
-
-ALTER TABLE ONLY public.page_filters DROP CONSTRAINT "PK_066730e267edae13a61a79b8bd3";
-DROP TABLE public.page_filters;
-
-
-CREATE TABLE public.page_filters (
-                                     name character varying NOT NULL,
-                                     "values" text NOT NULL
-);
-
+DELETE FROM public.page_filters;
 
 INSERT INTO public.page_filters VALUES ('sector', 'Agriculture;Forestry;Both');
 INSERT INTO public.page_filters VALUES ('type-of-stakeholder', 'Farmer/agricultural producers;Forester;Forest owner;Forest operator;Forest product processor;Farming association;Farming cooperative;Forest industry association;Trade association;NGO/Advisory Group;Data association/organisation/coalition;Data provider;Platform provider;Service/Information provider;Digital technology provider; Research institutes and research networks; National and European networks');
@@ -48,8 +36,3 @@ INSERT INTO public.page_filters VALUES ('advantages', 'Increased efficiency and
 INSERT INTO public.page_filters VALUES ('network-connectivity', 'Wired internet;Wireless internet;Cellular networks;Satellite internet;IoT networks;Fiber Optic Networks;Private Networks;Low-Power Wide-Area Network');
 
 
-ALTER TABLE ONLY public.page_filters
-    ADD CONSTRAINT "PK_066730e267edae13a61a79b8bd3" PRIMARY KEY (name);
-
-
-
diff --git a/api/data/mock-data.sql b/api/data/mock-data.sql
@@ -19,56 +19,13 @@ INSERT INTO survey.answers (survey_id, hierarchy_level_2, categorical_answer) VA
 INSERT INTO survey.answers (survey_id, hierarchy_level_2, categorical_answer) VALUES (2, 'Sector (Agri/Forestry/Both)', 'Agri');
 INSERT INTO survey.answers (survey_id, hierarchy_level_2, categorical_answer) VALUES (3, 'Sector (Agri/Forestry/Both)', 'Forestry');
 
--- MIGRATION
--- DROP SCHEMA IF EXISTS "survey" CASCADE;
-
 -- NEW DATA MODEL
-DROP TABLE IF EXISTS question_indicator_map;
-CREATE TABLE question_indicator_map (
-  indicator VARCHAR(256)
-  , question VARCHAR(256)
-  , PRIMARY KEY (indicator, question)
-);
-
+DELETE FROM question_indicator_map;
 INSERT INTO question_indicator_map (indicator, question) VALUES 
 ('sector', 'Sector (Agri/Forestry/Both)'),
 ('eu-member-state', 'Location (Country/Region)');
 
-CREATE OR REPLACE FUNCTION check_question_indicator_map()
-RETURNS TRIGGER AS $$
-BEGIN
-    -- Check if the combination exists in the map table
-    IF NOT EXISTS (
-        SELECT 1 
-        FROM question_indicator_map 
-        WHERE indicator = NEW.question_indicator AND question = NEW.question
-    ) THEN
-        RAISE EXCEPTION 'Invalid combination of indicator and question: (''%'' , ''%'')', NEW.question_indicator, NEW.question;
-    END IF;
-    RETURN NEW;
-END;
-$$ LANGUAGE plpgsql;
-
-DO $$
-BEGIN
-    -- Check if the trigger already exists
-    IF NOT EXISTS (
-        SELECT 1 
-        FROM pg_trigger 
-        WHERE tgname = 'validate_question_indicator'
-    ) THEN
-        -- Create the trigger if it doesn't exist
-        CREATE TRIGGER validate_question_indicator
-        BEFORE INSERT OR UPDATE ON survey_answers
-        FOR EACH ROW
-        EXECUTE FUNCTION check_question_indicator_map();
-    END IF;
-END $$;
-
-
--- DATA
 DELETE FROM survey_answers;
-
 INSERT INTO survey_answers (survey_id, question_indicator, question, answer, country_code) VALUES (1, 'eu-member-state', 'Location (Country/Region)', 'Spain', 'ESP');
 INSERT INTO survey_answers (survey_id, question_indicator, question, answer, country_code) VALUES (2, 'eu-member-state', 'Location (Country/Region)', 'Spain', 'ESP');
 INSERT INTO survey_answers (survey_id, question_indicator, question, answer, country_code) VALUES (3, 'eu-member-state', 'Location (Country/Region)', 'Spain', 'ESP');

diff --git a/api/data/schema.sql b/api/data/schema.sql
diff --git a/api/data/sections/sections-csv-parser.ts b/api/data/sections/sections-csv-parser.ts
@@ -31,7 +31,7 @@ const parseSectionsFromFile = async (
         if (indicator === '') continue;
 
         const section = row[3].trim() as string;
-        const sectionSlug = section.toLocaleLowerCase().replaceAll(' ', '-');
+        const sectionSlug = section.toLocaleLowerCase().replace(/ /g, '-');
 
         if (!(sectionSlug in sections)) {
           sections[sectionSlug] = {
@@ -48,7 +48,11 @@ const parseSectionsFromFile = async (
         for (let fieldIdx = 4; fieldIdx < 9; fieldIdx++) {
           const field = row[fieldIdx];
           if (field === 'TRUE') {
-            availableVisualizations.push(ROW_VISUALIZATION_MAP[fieldIdx]);
+            availableVisualizations.push(
+              ROW_VISUALIZATION_MAP[
+                fieldIdx as keyof typeof ROW_VISUALIZATION_MAP
+              ],
+            );
           }
         }
         const defaultVisualization = availableVisualizations[0];

diff --git a/api/package.json b/api/package.json
@@ -13,6 +13,7 @@
     "start:debug": "nest start --debug --watch",
     "lint": "eslint \"src/**/*.ts\" --fix",
     "test": "jest --config ./test/jest-config.json -i",
+    "typeorm": "pnpm exec ts-node -r tsconfig-paths/register node_modules/typeorm/cli.js",
     "transform:sections": "ts-node data/transform-sections.ts"
   },
   "dependencies": {
@@ -38,6 +39,7 @@
     "reflect-metadata": "^0.2.0",
     "rxjs": "^7.8.1",
     "shared": "workspace:*",
+    "typeorm": "catalog:",
     "uuid": "10.0.0",
     "zod": "catalog:"
   },
@@ -46,6 +48,7 @@
     "@nestjs/schematics": "^10.0.0",
     "@nestjs/testing": "^10.0.0",
     "@types/bcrypt": "5.0.2",
+    "@types/config": "3.3.5",
     "@types/express": "^4.17.17",
     "@types/jest": "^29.5.2",
     "@types/jsonapi-serializer": "3.6.8",

diff --git a/api/src/infrastructure/data-source-manager.ts b/api/src/infrastructure/data-source-manager.ts
@@ -14,7 +14,6 @@ export class DataSourceManager {
   ) {}
 
   public async loadInitialData(): Promise<void> {
-    const schemaSql = this.getInitialSchemaSqlCode();
     const sqlScripts: string[] = await Promise.all([
       this.getInitialSectionsSqlCode(),
       this.getFiltersSqlCode(),
@@ -24,7 +23,6 @@ export class DataSourceManager {
     const queryRunner = this.dataSource.createQueryRunner();
     try {
       await queryRunner.startTransaction();
-      await queryRunner.query(schemaSql);
       const sqlCodePromises = [];
       for (let sqlCodeIdx = 0; sqlCodeIdx < sqlScripts.length; sqlCodeIdx++) {
         sqlCodePromises.push(queryRunner.query(sqlScripts[sqlCodeIdx]));
@@ -39,16 +37,6 @@ export class DataSourceManager {
     }
   }
 
-  private getInitialSchemaSqlCode(): string {
-    const schemafilePath = `data/schema.sql`;
-    this.logger.log(
-      `Loading initial schema from "${schemafilePath}"`,
-      this.constructor.name,
-    );
-
-    return fs.readFileSync(schemafilePath, 'utf-8');
-  }
-
   private async getInitialSectionsSqlCode(): Promise<string> {
     const sectionsfilePath = `data/sections/chart-types.csv`;
     this.logger.log(

diff --git a/api/src/infrastructure/data-sources/postgres-data-source.ts b/api/src/infrastructure/data-sources/postgres-data-source.ts
@@ -0,0 +1,4 @@
+import { typeOrmConfig } from '@api/typeorm.config';
+import { DataSource } from 'typeorm';
+
+export const PostgresDataSource = new DataSource(typeOrmConfig);
diff --git a/api/src/infrastructure/sql-adapter.ts b/api/src/infrastructure/sql-adapter.ts
@@ -13,7 +13,7 @@ export class SQLAdapter {
     for (let idx = 0; idx < pageFilters.length; idx++) {
       const pageFilter = pageFilters[idx];
       // eslint-disable-next-line prettier/prettier
-      sqlCode += `INSERT INTO page_filters (name, values) VALUES ('${pageFilter.name.replaceAll("'", "''")}', '${pageFilter.values.join(';').replaceAll("'", "''")}') ON CONFLICT (name) DO UPDATE SET values = excluded.values;\n`;
+      sqlCode += `INSERT INTO page_filters (name, values) VALUES ('${pageFilter.name.replace(/'/g, "''")}', '${pageFilter.values.join(';').replace(/'/g, "''")}') ON CONFLICT (name) DO UPDATE SET values = excluded.values;\n`;
     }
     return sqlCode;
   }
@@ -22,7 +22,7 @@ export class SQLAdapter {
     let sqlCode: string = '';
     const keys = Object.keys(sections);
     for (const key of keys) {
-      const section = sections[key];
+      const section = sections[key as any];
       const { order, slug, name, description, widgets } = section;
 
       let widgetsSql = '';

diff --git a/api/src/migrations/1729163940111-initial-tables-and-indexes.ts b/api/src/migrations/1729163940111-initial-tables-and-indexes.ts
@@ -0,0 +1,93 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class InitialTablesAndIndexes1729163940111
+  implements MigrationInterface
+{
+  name = 'InitialSchema1729163940111';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TYPE "public"."base_widgets_default_visualization_enum" AS ENUM('single_value', 'map', 'horizontal_bar_chart', 'pie_chart', 'area_graph', 'filter', 'navigation')`,
+    );
+    await queryRunner.query(
+      `CREATE TYPE "public"."custom_widgets_default_visualization_enum" AS ENUM('single_value', 'map', 'horizontal_bar_chart', 'pie_chart', 'area_graph', 'filter', 'navigation')`,
+    );
+    await queryRunner.query(
+      `CREATE TABLE "sections" ("order" integer NOT NULL, "slug" character varying NOT NULL, "name" character varying NOT NULL, "description" character varying, "created_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "UQ_6585d4a8196db804deedc1b343a" UNIQUE ("slug"), CONSTRAINT "PK_a605d9121e6cc3e665f24314ad8" PRIMARY KEY ("order"))`,
+    );
+    await queryRunner.query(
+      `CREATE TABLE "base_widgets" ("id" SERIAL NOT NULL, "question" character varying, "indicator" character varying, "section_order" integer NOT NULL, "visualisations" text NOT NULL, "default_visualization" "public"."base_widgets_default_visualization_enum" NOT NULL, "updated_at" TIMESTAMP NOT NULL DEFAULT now(), "created_at" TIMESTAMP NOT NULL DEFAULT now(), "section_id" integer, CONSTRAINT "PK_93f5f3f72fcf9fb685e887c8064" PRIMARY KEY ("id"))`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "idx_section_widgets_section_order" ON "base_widgets" ("section_order") `,
+    );
+    await queryRunner.query(
+      `CREATE TABLE "custom_widgets" ("id" SERIAL NOT NULL, "name" character varying NOT NULL, "default_visualization" "public"."custom_widgets_default_visualization_enum" NOT NULL, "filters" jsonb NOT NULL, "updated_at" TIMESTAMP NOT NULL DEFAULT now(), "created_at" TIMESTAMP NOT NULL DEFAULT now(), "user_id" uuid, "widget_id" integer, CONSTRAINT "PK_a5d2205ca142399bde7d3f2b1cb" PRIMARY KEY ("id"))`,
+    );
+    await queryRunner.query(
+      `CREATE TABLE "users" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "email" character varying NOT NULL, "password" character varying NOT NULL, "created_at" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "UQ_97672ac88f789774dd47f7c8be3" UNIQUE ("email"), CONSTRAINT "PK_a3ffb1c0c8416b9fc6f907b7433" PRIMARY KEY ("id"))`,
+    );
+    await queryRunner.query(
+      `CREATE TABLE "api_events" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "timestamp" TIMESTAMP NOT NULL DEFAULT now(), "type" character varying NOT NULL, "associatedId" uuid, "data" jsonb, CONSTRAINT "PK_e3826ce4ae3f91c05bd37bb1502" PRIMARY KEY ("id"))`,
+    );
+    await queryRunner.query(
+      `CREATE TABLE "page_filters" ("name" character varying NOT NULL, "values" text NOT NULL, CONSTRAINT "PK_066730e267edae13a61a79b8bd3" PRIMARY KEY ("name"))`,
+    );
+    await queryRunner.query(
+      `CREATE TABLE "survey_answers" ("survey_id" character varying NOT NULL, "question_indicator" character varying NOT NULL, "question" character varying NOT NULL, "answer" character varying NOT NULL, "country_code" character varying NOT NULL, CONSTRAINT "PK_9474d624fb654ce2f69c74bb262" PRIMARY KEY ("survey_id", "question_indicator"))`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "idx_survey_answers_country_code" ON "survey_answers" ("country_code") `,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "idx_survey_answers_question_answer" ON "survey_answers" ("question_indicator", "answer") `,
+    );
+    await queryRunner.query(
+      `CREATE TABLE "question_indicator_map" ("indicator" character varying NOT NULL, "question" character varying NOT NULL, CONSTRAINT "PK_75e60c595a8a13cfe41e4710633" PRIMARY KEY ("indicator", "question"))`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "base_widgets" ADD CONSTRAINT "FK_ec4ecfd98ca886865a3f579994f" FOREIGN KEY ("section_id") REFERENCES "sections"("order") ON DELETE NO ACTION ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "custom_widgets" ADD CONSTRAINT "FK_dac539060d0b71bf4aeeb52b205" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "custom_widgets" ADD CONSTRAINT "FK_ba78a479ff764c0aa29e8f97ace" FOREIGN KEY ("widget_id") REFERENCES "base_widgets"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `DROP TYPE "public"."base_widgets_default_visualization_enum"`,
+    );
+    await queryRunner.query(
+      `DROP TYPE "public"."custom_widgets_default_visualization_enum"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "custom_widgets" DROP CONSTRAINT "FK_ba78a479ff764c0aa29e8f97ace"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "custom_widgets" DROP CONSTRAINT "FK_dac539060d0b71bf4aeeb52b205"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "base_widgets" DROP CONSTRAINT "FK_ec4ecfd98ca886865a3f579994f"`,
+    );
+    await queryRunner.query(`DROP TABLE "question_indicator_map"`);
+    await queryRunner.query(
+      `DROP INDEX "public"."idx_survey_answers_question_answer"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."idx_survey_answers_country_code"`,
+    );
+    await queryRunner.query(`DROP TABLE "survey_answers"`);
+    await queryRunner.query(`DROP TABLE "page_filters"`);
+    await queryRunner.query(`DROP TABLE "api_events"`);
+    await queryRunner.query(`DROP TABLE "users"`);
+    await queryRunner.query(`DROP TABLE "custom_widgets"`);
+    await queryRunner.query(
+      `DROP INDEX "public"."idx_section_widgets_section_order"`,
+    );
+    await queryRunner.query(`DROP TABLE "base_widgets"`);
+    await queryRunner.query(`DROP TABLE "sections"`);
+  }
+}