log all services in http-tests

.github/workflows/plugin-ci.yml

@@ -49,9 +49,9 @@ jobs:
       matrix:
         include:
           - PKGPATH: ./functions/
-            DESTPATH: /usr/src/plugin/node_modules/@data2evidence/d2e-functions
+            DESTPATH: /usr/src/data/plugins/node_modules/@data2evidence/d2e-functions
           - PKGPATH: ./fhir_functions/
-            DESTPATH: /usr/src/plugin/node_modules/@data2evidence/fhir
+            DESTPATH: /usr/src/data/plugins/node_modules/@data2evidence/fhir
 
     steps:
       - uses: actions/checkout@v4

docker-compose.yml

@@ -47,6 +47,7 @@ x-envs:
     PG__PORT: ${PG_PORT:-5432}
     PG__IDLE_TIMEOUT_IN_MS: 1000
     PG__DIALECT: postgresql
+    PG__SSL: ${PG__SSL:-false}
   fhir: &x-fhir
     FHIR__CLIENT_ID: db6b2304-f236-45ec-b10c-a852681e7129
     FHIR__CLIENT_SECRET: ${FHIR__CLIENT_SECRET}
@@ -118,7 +119,7 @@ services:
       data:
         priority: 10
     volumes:
-      - trex:/usr/src/plugin/node_modules
+      - trex:/usr/src/data
     healthcheck:
       interval: 60s
       retries: 30
@@ -165,6 +166,7 @@ services:
             "PG__DB_NAME": "alp",
             "PG__DEBUG": "${PG_DEBUG:-1}",
             "PG__HOST": "${PG_HOST:-${PROJECT_NAME:-d2e}-minerva-postgres-1}",
+            "PG__SSL": "${PG__SSL:-false}",
             "PG__MAX_POOL": 10,
             "PG__MIN_POOL": 0,
             "PG__PORT": ${PG_PORT:-5432},
@@ -442,7 +444,8 @@ services:
          "port": "${PG_PORT:-5432}",
          "query_timeout": 60000,
          "statement_timeout": 60000,
-         "user": "${PG_SUPER_USER:-postgres}"
+         "user": "${PG_SUPER_USER:-postgres}",
+         "ssl": false
         }
       POSTGRES_MANAGE_CONFIG: |-
         {
@@ -680,6 +683,7 @@ services:
       PG__PASSWORD: ${PG_SUPER_PASSWORD}
       PG__PORT: ${PG_PORT:-5432}
       PG__USER: ${PG__LOGTO_MANAGE_USER:-logto_postgres}
+      PG__SSL: ${PG__SSL:-false}
       # LOGTO__CUSTOM_JWT: '{"script": "/**\n* This function is called during the access token generation process to get custom claims for the JWT token.\n* Limit custom claims to under 50KB.\n*\n* @param {Object} payload - The input payload of the function.\n* @param {AccessTokenPayload} payload.token -The JWT token.\n* @param {Context} payload.context - Logto internal data that can be used to pass additional information\n* @param {EnvironmentVariables} [payload.environmentVariables] - The environment variables.\n*\n* @returns The custom claims.\n*/\n\n// @ts-ignore\nconst getCustomJwtClaims = async ({ token, context, environmentVariables, extra }) => {\n  return { ...extra };\n}", "tokenSample": {"aud": "http://localhost:3000/api/test", "gty": "authorization_code", "jti": "f1d3d2d1-1f2d-3d4e-5d6f-7d8a9d0e1d2", "kind": "AccessToken", "scope": "read write", "grantId": "grant_123", "clientId": "my_app", "accountId": "uid_123"}, "contextSample": {"user": {"id": "123", "name": "Foo Bar", "roles": [], "avatar": "https://example.com/avatar.png", "profile": {}, "username": "foo", "customData": {}, "identities": {}, "hasPassword": false, "primaryEmail": "[email protected]", "primaryPhone": "+1234567890", "applicationId": "my-app", "organizations": [], "ssoIdentities": [], "organizationRoles": [], "mfaVerificationFactors": []}}}'
       LOGTO__CLIENT_APPS: '[{"name":"alp-svc","description":"alp-svc","type":"MachineToMachine", "id": "${LOGTO__ALP_SVC__CLIENT_ID}", "secret": "${LOGTO__ALP_SVC__CLIENT_SECRET}"},{"name":"alp-data","description":"alp-data","type":"MachineToMachine", "id": "${LOGTO__ALP_DATA__CLIENT_ID}", "secret": "${LOGTO__ALP_DATA__CLIENT_SECRET}"},{"name":"alp-app","description":"alp-app","type":"Traditional", "id": "${LOGTO__ALP_APP__CLIENT_ID}", "secret": "${LOGTO__ALP_APP__CLIENT_SECRET}", "oidcClientMetadata":{"redirectUris":["https://${CADDY__ALP__PUBLIC_FQDN:-localhost}${PORT:+:${PORT:-443}}/portal/login-callback","https://localhost:4000/portal/login-callback","https://localhost:8081"],"postLogoutRedirectUris":["https://${CADDY__ALP__PUBLIC_FQDN:-localhost}${PORT:+:${PORT:-443}}/portal","https://localhost:4000/portal","https://localhost:8081"]},"customClientMetadata":{"corsAllowedOrigins":[],"refreshTokenTtlInDays":14,"alwaysIssueRefreshToken":true,"rotateRefreshToken":true}}]'
       LOGTO__RESOURCE: '{"name":"alp-default","indicator":"https://alp-default","accessTokenTtl":3600}'
@@ -745,7 +749,8 @@ services:
 
   alp-logto-seed: 
     container_name: &c37 alp-logto-seed
-    image: logto-schema-logto:latest
+    image: ghcr.io/data2evidence/d2e-logto:${DOCKER_TAG_NAME:-develop}
+    platform: linux/amd64
     entrypoint: 
       - sh
       - -c

functions/alp-db-credentials/import.json

@@ -38,6 +38,7 @@
   "./src/common/data-source/migrations/1700709870313-create-db-credential": "./src/common/data-source/migrations/1700709870313-create-db-credential.ts",
   "./src/common/data-source/migrations/1706832297709-add-vocab-schema": "./src/common/data-source/migrations/1706832297709-add-vocab-schema.ts",
   "./src/common/data-source/migrations/1739258149345-add-authentication-mode": "./src/common/data-source/migrations/1739258149345-add-authentication-mode.ts",
+  "./src/common/data-source/migrations/1739349524222-create-db-publication": "./src/common/data-source/migrations/1739349524222-create-db-publication.ts",
   "./src/common/data-source/migration-data-source": "./src/common/data-source/migration-data-source.ts",
   "./src/common/data-source/data-source": "./src/common/data-source/data-source.ts",
   "./src/common/hook": "./src/common/hook/index.ts",
@@ -48,13 +49,15 @@
   "./src/db/repository/db-credential.repository": "./src/db/repository/db-credential.repository.ts",
   "./src/db/repository": "./src/db/repository/index.ts",
   "./src/db/repository/db-extra.repository": "./src/db/repository/db-extra.repository.ts",
+  "./src/db/repository/db-publication.repository": "./src/db/repository/db-publication.repository.ts",
   "./src/db/repository/db.repository": "./src/db/repository/db.repository.ts",
   "./src/db/db.router.spec": "./src/db/db.router.spec.ts",
   "./src/db/db.router": "./src/db/db.router.ts",
   "./src/db/entity/db.entity": "./src/db/entity/db.entity.ts",
   "./src/db/entity/db-credential.entity": "./src/db/entity/db-credential.entity.ts",
   "./src/db/entity/db-extra.entity": "./src/db/entity/db-extra.entity.ts",
   "./src/db/entity/db-vocab-schema.entity": "./src/db/entity/db-vocab-schema.entity.ts",
+  "./src/db/entity/db-publication.entity": "./src/db/entity/db-publication.entity.ts",
   "./src/db/entity": "./src/db/entity/index.ts",
   "./src/db/db.service": "./src/db/db.service.ts",
   "./src/vocab/vocab.router": "./src/vocab/vocab.router.ts",

functions/alp-db-credentials/src/common/data-source/data-source.ts

@@ -7,19 +7,20 @@ import { Database } from '../../db/entity/db.entity'
 import { DbCredential } from '../../db/entity/db-credential.entity'
 import { DbExtra } from '../../db/entity/db-extra.entity'
 import { DbVocabSchema } from '../../db/entity/db-vocab-schema.entity'
+import { DbPublication } from '../../db/entity/db-publication.entity'
 
 const logger = createLogger('DataSource')
 
 export const getSsl = (): boolean | TlsOptions => {
+  let ssl: any = JSON.parse(env.PG_SSL.toLowerCase())
+
   if (env.PG_CA_ROOT_CERT) {
-    return {
+    ssl = {
       rejectUnauthorized: true,
       ca: env.PG_CA_ROOT_CERT
     }
-  } else if (env.NODE_ENV === 'production') {
-    logger.warn('PG_CA_ROOT_CERT is undefined')
   }
-  return false
+  return ssl
 }
 
 export const getLogLevels = (): LogLevel[] => {
@@ -40,7 +41,7 @@ export const dataSourceOptions: DataSourceOptions = {
   ssl: getSsl(),
   poolSize: env.PG_MAX_POOL,
   logging: getLogLevels(),
-  entities: [Audit, Database, DbCredential, DbExtra, DbVocabSchema]
+  entities: [Audit, Database, DbCredential, DbExtra, DbVocabSchema, DbPublication]
 }
 
 const dataSource = new DataSource(dataSourceOptions)

functions/alp-db-credentials/src/common/data-source/migration-data-source.ts

@@ -6,13 +6,15 @@ import { Database } from '../../db/entity/db.entity'
 import { DbCredential } from '../../db/entity/db-credential.entity'
 import { DbExtra } from '../../db/entity/db-extra.entity'
 import { DbVocabSchema } from '../../db/entity/db-vocab-schema.entity'
+import { DbPublication } from "../../db/entity/db-publication.entity";
 import { CreateDbCredential1700709870313 } from './migrations/1700709870313-create-db-credential'
 import { CreateDbExtra1701667241462 } from './migrations/1701667241462-create-db-extra'
 import { AddUserScopeToDbCredential1703139369074 } from './migrations/1703139369074-add-user-scope-to-db-credential'
 import { UpdateDbExtra1704846290461 } from './migrations/1704846290461-update-db-extra'
 import { AddCodeToDb1706063894993 } from './migrations/1706063894993-add-code-to-db'
 import { AddVocabSchema1706832297709 } from './migrations/1706832297709-add-vocab-schema'
 import { AddAuthenticationMode1739258149345 } from './migrations/1739258149345-add-authentication-mode'
+import { CreateDbPublication1739349524222 } from './migrations/1739349524222-create-db-publication'
 
 const migrationDataSourceOptions: DataSourceOptions = {
   type: 'postgres',
@@ -25,15 +27,16 @@ const migrationDataSourceOptions: DataSourceOptions = {
   ssl: getSsl(),
   poolSize: env.PG_MAX_POOL,
   logging: getLogLevels(),
-  entities: [Audit, Database, DbCredential, DbExtra, DbVocabSchema],
+  entities: [Audit, Database, DbCredential, DbExtra, DbVocabSchema, DbPublication],
   migrations: [
     CreateDbCredential1700709870313,
     CreateDbExtra1701667241462,
     AddUserScopeToDbCredential1703139369074,
     UpdateDbExtra1704846290461,
     AddCodeToDb1706063894993,
     AddVocabSchema1706832297709,
-    AddAuthenticationMode1739258149345
+    AddAuthenticationMode1739258149345,
+    CreateDbPublication1739349524222
   ]
 }
 

functions/alp-db-credentials/src/common/data-source/migrations/1739349524222-create-db-publication.ts

@@ -0,0 +1,21 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class CreateDbPublication1739349524222 implements MigrationInterface {
+  name = 'CreateDbPublication1739349524222'
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TABLE "db_credentials_mgr"."db_publication" ("id" SERIAL NOT NULL, "publication" character varying NOT NULL, "slot" character varying NOT NULL, "db_id" uuid NOT NULL, "created_by" character varying NOT NULL, "created_date" TIMESTAMP NOT NULL DEFAULT now(), "modified_by" character varying NOT NULL, "modified_date" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "UQ_2c00d40454a74a07284f53c3e41" UNIQUE ("publication", "db_id"), CONSTRAINT "PK_692b25da07a6fce963c98a1c445" PRIMARY KEY ("id"))`
+    )
+    await queryRunner.query(
+      `ALTER TABLE "db_credentials_mgr"."db_publication" ADD CONSTRAINT "FK_90722678a5543b197d244e87fab" FOREIGN KEY ("db_id") REFERENCES "db_credentials_mgr"."db"("id") ON DELETE CASCADE ON UPDATE NO ACTION`
+    )
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "db_credentials_mgr"."db_publication" DROP CONSTRAINT "FK_90722678a5543b197d244e87fab"`
+    )
+    await queryRunner.query(`DROP TABLE "db_credentials_mgr"."db_publication"`)
+  }
+}

functions/alp-db-credentials/src/db/db.service.ts

@@ -1,10 +1,17 @@
 import { Service } from 'typedi'
 import { v4 as uuidv4 } from 'uuid'
 import { createLogger } from '../logger'
-import { DbRepository, DbExtraRepository, DbCredentialRepository, DbVocabSchemaRepository } from './repository'
+import {
+  DbRepository,
+  DbExtraRepository,
+  DbCredentialRepository,
+  DbPublicationRepository,
+  DbVocabSchemaRepository
+} from './repository'
 import { DbDialect, IDbCredentialDto, IDbCredentialUpdateDto, IDbDto, IDbExtraDto, IDbUpdateDto } from '../types'
 import { getReqContext } from '../common/hook'
 import { SERVICE_SCOPE } from '../common/const'
+import { IDbPublicationDto } from '../types'
 
 @Service()
 export class DbService {
@@ -14,7 +21,8 @@ export class DbService {
     private readonly dbRepo: DbRepository,
     private readonly dbExtraRepo: DbExtraRepository,
     private readonly credentialRepo: DbCredentialRepository,
-    private readonly vocabSchemaRepo: DbVocabSchemaRepository
+    private readonly vocabSchemaRepo: DbVocabSchemaRepository,
+    private readonly publicationRepo: DbPublicationRepository
   ) {}
 
   async list() {
@@ -32,13 +40,15 @@ export class DbService {
       )
       .leftJoinAndSelect('db.vocabSchemas', 'dbVocabSchema')
       .leftJoinAndSelect('db.extra', 'dbExtra')
+      .leftJoinAndSelect('db.publications', 'dbPublication')
     const result = await query.select(this.getDbColumns(isClientCredentials)).getMany()
     return result.map(r => {
-      const { extra, vocabSchemas, ...entity } = r
+      const { extra, vocabSchemas, publications, ...entity } = r
       return {
         ...entity,
         extra,
-        vocabSchemas: vocabSchemas.map(vocabSchema => vocabSchema.name)
+        vocabSchemas: vocabSchemas.map(vocabSchema => vocabSchema.name),
+        publications
       }
     })
   }
@@ -63,11 +73,11 @@ export class DbService {
     if (grantType !== 'client_credentials') {
       db.credentials.forEach(c => {
         c.password = maskedValue
-        if(c !== undefined) delete c.salt
+        if (c !== undefined) delete c.salt
       })
     }
 
-    const extra = db.extra.find(ext => (ext !== undefined && ext.serviceScope === serviceScope))?.value
+    const extra = db.extra.find(ext => ext !== undefined && ext.serviceScope === serviceScope)?.value
 
     return {
       ...db,
@@ -91,37 +101,40 @@ export class DbService {
 
   async create(dbDto: IDbDto) {
     const dbId = uuidv4()
-    const { credentials, extra, vocabSchemas, ...newDbDto } = dbDto
+    const { credentials, extra, vocabSchemas, publications, ...newDbDto } = dbDto
 
     const credEntities = this.mapCredentialsToEntity(credentials, dbId)
     const entity = this.dbRepo.create({
       ...newDbDto,
       id: dbId,
       extra: this.mapExtraToEntity(extra, dbId),
       credentials: credEntities,
-      vocabSchemas: this.mapVocabSchemasToEntity(vocabSchemas, dbId)
+      vocabSchemas: this.mapVocabSchemasToEntity(vocabSchemas, dbId),
+      publications: this.mapPublicationsToEntity(publications, dbId)
     })
     await this.dbRepo.save(this.addOwner(entity))
     this.logger.debug(`Created db: ${JSON.stringify(entity)}`)
     return entity.id
   }
 
   async update(dbDto: IDbUpdateDto) {
-    const { id, name, port, host, vocabSchemas, extra } = dbDto
+    const { id, name, port, host, vocabSchemas, extra, publications } = dbDto
 
-    const existingDb = await this.dbRepo
+    const existingDb = (await this.dbRepo
       .createQueryBuilder('db')
       .leftJoinAndSelect('db.vocabSchemas', 'vocabSchema')
       .leftJoinAndSelect('db.extra', 'dbExtra')
+      .leftJoinAndSelect('db.publications', 'dbPublication')
       .where('db.id = :id', { id })
-      .getOne() as { vocabSchemas, extra, name, host, port }
+      .getOne()) as { vocabSchemas; extra; name; host; port; publications }
 
     const {
       vocabSchemas: existingVocabSchemaEntities,
       extra: existingExtraEntities,
       name: existingName,
       host: existingHost,
-      port: existingPort
+      port: existingPort,
+      publications: existingPublicationEntities
     } = existingDb
 
     if (name !== existingName || host !== existingHost || port !== existingPort) {
@@ -152,16 +165,28 @@ export class DbService {
       await this.dbExtraRepo.delete({ dbId: id })
     }
 
+    if (publications) {
+      const pubEntities = this.mapPublicationsToEntity(publications, dbDto.id)
+      await this.publicationRepo.upsert(pubEntities, ['publication', 'dbId'])
+      existingPublicationEntities
+        .filter(o => pubEntities.find(n => o.publication === n.publication) === undefined)
+        .forEach(async existingPublication => {
+          await this.publicationRepo.delete({ publication: existingPublication.publication, dbId: id })
+        })
+    } else {
+      await this.publicationRepo.delete({ dbId: id })
+    }
+
     this.logger.debug(`Updated db: ${JSON.stringify(dbDto)}`)
     return id
   }
   async updateCredentials(dbDto: IDbCredentialUpdateDto) {
     const { id, credentials } = dbDto
-    const existingDb = await this.dbRepo
+    const existingDb = (await this.dbRepo
       .createQueryBuilder('db')
       .leftJoinAndSelect('db.credentials', 'dbCredential')
       .where('db.id = :id', { id })
-      .getOne() as { credentials }
+      .getOne()) as { credentials }
 
     const { credentials: existingCredEntities } = existingDb
     if (credentials) {
@@ -200,7 +225,9 @@ export class DbService {
       'dbCredential.serviceScope',
       'dbVocabSchema.name',
       'dbExtra.value',
-      'dbExtra.serviceScope'
+      'dbExtra.serviceScope',
+      'dbPublication.publication',
+      'dbPublication.slot'
     ]
     if (hasSecret) {
       return [...baseColumns, 'dbCredential.password', 'dbCredential.salt']
@@ -247,4 +274,14 @@ export class DbService {
       return this.vocabSchemaRepo.create(entity)
     })
   }
+
+  private mapPublicationsToEntity(publications: IDbPublicationDto[], dbId: string) {
+    return publications?.map(pub => {
+      const pubEntity = this.publicationRepo.create({
+        ...pub,
+        dbId: dbId
+      })
+      return this.addOwner(pubEntity)
+    })
+  }
 }

functions/alp-db-credentials/src/db/dto/db.dto.ts

@@ -19,7 +19,8 @@ import type {
   IDbCredentialUpdateDto,
   IDbDto,
   IDbExtraDto,
-  IDbUpdateDto
+  IDbUpdateDto,
+  IDbPublicationDto
 } from '../../types'
 import { SERVICE_SCOPES, DB_DIALECTS, SERVICE_SCOPE, USER_SCOPES, AuthenticationMode } from '../../common/const'
 import { IsExistingDb, IsValidSchema, IsValidSchemaUpdate } from '../../common/validator'
@@ -72,6 +73,12 @@ export class DbDto implements IDbDto {
   @ArrayMinSize(1)
   @IsValidSchema({ each: true })
   vocabSchemas: string[]
+
+  @IsOptional()
+  @IsArray()
+  @ValidateNested({ each: true })
+  @Type(() => DbPublicationDto)
+  publications: DbPublicationDto[]
 }
 
 export class DbUpdateDto implements IDbUpdateDto {
@@ -101,6 +108,12 @@ export class DbUpdateDto implements IDbUpdateDto {
   @ValidateNested()
   @Type(() => DbExtraDto)
   extra: DbExtraDto
+
+  @IsOptional()
+  @IsArray()
+  @ValidateNested({ each: true })
+  @Type(() => DbPublicationDto)
+  publications: DbPublicationDto[]
 }
 
 export class DbCredentialUpdateDto implements IDbCredentialUpdateDto {
@@ -142,3 +155,13 @@ export class DbCredentialDto implements IDbCredentialDto {
   @IsIn(SERVICE_SCOPES)
   serviceScope: string
 }
+
+export class DbPublicationDto implements IDbPublicationDto {
+  @IsNotEmpty()
+  @IsString()
+  publication: string
+
+  @IsNotEmpty()
+  @IsString()
+  slot: string
+}