Merge branch 'develop' into cleanupAfterRedisMigration

dm3-org · Sep 12, 2024 · bbb9844 · bbb9844
2 parents dd9ed94 + 5797454
commit bbb9844
Show file tree

Hide file tree

Showing 66 changed files with 3,443 additions and 840 deletions.
diff --git a/packages/backend/package.json b/packages/backend/package.json
@@ -31,7 +31,7 @@
     "start": "yarn prisma-init && node  ./dist/index.js",
     "start-inspect": "node --inspect=0.0.0.0:9229  ./dist/index.js",
     "test": "yarn run before:tests && DATABASE_URL='postgresql://prisma:prisma@localhost:5433/tests?schema=public' yarn jest --coverage --runInBand --transformIgnorePatterns 'node_modules/(?!(dm3-lib-\\w*)/)'",
-    "build": "yarn prisma generate && yarn tsc && cp ./config.yml ./dist/config.yml | true",
+    "build": "yarn prisma generate && yarn tsc | true",
     "build:schema": "sh ./schemas.sh",
     "createDeliveryServiceProfile": "node --no-warnings ./cli.js",
     "before:tests": "docker compose -f docker-compose.test.yml up -d && DATABASE_URL='postgresql://prisma:prisma@localhost:5433/tests?schema=public' yarn prisma-init"

diff --git a/packages/backend/src/config/getDeliveryServiceProperties.test.ts b/packages/backend/src/config/getDeliveryServiceProperties.test.ts
diff --git a/packages/backend/src/config/getDeliveryServiceProperties.ts b/packages/backend/src/config/getDeliveryServiceProperties.ts
diff --git a/packages/delivery-service/package.json b/packages/delivery-service/package.json
@@ -27,6 +27,7 @@
     "start-inspect": "node --inspect=0.0.0.0:9229  ./dist/index.js",
     "test": "yarn run before:tests &&  jest --coverage --runInBand --transformIgnorePatterns 'node_modules/(?!(dm3-lib-\\w*)/)' ",
     "build": "yarn tsc ",
+    "dev": "nodemon --exec 'ts-node ./src/index.ts' --watch src --ext ts",
     "createDeliveryServiceProfile": "node --no-warnings ./cli.js",
     "before:tests": "docker compose -f docker-compose.test.yml up -d",
     "after:tests": "docker compose -f docker-compose.test.yml down"
@@ -45,6 +46,7 @@
     "babel-preset-env": "^1.7.0",
     "jest": "^29.2.2",
     "jest-mock-extended": "2.0.4",
+    "nodemon": "^3.1.4",
     "prettier": "^2.6.2",
     "superagent": "^8.0.3",
     "supertest": "^6.3.1",

diff --git a/packages/delivery-service/src/config/getDeliveryServiceProperties.test.ts b/packages/delivery-service/src/config/getDeliveryServiceProperties.test.ts
@@ -21,12 +21,16 @@ describe('ReadDeliveryServiceProperties', () => {
             messageTTL: 12345,
             sizeLimit: 456,
             notificationChannel: [],
+            metricsCollectionIntervalInSeconds: 600,
+            metricsRetentionDurationInSeconds: 172800,
         });
 
         expect(config).toStrictEqual({
             messageTTL: 12345,
             sizeLimit: 456,
             notificationChannel: [],
+            metricsCollectionIntervalInSeconds: 600,
+            metricsRetentionDurationInSeconds: 172800,
         });
     });
 
@@ -37,6 +41,8 @@ describe('ReadDeliveryServiceProperties', () => {
                 messageTTL: 12345,
                 sizeLimit: 456,
                 notificationChannel: [],
+                metricsCollectionIntervalInSeconds: 900,
+                metricsRetentionDurationInSeconds: 259200,
             }),
             { encoding: 'utf-8' },
         );
@@ -46,8 +52,11 @@ describe('ReadDeliveryServiceProperties', () => {
             messageTTL: 12345,
             sizeLimit: 456,
             notificationChannel: [],
+            metricsCollectionIntervalInSeconds: 900,
+            metricsRetentionDurationInSeconds: 259200,
         });
     });
+
     it('Adds default properties if config.yml is not fully specified', () => {
         writeFileSync(
             path,
@@ -68,6 +77,7 @@ describe('ReadDeliveryServiceProperties', () => {
                         },
                     },
                 ],
+                metricsCollectionIntervalInSeconds: 1200,
             }),
             { encoding: 'utf-8' },
         );
@@ -91,8 +101,11 @@ describe('ReadDeliveryServiceProperties', () => {
                     },
                 },
             ],
+            metricsCollectionIntervalInSeconds: 1200,
+            metricsRetentionDurationInSeconds: 60 * 60 * 24 * 10,
         });
     });
+
     it('Adds email channel from config.yml file & rest from default properties', () => {
         writeFileSync(
             path,
@@ -135,6 +148,8 @@ describe('ReadDeliveryServiceProperties', () => {
                     },
                 },
             ],
+            metricsCollectionIntervalInSeconds: 60 * 60 * 24,
+            metricsRetentionDurationInSeconds: 60 * 60 * 24 * 10,
         });
     });
 
@@ -170,6 +185,28 @@ describe('ReadDeliveryServiceProperties', () => {
                     },
                 },
             ],
+            metricsCollectionIntervalInSeconds: 60 * 60 * 24,
+            metricsRetentionDurationInSeconds: 60 * 60 * 24 * 10,
+        });
+    });
+
+    it('Uses default values for metrics properties if not specified', () => {
+        writeFileSync(
+            path,
+            stringify({
+                messageTTL: 54321,
+                sizeLimit: 789,
+            }),
+            { encoding: 'utf-8' },
+        );
+        const config = getDeliveryServiceProperties(path);
+
+        expect(config).toStrictEqual({
+            messageTTL: 54321,
+            sizeLimit: 789,
+            notificationChannel: [],
+            metricsCollectionIntervalInSeconds: 60 * 60 * 24,
+            metricsRetentionDurationInSeconds: 60 * 60 * 24 * 10,
         });
     });
 });
diff --git a/packages/delivery-service/src/config/getDeliveryServiceProperties.ts b/packages/delivery-service/src/config/getDeliveryServiceProperties.ts
@@ -10,6 +10,8 @@ const DEFAULT_DELIVERY_SERVICE_PROPERTIES: DeliveryServiceProperties = {
     //100Kb
     sizeLimit: 100000,
     notificationChannel: [],
+    metricsCollectionIntervalInSeconds: 60 * 60 * 24, // 1 day
+    metricsRetentionDurationInSeconds: 60 * 60 * 24 * 10, // 10 days
 };
 
 export function getDeliveryServiceProperties(

diff --git a/packages/delivery-service/src/index.ts b/packages/delivery-service/src/index.ts
@@ -28,6 +28,7 @@ import { Profile } from './profile/profile';
 import RpcProxy from './rpc/rpc-proxy';
 import { WebSocketManager } from './ws/WebSocketManager';
 import { socketAuth } from './socketAuth';
+import Metrics from './metrics';
 
 const app = express();
 app.use(express.json({ limit: '50mb' }));
@@ -139,10 +140,7 @@ app.use(bodyParser.json());
         return res.status(200).send('Hello DM3');
     });
 
-    //Auth
-    //socketAuth
-    //restAuth
-
+    app.use('/metrics', Metrics(db, deliveryServiceProperties));
     app.use('/auth', Authenticate(db, serverSecret, web3Provider));
     app.use('/profile', Profile(db, web3Provider, serverSecret));
     app.use('/delivery', Delivery(web3Provider, db, serverSecret));

diff --git a/packages/delivery-service/src/message/MessageProcessor.ts b/packages/delivery-service/src/message/MessageProcessor.ts
@@ -174,6 +174,7 @@ export class MessageProcessor {
                     deliveryInformation,
                     this.db.getUsersNotificationChannels,
                 );
+                await this.db.countNotification(this.deliveryServiceProperties);
             } catch (err) {
                 console.log(
                     'Unable to send Notification. There might be an error in the config.yml. Message has been received regardless',