WIP Victoria Metrics (#689)

.env.example

@@ -11,8 +11,6 @@ SERVER_DOMAIN=127.0.0.1
 
 patreon_client_ID=<INSERT IF NEEDED>
 patreon_client_secret=<INSERT IF NEEDED>
-patreon_creator_access_token=<INSERT IF NEEDED>
-patreon_creator_refresh_token=<INSERT IF NEEDED>
 patreon_redirectURL=<INSERT IF NEEDED>
 
 MAILGUN_API_KEY=<INSERT IF NEEDED>
@@ -35,3 +33,5 @@ S3_PUBLIC_FILE_LINK_PREFIX=http://localhost:9000/proavalon/
 S3_BUCKET_NAME=proavalon
 S3_ENDPOINT=http://127.0.0.1:9000
 S3_REGION=auto
+
+VM_IMPORT_PROMETHEUS_URL=http://localhost:8428/api/v1/import/prometheus

README.md

@@ -55,6 +55,14 @@ $ yarn
 3. Create a Bucket named `proavalon` in MinIO.
 4. Set its Access Policy to Public.
 
+### Setting up Grafana
+
+1. Access Grafana via localhost:5000
+2. Connect `victoria-metrics`
+    a. From the main menu: Connections -> Data sources -> Add new data source. 
+    b. Add a Prometheus time series database. 
+    c. Under Connection, paste `http://victoria-metrics:8428` then Save & test.
+
 ### Stopping
 
 1. Stop the server with `Ctrl+C`.

docker-compose.yml

@@ -35,3 +35,28 @@ services:
       MINIO_ROOT_USER: admin
       MINIO_ROOT_PASSWORD: password
     command: server /data --console-address ":9001"
+
+  victoria-metrics:
+    image: victoriametrics/victoria-metrics:latest
+    ports:
+      - 8428:8428
+    volumes:
+      - ./docker-compose-data/victoria-metrics:/data
+    restart: always
+    command:
+      - -storageDataPath=/data
+      - -retentionPeriod=100y
+
+  grafana:
+    image: grafana/grafana:latest
+    ports:
+      - 5000:3000
+    volumes:
+      - ./docker-compose-data/grafana:/var/lib/grafana
+    restart: always
+    environment:
+      GF_SECURITY_ADMIN_USER: admin
+      GF_SECURITY_ADMIN_PASSWORD: pass
+    user: "$UID:$GID"
+    depends_on:
+      - victoria-metrics

package.json

@@ -51,6 +51,7 @@
     "passport-local-mongoose": "^6.1.0",
     "passport.socketio": "^3.7.0",
     "patreon": "^0.4.1",
+    "prom-client": "^15.1.2",
     "react": "^17.0.2",
     "react-autosuggest": "^10.1.0",
     "react-dom": "^17.0.2",

src/app.ts

@@ -35,6 +35,7 @@ import staticifyFactory from 'staticify';
 // Create a MongoDB session store
 import MongoDBStoreFactory from 'connect-mongodb-session';
 import { SESSIONS_COLLECTION_NAME } from './constants';
+import { promAgent } from './clients/victoriaMetrics/promAgent';
 
 const assetsPath = path.join(__dirname, '../assets');
 
@@ -212,3 +213,10 @@ io.use(
 );
 
 socketServer(io);
+
+// Periodically push metrics every 15 seconds to VictoriaMetrics
+if (process.env.ENV === 'local' || process.env.ENV === 'prod') {
+  setInterval(async () => {
+    await promAgent.pushMetrics();
+  }, 15000);
+}

src/clients/victoriaMetrics/promAgent.ts

@@ -0,0 +1,83 @@
+import promClient from 'prom-client';
+import { sendToDiscordAdmins } from '../discord';
+
+const MAX_PUSH_METRICS_ERRORS = 5;
+const FORGET_ERROR_TIME_THRESHOLD = 60 * 60 * 1000; // 1 hour
+
+export class PromAgent {
+  private metricNames = new Set<string>();
+  private pushMetricsErrorsTimestamps: number[] = [];
+  private readonly dupeMetricErrorHandler: (metricName: string) => void;
+
+  constructor(dupeMetricErrorHandler: (metricName: string) => void) {
+    this.dupeMetricErrorHandler = dupeMetricErrorHandler;
+  }
+
+  public registerMetric(metricName: string) {
+    this.metricNames.has(metricName)
+      ? this.dupeMetricErrorHandler(metricName)
+      : this.metricNames.add(metricName);
+  }
+
+  public async pushMetrics() {
+    let metrics: string;
+
+    try {
+      metrics = await promClient.register.metrics(); // Will call any collect() functions for gauges
+    } catch (e) {
+      // Exit program if non-initialised labels are used in collect() function
+      if (e.message.includes('label')) {
+        sendToDiscordAdmins(e.message);
+        console.error(e);
+        process.exit(1);
+      }
+
+      throw e;
+    }
+
+    if (!process.env.VM_IMPORT_PROMETHEUS_URL) {
+      console.error(`Missing environment variable: VM_IMPORT_PROMETHEUS_URL`);
+      process.exit(1);
+    }
+
+    const response = await fetch(process.env.VM_IMPORT_PROMETHEUS_URL, {
+      method: 'POST',
+      body: metrics,
+      headers: {
+        'Content-Type': 'text/plain',
+      },
+    });
+
+    if (!response.ok) {
+      const errMsg = `Failed to push metrics: status=${response.status} text=${response.statusText}`;
+      console.error(errMsg);
+
+      // Alert Discord while errors are less than MAX_PUSH_METRICS_ERRORS
+      const now = Date.now();
+      this.pushMetricsErrorsTimestamps.push(now);
+
+      while (this.pushMetricsErrorsTimestamps.length > 0) {
+        const timeDiff = now - this.pushMetricsErrorsTimestamps[0];
+
+        if (timeDiff > FORGET_ERROR_TIME_THRESHOLD) {
+          this.pushMetricsErrorsTimestamps.unshift();
+        } else {
+          break;
+        }
+      }
+
+      if (this.pushMetricsErrorsTimestamps.length <= MAX_PUSH_METRICS_ERRORS) {
+        sendToDiscordAdmins(errMsg);
+      }
+    }
+  }
+}
+
+const dupeMetricErrorHandler = (metricName: string) => {
+  const errMsg = `Error metric name already exists: ${metricName}`;
+  sendToDiscordAdmins(errMsg);
+  console.error(errMsg);
+  process.exit(1);
+};
+
+export const promAgent = new PromAgent(dupeMetricErrorHandler);

src/clients/victoriaMetrics/promMetricGauge.ts

@@ -0,0 +1,19 @@
+import promClient, { Gauge } from 'prom-client';
+import { promAgent } from './promAgent';
+
+interface GaugeConfig {
+  name: string;
+  help: string;
+  labelNames?: string[];
+  collect?: () => void; // Refer to prom-client docs on how this should be used.
+}
+
+export class PromMetricGauge {
+  private gauge: Gauge;
+
+  constructor(gaugeConfig: GaugeConfig) {
+    promAgent.registerMetric(gaugeConfig.name);
+
+    this.gauge = new promClient.Gauge(gaugeConfig);
+  }
+}

src/clients/victoriaMetrics/tests/promAgent.test.ts

@@ -0,0 +1,25 @@
+import { PromAgent } from '../promAgent';
+
+describe('PromAgent', () => {
+  let promAgent: PromAgent;
+  let dupeMetricErrorHandler: jest.Mock;
+
+  beforeEach(() => {
+    dupeMetricErrorHandler = jest.fn();
+    promAgent = new PromAgent(dupeMetricErrorHandler);
+  });
+
+  it(`Adds unique metric names.`, () => {
+    promAgent.registerMetric('metric_name_1');
+    promAgent.registerMetric('metric_name_2');
+
+    expect(dupeMetricErrorHandler).not.toHaveBeenCalled();
+  });
+
+  it('Throws an error when adding a duplicate metric name.', () => {
+    promAgent.registerMetric('metric_name_1');
+    promAgent.registerMetric('metric_name_1');
+
+    expect(dupeMetricErrorHandler).toHaveBeenCalledTimes(1);
+  });
+});

src/sockets/sockets.ts

@@ -44,8 +44,7 @@ import { Role } from '../gameplay/roles/types';
 import { Phase } from '../gameplay/phases/types';
 import { Card } from '../gameplay/cards/types';
 import { TOCommandsImported } from './commands/tournamentOrganisers';
-import { PatreonAgent } from '../clients/patreon/patreonAgent';
-import { PatreonController } from '../clients/patreon/patreonController';
+import { PromMetricGauge } from '../clients/victoriaMetrics/promMetricGauge';
 
 const chatSpamFilter = new ChatSpamFilter();
 const createRoomFilter = new CreateRoomFilter();
@@ -60,6 +59,14 @@ if (process.env.NODE_ENV !== 'test') {
   }, 1000);
 }
 
+const onlinePlayersMetric = new PromMetricGauge({
+  name: `online_players_total`,
+  help: `Number of online players.`,
+  collect() {
+    this.set(allSockets.length);
+  },
+});
+
 const quote = new Quote();
 
 const dateResetRequired = 1543480412695;

yarn.lock

@@ -1904,6 +1904,11 @@
     "@nodelib/fs.scandir" "2.1.5"
     fastq "^1.6.0"
 
+"@opentelemetry/api@^1.4.0":
+  version "1.9.0"
+  resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe"
+  integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==
+
 "@pkgjs/parseargs@^0.11.0":
   version "0.11.0"
   resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33"
@@ -3804,6 +3809,11 @@ [email protected]:
   dependencies:
     underscore "~1.4.4"
 
+[email protected]:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.2.tgz#49f896d6e858a4a499df85c38fb399b9aff840f8"
+  integrity sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==
+
 [email protected]:
   version "0.0.5"
   resolved "https://registry.yarnpkg.com/blob/-/blob-0.0.5.tgz#d680eeef25f8cd91ad533f5b01eed48e64caf683"
@@ -7587,6 +7597,14 @@ progress@^2.0.0:
   resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
   integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
 
+prom-client@^15.1.2:
+  version "15.1.2"
+  resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-15.1.2.tgz#78d79f12c35d395ca97edf7111c18210cf07f815"
+  integrity sha512-on3h1iXb04QFLLThrmVYg1SChBQ9N1c+nKAjebBjokBqipddH3uxmOUcEkTnzmJ8Jh/5TSUnUqS40i2QB2dJHQ==
+  dependencies:
+    "@opentelemetry/api" "^1.4.0"
+    tdigest "^0.1.1"
+
 prompts@^2.0.1:
   version "2.4.2"
   resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.2.tgz#7b57e73b3a48029ad10ebd44f74b01722a4cb069"
@@ -8648,6 +8666,13 @@ tapable@^2.1.1, tapable@^2.2.0:
   resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0"
   integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==
 
+tdigest@^0.1.1:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.2.tgz#96c64bac4ff10746b910b0e23b515794e12faced"
+  integrity sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==
+  dependencies:
+    bintrees "1.0.2"
+
 terser-webpack-plugin@^5.3.10:
   version "5.3.10"
   resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-5.3.10.tgz#904f4c9193c6fd2a03f693a2150c62a92f40d199"