feat: collect deprecated images metric (CR-26713) (#138)

Author: vasil-cf

.dockerignore

@@ -8,3 +8,4 @@ lib/state.json
 .eslintrc.json
 test
 .eslintignore
+dist

.eslintignore

@@ -3,6 +3,7 @@
 # Ignore built files except build/index.js
 build/*
 !build/index.js
+dist/*
 coverage/*
 
 # Directory for instrumented libs generated by jscoverage/JSCover

.eslintrc.json

@@ -14,7 +14,7 @@
       "impliedStrict": true
     }
   },
-  
+
   "plugins": [
     "chai-friendly",
     "import",
@@ -85,9 +85,45 @@
 
     "node/no-unsupported-features": "error",
     "node/process-exit-as-throw": "error",
-    "node/shebang": "warn",
+    "node/shebang": "off",
     "node/no-deprecated-api": "warn",
     "no-useless-constructor": "warn",
     "no-return-await": "off"
-  }
+  },
+
+  "overrides": [
+    {
+      "files": ["**/*.ts"],
+      "parser": "@typescript-eslint/parser",
+      "plugins": ["@typescript-eslint/eslint-plugin"],
+      "extends": ["plugin:@typescript-eslint/recommended"],
+      "parserOptions": {
+        "project": "tsconfig.json",
+        "sourceType": "module"
+      },
+      "env": {
+        "node": true
+      },
+      "rules": {
+        "no-restricted-syntax": "warn",
+        "import/prefer-default-export": "off",
+        "node/no-unsupported-features": "off",
+        "node/no-unsupported-features/es-builtins": "error",
+        "node/no-unsupported-features/es-syntax": ["error", {
+          "ignores": ["modules"]
+        }],
+        "node/no-unsupported-features/node-builtins": "error",
+        "no-empty-function":  "warn",
+        "lines-between-class-members": "off",
+        "@typescript-eslint/no-explicit-any": "warn",
+        "@typescript-eslint/no-var-requires": "warn",
+        "@typescript-eslint/no-unused-vars": ["error", {
+          "argsIgnorePattern": "^_",
+          "varsIgnorePattern": "^_",
+          "caughtErrorsIgnorePattern": "^_"
+        }
+        ]
+      }
+    }
+  ]
 }

.gitignore

@@ -10,3 +10,4 @@ state.json
 .idea
 .vscode
 lib/manual-test.js
+dist/

Dockerfile

@@ -3,22 +3,30 @@ RUN adduser --disabled-password -home /home/cfu -shell /bin/bash cfu
 WORKDIR /root/cf-runtime
 COPY package.json yarn.lock ./
 
-FROM base AS dependencies
+FROM base AS build-dependencies
 RUN apt-get update \
     && apt upgrade -y \
     && apt-get install -y \
     g++ \
     git \
     make \
     python3
+
+FROM build-dependencies AS build
+RUN yarn install --frozen-lockfile
+COPY . .
+RUN yarn build
+
+FROM build-dependencies AS prod-dependencies
 RUN yarn install --frozen-lockfile --production
 
 FROM base AS production
-COPY --from=dependencies /root/cf-runtime/node_modules ./node_modules
+COPY --from=prod-dependencies /root/cf-runtime/node_modules ./node_modules
+COPY --from=build /root/cf-runtime/dist ./dist
 COPY . .
 
 #purpose of security
 RUN npm -g uninstall npm
 
 USER cfu
-CMD ["node", "lib/index.js"]
+CMD ["node", "dist/index.js"]

lib/@types/index.d.ts

@@ -0,0 +1 @@
+declare module 'cf-logs';

lib/ContainerLogger.js

@@ -4,9 +4,13 @@ const promiseRetry = require('promise-retry');
 const logger = require('cf-logs').Logger('codefresh:containerLogger');
 const CFError = require('cf-errors');
 const { Transform } = require('stream');
+
 const _ = require('lodash');
 const { LoggerStrategy } = require('./enums');
 
+// eslint-disable-next-line import/no-unresolved
+const { DeprecatedImagesInterceptorStream } = require('./metric/deprecated-images/deprecated-images-interceptor.stream');
+
 const CONTAINER_START_RETRY_TIMEOUT_SECONDS = 1;
 const CONTAINER_START_RETRY_LIMIT = 10;
 const BUFFER_SIZE = 2 * 1024 * 1024; // 2 MiB
@@ -131,6 +135,7 @@ class ContainerLogger extends EventEmitter {
         // { end = false } on the stepLoggerWritableStream because there is only one instance of it for all the steps.
         this.handledStreams++;
         let stdoutStream = stdout
+            .pipe(new DeprecatedImagesInterceptorStream())
             .pipe(this._logSizeLimitStream())
             .pipe(this.stepLogger.createMaskingStream());
 
@@ -148,6 +153,7 @@ class ContainerLogger extends EventEmitter {
 
         this.handledStreams++;
         let stderrStream = stderr
+            .pipe(new DeprecatedImagesInterceptorStream())
             .pipe(this._logSizeLimitStream())
             .pipe(this._errorTransformerStream())
             .pipe(this.stepLogger.createMaskingStream());
@@ -180,27 +186,38 @@ class ContainerLogger extends EventEmitter {
 
     _handleTtyStream(stream, isError) {
         this.handledStreams++;
-        stream.on('end', this._handleFinished.bind(this));
+        const deprecatedImagesInterceptor = new DeprecatedImagesInterceptorStream(true);
+        stream.on('end', () => {
+            this._handleFinished();
+            deprecatedImagesInterceptor.end();
+        });
         stream.on('data', (chunk) => {
+            deprecatedImagesInterceptor.write(chunk);
             this._logMessage(Buffer.from(chunk).toString('utf-8'), isError);
         });
         logger.info(`Listening on stream 'data' event for container: ${this.containerId}`);
     }
 
     _handleNonTtyStream(stream, isError) {
         this.handledStreams++;
+        const deprecatedImagesInterceptor = new DeprecatedImagesInterceptorStream(true);
         stream.on('readable', () => {
             let header = stream.read(8);
             while (header !== null) {
+                deprecatedImagesInterceptor.write(header);
                 const payload = stream.read(header.readUInt32BE(4));
                 if (payload === null) {
                     break;
                 }
+                deprecatedImagesInterceptor.write(payload);
                 this._logMessage(Buffer.from(payload).toString('utf8'), isError);
                 header = stream.read(8);
             }
         });
-        stream.on('end', this._handleFinished.bind(this));
+        stream.on('end', () => {
+            this._handleFinished();
+            deprecatedImagesInterceptor.end();
+        });
         logger.info(`Listening on stream 'readable' event for container: ${this.containerId}`);
     }
 

lib/forever.ps1

@@ -1,4 +1,4 @@
 while ($true) {
     Start-Sleep -s 1
-    & node lib/index.js
+    & node dist/index.js
 }

lib/http-server/index.ts

@@ -0,0 +1,102 @@
+import fastify from 'fastify';
+import cfLogs from 'cf-logs';
+
+import { saveServerAddress } from '../helpers';
+
+// eslint-disable-next-line import/no-unresolved
+import deprecatedImagesCollector from '../metric/deprecated-images/deprecated-images.collector';
+
+const logger = cfLogs.Logger('codefresh:containerLogger');
+
+export class HttpServer {
+
+    private readonly host;
+    private readonly port;
+    private readonly server;
+
+    constructor(private taskLogger: any) {
+        try {
+            this.host = process.env.HOST || '0.0.0.0';
+            this.port = +(process.env.PORT || 8080);
+            this.server = fastify();
+
+            this.initSecrets();
+            this.initDeprecatedImages();
+        } catch (error) {
+            logger.error(`could not initialize server for engine's requests due to error: ${error}`);
+            throw error;
+        }
+    }
+
+    private initSecrets() {
+        const secretsOptions = {
+            schema: {
+                body: {
+                    type: 'object',
+                    required: ['key', 'value'],
+                    properties: {
+                        key: { type: 'string' },
+                        value: { type: 'string' },
+                    },
+                },
+            },
+        };
+
+        this.server.post('/secrets', secretsOptions, async (request, reply) => {
+            try {
+                const { body }: { body: any } = request;
+                const { secret } = body;
+                logger.info(`got request to add new mask: ${secret.key}`);
+                this.taskLogger.addNewMask(secret);
+                reply.code(201);
+                return 'secret added';
+            } catch (err) {
+                logger.info(`could not create new mask for due to error: ${err}`);
+                reply.code(500);
+                throw err;
+            }
+        });
+    }
+
+    private initDeprecatedImages() {
+        this.server.get('/deprecated-images', async () => {
+            logger.info(`got request to retrieve deprecated images`);
+            return deprecatedImagesCollector.consumeAll();
+        });
+
+        this.server.post<{ Body: { count: number } }>('/deprecated-images/ack', async (request, reply) => {
+            const { count } = request.body;
+
+            if (typeof count !== 'number' || count < 1) {
+                return reply.status(400).send({ error: 'You must provide a valid count (positive integer).' });
+            }
+
+            deprecatedImagesCollector.destroyConsumed(count);
+
+            return reply.status(200).send({ count });
+        });
+    }
+
+    async start() {
+        let address: string;
+        try {
+            address = await this.server.listen({
+                host: this.host,
+                port: this.port,
+            });
+
+            logger.info(`listening for engine's requests on ${address}`);
+        } catch (error) {
+            logger.error(`could not start server for engine updates due to error: ${error}`);
+            throw error;
+        }
+
+        try {
+            await saveServerAddress(address);
+        } catch (error) {
+            logger.error(`could not save server address due to error: ${error}`);
+            throw error;
+        }
+    }
+
+}

lib/isReady.js

@@ -23,7 +23,7 @@ function isContainerLoggerReady(state) {
 
 (() => {
     const containerId = process.argv[2];
-    const state = JSON.parse(readFileSync('./lib/state.json').toString('utf-8'));
+    const state = JSON.parse(readFileSync('./dist/state.json').toString('utf-8'));
     let isReady = false;
     if (containerId) {
         isReady = isContainerReady(state, containerId);

lib/isReady.ps1

@@ -6,7 +6,7 @@ $CONTAINER_ID=$args[0]
 
 if ( $CONTAINER_ID ) {
     echo "checking if container:$CONTAINER_ID exists"
-    if (select-string -Pattern $CONTAINER_ID -Path ./lib/state.json) {
+    if (select-string -Pattern $CONTAINER_ID -Path ./dist/state.json) {
         echo "container $CONTAINER_ID is ready"
         Exit 0
     } else {
@@ -15,7 +15,7 @@ if ( $CONTAINER_ID ) {
     }
 } else {
     echo "checking if container logger is ready"
-    if (select-string -Pattern "ready" -Path ./lib/state.json) {
+    if (select-string -Pattern "ready" -Path ./dist/state.json) {
         echo "ready"
         Exit 0
     } else {

lib/isReady.sh

@@ -7,10 +7,10 @@ CONTAINER_ID=$1
 
 if [ -n "$CONTAINER_ID" ]; then
    echo "checking if container: $CONTAINER_ID exists"
-   grep -q $CONTAINER_ID ./lib/state.json
+   grep -q $CONTAINER_ID ./dist/state.json
 else
    echo "checking if container logger is ready"
-   grep -q "ready" ./lib/state.json
+   grep -q "ready" ./dist/state.json
 fi