Feature/google users groups

.github/workflows/feature-branch.yml

@@ -70,6 +70,21 @@ jobs:
         run: |
           echo -e "$CONTENTS" > src/app-config.d/github-app-credentials.yaml
 
+      # Couldn't store JSON creds while retaining proper formatting so going to do this in two steps
+      - name: Write Google Admin API creds
+        id: google-admin-creds
+        shell: bash
+        env:
+          CONTENTS: ${{ secrets.CATALOG_GOOGLE_JWT_KEYS }}
+        run: |
+          echo -e "$CONTENTS" > src/app-config.d/credentials/google-jwt.keys.yaml
+
+      - name: Convert Google Admin API creds to YAML
+        id: google-admin-creds-yaml
+        shell: bash
+        run: |
+          yq -p yaml -o json src/app-config.d/credentials/google-jwt.keys.yaml > src/app-config.d/credentials/google-jwt.keys.json
+
       - name: Compile Typescript
         id: compile
         shell: bash

.github/workflows/main.yml

@@ -68,7 +68,22 @@ jobs:
         env:
           CONTENTS: ${{ secrets.X_GITHUB_APP_CREDS }}
         run: |
-          echo -e "$CONTENTS" > src/app-config.d/github-app-credentials.yaml
+          echo -e "$CONTENTS" > src/app-config.d/credentials/github-app-credentials.yaml
+
+      # Couldn't store JSON creds while retaining proper formatting so going to do this in two steps
+      - name: Write Google Admin API creds
+        id: google-admin-creds
+        shell: bash
+        env:
+          CONTENTS: ${{ secrets.CATALOG_GOOGLE_JWT_KEYS }}
+        run: |
+          echo -e "$CONTENTS" > src/app-config.d/credentials/google-jwt.keys.yaml
+
+      - name: Convert Google Admin API creds to YAML
+        id: google-admin-creds-yaml
+        shell: bash
+        run: |
+          yq -p yaml -o json src/app-config.d/credentials/google-jwt.keys.yaml > src/app-config.d/credentials/google-jwt.keys.json
 
       - name: Compile Typescript
         id: compile

.gitignore

@@ -76,3 +76,6 @@ codepipeline-config-*.yaml
 
 # AWS SAM
 .aws-sam/
+
+# Backstage credentials
+src/app-config.d/credentials

src/app-config.production.yaml

@@ -52,10 +52,19 @@ integrations:
   github:
     - host: github.com
       apps:
-        - $include: /app/app-config.d/github-app-credentials.yaml
+        - $include: /app/app-config.d/credentials/github-app-credentials.yaml
 
 catalog:
   providers:
+    google:
+      auth:
+        adminAccountEmail: tom@serverlessops.io
+        clientCredentials:
+          $include: /app/app-config.d/credentials/google-jwt.keys.yaml
+      schedule:
+        initialDelay: { seconds: 5 }
+        frequency: { minutes: 1 }
+        timeout: { seconds: 30 }
     serverlessops-catalog:
       baseUrl: ${SERVERLESSOPS_CATALOG_API_URL}
       namespace: default
@@ -78,14 +87,6 @@ catalog:
   # See https://backstage.io/docs/features/software-catalog/#adding-components-to-the-catalog for more details
   # on how to get entities into the catalog.
   locations:
-    - type: file
-      target: /app/app-config.d/catalogs/users.yaml
-      rules:
-        - allow: [User]
-    - type: file
-      target: /app/app-config.d/catalogs/groups.yaml
-      rules:
-        - allow: [Groups]
     - type: file
       target: /app/app-config.d/scaffolders/domain-add.yaml
       rules:

src/packages/backend/package.json

@@ -41,6 +41,7 @@
     "@backstage/plugin-search-backend-module-techdocs": "^0.2.2",
     "@backstage/plugin-search-backend-node": "^1.3.2",
     "@backstage/plugin-techdocs-backend": "^1.10.13",
+    "@internal/backstage-plugin-catalog-backend-module-google": "workspace:^",
     "@internal/backstage-plugin-catalog-backend-module-serverlessops-catalog": "workspace:^",
     "@internal/backstage-plugin-scaffolder-backend-module-serverlessops": "workspace:^",
     "app": "link:../app",

src/packages/backend/src/index.ts

@@ -56,4 +56,5 @@ backend.add(import('@backstage/plugin-scaffolder-backend-module-github'));
 
 backend.add(import('@internal/backstage-plugin-catalog-backend-module-serverlessops-catalog'));
 backend.add(import('@internal/backstage-plugin-scaffolder-backend-module-serverlessops'));
+backend.add(import('@internal/backstage-plugin-catalog-backend-module-google'));
 backend.start();

src/plugins/plugin-catalog-backend-module-google/.eslintrc.js

@@ -0,0 +1 @@
+module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);

src/plugins/plugin-catalog-backend-module-google/README.md

@@ -0,0 +1,5 @@
+# @internal/backstage-plugin-plugin-catalog-backend-module-google
+
+The google backend module for the plugin-catalog plugin.
+
+_This plugin was created through the Backstage CLI_

src/plugins/plugin-catalog-backend-module-google/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@internal/backstage-plugin-catalog-backend-module-google",
+  "description": "The google backend module for the plugin-catalog plugin.",
+  "version": "0.1.0",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "license": "Apache-2.0",
+  "private": true,
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "types": "dist/index.d.ts"
+  },
+  "backstage": {
+    "role": "backend-plugin-module"
+  },
+  "scripts": {
+    "start": "backstage-cli package start",
+    "build": "backstage-cli package build",
+    "lint": "backstage-cli package lint",
+    "test": "backstage-cli package test",
+    "clean": "backstage-cli package clean",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack"
+  },
+  "dependencies": {
+    "@backstage/backend-plugin-api": "^1.0.0",
+    "@backstage/catalog-model": "^1.7.2",
+    "@backstage/config": "^1.3.1",
+    "@backstage/plugin-catalog-node": "^1.15.0",
+    "google-auth-library": "^9.15.0",
+    "googleapis": "^144.0.0"
+  },
+  "devDependencies": {
+    "@backstage/backend-test-utils": "^1.0.0",
+    "@backstage/cli": "^0.27.1"
+  },
+  "files": [
+    "dist"
+  ]
+}

src/plugins/plugin-catalog-backend-module-google/src/index.ts

@@ -0,0 +1,8 @@
+/***/
+/**
+ * The google backend module for the plugin-catalog plugin.
+ *
+ * @packageDocumentation
+ */
+
+export { pluginCatalogModuleGoogle as default } from './module';

src/plugins/plugin-catalog-backend-module-google/src/module.ts

@@ -0,0 +1,64 @@
+import {
+    coreServices,
+    createBackendModule,
+    SchedulerServiceTaskRunner,
+} from '@backstage/backend-plugin-api';
+import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'
+import {
+    GoogleGroupProvider,
+    GoogleUserProvider,
+    ProviderConfig
+} from './provider'
+
+export const pluginCatalogModuleGoogle = createBackendModule({
+    pluginId: 'catalog',
+    moduleId: 'google',
+    register(reg) {
+        reg.registerInit({
+            deps: {
+                catalog: catalogProcessingExtensionPoint,
+                logger: coreServices.logger,
+                rootConfig: coreServices.rootConfig,
+                scheduler: coreServices.scheduler
+            },
+            async init({ catalog, rootConfig, logger, scheduler }) {
+                logger.info('Initializing Google User & Group Providers');
+
+                const config = rootConfig.get('catalog.providers.google') as ProviderConfig | undefined
+
+                // Create a scheduled task runner
+                const schedule = config?.schedule ?
+                    config?.schedule
+                    : {
+                        initialDelay: { seconds: 0 },
+                        frequency: { hours: 1 },
+                        timeout: { seconds: 60 }
+                    }
+
+                const taskRunner: SchedulerServiceTaskRunner =
+                    scheduler.createScheduledTaskRunner(schedule);
+
+                // Initialize the group provider
+                catalog.addEntityProvider(
+                    GoogleGroupProvider.fromConfig(
+                        rootConfig,
+                        {
+                            logger,
+                            taskRunner
+                        }
+                    )
+                )
+                // Initialize the user provider
+                catalog.addEntityProvider(
+                    GoogleUserProvider.fromConfig(
+                        rootConfig,
+                        {
+                            logger,
+                            taskRunner
+                        }
+                    )
+                )
+            },
+        });
+    },
+});

src/plugins/plugin-catalog-backend-module-google/src/provider/GoogleBaseProvider.test.ts

@@ -0,0 +1,78 @@
+import { mockServices } from '@backstage/backend-test-utils'
+import { GoogleBaseProvider } from './GoogleBaseProvider'
+import * as creds from '../../../../app-config.d/credentials/google-jwt.keys.json' /* eslint @backstage/no-relative-monorepo-imports: off */
+
+describe('GoogleUserProvider', () => {
+    let mockConfig: any
+    let provider: GoogleBaseProvider
+    let mockConnection: any
+    let mockTaskRunner: any
+
+    beforeEach(() => {
+        mockConnection = {
+            applyMutation: jest.fn()
+        }
+        mockTaskRunner = {
+            run: jest.fn()
+        }
+
+        mockConfig = {
+            auth: {
+                adminAccountEmail: "tom@serverlessops.io",
+                clientCredentials: creds
+            },
+            schedule: {
+                initialDelay: { seconds: 5 },
+                frequency: { minutes: 1 },
+                timeout: { seconds: 30 },
+            }
+        }
+
+        provider = new GoogleBaseProvider(
+            mockConfig,
+            {
+                logger: mockServices.rootLogger(),
+                taskRunner: mockTaskRunner
+            }
+        )
+    })
+
+    describe('getProviderName()', () => {
+        describe('should succeed', () => {
+            test('provider name', () => {
+                expect(provider.getProviderName()).toBe('google-base')
+            })
+        })
+    })
+
+    describe('getCredentials()', () => {
+        describe('should succeed', () => {
+            test('when getting JWT', () => {
+                const credentials = provider.getCredentials(
+                    mockConfig.auth.adminAccountEmail,
+                    mockConfig.auth.clientCredentials,
+                    []
+                )
+                expect(credentials).toBeDefined()
+            })
+        })
+    })
+
+    describe('connect()', () => {
+        describe('should succeed when', () => {
+            test('connects and runs successfully', async () => {
+                await provider.connect(mockConnection)
+                expect(mockTaskRunner.run).toHaveBeenCalled()
+            })
+        })
+    })
+
+    describe('run()', () => {
+        describe('should succeed when', () => {
+            test('runs successfully', async () => {
+                await provider.connect(mockConnection)
+                expect( async () => await provider.run()).not.toThrow()
+            }, 10 * 1000)
+        })
+    })
+})