Merge pull request #1696 from Azure/jcotillo/generate_onboarded_report

Ggenerate onboarded report

Author: jorgecotillo

azure-pipelines-generate-onboarded-report.yml

@@ -0,0 +1,69 @@
+pr: none
+trigger: none
+
+schedules:
+- cron: "0 6 * * 5"
+  branches:
+    include:
+    - master
+  displayName: Weekly report of onboarded RPs
+
+jobs:
+- job: Generate
+  timeoutInMinutes: 180
+
+  pool:
+    vmImage: 'ubuntu-latest'
+
+  steps:
+  - checkout: self
+    clean: true
+
+  - task: DownloadSecureFile@1
+    name: schemasDeployKey
+    inputs:
+      secureFile: schemas_rsa
+    displayName: "Download GitHub Deploy Key"
+  
+  - task: NodeTool@0
+    inputs:
+      versionSpec: '14.x'
+
+  - script: |
+      set -Eeuxo pipefail
+      mkdir ~/.ssh && mv $(schemasDeployKey.secureFilePath) ~/.ssh/schemas_rsa
+      chmod 700 ~/.ssh
+      chmod 600 ~/.ssh/schemas_rsa
+      git config user.name "Autogenerator Pipeline"
+      git config user.email "[email protected]"
+      git config core.sshCommand "ssh -i ~/.ssh/schemas_rsa -F /dev/null"
+      git remote set-url origin [email protected]:Azure/azure-resource-manager-schemas.git
+      ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
+      git fetch
+    displayName: Configure git
+
+  - script: |
+      set -Eeuxo pipefail
+      git checkout $(Build.SourceBranchName)
+    displayName: Checkout repo
+
+  - script: |
+      set -Eeuxo pipefail
+      cd generator
+      npm install
+    displayName: 'Install packages'
+
+  - script: |
+      set -Eeuxo pipefail
+      cd generator
+      npm run generate-onboarded-report
+    displayName: 'Generate report'
+
+  - script: |
+      set -Eeuxo pipefail
+      git add --all onboarded-report
+      if ! git diff-index --quiet HEAD --; then
+        git commit -m "Generated report"
+        git push origin $(Build.SourceBranchName)
+      fi
+    displayName: Commit report

generator/cmd/generateonboardedreport.ts

@@ -0,0 +1,56 @@
+import * as constants from '../constants';
+import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs';
+import chalk from 'chalk';
+import { findAutogenEntries } from '../autogenlist';
+import { executeSynchronous, lowerCaseEquals, writeJsonFile, safeMkdir } from '../utils';
+import { getApiVersionsByNamespace } from '../generate';
+import { keys, partition } from 'lodash';
+import path from 'path';
+
+executeSynchronous(async () => {
+    const basePaths = await cloneAndGenerateBasePaths(constants.specsRepoPath, constants.specsRepoUri, constants.specsRepoCommitHash);
+
+    let allBasePaths = [];
+
+    for (const basePath of basePaths) {
+        const readme = await validateAndReturnReadmePath(constants.specsRepoPath, basePath);
+        const namespaces = keys(await getApiVersionsByNamespace(readme));
+        const autogenlistEntries = findAutogenEntries(basePath);
+
+        const [autogened, unautogened] = partition(
+            namespaces,
+            n => autogenlistEntries.findIndex(w => lowerCaseEquals(w.namespace, n)) > -1);
+        
+        if (unautogened.length > 0 && autogened.length > 0) {
+            // For partial autogeneration only, add two items
+            // one item containing resource types not onboarded for autogeneration
+            // and the other item containing resource types onboarded for autogeneration
+            allBasePaths.push({
+                'basePath': basePath,
+                'onboardedToAutogen': 'no',
+                'missing': unautogened,
+                'included': []
+            });
+
+            allBasePaths.push({
+                'basePath': basePath,
+                'onboardedToAutogen': 'yes',
+                'missing': [],
+                'included': autogened
+            });
+        }
+        else {
+            // unautogened.length === 0 means all resource types are onboarded for autogeneration
+            allBasePaths.push({
+                'basePath': basePath,
+                'onboardedToAutogen': unautogened.length === 0 ? 'yes' : 'no',
+                'missing': unautogened,
+                'onboarded': []
+            });
+        }
+    }
+
+    const autogenResultPath = path.join(constants.autogenResultPath, 'result.json');
+    await safeMkdir(constants.autogenResultPath);
+    await writeJsonFile(autogenResultPath, allBasePaths);
+});

generator/constants.ts

@@ -12,6 +12,7 @@ export const autoRestVerboseOutput = false;
 
 export const schemasBaseUri = 'https://schema.management.azure.com/schemas';
 export const schemasBasePath = path.join(generatorRoot, 'schemas');
+export const autogenResultPath = path.join(generatorRoot, 'onboarded-report');
 export const resourceGroupRootSchema = {
     file: path.join(schemasBasePath, 'common/autogeneratedResources.json'),
     jsonPath: 'allOf[1].oneOf'

generator/package.json

@@ -8,6 +8,7 @@
     "npm": ">=6.0.0"
   },
   "scripts": {
+    "generate-onboarded-report": "ts-node cmd/getrpsonboarded",
     "list-basepaths": "ts-node cmd/listbasepaths",
     "list-resources": "ts-node cmd/listresources",
     "generate-all": "ts-node cmd/generateall",