Merge pull request #1561 from openapi-env-test/master

Refactor sdkautomation scripts

Author: leniatgh

.sdkauto/generateScript.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+PARAMS=$(cat $1 | jq '{localPath: .specFolder, readmeFiles: .relatedReadmeMdFiles, $outputPath}' -c --arg outputPath $2)
+
+echo $PARAMS
+
+pushd generator
+
+npm run generate-all $PARAMS

.sdkauto/initScript.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+pushd generator
+
+npm install
+
+popd

.vscode/launch.json

@@ -34,7 +34,20 @@
       "name": "Generate all",
       "runtimeArgs": ["-r", "ts-node/register"],
       "cwd": "${workspaceFolder}/generator",
-      "args": ["${workspaceFolder}/generator/cmd/generateall.ts"]
+      "args": [
+        "${workspaceFolder}/generator/cmd/generateall.ts"
+      ]
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Generate all - local",
+      "runtimeArgs": ["-r", "ts-node/register"],
+      "cwd": "${workspaceFolder}/generator",
+      "args": [
+        "${workspaceFolder}/generator/cmd/generateall.ts",
+        "{\"localPath\":\"/home/me/workspace/azure-rest-api-specs\",\"readmeFiles\":[\"specification/signalr/resource-manager/readme.md\",\"specification/appplatform/resource-manager/readme.md\"],\"outputPath\":\"../generateOutput.json\"}"
+      ]
     }
   ],
   "inputs": [

generator/cmd/generateall.ts

@@ -1,14 +1,18 @@
 import * as constants from '../constants';
-import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs';
+import { cloneAndGenerateBasePaths, getPackageString, resolveAbsolutePath, validateAndReturnReadmePath } from '../specs';
 import { SchemaConfiguration, generateSchemas, clearAutogeneratedSchemaRefs, saveAutogeneratedSchemaRefs } from '../generate';
 import { getAutogenlist } from '../autogenlist';
 import chalk from 'chalk';
-import { flatten, chunk } from 'lodash';
-import { executeSynchronous, chunker } from '../utils';
+import { flatten } from 'lodash';
+import { executeSynchronous, chunker, writeJsonFile } from '../utils';
+import { Package } from '../models';
 
 interface GenerateAllParams {
     batchCount?: number,
     batchIndex?: number,
+    localPath?: string,
+    readmeFiles?: string[],
+    outputPath?: string,
 }
 
 function parseParams(): GenerateAllParams {
@@ -27,29 +31,62 @@ executeSynchronous(async () => {
         filteredAutogenlist = chunker(filteredAutogenlist, params.batchCount)[params.batchIndex];
     }
 
-    await cloneAndGenerateBasePaths(constants.specsRepoPath, constants.specsRepoUri, constants.specsRepoCommitHash);
+    let localPath = params.localPath;
+    if (!localPath) {
+        localPath = constants.specsRepoPath;
+        await cloneAndGenerateBasePaths(localPath, constants.specsRepoUri, constants.specsRepoCommitHash);
+    } else {
+        localPath = await resolveAbsolutePath(localPath);
+    }
+
+    if (!!params.readmeFiles) {
+        filteredAutogenlist = filteredAutogenlist.filter(c => {
+            let r = params.readmeFiles?.find(f => f.startsWith('specification/' + c.basePath));
+            if (!!r) {
+                c.readmeFile = r;
+                return true;
+            }
+            return false;
+        }
+        );
+    }
 
     await clearAutogeneratedSchemaRefs(filteredAutogenlist);
 
     const schemaConfigs: SchemaConfiguration[] = [];
     const errors = [];
+    const packages: Package[] = [];
     for (const autogenlistConfig of filteredAutogenlist) {
+        let pkg = {
+            path: ['schemas']
+        } as Package;
         try {
-            const readme = await validateAndReturnReadmePath(autogenlistConfig.basePath);
+            const readme = await validateAndReturnReadmePath(localPath, autogenlistConfig.readmeFile || autogenlistConfig.basePath);
+            pkg.packageName = getPackageString(readme);
 
             const newConfigs = await generateSchemas(readme, autogenlistConfig);
             schemaConfigs.push(...newConfigs);
+            pkg.result = 'succeeded';
         } catch(error) {
+            pkg.packageName = autogenlistConfig.basePath;
+            pkg.result = 'failed';
             console.log(chalk.red(`Caught exception processing autogenlist entry ${autogenlistConfig.basePath}.`));
             console.log(chalk.red(error));
 
             errors.push(error);
         }
+        packages.push(pkg);
     }
 
     await saveAutogeneratedSchemaRefs(flatten(schemaConfigs));
 
-    if (errors.length > 0) {
-        throw new Error(`Autogeneration failed with ${errors.length} errors. See logs for detailed information.`);
+    if (!!params.outputPath) {
+        const outputPath = await resolveAbsolutePath(params.outputPath);
+        await writeJsonFile(outputPath, { packages });
+    } else {
+        if (errors.length > 0) {
+            throw new Error(`Autogeneration failed with ${errors.length} errors. See logs for detailed information.`);
+        }
     }
+
 });

generator/cmd/generatesingle.ts

@@ -1,5 +1,5 @@
 import * as constants from '../constants';
-import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs';
+import { cloneAndGenerateBasePaths, resolveAbsolutePath, validateAndReturnReadmePath, getPackageString } from '../specs';
 import { generateSchemas, saveAutogeneratedSchemaRefs } from '../generate';
 import process from 'process';
 import { findAutogenEntries } from '../autogenlist';
@@ -8,11 +8,17 @@ import { executeSynchronous } from '../utils';
 
 executeSynchronous(async () => {
     const basePath = process.argv[2];
-    await cloneAndGenerateBasePaths(constants.specsRepoPath, constants.specsRepoUri, constants.specsRepoCommitHash);
+    let localPath = process.argv[3];
+    if (!localPath) {
+        localPath = constants.specsRepoPath;
+        await cloneAndGenerateBasePaths(localPath, constants.specsRepoUri, constants.specsRepoCommitHash);
+    } else {
+        localPath = await resolveAbsolutePath(localPath);
+    }
 
     let readme = '';
     try {
-        readme = await validateAndReturnReadmePath(basePath);
+        readme = await validateAndReturnReadmePath(localPath, basePath);
     } catch {
         throw new Error(`Unable to find a readme under '${basePath}'. Please try running 'npm run list-basepaths' to find the list of valid paths.`);
     }

generator/cmd/listbasepaths.ts

@@ -10,7 +10,7 @@ executeSynchronous(async () => {
     const basePaths = await cloneAndGenerateBasePaths(constants.specsRepoPath, constants.specsRepoUri, constants.specsRepoCommitHash);
 
     for (const basePath of basePaths) {
-        const readme = await validateAndReturnReadmePath(basePath);
+        const readme = await validateAndReturnReadmePath(constants.specsRepoPath, basePath);
         const namespaces = keys(await getApiVersionsByNamespace(readme));
         const autogenlistEntries = findAutogenEntries(basePath);
 

generator/cmd/postprocessor.ts

@@ -4,8 +4,7 @@ import { findRecursive, findDirRecursive, executeCmd, rmdirRecursive, lowerCaseC
 import * as constants from './constants';
 import chalk from 'chalk';
 import { ScopeType, AutogenlistConfig } from './models';
-import { resetFile } from './git';
-import { get, set, flatten, uniq, concat, Dictionary, groupBy, keys, difference, pickBy, flatMap, values, uniqBy } from 'lodash';
+import { get, set, flatten, uniq, concat, Dictionary, groupBy, keys, difference, pickBy } from 'lodash';
 
 const autorestBinary = os.platform() === 'win32' ? 'autorest.cmd' : 'autorest';
 const apiVersionRegex = /^\d{4}-\d{2}-\d{2}(|-preview)$/;
@@ -76,31 +75,6 @@ export async function generateSchemas(readme: string, autogenlistConfig?: Autoge
     return schemaConfigs;
 }
 
-export async function schemaPostProcess(schemaPath: string, isNew: boolean, autogenlistConfig?: AutogenlistConfig) {
-    const namespace = path.basename(schemaPath.substring(0, schemaPath.lastIndexOf(path.extname(schemaPath))));
-    const apiVersion = path.basename(path.resolve(`${schemaPath}/..`));
-    const schemaConfig = await generateSchemaConfig(schemaPath, namespace, apiVersion, autogenlistConfig);
-    const unknownScopeResources = schemaConfig.references.filter(x => x.scope & ScopeType.Unknown);
-    if (autogenlistConfig && unknownScopeResources.length > 0) {
-        throw new Error(`Unable to determine scope for resource types ${unknownScopeResources.map(x => x.type).join(', ')} for file ${schemaPath}`);
-    }
-
-    await saveSchemaFile(schemaPath, schemaConfig);
-    const schemaPathNew = path.join(constants.schemasBasePath, schemaConfig.relativePath);
-
-    if (schemaPathNew !== schemaPath) {
-        if (isNew) {
-            console.log('Delete file: ' + chalk.red(schemaPath));
-            safeUnlink(schemaPath);
-        } else {
-            console.log('Reset file: ' + chalk.green(schemaPath));
-            resetFile(path.dirname(schemaPath), path.basename(schemaPath));
-        }
-    }
-
-    return schemaConfig;
-}
-
 async function handleGeneratedSchema(readme: string, schemaPath: string, autogenlistConfig?: AutogenlistConfig) {
     const namespace = path.basename(schemaPath.substring(0, schemaPath.lastIndexOf(path.extname(schemaPath))));
 

generator/generate.ts

@@ -31,7 +31,3 @@ export async function cloneGitRepo(localPath: string, remoteUri: string, commitH
     await executeCmd(localPath, 'git', ['fetch', '-q']);
     await executeCmd(localPath, 'git', ['checkout', '-q', commitHash]);
 }
-
-export async function resetFile(localPath: string, fileName: string) {
-    await executeCmd(localPath, 'git', ['checkout', '-q', '--', fileName]);
-}

generator/git.ts

@@ -11,6 +11,7 @@ export enum ScopeType {
 export interface AutogenlistConfig {
   basePath: string,
   namespace: string,
+  readmeFile?: string,
   suffix?: string,
   resourceConfig?: AutogenlistResourceConfig[],
   postProcessor?: SchemaPostProcessor,
@@ -23,4 +24,10 @@ export interface AutogenlistResourceConfig {
 
 export interface SchemaPostProcessor {
   (namespace: string, apiVersion: string, schema: any): void,
+}
+
+export interface Package {
+  packageName?: string,
+  path: string[],
+  result: 'succeeded' | 'failed' | 'warning'
 }