From 8cb8862eba801b51335eec4bc1d6ad865ea67e39 Mon Sep 17 00:00:00 2001 From: Marco Franceschi Date: Mon, 25 Apr 2022 18:06:37 -0400 Subject: [PATCH 1/4] feat: Included new utils to handle schema migrations --- src/utils/index.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/utils/index.ts b/src/utils/index.ts index b30ec4d..69a2960 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -150,8 +150,19 @@ export function deleteFolder(dirPath: string): void { fs.rmSync(dirPath, { recursive: true }) } +export const getStoredSchema = (dirPath: string): string => { + try { + const schemaPath = path.normalize(`${dirPath}/cg/schema.graphql`) + const schema = fs.readFileSync(schemaPath, 'utf8') + return schema + } catch (error) { + // Return an empty string if a schema was not found + return '' + } +} + export const sleep = (ms: number): Promise => - new Promise(resolve => setTimeout(resolve, ms)) + new Promise(resolve => setTimeout(resolve, ms * 1000)) export const calculateBackoff = (n: number): number => { const temp = Math.min( @@ -248,3 +259,6 @@ export const getNextPort = async (port: number): Promise => { const availablePort = await detect(port) return String(availablePort) } + +export const cleanString = (dirtyString: string): string => + dirtyString.replace(/(\r\n|\n|\r)/gm, '').replace(/\s+/g, '') From 91ed85a57b554214d5872556522726943be567e3 Mon Sep 17 00:00:00 2001 From: Marco Franceschi Date: Mon, 25 Apr 2022 18:10:08 -0400 Subject: [PATCH 2/4] feat: Created schema file when use the setSchema method --- src/storage/dgraph/index.ts | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/storage/dgraph/index.ts b/src/storage/dgraph/index.ts index b940b28..c1944c5 100644 --- a/src/storage/dgraph/index.ts +++ b/src/storage/dgraph/index.ts @@ -3,8 +3,9 @@ import { StorageEngineConfig, StorageEngine, GraphQLInputData, - GraphQLQueryData + GraphQLQueryData, } from '@cloudgraph/sdk' +import { isEmpty } from 'lodash' import DGraphClientWrapper from './base' import { @@ -12,6 +13,7 @@ import { processGQLExecutionResult, UPDATE_SCHEMA_QUERY, } from './utils' +import { fileUtils, sleep } from '../../utils' export default class DgraphEngine extends DGraphClientWrapper @@ -80,11 +82,15 @@ export default class DgraphEngine }) } - async setSchema(schemas: string[]): Promise { + async setSchema( + schemas: string[], + config?: { overwrite: string } + ): Promise { + const schema = schemas.join() const data = { query: UPDATE_SCHEMA_QUERY, variables: { - schema: schemas.join(), + schema, }, } try { @@ -99,8 +105,16 @@ export default class DgraphEngine resData, errors, }) + + if (isEmpty(errors) && config?.overwrite) { + fileUtils.writeGraphqlSchemaToFile( + `${config.overwrite}/cg/schema.graphql`, + schema + ) + } }) .catch(error => Promise.reject(error)) + sleep(3) } catch (error: any) { const { response: { data: resData, errors }, @@ -181,8 +195,7 @@ export default class DgraphEngine /** * Executes mutations sequentially into Dgraph */ - async run(dropData = true): Promise { - dropData && (await this.dropData()) + async run(): Promise { for (const mutation of this.axiosPromises) { try { await mutation() From 0f7a2a4c767652395bcfa76c16c590ff1643193a Mon Sep 17 00:00:00 2001 From: Marco Franceschi Date: Mon, 25 Apr 2022 18:11:23 -0400 Subject: [PATCH 3/4] feat: Evaluates when the schema changes before drop data --- src/commands/scan.ts | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/commands/scan.ts b/src/commands/scan.ts index b62e856..fe3ce32 100644 --- a/src/commands/scan.ts +++ b/src/commands/scan.ts @@ -5,7 +5,7 @@ import { Opts, pluginMap, PluginType, StorageEngine } from '@cloudgraph/sdk' import { range } from 'lodash' import Command from './base' -import { fileUtils } from '../utils' +import { cleanString, fileUtils, getStoredSchema } from '../utils' import DgraphEngine from '../storage/dgraph' import { scanReport } from '../reports' import { loadAllData, processConnectionsBetweenEntities } from '../utils/data' @@ -91,6 +91,14 @@ export default class Scan extends Command { } } + // Indicates when schema has new changes + private schemaHasChange(oldSchema: string, newSchema: string): boolean { + return !!Buffer.compare( + Buffer.from(cleanString(oldSchema), 'utf8'), + Buffer.from(cleanString(newSchema), 'utf8') + ) + } + async run(): Promise { const { argv, flags } = await this.parse(Scan) const { dev: devMode } = flags as { @@ -227,8 +235,14 @@ export default class Scan extends Command { if (storageEngine instanceof DgraphEngine) { await storageEngine.validateSchema(schema, dataFolder) } - await storageEngine.dropAll() // Delete schema before change it - await storageEngine.setSchema(schema) + + const currentSchema = getStoredSchema(dataDir) + + // Only drops and changes data when the schema changes + if (this.schemaHasChange(currentSchema, schema.join())) { + await storageEngine.dropAll() // Delete schema before change it + await storageEngine.setSchema(schema, { overwrite: dataDir }) + } } catch (error: any) { this.logger.error( `There was an issue pushing schema for providers: ${allProviders.join( From f2402e3a06cde5504411ee82fecb249de8391955 Mon Sep 17 00:00:00 2001 From: Marco Franceschi Date: Mon, 25 Apr 2022 18:12:05 -0400 Subject: [PATCH 4/4] fix: Ignored ruleMetadata insertions on scan report --- src/reports/scan-report.ts | 1 + src/storage/dgraph/index.ts | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/reports/scan-report.ts b/src/reports/scan-report.ts index b51fab8..99378fe 100644 --- a/src/reports/scan-report.ts +++ b/src/reports/scan-report.ts @@ -29,6 +29,7 @@ const servicesToIgnore = [ /^account$/, /^tag$/, /^label$/, + /ruleMetadata$/, /^billing$/, /Findings$/, ] diff --git a/src/storage/dgraph/index.ts b/src/storage/dgraph/index.ts index c1944c5..fcdde5a 100644 --- a/src/storage/dgraph/index.ts +++ b/src/storage/dgraph/index.ts @@ -107,10 +107,7 @@ export default class DgraphEngine }) if (isEmpty(errors) && config?.overwrite) { - fileUtils.writeGraphqlSchemaToFile( - `${config.overwrite}/cg/schema.graphql`, - schema - ) + fileUtils.writeGraphqlSchemaToFile(`${config.overwrite}/cg`, schema) } }) .catch(error => Promise.reject(error))