From 94b2b651ddae002b0f08aeba84a475f08718a0c7 Mon Sep 17 00:00:00 2001 From: oschwartz10612 Date: Tue, 11 Jan 2022 22:50:03 -0500 Subject: [PATCH 1/4] Add secrets manager to config --- package.json | 1 + src/common/config.ts | 64 ++++++++++++++++++++++---- src/integrations/csv-export/setup.ts | 4 +- src/integrations/csv-import/setup.ts | 4 +- src/integrations/google/setup.ts | 7 +-- src/integrations/plaid/accountSetup.ts | 4 +- src/integrations/plaid/setup.ts | 4 +- src/scripts/cli.ts | 2 +- src/scripts/fetch.ts | 2 +- src/scripts/lambda.ts | 7 +++ src/scripts/migrate.ts | 4 +- 11 files changed, 80 insertions(+), 23 deletions(-) create mode 100644 src/scripts/lambda.ts diff --git a/package.json b/package.json index 68071652..d75fe811 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "tsconfig.json" ], "dependencies": { + "@aws-sdk/client-secrets-manager": "^3.46.0", "@types/body-parser": "^1.19.0", "@types/express": "^4.17.3", "@types/glob": "^7.1.2", diff --git a/src/common/config.ts b/src/common/config.ts index 0544140f..d04b1942 100644 --- a/src/common/config.ts +++ b/src/common/config.ts @@ -10,6 +10,10 @@ import { Definition, CompilerOptions, PartialArgs, getProgramFromFiles, generate import Ajv from 'ajv' import { BalanceConfig } from '../types/balance' import { jsonc } from 'jsonc' +import { SecretsManagerClient, PutSecretValueCommand, GetSecretValueCommand } from "@aws-sdk/client-secrets-manager"; + +const REGION = "us-east-1"; +const secretsManagerClient = new SecretsManagerClient({ region: REGION }); const DEFAULT_CONFIG_FILE = '~/mintable.jsonc' const DEFAULT_CONFIG_VAR = 'MINTABLE_CONFIG' @@ -37,7 +41,12 @@ export interface EnvironmentConfig { variable: string } -export type ConfigSource = FileConfig | EnvironmentConfig +export interface SecretManagerConfig { + type: 'secretManager' + secretName: string +} + +export type ConfigSource = FileConfig | EnvironmentConfig | SecretManagerConfig export interface Config { integrations: { [id: string]: IntegrationConfig } @@ -58,6 +67,11 @@ export const getConfigSource = (): ConfigSource => { return { type: 'environment', variable: DEFAULT_CONFIG_VAR } } + if (process.env.SECRET_MANAGER_ARN) { + logInfo(`Using Secrets Manager.`) + return { type: 'secretManager', secretName: process.env.SECRET_MANAGER_NAME } + } + // Default to DEFAULT_CONFIG_FILE const path = DEFAULT_CONFIG_FILE.replace(/^~(?=$|\/|\\)/, os.homedir()) logInfo(`Using default configuration file \`${path}.\``) @@ -65,7 +79,7 @@ export const getConfigSource = (): ConfigSource => { return { type: 'file', path: path } } -export const readConfig = (source: ConfigSource, checkExists?: boolean): string => { +export const readConfig = async (source: ConfigSource, checkExists?: boolean): Promise => { if (source.type === 'file') { try { const config = fs.readFileSync(source.path, 'utf8') @@ -98,6 +112,28 @@ export const readConfig = (source: ConfigSource, checkExists?: boolean): string } } } + if (source.type === 'secretManager') { + try { + + const secret = await secretsManagerClient.send(new GetSecretValueCommand({ + SecretId: source.secretName + })); + const config = secret.SecretString; + + if (config === undefined) { + throw `Unable to get config from Secrets Manager.` + } + + logInfo('Successfully retrieved configuration variable.') + return config + } catch (e) { + if (!checkExists) { + logInfo('Unable to read config variable from secrets manager.') + } else { + logError('Unable to read config variable from secrets manager', e) + } + } + } } export const parseConfig = (configString: string): Object => { @@ -159,15 +195,15 @@ export const validateConfig = (parsedConfig: Object): Config => { return validatedConfig } -export const getConfig = (): Config => { +export const getConfig = async (): Promise => { const configSource = getConfigSource() - const configString = readConfig(configSource) + const configString = await readConfig(configSource) const parsedConfig = parseConfig(configString) const validatedConfig = validateConfig(parsedConfig) return validatedConfig } -export const writeConfig = (source: ConfigSource, config: Config): void => { +export const writeConfig = async (source: ConfigSource, config: Config): Promise => { if (source.type === 'file') { try { fs.writeFileSync(source.path, jsonc.stringify(config, null, 2)) @@ -181,23 +217,35 @@ export const writeConfig = (source: ConfigSource, config: Config): void => { 'Node does not have permissions to modify global environment variables. Please use file-based configuration to make changes.' ) } + if (source.type === 'secretManager') { + try { + const putSecretValue = new PutSecretValueCommand({ + SecretId: source.secretName, + SecretString: jsonc.stringify(config, null, 2) + }); + await secretsManagerClient.send(putSecretValue); + logInfo('Successfully wrote configuration file.') + } catch (e) { + logError('Unable to write configuration file.', e) + } + } } type ConfigTransformer = (oldConfig: Config) => Config -export const updateConfig = (configTransformer: ConfigTransformer, initialize?: boolean): Config => { +export const updateConfig = async (configTransformer: ConfigTransformer, initialize?: boolean): Promise => { let newConfig: Config const configSource = getConfigSource() if (initialize) { newConfig = configTransformer(DEFAULT_CONFIG) } else { - const configString = readConfig(configSource) + const configString = await readConfig(configSource) const oldConfig = parseConfig(configString) as Config newConfig = configTransformer(oldConfig) } const validatedConfig = validateConfig(newConfig) - writeConfig(configSource, validatedConfig) + await writeConfig(configSource, validatedConfig) return validatedConfig } diff --git a/src/integrations/csv-export/setup.ts b/src/integrations/csv-export/setup.ts index d953fef0..3c29c208 100644 --- a/src/integrations/csv-export/setup.ts +++ b/src/integrations/csv-export/setup.ts @@ -45,7 +45,7 @@ export default async () => { } ]) - updateConfig(config => { + await updateConfig(config => { let CSVExportConfig = (config.integrations[IntegrationId.CSVExport] as CSVExportConfig) || defaultCSVExportConfig @@ -62,7 +62,7 @@ export default async () => { }) logInfo('Successfully set up CSV Export Integration.') - return resolve() + return resolve(1) } catch (e) { logError('Unable to set up CSV Export Integration.', e) return reject() diff --git a/src/integrations/csv-import/setup.ts b/src/integrations/csv-import/setup.ts index 5006b22e..169dce3e 100644 --- a/src/integrations/csv-import/setup.ts +++ b/src/integrations/csv-import/setup.ts @@ -61,7 +61,7 @@ export default async () => { integration: IntegrationId.CSVImport } - updateConfig(config => { + await updateConfig(config => { let CSVImportConfig = (config.integrations[IntegrationId.CSVImport] as CSVImportConfig) || defaultCSVImportConfig @@ -78,7 +78,7 @@ export default async () => { ) logInfo('Successfully set up CSV Import Integration.') - return resolve() + return resolve(1) } catch (e) { logError('Unable to set up CSV Import Integration.', e) return reject() diff --git a/src/integrations/google/setup.ts b/src/integrations/google/setup.ts index f1352b17..dadfe5b0 100644 --- a/src/integrations/google/setup.ts +++ b/src/integrations/google/setup.ts @@ -47,7 +47,7 @@ export default async () => { } ]) - updateConfig(config => { + await updateConfig(config => { let googleConfig = (config.integrations[IntegrationId.Google] as GoogleConfig) || defaultGoogleConfig googleConfig.name = credentials.name @@ -63,7 +63,8 @@ export default async () => { return config }) - const google = new GoogleIntegration(getConfig()) + const newConfig = await getConfig(); + const google = new GoogleIntegration(newConfig); open(google.getAuthURL()) console.log('\n\t5. A link will open in your browser asking you to sign in') @@ -87,7 +88,7 @@ export default async () => { await google.saveAccessTokens(tokens) logInfo('Successfully set up Google Integration.') - return resolve() + return resolve(1) } catch (e) { logError('Unable to set up Plaid Integration.', e) return reject() diff --git a/src/integrations/plaid/accountSetup.ts b/src/integrations/plaid/accountSetup.ts index 9a08dd62..774a2827 100644 --- a/src/integrations/plaid/accountSetup.ts +++ b/src/integrations/plaid/accountSetup.ts @@ -13,7 +13,7 @@ export default async () => { console.log('\t2. Sign in with your banking provider for each account you wish to link.') console.log("\t3. Click 'Done Linking Accounts' in your browser when you are finished.\n") - const config = getConfig() + const config = await getConfig() const plaidConfig = config.integrations[IntegrationId.Plaid] as PlaidConfig const plaid = new PlaidIntegration(config) @@ -22,7 +22,7 @@ export default async () => { await plaid.accountSetup() logInfo('Successfully set up Plaid Account(s).') - return resolve() + return resolve(1) } catch (e) { logError('Unable to set up Plaid Account(s).', e) return reject() diff --git a/src/integrations/plaid/setup.ts b/src/integrations/plaid/setup.ts index e39862f5..0d7891f7 100644 --- a/src/integrations/plaid/setup.ts +++ b/src/integrations/plaid/setup.ts @@ -60,7 +60,7 @@ export default async () => { } ]) - updateConfig(config => { + await updateConfig(config => { let plaidConfig = (config.integrations[IntegrationId.Plaid] as PlaidConfig) || defaultPlaidConfig plaidConfig.name = credentials.name @@ -74,7 +74,7 @@ export default async () => { }) logInfo('Successfully set up Plaid Integration.') - return resolve() + return resolve(1) } catch (e) { logError('Unable to set up Plaid Integration.', e) return reject() diff --git a/src/scripts/cli.ts b/src/scripts/cli.ts index 0e71a330..d5398377 100755 --- a/src/scripts/cli.ts +++ b/src/scripts/cli.ts @@ -63,7 +63,7 @@ import { logError } from '../common/logging' logError('Config update cancelled by user.') } } - updateConfig(config => config, true) + await updateConfig(config => config, true) await plaid() await google() await accountSetup() diff --git a/src/scripts/fetch.ts b/src/scripts/fetch.ts index a80562cb..0ed436fe 100644 --- a/src/scripts/fetch.ts +++ b/src/scripts/fetch.ts @@ -10,7 +10,7 @@ import { CSVExportIntegration } from '../integrations/csv-export/csvExportIntegr import { Transaction, TransactionRuleCondition, TransactionRule } from '../types/transaction' export default async () => { - const config = getConfig() + const config = await getConfig() // Start date to fetch transactions, default to 2 months of history let startDate = config.transactions.startDate diff --git a/src/scripts/lambda.ts b/src/scripts/lambda.ts new file mode 100644 index 00000000..7e7ffe80 --- /dev/null +++ b/src/scripts/lambda.ts @@ -0,0 +1,7 @@ +import transact from './fetch' + +export const lambdaHandler = async (event: any, context: any) => { + + await transact() + +}; \ No newline at end of file diff --git a/src/scripts/migrate.ts b/src/scripts/migrate.ts index 700ab821..e9d2821c 100644 --- a/src/scripts/migrate.ts +++ b/src/scripts/migrate.ts @@ -14,10 +14,10 @@ export const getOldConfig = (): ConfigSource => { logError('You need to specify the --old-config-file argument.') } -export default () => { +export default async () => { try { const oldConfigSource = getOldConfig() - const oldConfigString = readConfig(oldConfigSource) + const oldConfigString = await readConfig(oldConfigSource) let oldConfig = parseConfig(oldConfigString) const deprecatedProperties = ['HOST', 'PORT', 'CATEGORY_OVERRIDES', 'DEBUG', 'CREATE_BALANCES_SHEET', 'DEBUG'] From 328a1868e01428aea3b82750972ad7df61ab1cca Mon Sep 17 00:00:00 2001 From: oschwartz10612 Date: Tue, 11 Jan 2022 22:51:20 -0500 Subject: [PATCH 2/4] Add template From a34c0d919cdd4bb354da11b7c6340fbfb2e42cff Mon Sep 17 00:00:00 2001 From: oschwartz10612 Date: Tue, 11 Jan 2022 23:20:54 -0500 Subject: [PATCH 3/4] Add initial documentation --- README.md | 4 +++ docs/README.md | 82 ++++++++++++++++++++++++++++++++++++++------------ template.yaml | 22 ++++++++++++++ 3 files changed, 88 insertions(+), 20 deletions(-) create mode 100644 template.yaml diff --git a/README.md b/README.md index 1841c1ba..50abf915 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,10 @@ Nope. You can [import transactions from a CSV bank statement](./docs/README.md#m Nope. You can [export your account balances & transactions to a CSV file](./docs/README.md#on-your-local-machine--via-csv-files) exclusively on your local machine. +**Do I have to use AWS Secrets Manager?** + +Nope. You can keep your config information in a file exclusively on your local machine. + **Do I have to manually run this every time I want new transactions in my spreadsheet?** Nope. You can automate it for free using [BitBar](./docs/README.md#automatically-in-your-macs-menu-bar--via-bitbar), [`cron`](./docs/README.md#automatically-in-your-local-machines-terminal--via-cron), or [GitHub Actions](./docs/README.md#automatically-in-the-cloud--via-github-actions). diff --git a/docs/README.md b/docs/README.md index 055a8cf2..7a426671 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,26 +2,29 @@ #### Table of Contents -- [Overview](#overview) -- [Installation](#installation) - - [Creating a Fresh Installation](#creating-a-fresh-installation) - - [Migrating from `v1.x.x`](#migrating-from-v1xx) -- [Importing Account Balances & Transactions](#importing-account-balances--transactions) - - [Automatically – in the cloud – via Plaid](#automatically-in-the-cloud--via-plaid) - - [Manually – on your local machine – via CSV bank statements](#manually--on-your-local-machine--via-csv-bank-statements) -- [Exporting Account Balances & Transactions](#exporting-account-balances--transactions) - - [In the cloud – via Google Sheets](#in-the-cloud-via-google-sheets) - - [On your local machine – via CSV files](#on-your-local-machine--via-csv-files) -- [Updating Transactions/Accounts](#updating-transactionsaccounts) - - [Manually – in your local machine's terminal](#manually-in-your-local-machines-terminal) - - [Automatically – in your Mac's Menu Bar – via BitBar](#automatically-in-your-macs-menu-bar--via-bitbar) - - [Automatically – in your local machine's terminal – via `cron`](#automatically-in-your-local-machines-terminal--via-cron) - - [Automatically – in the cloud – via GitHub Actions](#automatically-in-the-cloud--via-github-actions) -- [Transaction Rules](#transaction-rules) - - [Transaction `filter` Rules](#transaction-filter-rules) - - [Transaction `override` Rules](#transaction-override-rules) -- [Development](#development) -- [Contributing](#contributing) +- [Documentation](#documentation) + - [Table of Contents](#table-of-contents) + - [Overview](#overview) + - [Installation](#installation) + - [Creating a Fresh Installation](#creating-a-fresh-installation) + - [Migrating from `v1.x.x`](#migrating-from-v1xx) + - [Importing Account Balances & Transactions](#importing-account-balances--transactions) + - [Automatically – in the cloud – via Plaid](#automatically-in-the-cloud--via-plaid) + - [Manually – on your local machine – via CSV bank statements](#manually--on-your-local-machine--via-csv-bank-statements) + - [Exporting Account Balances & Transactions](#exporting-account-balances--transactions) + - [In the cloud – via Google Sheets](#in-the-cloud-via-google-sheets) + - [On your local machine – via CSV files](#on-your-local-machine--via-csv-files) + - [Updating Transactions/Accounts](#updating-transactionsaccounts) + - [Manually – in your local machine's terminal](#manually-in-your-local-machines-terminal) + - [Automatically – in your Mac's Menu Bar – via BitBar](#automatically-in-your-macs-menu-bar--via-bitbar) + - [Automatically – in your local machine's terminal – via `cron`](#automatically-in-your-local-machines-terminal--via-cron) + - [Automatically – in the cloud – via GitHub Actions](#automatically-in-the-cloud--via-github-actions) + - [Automatically – in the cloud – via AWS Lambda Functions](#automatically-in-the-cloud--via-aws-lambda-functions) + - [Transaction Rules](#transaction-rules) + - [Transaction `filter` Rules](#transaction-filter-rules) + - [Transaction `override` Rules](#transaction-override-rules) + - [Development](#development) + - [Contributing](#contributing) ## Overview @@ -239,6 +242,45 @@ In the **Actions** tab of your repo, the **Fetch** workflow will now update your > **Note:** The minimum interval supported by GitHub Actions is every 5 minutes. +### Automatically – in the cloud – via AWS Lambda Functions + +You can use AWS Lambda functions to run Mintable automatically in the cloud: + +> **Note:** This requires an AWS account with billing enabled, but Lambda invocations and your secrets manager secret will be covered under the free tier. +> **Note:** Some of these steps can be skipped if you have already setup an AWS account and are authenticated with the CLI. + +1. Install mintable normally and setup your accounts. +2. Fork [this repo](https://github.com/kevinschaich/mintable) and open the directory. +3. [Install AWS cli](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html). +4. [Install AWS SAM](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html). +5. [Create an AWS account](https://aws.amazon.com/premiumsupport/knowledge-center/create-and-activate-aws-account/). +6. Go to [AWS secrets manager](https://console.aws.amazon.com/secretsmanager/home). + 1. Select **Store a new secret** >> **Other type of secret** >> **Plaintext** + 2. Open your mintable generated config file and copy the contents. Then, paste over the contents in the box on AWS. + 3. Select **Next** and give your secret a name in **Secret name**. Select **Next** >> **Next** >> **Store**. + 4. Select your new secret and copy the **Secret ARN**. + 5. Paste your copied ARN into SECRET_MANAGER_ARN in the `template.yaml` file. +7. Go to [AWS IAM](https://console.aws.amazon.com/iamv2/home#/roles). +8. Select **Create role** >> **AWS Service** >> **Lambda** >> **Next: Permissions** + 1. Search **SecretsManagerReadWrite** and select the option. + 2. Select **Next: Tags** >> **Next: Review** and give your role a name. + 3. Select **Create Role**. + 4. Select your new role from the list. Copy the **Role ARN** and paste it into `template.yaml` file next to role. +9. [Create access keys for your root user](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_root-user.html#id_root-user_manage_add-key). +10. Authenticate with the AWS cli and paste your keys: + ``` + $ aws configure + AWS Access Key ID []: + AWS Secret Access Key []: + Default region name []: us-east-1 + Default output format [json]: + + ``` +15. Open a terminal in the cloned repo. +16. Run `sam deploy --guided` and follow the prompts accordingly. + +Done! Look for you lambda in AWS. + --- ## Transaction Rules diff --git a/template.yaml b/template.yaml new file mode 100644 index 00000000..4829df27 --- /dev/null +++ b/template.yaml @@ -0,0 +1,22 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: 'AWS::Serverless-2016-10-31' +Description: An AWS Serverless Specification template describing your function. + mintable: + Type: 'AWS::Serverless::Function' + Properties: + CodeUri: . + Handler: lib/scripts/lambda.lambdaHandler + Runtime: nodejs14.x + Description: '' + MemorySize: 512 + Timeout: 60 + Role: + Environment: + Variables: + SECRET_MANAGER_ARN: + Events: + ScheduledEvent: + Type: Schedule + Properties: + Schedule: rate(1 days) + Enabled: True \ No newline at end of file From ba60f37f3bace64eb8dbf13503bb6f23071fbc6f Mon Sep 17 00:00:00 2001 From: Owen Schwartz Date: Tue, 11 Jan 2022 23:38:28 -0500 Subject: [PATCH 4/4] Update readme with lambda option --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 50abf915..5cdbda11 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Nope. You can keep your config information in a file exclusively on your local m **Do I have to manually run this every time I want new transactions in my spreadsheet?** -Nope. You can automate it for free using [BitBar](./docs/README.md#automatically-in-your-macs-menu-bar--via-bitbar), [`cron`](./docs/README.md#automatically-in-your-local-machines-terminal--via-cron), or [GitHub Actions](./docs/README.md#automatically-in-the-cloud--via-github-actions). +Nope. You can automate it for free using [BitBar](./docs/README.md#automatically-in-your-macs-menu-bar--via-bitbar), [`cron`](./docs/README.md#automatically-in-your-local-machines-terminal--via-cron), [GitHub Actions](./docs/README.md#automatically-in-the-cloud--via-github-actions), or [AWS Lambdas](./docs/README.md#automatically-in-the-cloud--via-aws-lambda-functions). **It's not working!**