diff --git a/.gitignore b/.gitignore index 86536db3..7193d61e 100644 --- a/.gitignore +++ b/.gitignore @@ -115,7 +115,7 @@ web_modules/ # dotenv environment variables file .env -.env.test +.env.* # parcel-bundler cache (https://parceljs.org/) .cache @@ -385,4 +385,8 @@ archive/ _build/ # Pytest Setup -/module/ \ No newline at end of file +/module/ + +# Testing +seedfarmer.yaml +tmp-metadata diff --git a/CHANGELOG.md b/CHANGELOG.md index dc13aae7..75fc40d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Ch ### New +- support list of env files using `--env-file` + ### Changes - added `--update-seedkit` support to `apply` - SeedFarmer will no longer try to update the seedkit on every request diff --git a/docs/source/manifests.md b/docs/source/manifests.md index e67270e5..3fc2d634 100644 --- a/docs/source/manifests.md +++ b/docs/source/manifests.md @@ -364,6 +364,8 @@ In this example, the `glue-db-suffix` parameter will be exposed to the CodeBuild ### Environment Variables `SeedFarmer` supports using [Dotenv](https://github.com/theskumar/python-dotenv) for dynamic replacement. When a file named `.env` is placed at the projecr root (where `seedfarmer.yaml` resides), any value in a manifest with a key of `envVariable` will be matched and replaced with the corresponding environment variable. You can pass in overriding `.env` files by using the `--env-file` on CLI command invocation. +`SeedFarmer` also supports passing multiple `.env`, by using `--env-file` multiple times. For example: `seedfarmer apply --env-file .env.shared --env-file .env.secret`. If the same value is present in multiple `.env` files, subsequent files will override the value from the previous one. In the aforementioned example, values from `.env.secret` will override values from `.env.shared`. + ```yaml name: opensearch path: modules/core/opensearch/ diff --git a/seedfarmer/__main__.py b/seedfarmer/__main__.py index a8d0664d..a3bffc10 100644 --- a/seedfarmer/__main__.py +++ b/seedfarmer/__main__.py @@ -14,16 +14,15 @@ import logging -import os -from typing import Optional +from typing import List, Optional import click -from dotenv import load_dotenv import seedfarmer from seedfarmer import DEBUG_LOGGING_FORMAT, commands, config, enable_debug from seedfarmer.cli_groups import bootstrap, init, list, metadata, projectpolicy, remove, store from seedfarmer.output_utils import print_bolded +from seedfarmer.utils import load_dotenv_files _logger: logging.Logger = logging.getLogger(__name__) @@ -65,8 +64,10 @@ def version() -> None: ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -116,7 +117,7 @@ def apply( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], debug: bool, dry_run: bool, show_manifest: bool, @@ -129,7 +130,7 @@ def apply( enable_debug(format=DEBUG_LOGGING_FORMAT) # Load environment variables from .env file if it exists - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + load_dotenv_files(config.OPS_ROOT, env_files) _logger.info("Apply request with manifest %s", spec) if dry_run: @@ -186,8 +187,10 @@ def apply( ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -217,7 +220,7 @@ def destroy( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], debug: bool, enable_session_timeout: bool, session_timeout_interval: int, @@ -227,7 +230,7 @@ def destroy( enable_debug(format=DEBUG_LOGGING_FORMAT) # Load environment variables from .env file if it exists - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + load_dotenv_files(config.OPS_ROOT, env_files) # MUST use seedfarmer.yaml so we can initialize codeseeder configs project = config.PROJECT diff --git a/seedfarmer/cli_groups/_list_group.py b/seedfarmer/cli_groups/_list_group.py index 05180eac..cb705229 100644 --- a/seedfarmer/cli_groups/_list_group.py +++ b/seedfarmer/cli_groups/_list_group.py @@ -14,12 +14,10 @@ import json import logging -import os import sys -from typing import Optional +from typing import List, Optional import click -from dotenv import load_dotenv import seedfarmer.mgmt.build_info as bi import seedfarmer.mgmt.deploy_utils as du @@ -33,6 +31,7 @@ print_manifest_inventory, ) from seedfarmer.services.session_manager import SessionManager +from seedfarmer.utils import load_dotenv_files _logger: logging.Logger = logging.getLogger(__name__) @@ -110,8 +109,10 @@ def list() -> None: ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -128,7 +129,7 @@ def list_dependencies( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], debug: bool, ) -> None: if debug: @@ -137,7 +138,8 @@ def list_dependencies( if project is None: project = _load_project() - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + + load_dotenv_files(config.OPS_ROOT, env_files=env_files) SessionManager().get_or_create(project_name=project, profile=profile, region_name=region, qualifier=qualifier) dep_manifest = du.generate_deployed_manifest(deployment_name=deployment, skip_deploy_spec=True) @@ -203,8 +205,10 @@ def list_dependencies( ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -221,7 +225,7 @@ def list_deployspec( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], debug: bool, ) -> None: if debug: @@ -230,7 +234,8 @@ def list_deployspec( if project is None: project = _load_project() - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + + load_dotenv_files(config.OPS_ROOT, env_files=env_files) session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier @@ -310,8 +315,10 @@ def list_deployspec( ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -328,7 +335,7 @@ def list_module_metadata( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], export_local_env: bool, debug: bool, ) -> None: @@ -338,7 +345,8 @@ def list_module_metadata( if project is None: project = _load_project() - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + + load_dotenv_files(config.OPS_ROOT, env_files=env_files) session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier @@ -407,8 +415,10 @@ def list_module_metadata( ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -423,7 +433,7 @@ def list_all_module_metadata( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], debug: bool, ) -> None: if debug: @@ -432,7 +442,8 @@ def list_all_module_metadata( if project is None: project = _load_project() - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + + load_dotenv_files(config.OPS_ROOT, env_files=env_files) session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier @@ -500,8 +511,10 @@ def list_all_module_metadata( ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -516,7 +529,7 @@ def list_modules( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], debug: bool, ) -> None: if debug: @@ -525,7 +538,9 @@ def list_modules( if project is None: project = _load_project() - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + + load_dotenv_files(config.OPS_ROOT, env_files=env_files) + SessionManager().get_or_create(project_name=project, profile=profile, region_name=region, qualifier=qualifier) dep_manifest = du.generate_deployed_manifest(deployment_name=deployment, skip_deploy_spec=True) @@ -647,8 +662,10 @@ def list_deployments( ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -666,7 +683,7 @@ def list_build_env_params( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], export_local_env: str, debug: bool, ) -> None: @@ -678,7 +695,8 @@ def list_build_env_params( if project is None: project = _load_project() - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + + load_dotenv_files(config.OPS_ROOT, env_files=env_files) session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier diff --git a/seedfarmer/utils.py b/seedfarmer/utils.py index 54e9417a..d95a6c17 100644 --- a/seedfarmer/utils.py +++ b/seedfarmer/utils.py @@ -14,11 +14,13 @@ import hashlib import logging -from typing import Optional +import os +from typing import List, Optional import humps import yaml from boto3 import Session +from dotenv import dotenv_values, load_dotenv from seedfarmer.services._service_utils import get_region, get_sts_identity_info @@ -158,3 +160,26 @@ def get_deployment_role_arn( def valid_qualifier(qualifer: str) -> bool: return True if ((len(qualifer) <= 6) and qualifer.isalnum()) else False + + +def load_dotenv_files(root_path: str, env_files: List[str]) -> None: + """ + Load the environment variables from the .env files + + Parameters + ---------- + root_path : str + The path to the root of the project + env_files : List[str] + The list of the .env files to load + """ + loaded_values = {} + + for env_file in env_files: + _logger.info("Loading environment variables from %s", env_file) + dotenv_path = os.path.join(root_path, env_file) + + load_dotenv(dotenv_path=dotenv_path, verbose=True, override=True) + loaded_values.update(dotenv_values(dotenv_path, verbose=True)) + + _logger.debug("Loaded environment variables: %s", loaded_values) diff --git a/test/unit-test/test_cli_arg.py b/test/unit-test/test_cli_arg.py index 91c92db8..5006205b 100644 --- a/test/unit-test/test_cli_arg.py +++ b/test/unit-test/test_cli_arg.py @@ -48,6 +48,32 @@ def aws_credentials(): os.environ["MOTO_ACCOUNT_ID"] = "123456789012" +@pytest.fixture(scope="function") +def env_file(): + path = os.path.join(_OPS_ROOT, ".env") + + with open(path, "w") as f: + f.write("PRIMARY_ACCOUNT=123456789012\n") + f.write("VPCID=vpc-123456\n") + + yield path + + os.remove(path) + + +@pytest.fixture(scope="function") +def env_file2(): + path = os.path.join(_OPS_ROOT, ".env.test2") + + with open(path, "w") as f: + f.write("PRIMARY_ACCOUNT=000000000000\n") + f.write("SECONDARY_ACCOUNT=123456789012\n") + + yield path + + os.remove(path) + + @pytest.fixture(scope="function") def sts_client(aws_credentials): with mock_sts(): @@ -161,6 +187,55 @@ def test_apply_deployment(mocker): command_output = _test_command(sub_command=apply, options=[deployment_manifest, "--debug"], exit_code=0) +@pytest.mark.first +@pytest.mark.apply_working_module +def test_apply_deployment__env_variables_no_env_file(mocker, env_file): + # Deploys a functioning module + mocker.patch("seedfarmer.__main__.commands.apply", return_value=None) + mocker.patch.dict(os.environ, {}, clear=True) + + deployment_manifest = f"{_TEST_ROOT}/manifests/module-test/deployment.yaml" + _test_command(sub_command=apply, options=[deployment_manifest, "--debug"], exit_code=0) + + assert os.environ == {"PRIMARY_ACCOUNT": "123456789012", "VPCID": "vpc-123456"} + + +@pytest.mark.first +@pytest.mark.apply_working_module +def test_apply_deployment__env_variables_single_env_file(mocker, env_file): + # Deploys a functioning module + mocker.patch("seedfarmer.__main__.commands.apply", return_value=None) + mocker.patch.dict(os.environ, {}, clear=True) + + deployment_manifest = f"{_TEST_ROOT}/manifests/module-test/deployment.yaml" + _test_command(sub_command=apply, options=[deployment_manifest, "--debug", "--env-file", env_file], exit_code=0) + + assert os.environ == {"PRIMARY_ACCOUNT": "123456789012", "VPCID": "vpc-123456"} + + +@pytest.mark.first +@pytest.mark.apply_working_module +@pytest.mark.parametrize("reverse_order", [False, True]) +def test_apply_deployment__env_variables_multiple_env_files(mocker, reverse_order, env_file, env_file2): + # Deploys a functioning module + mocker.patch("seedfarmer.__main__.commands.apply", return_value=None) + mocker.patch.dict(os.environ, {}, clear=True) + + deployment_manifest = f"{_TEST_ROOT}/manifests/module-test/deployment.yaml" + + env_files = [env_file, env_file2] + if reverse_order: + env_files = env_files[::-1] + + _test_command(sub_command=apply, options=[deployment_manifest, "--debug", "--env-file", env_files[0], "--env-file", env_files[1]], exit_code=0) + + assert os.environ == { + "PRIMARY_ACCOUNT": "123456789012" if reverse_order else "000000000000", + "SECONDARY_ACCOUNT": "123456789012", + "VPCID": "vpc-123456", + } + + @pytest.mark.destroy def test_destroy_deployment_dry_run(mocker): # Destroy a functioning module @@ -175,6 +250,48 @@ def test_destroy_deployment(mocker): command_output = _test_command(sub_command=destroy, options=["myapp", "--debug"], exit_code=0) +@pytest.mark.destroy +def test_destroy__deployment_env_variables_no_env_file(mocker, env_file): + # Destroy a functioning module + mocker.patch("seedfarmer.__main__.commands.destroy", return_value=None) + mocker.patch.dict(os.environ, {}, clear=True) + + _test_command(sub_command=destroy, options=["myapp", "--debug"], exit_code=0) + + assert os.environ == {"PRIMARY_ACCOUNT": "123456789012", "VPCID": "vpc-123456"} + + +@pytest.mark.destroy +def test_destroy__deployment_env_variables_single_env_file(mocker, env_file): + # Destroy a functioning module + mocker.patch("seedfarmer.__main__.commands.destroy", return_value=None) + mocker.patch.dict(os.environ, {}, clear=True) + + _test_command(sub_command=destroy, options=["myapp", "--debug", "--env-file", env_file], exit_code=0) + + assert os.environ == {"PRIMARY_ACCOUNT": "123456789012", "VPCID": "vpc-123456"} + + +@pytest.mark.destroy +@pytest.mark.parametrize("reverse_order", [False, True]) +def test_destroy__deployment_env_variables_multiple_env_files(mocker, reverse_order, env_file, env_file2): + # Destroy a functioning module + mocker.patch("seedfarmer.__main__.commands.destroy", return_value=None) + mocker.patch.dict(os.environ, {}, clear=True) + + env_files = [env_file, env_file2] + if reverse_order: + env_files = env_files[::-1] + + _test_command(sub_command=destroy, options=["myapp", "--debug", "--env-file", env_files[0], "--env-file", env_files[1]], exit_code=0) + + assert os.environ == { + "PRIMARY_ACCOUNT": "123456789012" if reverse_order else "000000000000", + "SECONDARY_ACCOUNT": "123456789012", + "VPCID": "vpc-123456", + } + + @pytest.mark.bootstrap def test_bootstrap_toolchain_only(mocker): # Bootstrap an Account As Target