From 6b3ad40d60d6e7e2bbac53bc0a7c6a78f8b7ea41 Mon Sep 17 00:00:00 2001 From: Leon Luttenberger Date: Wed, 21 Feb 2024 13:25:38 -0600 Subject: [PATCH 01/10] adapt gitingore for running unit tests --- .gitignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 86536db3..2507aa43 100644 --- a/.gitignore +++ b/.gitignore @@ -385,4 +385,8 @@ archive/ _build/ # Pytest Setup -/module/ \ No newline at end of file +/module/ + +# Testing +seedfarmer.yaml +tmp-metadata \ No newline at end of file From ace953797ffd36ff74544f1d3366a954908c37ab Mon Sep 17 00:00:00 2001 From: Leon Luttenberger Date: Thu, 22 Feb 2024 10:26:01 -0600 Subject: [PATCH 02/10] add ability to specify list of env files --- .gitignore | 2 +- seedfarmer/__main__.py | 24 ++++++---- seedfarmer/cli_groups/_list_group.py | 66 ++++++++++++++++++++-------- 3 files changed, 64 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index 2507aa43..631e929e 100644 --- a/.gitignore +++ b/.gitignore @@ -115,7 +115,7 @@ web_modules/ # dotenv environment variables file .env -.env.test +.env.* # parcel-bundler cache (https://parceljs.org/) .cache diff --git a/seedfarmer/__main__.py b/seedfarmer/__main__.py index de38b8cc..8a80b845 100644 --- a/seedfarmer/__main__.py +++ b/seedfarmer/__main__.py @@ -15,10 +15,10 @@ import logging import os -from typing import Optional +from typing import List, Optional import click -from dotenv import load_dotenv +from dotenv import load_dotenv, dotenv_values import seedfarmer from seedfarmer import DEBUG_LOGGING_FORMAT, commands, config, enable_debug @@ -65,8 +65,10 @@ def version() -> None: ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -109,7 +111,7 @@ def apply( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], debug: bool, dry_run: bool, show_manifest: bool, @@ -121,7 +123,9 @@ def apply( enable_debug(format=DEBUG_LOGGING_FORMAT) # Load environment variables from .env file if it exists - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + for env_file in env_files: + _logger.info("Loading environment variables from %s", env_file) + load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) _logger.info("Apply request with manifest %s", spec) if dry_run: @@ -177,8 +181,10 @@ def apply( ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -208,7 +214,7 @@ def destroy( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], debug: bool, enable_session_timeout: bool, session_timeout_interval: int, @@ -218,7 +224,9 @@ def destroy( enable_debug(format=DEBUG_LOGGING_FORMAT) # Load environment variables from .env file if it exists - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + for env_file in env_files: + _logger.info("Loading environment variables from %s", env_file) + load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) # MUST use seedfarmer.yaml so we can initialize codeseeder configs project = config.PROJECT diff --git a/seedfarmer/cli_groups/_list_group.py b/seedfarmer/cli_groups/_list_group.py index 05180eac..8feaceb6 100644 --- a/seedfarmer/cli_groups/_list_group.py +++ b/seedfarmer/cli_groups/_list_group.py @@ -16,7 +16,7 @@ import logging import os import sys -from typing import Optional +from typing import List, Optional import click from dotenv import load_dotenv @@ -110,8 +110,10 @@ def list() -> None: ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -128,7 +130,7 @@ def list_dependencies( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], debug: bool, ) -> None: if debug: @@ -137,7 +139,9 @@ def list_dependencies( if project is None: project = _load_project() - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + + for env_file in env_files: + load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) SessionManager().get_or_create(project_name=project, profile=profile, region_name=region, qualifier=qualifier) dep_manifest = du.generate_deployed_manifest(deployment_name=deployment, skip_deploy_spec=True) @@ -203,8 +207,10 @@ def list_dependencies( ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -221,7 +227,7 @@ def list_deployspec( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], debug: bool, ) -> None: if debug: @@ -230,7 +236,9 @@ def list_deployspec( if project is None: project = _load_project() - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + + for env_file in env_files: + load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier @@ -310,8 +318,10 @@ def list_deployspec( ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -328,7 +338,7 @@ def list_module_metadata( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], export_local_env: bool, debug: bool, ) -> None: @@ -338,7 +348,10 @@ def list_module_metadata( if project is None: project = _load_project() - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + + for env_file in env_files: + load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier @@ -407,8 +420,10 @@ def list_module_metadata( ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -423,7 +438,7 @@ def list_all_module_metadata( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], debug: bool, ) -> None: if debug: @@ -432,7 +447,10 @@ def list_all_module_metadata( if project is None: project = _load_project() - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + + for env_file in env_files: + load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier @@ -500,8 +518,10 @@ def list_all_module_metadata( ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -516,7 +536,7 @@ def list_modules( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], debug: bool, ) -> None: if debug: @@ -525,7 +545,10 @@ def list_modules( if project is None: project = _load_project() - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + + for env_file in env_files: + load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + SessionManager().get_or_create(project_name=project, profile=profile, region_name=region, qualifier=qualifier) dep_manifest = du.generate_deployed_manifest(deployment_name=deployment, skip_deploy_spec=True) @@ -647,8 +670,10 @@ def list_deployments( ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -666,7 +691,7 @@ def list_build_env_params( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], export_local_env: str, debug: bool, ) -> None: @@ -678,7 +703,10 @@ def list_build_env_params( if project is None: project = _load_project() - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + + for env_file in env_files: + load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier From 86cf2c26ff6ff715bf63f1db758211dc3d763d4c Mon Sep 17 00:00:00 2001 From: Leon Luttenberger Date: Thu, 22 Feb 2024 10:26:37 -0600 Subject: [PATCH 03/10] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 740ebe35..d3d06706 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Ch ### New +- support list of env files using `--env-file` + ### Changes ### Fixes From f67ce19c8bafbd446fcd392ad3dd842669e7c7d1 Mon Sep 17 00:00:00 2001 From: Leon Luttenberger Date: Thu, 22 Feb 2024 11:06:25 -0600 Subject: [PATCH 04/10] add unit tests --- test/unit-test/test_cli_arg.py | 75 ++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/test/unit-test/test_cli_arg.py b/test/unit-test/test_cli_arg.py index 91c92db8..8069772b 100644 --- a/test/unit-test/test_cli_arg.py +++ b/test/unit-test/test_cli_arg.py @@ -161,6 +161,47 @@ def test_apply_deployment(mocker): command_output = _test_command(sub_command=apply, options=[deployment_manifest, "--debug"], exit_code=0) +@pytest.mark.first +@pytest.mark.apply_working_module +def test_apply_deployment__env_variables_no_env_file(mocker, caplog): + # Deploys a functioning module + mocker.patch("seedfarmer.__main__.commands.apply", return_value=None) + deployment_manifest = f"{_TEST_ROOT}/manifests/module-test/deployment.yaml" + + _test_command(sub_command=apply, options=[deployment_manifest, "--debug"], exit_code=0) + + env_messages = [msg for msg in caplog.messages if msg.startswith("Loading environment variables from ")] + assert env_messages == ["Loading environment variables from .env"] + + +@pytest.mark.first +@pytest.mark.apply_working_module +def test_apply_deployment__env_variables_single_env_file(mocker, caplog): + # Deploys a functioning module + mocker.patch("seedfarmer.__main__.commands.apply", return_value=None) + deployment_manifest = f"{_TEST_ROOT}/manifests/module-test/deployment.yaml" + + env_file = ".env.test" + _test_command(sub_command=apply, options=[deployment_manifest, "--debug", "--env-file", env_file], exit_code=0) + + env_messages = [msg for msg in caplog.messages if msg.startswith("Loading environment variables from ")] + assert env_messages == [f"Loading environment variables from {env_file}"] + + +@pytest.mark.first +@pytest.mark.apply_working_module +def test_apply_deployment__env_variables_multiple_env_files(mocker, caplog): + # Deploys a functioning module + mocker.patch("seedfarmer.__main__.commands.apply", return_value=None) + deployment_manifest = f"{_TEST_ROOT}/manifests/module-test/deployment.yaml" + + env_files = [".env.test", ".env.test2"] + _test_command(sub_command=apply, options=[deployment_manifest, "--debug", "--env-file", env_files[0], "--env-file", env_files[1]], exit_code=0) + + env_messages = [msg for msg in caplog.messages if msg.startswith("Loading environment variables from ")] + assert env_messages == [f"Loading environment variables from {env_file}" for env_file in env_files] + + @pytest.mark.destroy def test_destroy_deployment_dry_run(mocker): # Destroy a functioning module @@ -175,6 +216,40 @@ def test_destroy_deployment(mocker): command_output = _test_command(sub_command=destroy, options=["myapp", "--debug"], exit_code=0) +@pytest.mark.destroy +def test_destroy__deployment_env_variables_no_env_file(mocker, caplog): + # Destroy a functioning module + mocker.patch("seedfarmer.__main__.commands.destroy", return_value=None) + _test_command(sub_command=destroy, options=["myapp", "--debug"], exit_code=0) + + env_messages = [msg for msg in caplog.messages if msg.startswith("Loading environment variables from ")] + assert env_messages == ["Loading environment variables from .env"] + + +@pytest.mark.destroy +def test_destroy__deployment_env_variables_single_env_file(mocker, caplog): + # Destroy a functioning module + mocker.patch("seedfarmer.__main__.commands.destroy", return_value=None) + + env_file = ".env.test" + _test_command(sub_command=destroy, options=["myapp", "--debug", "--env-file", env_file], exit_code=0) + + env_messages = [msg for msg in caplog.messages if msg.startswith("Loading environment variables from ")] + assert env_messages == [f"Loading environment variables from {env_file}"] + + +@pytest.mark.destroy +def test_destroy__deployment_env_variables_multiple_env_files(mocker, caplog): + # Destroy a functioning module + mocker.patch("seedfarmer.__main__.commands.destroy", return_value=None) + + env_files = [".env.test", ".env.test2"] + _test_command(sub_command=destroy, options=["myapp", "--debug", "--env-file", env_files[0], "--env-file", env_files[1]], exit_code=0) + + env_messages = [msg for msg in caplog.messages if msg.startswith("Loading environment variables from ")] + assert env_messages == [f"Loading environment variables from {env_file}" for env_file in env_files] + + @pytest.mark.bootstrap def test_bootstrap_toolchain_only(mocker): # Bootstrap an Account As Target From 235065c4eef05b158d55c40f4f9551d47c9b3677 Mon Sep 17 00:00:00 2001 From: Leon Luttenberger Date: Thu, 22 Feb 2024 11:10:01 -0600 Subject: [PATCH 05/10] fix formatting --- .gitignore | 2 +- seedfarmer/__main__.py | 2 +- seedfarmer/cli_groups/_list_group.py | 3 --- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 631e929e..7193d61e 100644 --- a/.gitignore +++ b/.gitignore @@ -389,4 +389,4 @@ _build/ # Testing seedfarmer.yaml -tmp-metadata \ No newline at end of file +tmp-metadata diff --git a/seedfarmer/__main__.py b/seedfarmer/__main__.py index 8a80b845..887384f1 100644 --- a/seedfarmer/__main__.py +++ b/seedfarmer/__main__.py @@ -18,7 +18,7 @@ from typing import List, Optional import click -from dotenv import load_dotenv, dotenv_values +from dotenv import load_dotenv import seedfarmer from seedfarmer import DEBUG_LOGGING_FORMAT, commands, config, enable_debug diff --git a/seedfarmer/cli_groups/_list_group.py b/seedfarmer/cli_groups/_list_group.py index 8feaceb6..29eccce1 100644 --- a/seedfarmer/cli_groups/_list_group.py +++ b/seedfarmer/cli_groups/_list_group.py @@ -352,7 +352,6 @@ def list_module_metadata( for env_file in env_files: load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) - session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier ) @@ -451,7 +450,6 @@ def list_all_module_metadata( for env_file in env_files: load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) - session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier ) @@ -707,7 +705,6 @@ def list_build_env_params( for env_file in env_files: load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) - session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier ) From 11b7e1e0c053295724447f497c8ba61afc638cbd Mon Sep 17 00:00:00 2001 From: Leon Luttenberger Date: Thu, 22 Feb 2024 12:02:20 -0600 Subject: [PATCH 06/10] add debug logging for the final resolution of env parameters --- seedfarmer/__main__.py | 9 +++------ seedfarmer/cli_groups/_list_group.py | 19 +++++++------------ seedfarmer/utils.py | 28 +++++++++++++++++++++++++++- 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/seedfarmer/__main__.py b/seedfarmer/__main__.py index 5f30db1d..15f0eceb 100644 --- a/seedfarmer/__main__.py +++ b/seedfarmer/__main__.py @@ -22,6 +22,7 @@ import seedfarmer from seedfarmer import DEBUG_LOGGING_FORMAT, commands, config, enable_debug +from seedfarmer.utils import load_dotenv_files from seedfarmer.cli_groups import bootstrap, init, list, metadata, projectpolicy, remove, store from seedfarmer.output_utils import print_bolded @@ -131,9 +132,7 @@ def apply( enable_debug(format=DEBUG_LOGGING_FORMAT) # Load environment variables from .env file if it exists - for env_file in env_files: - _logger.info("Loading environment variables from %s", env_file) - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + load_dotenv_files(config.OPS_ROOT, env_files) _logger.info("Apply request with manifest %s", spec) if dry_run: @@ -233,9 +232,7 @@ def destroy( enable_debug(format=DEBUG_LOGGING_FORMAT) # Load environment variables from .env file if it exists - for env_file in env_files: - _logger.info("Loading environment variables from %s", env_file) - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + load_dotenv_files(config.OPS_ROOT, env_files) # MUST use seedfarmer.yaml so we can initialize codeseeder configs project = config.PROJECT diff --git a/seedfarmer/cli_groups/_list_group.py b/seedfarmer/cli_groups/_list_group.py index 29eccce1..22a6eb86 100644 --- a/seedfarmer/cli_groups/_list_group.py +++ b/seedfarmer/cli_groups/_list_group.py @@ -25,6 +25,7 @@ import seedfarmer.mgmt.deploy_utils as du import seedfarmer.mgmt.module_info as mi from seedfarmer import DEBUG_LOGGING_FORMAT, commands, config, enable_debug +from seedfarmer.utils import load_dotenv_files from seedfarmer.output_utils import ( print_bolded, print_dependency_list, @@ -140,8 +141,7 @@ def list_dependencies( if project is None: project = _load_project() - for env_file in env_files: - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + load_dotenv_files(config.OPS_ROOT, env_files=env_files) SessionManager().get_or_create(project_name=project, profile=profile, region_name=region, qualifier=qualifier) dep_manifest = du.generate_deployed_manifest(deployment_name=deployment, skip_deploy_spec=True) @@ -237,8 +237,7 @@ def list_deployspec( if project is None: project = _load_project() - for env_file in env_files: - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + load_dotenv_files(config.OPS_ROOT, env_files=env_files) session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier @@ -349,8 +348,7 @@ def list_module_metadata( if project is None: project = _load_project() - for env_file in env_files: - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + load_dotenv_files(config.OPS_ROOT, env_files=env_files) session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier @@ -447,8 +445,7 @@ def list_all_module_metadata( if project is None: project = _load_project() - for env_file in env_files: - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + load_dotenv_files(config.OPS_ROOT, env_files=env_files) session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier @@ -544,8 +541,7 @@ def list_modules( if project is None: project = _load_project() - for env_file in env_files: - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + load_dotenv_files(config.OPS_ROOT, env_files=env_files) SessionManager().get_or_create(project_name=project, profile=profile, region_name=region, qualifier=qualifier) @@ -702,8 +698,7 @@ def list_build_env_params( if project is None: project = _load_project() - for env_file in env_files: - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + load_dotenv_files(config.OPS_ROOT, env_files=env_files) session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier diff --git a/seedfarmer/utils.py b/seedfarmer/utils.py index 54e9417a..c9251244 100644 --- a/seedfarmer/utils.py +++ b/seedfarmer/utils.py @@ -14,11 +14,14 @@ import hashlib import logging -from typing import Optional +import os +from typing import List, Optional import humps import yaml from boto3 import Session +from dotenv import dotenv_values, load_dotenv + from seedfarmer.services._service_utils import get_region, get_sts_identity_info @@ -158,3 +161,26 @@ def get_deployment_role_arn( def valid_qualifier(qualifer: str) -> bool: return True if ((len(qualifer) <= 6) and qualifer.isalnum()) else False + + +def load_dotenv_files(root_path: str, env_files: List[str]) -> None: + """ + Load the environment variables from the .env files + + Parameters + ---------- + root_path : str + The path to the root of the project + env_files : List[str] + The list of the .env files to load + """ + loaded_values = {} + + for env_file in env_files: + _logger.info("Loading environment variables from %s", env_file) + dotenv_path = os.path.join(root_path, env_file) + + load_dotenv(dotenv_path=dotenv_path, verbose=True, override=True) + loaded_values.update(dotenv_values(dotenv_path, verbose=True)) + + _logger.debug("Loaded environment variables: %s", loaded_values) From a2b2c5d0ad11c0dcfa3da5489953907d3cfcb142 Mon Sep 17 00:00:00 2001 From: Leon Luttenberger Date: Thu, 22 Feb 2024 12:05:54 -0600 Subject: [PATCH 07/10] fix formatting --- seedfarmer/__main__.py | 4 +--- seedfarmer/cli_groups/_list_group.py | 4 +--- seedfarmer/utils.py | 1 - 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/seedfarmer/__main__.py b/seedfarmer/__main__.py index 15f0eceb..a3bffc10 100644 --- a/seedfarmer/__main__.py +++ b/seedfarmer/__main__.py @@ -14,17 +14,15 @@ import logging -import os from typing import List, Optional import click -from dotenv import load_dotenv import seedfarmer from seedfarmer import DEBUG_LOGGING_FORMAT, commands, config, enable_debug -from seedfarmer.utils import load_dotenv_files from seedfarmer.cli_groups import bootstrap, init, list, metadata, projectpolicy, remove, store from seedfarmer.output_utils import print_bolded +from seedfarmer.utils import load_dotenv_files _logger: logging.Logger = logging.getLogger(__name__) diff --git a/seedfarmer/cli_groups/_list_group.py b/seedfarmer/cli_groups/_list_group.py index 22a6eb86..cb705229 100644 --- a/seedfarmer/cli_groups/_list_group.py +++ b/seedfarmer/cli_groups/_list_group.py @@ -14,18 +14,15 @@ import json import logging -import os import sys from typing import List, Optional import click -from dotenv import load_dotenv import seedfarmer.mgmt.build_info as bi import seedfarmer.mgmt.deploy_utils as du import seedfarmer.mgmt.module_info as mi from seedfarmer import DEBUG_LOGGING_FORMAT, commands, config, enable_debug -from seedfarmer.utils import load_dotenv_files from seedfarmer.output_utils import ( print_bolded, print_dependency_list, @@ -34,6 +31,7 @@ print_manifest_inventory, ) from seedfarmer.services.session_manager import SessionManager +from seedfarmer.utils import load_dotenv_files _logger: logging.Logger = logging.getLogger(__name__) diff --git a/seedfarmer/utils.py b/seedfarmer/utils.py index c9251244..d95a6c17 100644 --- a/seedfarmer/utils.py +++ b/seedfarmer/utils.py @@ -22,7 +22,6 @@ from boto3 import Session from dotenv import dotenv_values, load_dotenv - from seedfarmer.services._service_utils import get_region, get_sts_identity_info _logger: logging.Logger = logging.getLogger(__name__) From dbc0058d551c37e72ce3a0ad40f78b469e71fd7c Mon Sep 17 00:00:00 2001 From: Leon Luttenberger Date: Thu, 22 Feb 2024 12:36:41 -0600 Subject: [PATCH 08/10] check for environment variable values in unit tests --- test/unit-test/test_cli_arg.py | 82 ++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 24 deletions(-) diff --git a/test/unit-test/test_cli_arg.py b/test/unit-test/test_cli_arg.py index 8069772b..5c3a8729 100644 --- a/test/unit-test/test_cli_arg.py +++ b/test/unit-test/test_cli_arg.py @@ -48,6 +48,32 @@ def aws_credentials(): os.environ["MOTO_ACCOUNT_ID"] = "123456789012" +@pytest.fixture(scope="function") +def env_file(): + path = os.path.join(_OPS_ROOT, ".env") + + with open(path, "w") as f: + f.write("PRIMARY_ACCOUNT=123456789012\n") + f.write("VPCID=vpc-123456\n") + + yield path + + os.remove(path) + + +@pytest.fixture(scope="function") +def env_file2(): + path = os.path.join(_OPS_ROOT, ".env.test2") + + with open(path, "w") as f: + f.write("PRIMARY_ACCOUNT=000000000000\n") + f.write("SECONDARY_ACCOUNT=123456789012\n") + + yield path + + os.remove(path) + + @pytest.fixture(scope="function") def sts_client(aws_credentials): with mock_sts(): @@ -163,43 +189,47 @@ def test_apply_deployment(mocker): @pytest.mark.first @pytest.mark.apply_working_module -def test_apply_deployment__env_variables_no_env_file(mocker, caplog): +def test_apply_deployment__env_variables_no_env_file(mocker, env_file): # Deploys a functioning module mocker.patch("seedfarmer.__main__.commands.apply", return_value=None) - deployment_manifest = f"{_TEST_ROOT}/manifests/module-test/deployment.yaml" + mocker.patch.dict(os.environ, {}, clear=True) + deployment_manifest = f"{_TEST_ROOT}/manifests/module-test/deployment.yaml" _test_command(sub_command=apply, options=[deployment_manifest, "--debug"], exit_code=0) - env_messages = [msg for msg in caplog.messages if msg.startswith("Loading environment variables from ")] - assert env_messages == ["Loading environment variables from .env"] + assert os.environ == {"PRIMARY_ACCOUNT": "123456789012", "VPCID": "vpc-123456"} @pytest.mark.first @pytest.mark.apply_working_module -def test_apply_deployment__env_variables_single_env_file(mocker, caplog): +def test_apply_deployment__env_variables_single_env_file(mocker, env_file): # Deploys a functioning module mocker.patch("seedfarmer.__main__.commands.apply", return_value=None) - deployment_manifest = f"{_TEST_ROOT}/manifests/module-test/deployment.yaml" + mocker.patch.dict(os.environ, {}, clear=True) - env_file = ".env.test" + deployment_manifest = f"{_TEST_ROOT}/manifests/module-test/deployment.yaml" _test_command(sub_command=apply, options=[deployment_manifest, "--debug", "--env-file", env_file], exit_code=0) - env_messages = [msg for msg in caplog.messages if msg.startswith("Loading environment variables from ")] - assert env_messages == [f"Loading environment variables from {env_file}"] + assert os.environ == {"PRIMARY_ACCOUNT": "123456789012", "VPCID": "vpc-123456"} @pytest.mark.first @pytest.mark.apply_working_module -def test_apply_deployment__env_variables_multiple_env_files(mocker, caplog): +def test_apply_deployment__env_variables_multiple_env_files(mocker, env_file, env_file2): # Deploys a functioning module mocker.patch("seedfarmer.__main__.commands.apply", return_value=None) + mocker.patch.dict(os.environ, {}, clear=True) + deployment_manifest = f"{_TEST_ROOT}/manifests/module-test/deployment.yaml" - env_files = [".env.test", ".env.test2"] + env_files = [env_file, env_file2] _test_command(sub_command=apply, options=[deployment_manifest, "--debug", "--env-file", env_files[0], "--env-file", env_files[1]], exit_code=0) - env_messages = [msg for msg in caplog.messages if msg.startswith("Loading environment variables from ")] - assert env_messages == [f"Loading environment variables from {env_file}" for env_file in env_files] + assert os.environ == { + "PRIMARY_ACCOUNT": "000000000000", + "SECONDARY_ACCOUNT": "123456789012", + "VPCID": "vpc-123456", + } @pytest.mark.destroy @@ -217,37 +247,41 @@ def test_destroy_deployment(mocker): @pytest.mark.destroy -def test_destroy__deployment_env_variables_no_env_file(mocker, caplog): +def test_destroy__deployment_env_variables_no_env_file(mocker, env_file): # Destroy a functioning module mocker.patch("seedfarmer.__main__.commands.destroy", return_value=None) + mocker.patch.dict(os.environ, {}, clear=True) + _test_command(sub_command=destroy, options=["myapp", "--debug"], exit_code=0) - env_messages = [msg for msg in caplog.messages if msg.startswith("Loading environment variables from ")] - assert env_messages == ["Loading environment variables from .env"] + assert os.environ == {"PRIMARY_ACCOUNT": "123456789012", "VPCID": "vpc-123456"} @pytest.mark.destroy -def test_destroy__deployment_env_variables_single_env_file(mocker, caplog): +def test_destroy__deployment_env_variables_single_env_file(mocker, env_file): # Destroy a functioning module mocker.patch("seedfarmer.__main__.commands.destroy", return_value=None) + mocker.patch.dict(os.environ, {}, clear=True) - env_file = ".env.test" _test_command(sub_command=destroy, options=["myapp", "--debug", "--env-file", env_file], exit_code=0) - env_messages = [msg for msg in caplog.messages if msg.startswith("Loading environment variables from ")] - assert env_messages == [f"Loading environment variables from {env_file}"] + assert os.environ == {"PRIMARY_ACCOUNT": "123456789012", "VPCID": "vpc-123456"} @pytest.mark.destroy -def test_destroy__deployment_env_variables_multiple_env_files(mocker, caplog): +def test_destroy__deployment_env_variables_multiple_env_files(mocker, env_file, env_file2): # Destroy a functioning module mocker.patch("seedfarmer.__main__.commands.destroy", return_value=None) + mocker.patch.dict(os.environ, {}, clear=True) - env_files = [".env.test", ".env.test2"] + env_files = [env_file, env_file2] _test_command(sub_command=destroy, options=["myapp", "--debug", "--env-file", env_files[0], "--env-file", env_files[1]], exit_code=0) - env_messages = [msg for msg in caplog.messages if msg.startswith("Loading environment variables from ")] - assert env_messages == [f"Loading environment variables from {env_file}" for env_file in env_files] + assert os.environ == { + "PRIMARY_ACCOUNT": "000000000000", + "SECONDARY_ACCOUNT": "123456789012", + "VPCID": "vpc-123456", + } @pytest.mark.bootstrap From 636ebbb2ea4820607af1df33be70c40f471c9c2b Mon Sep 17 00:00:00 2001 From: Leon Luttenberger Date: Thu, 22 Feb 2024 12:42:44 -0600 Subject: [PATCH 09/10] add reverse order for env file resolution --- test/unit-test/test_cli_arg.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/test/unit-test/test_cli_arg.py b/test/unit-test/test_cli_arg.py index 5c3a8729..5006205b 100644 --- a/test/unit-test/test_cli_arg.py +++ b/test/unit-test/test_cli_arg.py @@ -215,7 +215,8 @@ def test_apply_deployment__env_variables_single_env_file(mocker, env_file): @pytest.mark.first @pytest.mark.apply_working_module -def test_apply_deployment__env_variables_multiple_env_files(mocker, env_file, env_file2): +@pytest.mark.parametrize("reverse_order", [False, True]) +def test_apply_deployment__env_variables_multiple_env_files(mocker, reverse_order, env_file, env_file2): # Deploys a functioning module mocker.patch("seedfarmer.__main__.commands.apply", return_value=None) mocker.patch.dict(os.environ, {}, clear=True) @@ -223,10 +224,13 @@ def test_apply_deployment__env_variables_multiple_env_files(mocker, env_file, en deployment_manifest = f"{_TEST_ROOT}/manifests/module-test/deployment.yaml" env_files = [env_file, env_file2] + if reverse_order: + env_files = env_files[::-1] + _test_command(sub_command=apply, options=[deployment_manifest, "--debug", "--env-file", env_files[0], "--env-file", env_files[1]], exit_code=0) assert os.environ == { - "PRIMARY_ACCOUNT": "000000000000", + "PRIMARY_ACCOUNT": "123456789012" if reverse_order else "000000000000", "SECONDARY_ACCOUNT": "123456789012", "VPCID": "vpc-123456", } @@ -269,16 +273,20 @@ def test_destroy__deployment_env_variables_single_env_file(mocker, env_file): @pytest.mark.destroy -def test_destroy__deployment_env_variables_multiple_env_files(mocker, env_file, env_file2): +@pytest.mark.parametrize("reverse_order", [False, True]) +def test_destroy__deployment_env_variables_multiple_env_files(mocker, reverse_order, env_file, env_file2): # Destroy a functioning module mocker.patch("seedfarmer.__main__.commands.destroy", return_value=None) mocker.patch.dict(os.environ, {}, clear=True) env_files = [env_file, env_file2] + if reverse_order: + env_files = env_files[::-1] + _test_command(sub_command=destroy, options=["myapp", "--debug", "--env-file", env_files[0], "--env-file", env_files[1]], exit_code=0) assert os.environ == { - "PRIMARY_ACCOUNT": "000000000000", + "PRIMARY_ACCOUNT": "123456789012" if reverse_order else "000000000000", "SECONDARY_ACCOUNT": "123456789012", "VPCID": "vpc-123456", } From d2d597ff3a9bd79ce448918e1b376412ef2ad1a7 Mon Sep 17 00:00:00 2001 From: Leon Luttenberger Date: Thu, 22 Feb 2024 12:51:16 -0600 Subject: [PATCH 10/10] update docs/source/manifests.md --- docs/source/manifests.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/manifests.md b/docs/source/manifests.md index e67270e5..3fc2d634 100644 --- a/docs/source/manifests.md +++ b/docs/source/manifests.md @@ -364,6 +364,8 @@ In this example, the `glue-db-suffix` parameter will be exposed to the CodeBuild ### Environment Variables `SeedFarmer` supports using [Dotenv](https://github.com/theskumar/python-dotenv) for dynamic replacement. When a file named `.env` is placed at the projecr root (where `seedfarmer.yaml` resides), any value in a manifest with a key of `envVariable` will be matched and replaced with the corresponding environment variable. You can pass in overriding `.env` files by using the `--env-file` on CLI command invocation. +`SeedFarmer` also supports passing multiple `.env`, by using `--env-file` multiple times. For example: `seedfarmer apply --env-file .env.shared --env-file .env.secret`. If the same value is present in multiple `.env` files, subsequent files will override the value from the previous one. In the aforementioned example, values from `.env.secret` will override values from `.env.shared`. + ```yaml name: opensearch path: modules/core/opensearch/