diff --git a/CHANGELOG.md b/CHANGELOG.md index af2770d6..86e82c12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,22 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Ch ### Fixes + +## v3.1.0 (2024-01-22) + +### New +- adding AWS curated codebuild iamge override with opinionated runtimes + +### Changes +- updating pydantic support from 1.X.X to 2.5.3 +- adding seedfarmer verions check support with `seedfarmer.yaml` +- updating `aws-codeseeder` dependency top 0.11.0 + +### Fixes +- update `manifests/examples/` to point to an updated release branch +- Docs - manifest name description (seed-farmer/docs/source/manifests.md) needed correction +- Docs - added definition of `nameGenerator` for deployment manifest (seed-farmer/docs/source/manifests.md) + ## v3.0.1 (2023-11-10) ### New diff --git a/VERSION b/VERSION index cb2b00e4..fd2a0186 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0.1 +3.1.0 diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 74c8f079..6e803a89 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -12,13 +12,13 @@ babel==2.12.1 # via sphinx certifi==2023.7.22 # via - # -r docs/requirements-docs.in + # -r requirements-docs.in # requests charset-normalizer==3.1.0 # via requests docutils==0.18.1 # via - # -r docs/requirements-docs.in + # -r requirements-docs.in # myst-parser # sphinx # sphinx-rtd-theme @@ -26,7 +26,9 @@ idna==3.4 # via requests imagesize==1.4.1 # via sphinx -jinja2==3.1.2 +importlib-metadata==7.0.1 + # via sphinx +jinja2==3.1.3 # via # myst-parser # sphinx @@ -44,14 +46,14 @@ mdit-py-plugins==0.3.5 mdurl==0.1.2 # via markdown-it-py myst-parser==1.0.0 - # via -r docs/requirements-docs.in + # via -r requirements-docs.in packaging==23.1 # via sphinx pygments==2.15.1 # via sphinx pyyaml==5.4 # via - # -r docs/requirements-docs.in + # -r requirements-docs.in # myst-parser # sphinx-autoapi requests==2.31.0 @@ -65,46 +67,50 @@ sphinx==6.2.1 # sphinx-rtd-theme # sphinxcontrib-jquery sphinx-autoapi==2.1.0 - # via -r docs/requirements-docs.in + # via -r requirements-docs.in sphinx-rtd-theme==1.2.1 - # via -r docs/requirements-docs.in + # via -r requirements-docs.in sphinxcontrib-applehelp==1.0.4 # via - # -r docs/requirements-docs.in + # -r requirements-docs.in # sphinx sphinxcontrib-devhelp==1.0.2 # via - # -r docs/requirements-docs.in + # -r requirements-docs.in # sphinx sphinxcontrib-htmlhelp==2.0.1 # via - # -r docs/requirements-docs.in + # -r requirements-docs.in # sphinx sphinxcontrib-jquery==4.1 # via - # -r docs/requirements-docs.in + # -r requirements-docs.in # sphinx-rtd-theme sphinxcontrib-jsmath==1.0.1 # via - # -r docs/requirements-docs.in + # -r requirements-docs.in # sphinx sphinxcontrib-qthelp==1.0.3 # via - # -r docs/requirements-docs.in + # -r requirements-docs.in # sphinx sphinxcontrib-serializinghtml==1.1.5 # via - # -r docs/requirements-docs.in + # -r requirements-docs.in # sphinx typing-extensions==4.5.0 - # via -r docs/requirements-docs.in + # via + # -r requirements-docs.in + # astroid unidecode==1.3.6 # via sphinx-autoapi urllib3==1.26.18 # via - # -r docs/requirements-docs.in + # -r requirements-docs.in # requests wheel==0.38.1 - # via -r docs/requirements-docs.in + # via -r requirements-docs.in wrapt==1.15.0 # via astroid +zipp==3.17.0 + # via importlib-metadata diff --git a/docs/source/manifests.md b/docs/source/manifests.md index cf3edd74..e67270e5 100644 --- a/docs/source/manifests.md +++ b/docs/source/manifests.md @@ -9,6 +9,11 @@ The deployment manifest is the top level manifest and resides in the `modules` d ```yaml name: examples +nameGenerator: + prefix: myprefix + suffix: + valueFrom: + envVariable: SUFFIX_ENV_VARIABLE toolchainRegion: us-west-2 forceDependencyRedeploy: False groups: @@ -69,6 +74,12 @@ targetAccountMappings: ``` - **name** : this is the name of your deployment. There can be only one deployment with this name in a project. + - THIS CANNOT BE USED WITH `nameGenerator` +- **nameGenerator** : this supports dynamically generating a deployment name by concatenation of the following fields: + - **prefix** - the prefix string of the name + - **suffix** - the suffix string of the name + - Both of these fields support the use of [Environment Variables](envVariable) (see example above) + - THIS CANNOT BE USED WITH `name` - **toolchainRegion** :the designated region that the `toolchain` is created in - **forceDependencyRedeploy**: this is a boolean that tells seedfarmer to redeploy ALL dependency modules (see [Force Dependency Redeploy](force-redeploy)) - Default is `False` - **groups** : the relative path to the [`module manifests`](module_manifest) that define each module in the group. This sequential order is preserved in deployment, and reversed in destroy. @@ -80,14 +91,14 @@ targetAccountMappings: - **alias** - the logical name for an account, referenced by [`module manifests`](module_manifest) - **account** - the account id tied to the alias. This parameter also supports [Environment Variables](envVariable) - **default** - this designates this mapping as the default account for all modules unless otherwise specified. This is primarily for supporting migrating from `seedfarmer v1` to the current version. - - **codebuildImage** - a custom build image to use (see [Custom Build Image](custombuildimage)) + - **codebuildImage** - a custom build image to use (see [Build Image Override](buildimageoverride)) - **parametersGlobal** - these are parameters that apply to all region mappings unless otherwise overridden at the region level - **dockerCredentialsSecret** - please see [Docker Credentials Secret](dockerCredentialsSecret) - **permissionsBoundaryName** - the name of the [permissions boundary](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_boundaries.html) policy to apply to all module-specific roles created - **regionMappings** - section to define region-specific configurations for the defined account, this is a list - **region** - the region name - **default** - this designates this mapping as the default region for all modules unless otherwise specified. This is primarily for supporting migrating - - **codebuildImage** - a custom build image to use (see [Custom Build Image](custombuildimage)) + - **codebuildImage** - a custom build image to use (see [Build Image Override](buildimageoverride)) - **parametersRegional** - these are parameters that apply to all region mappings unless otherwise overridden at the region level - **dockerCredentialsSecret** - please see [Docker Credentials Secret](dockerCredentialsSecret) - This is a NAMED PARAMETER...in that `dockerCredentialsSecret` is recognized by `seed-farmer` @@ -183,10 +194,10 @@ network: envVariable: SECURITYGROUPS ``` The corresponding `.env` file would have the following defined (again, remember the lists!!): -```code +```bash VPCID="vpc-0c4cb9e06c9413222" -PRIVATESUBNETS=["subnet-0c36d3d5808f67a02","subnet-00fa1e71cddcf57d3"] -SECURITYGROUPS=["sg-049033188c114a3d2"] +PRIVATESUBNETS='["subnet-0c36d3d5808f67a02","subnet-00fa1e71cddcf57d3"]' +SECURITYGROUPS='["sg-049033188c114a3d2"]' ``` (dependency-management)= ### Dependency Management @@ -211,7 +222,7 @@ What does this mean? Well, lets take the following module deployment order: **This is an important feature to understand: redeployment is not discriminant.** SeedFarmer does not know how to assess what has changed in a module and its impact on downstream modules. Nor does it have the ability to know if a module can incur a redeployment (as opposed to a destroy and deploy process). That is up to you to determine with respect to the modules you are leveraging. ANY change to the source code (deployspec, modulestack, comments in cdk code, etc.) will indicate to SeedFarmer that the module needs to be redeployed, even if the underlying logic / artifact has not changed. Also, it is important to understand that this feature could put your deployment in an unusable state if the shared-responsibility model is not followed. - For example: lets say a deployment has a module (called `networking`) that deploys a VPC with public and private subnets that are restricted to a particular CIDR (as input). Then, downstream modules reference the metadata of `netowrking`. If a user were to change the CIDR references and redeploy the `networking` module, this has the potential to render the deployment in an unusable state: the process to change the CIDR's would trigger a destroy of the existing subnets...which would fail due to resources from other modules leveraging those subnets. The redeployment would fail, and the user would have to manually correct the state. + For example: lets say a deployment has a module (called `networking`) that deploys a VPC with public and private subnets that are restricted to a particular CIDR (as input). Then, downstream modules reference the metadata of `networking`. If a user were to change the CIDR references and redeploy the `networking` module, this has the potential to render the deployment in an unusable state: the process to change the CIDR's would trigger a destroy of the existing subnets...which would fail due to resources from other modules leveraging those subnets. The redeployment would fail, and the user would have to manually correct the state. (module_manifest)= ## Module Manifest @@ -244,13 +255,14 @@ dataFiles: - filePath: test1.txt - filePath: git::https://github.com/awslabs/idf-modules.git//modules/storage/buckets/deployspec.yaml?ref=release/1.0.0&depth=1 ``` -- **name** - the name of the group +- **name** - the name of the module + - this name must be unique in the group of the deployment - **path** - this element supports two sources of code: - - the relative path to the module code in the project + - the relative path to the module code in the project if deploying code from the local filesystem - a public Git Repository, leveraging the Terraform semantic as denoted [HERE](https://www.terraform.io/language/modules/sources#generic-git-repository) - **targetAccount** - the alias of the account from the [deployment manifest mappings](deployment_manifest) - **targetRegion** - the name of the region to deploy to - this overrides any mappings -- **codebuildImage** - a custom build image to use (see [Custom Build Image](custombuildimage)) +- **codebuildImage** - a custom build image to use (see [Build Image Override](buildimageoverride)) - **parameters** - the parameters section .... see [Parameters](parameters) - **dataFiles** - additional files to add to the bundle that are outside of the module code - this is LIST and EVERY element in the list must have the keyword **filePath** @@ -276,13 +288,30 @@ When using this feature, any change to these file(s) (modifying, add to manifest ***Iceburg, dead ahead!*** Heres the rub: if you deploy with data files sourced from a local filesystem, you MUST provide those same files in order to destroy the module(s)...we are not keeping them stored anywhere (much like the module source code). ***Iceburg missed us! (why is everthing so wet??)*** -(custombuildimage)= -## Custom Codebuild Image -`seed-farmer` is preconfigued to use the optimal build image and we recommend using it as-is (no need to leverage the `codebuildImage` manifest named paramter). But, we get it....no one wants to be boxed in.
+(buildimageoverride)= +## Codebuild Image Override +An AWS Codebuild complaint image is provided for use with `seed-farmer` and we recommend using it as-is (no need to leverage the `codebuildImage` manifest named paramter). But, we get it....no one wants to be boxed in.
+ USER BEWARE - this is a feature meant for advanced users...use at own risk! + +Users can override the default build image via one of the following: +- an AWS Curated Build Image +- a custom-built image + +#### AWS Curated Build Images +There are multiple [build images and available runtimes](https://docs.aws.amazon.com/codebuild/latest/userguide/available-runtimes.html) that are supported by AWS Codebuild. For `seed-farmer`, we currently support the following AWS Curated Images with the default runtimes installed: + +| AWS Curated Build Image | Confgured Runtimes| +| ----------- | ----------- | +|aws/codebuild/standard:6.0|nodejs:16| +||python:3.10| +||java:corretto17| +|aws/codebuild/standard:7.0|nodejs:18| +||python:3.11| +||java:corretto21| -### The Build Image -An AWS Codebuild complaint image is provided for use with `seed-farmer` and the CLI is configured by default to use this image. Advanced users have the option of building their own image and configuring their deployment to use it. If an end user wants to build their own image, it is STRONGLY encouraged to use [this Dockerfile from AWS public repos](https://github.com/awslabs/aws-codeseeder/blob/main/images/code-build-image/Dockerfile) as the base layer. `seed-farmer` leverages this as the base for its default image ([see HERE](https://github.com/awslabs/aws-codeseeder/blob/main/images/code-build-image/Dockerfile)). +#### Custom Build Images +If an end user wants to build their own image, it is STRONGLY encouraged to use [this Dockerfile from AWS public repos](https://github.com/awslabs/aws-codeseeder/blob/main/images/code-build-image/Dockerfile) as the base layer. `seed-farmer` leverages this as the base for its default image ([see HERE](https://github.com/awslabs/aws-codeseeder/blob/main/images/code-build-image/Dockerfile)). It is up to the module developer to verify all proper libraries are installed and available. ### Logic for Rules -- Application There are three (3) places to configure a custom build image: @@ -420,7 +449,7 @@ parameters: valueFrom: parameterValue: mygreatkey ``` -`seed-farrmer` will first look in the Regional Parameters for a matching key, and return a string object (all json convert to a string) represening the value. If not found, `seed-farrmer` will look in the Global Parameters for the same key and return that string-ified value. +`seed-farmer` will first look in the Regional Parameters for a matching key, and return a string object (all json convert to a string) represening the value. If not found, `seed-farrmer` will look in the Global Parameters for the same key and return that string-ified value. NOTE: the `network` section of the [deployment manifest](deployment_manifest) leverages Regional Parameters only! diff --git a/docs/source/project_development.md b/docs/source/project_development.md index 11f16f5f..c80bb626 100644 --- a/docs/source/project_development.md +++ b/docs/source/project_development.md @@ -27,13 +27,15 @@ It is important to have the ```seedfarmer.yaml``` at the root of your project. project: description: projectPolicyPath: +seedfarmer_version: ``` - **project** (REQUIRED) - this is the name of the project that all deployments will reference - **description** (OPTIONAL) - this is the description of the project - **projectPolicyPath** (OPTIONAL) - this allows advanced users change the project policy that has the basic minimim permissions seedfarmer needs - it consists of a path relative to the project root and MUST be a valid relative path - to synth the existing project policy, run `seedfarmer projectpolicy synth` - +- **seedfarmer_version** (OPTIONAL) - this specifies what is the minimum allowable version of `seed-farmer` the project supports + - if this value is set AND the runtime version of seedfarmer is greater, `seed-farmer` will exit immediately (project_initalization)= diff --git a/examples/exampleproject/manifests-isolated/examples/optional-modules-2.yaml b/examples/exampleproject/manifests-isolated/examples/optional-modules-2.yaml index 847e7d18..e9957715 100644 --- a/examples/exampleproject/manifests-isolated/examples/optional-modules-2.yaml +++ b/examples/exampleproject/manifests-isolated/examples/optional-modules-2.yaml @@ -1,12 +1,12 @@ # name: networking -# path: git::https://github.com/awslabs/seedfarmer-modules.git//modules/network/basic-cdk/?ref=release/1.0.0&depth=1 +# path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk/?ref=release/1.2.0&depth=1 # targetAccount: primary # parameters: # - name: internet-accessible # value: true # --- name: buckets -path: git::https://github.com/awslabs/seedfarmer-modules.git//modules/storage/buckets/?ref=release/1.0.0&depth=1 +path: git::https://github.com/awslabs/idf-modules.git//modules/storage/buckets/?ref=release/1.2.0&depth=1 targetAccount: secondary targetRegion: us-west-2 parameters: diff --git a/examples/exampleproject/manifests-isolated/examples/optional-modules.yaml b/examples/exampleproject/manifests-isolated/examples/optional-modules.yaml index 4051ee3f..149a47c4 100644 --- a/examples/exampleproject/manifests-isolated/examples/optional-modules.yaml +++ b/examples/exampleproject/manifests-isolated/examples/optional-modules.yaml @@ -1,12 +1,12 @@ # name: networking -# path: git::https://github.com/awslabs/seedfarmer-modules.git//modules/network/basic-cdk/?ref=release/1.0.0&depth=1 +# path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk/?ref=release/1.2.0&depth=1 # targetAccount: secondary # parameters: # - name: internet-accessible # value: true # --- name: buckets -path: git::https://github.com/awslabs/seedfarmer-modules.git//modules/storage/buckets/?ref=release/1.0.0&depth=1 +path: git::https://github.com/awslabs/idf-modules.git//modules/storage/buckets/?ref=release/1.2.0&depth=1 targetAccount: primary targetRegion: us-east-2 parameters: diff --git a/examples/exampleproject/manifests/examples/optional-modules.yaml b/examples/exampleproject/manifests/examples/optional-modules.yaml index 1436a66e..bfc7d4c0 100644 --- a/examples/exampleproject/manifests/examples/optional-modules.yaml +++ b/examples/exampleproject/manifests/examples/optional-modules.yaml @@ -1,11 +1,11 @@ name: networking -path: git::https://github.com/awslabs/seedfarmer-modules.git//modules/network/basic-cdk/?ref=release/1.0.0&depth=1 +path: git::https://github.com/awslabs/idf-modules.git//modules/network/basic-cdk/?ref=release/1.2.0&depth=1 parameters: - name: internet-accessible value: true --- name: buckets -path: git::https://github.com/awslabs/seedfarmer-modules.git//modules/storage/buckets/?ref=release/1.0.0&depth=1 +path: git::https://github.com/awslabs/idf-modules.git//modules/storage/buckets/?ref=release/1.2.0&depth=1 parameters: - name: encryption-type value: SSE \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d5f230c8..27de2b0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,7 @@ markers = [ "commands_parameters: marks all `commands_parameters` tests", "commands_modules: marks all `commands_modules` tests", "commands_deployment: marks all `commands_deployment` tests", + "commands_bootstrap: marks all `commands_bootstrap` tests", ] log_cli_level = "INFO" addopts = "-v --cov=. --cov-report=term --cov-report=html --cov-config=coverage.ini --cov-fail-under=80" diff --git a/requirements-dev.txt b/requirements-dev.txt index b566aafc..5f3d735c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -48,8 +48,10 @@ click==8.1.6 colorama==0.4.4 # via awscli coverage[toml]==7.2.7 - # via pytest-cov -cryptography==41.0.4 + # via + # coverage + # pytest-cov +cryptography==41.0.6 # via # moto # secretstorage @@ -61,6 +63,8 @@ docutils==0.16 # readme-renderer # sphinx # sphinx-rtd-theme +exceptiongroup==1.2.0 + # via pytest flake8==4.0.1 # via -r requirements-dev.in idna==3.4 @@ -114,7 +118,9 @@ mdurl==0.1.2 more-itertools==9.1.0 # via jaraco-classes moto[codebuild,iam,s3,secretsmanager,ssm,sts]==4.0.13 - # via -r requirements-dev.in + # via + # -r requirements-dev.in + # moto mypy==0.991 # via -r requirements-dev.in mypy-extensions==1.0.0 @@ -239,6 +245,16 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx +tomli==2.0.1 + # via + # black + # build + # check-manifest + # coverage + # mypy + # pip-tools + # pyproject-hooks + # pytest trove-classifiers==2023.7.6 # via pyroma twine==4.0.2 @@ -251,6 +267,8 @@ types-setuptools==57.4.18 # via -r requirements-dev.in typing-extensions==4.7.1 # via + # astroid + # black # mypy # myst-parser unidecode==1.3.6 diff --git a/requirements.txt b/requirements.txt index 839a6ef6..0e6238c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,9 +4,11 @@ # # pip-compile # +annotated-types==0.6.0 + # via pydantic arrow==1.2.3 # via jinja2-time -aws-codeseeder==0.10.2 +aws-codeseeder==0.11.0 # via seed-farmer (setup.py) binaryornot==0.4.4 # via cookiecutter @@ -46,7 +48,7 @@ gitdb==4.0.10 # via gitpython gitignore-parser==0.1.3 # via seed-farmer (setup.py) -gitpython==3.1.37 +gitpython==3.1.41 # via seed-farmer (setup.py) humanfriendly==10.0 # via @@ -55,7 +57,7 @@ humanfriendly==10.0 # property-manager idna==3.4 # via requests -jinja2==3.1.2 +jinja2==3.1.3 # via # cookiecutter # jinja2-time @@ -71,8 +73,10 @@ mypy-extensions==1.0.0 # via aws-codeseeder property-manager==3.0 # via executor -pydantic==1.10.7 +pydantic==2.5.3 # via seed-farmer (setup.py) +pydantic-core==2.14.6 + # via pydantic pygments==2.15.1 # via rich pyhumps==3.5.3 @@ -108,9 +112,10 @@ smmap==5.0.0 # via gitdb text-unidecode==1.3 # via python-slugify -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via # pydantic + # pydantic-core # seed-farmer (setup.py) urllib3==1.26.18 # via diff --git a/seedfarmer/__init__.py b/seedfarmer/__init__.py index 96a7fb27..90bc77f8 100644 --- a/seedfarmer/__init__.py +++ b/seedfarmer/__init__.py @@ -22,6 +22,7 @@ import yaml from aws_codeseeder import LOGGER, codeseeder from aws_codeseeder.codeseeder import CodeSeederConfig +from packaging.version import parse import seedfarmer.errors from seedfarmer.__metadata__ import __description__, __license__, __title__ @@ -90,6 +91,13 @@ def _load_config_data(self) -> None: if self._project_spec.project_policy_path else os.path.join(CLI_ROOT, DEFAULT_PROJECT_POLICY_PATH) ) + if self._project_spec.seedfarmer_version: + if parse(__version__) < parse(str(self._project_spec.seedfarmer_version)): + msg = ( + f"The seedfarmer.yaml specified a minimum version: " + f"{self._project_spec.seedfarmer_version} but you are using {__version__}" + ) + raise seedfarmer.errors.SeedFarmerException(msg) @codeseeder.configure(self._project_spec.project.lower(), deploy_if_not_exists=True) def configure(configuration: CodeSeederConfig) -> None: @@ -104,7 +112,6 @@ def configure(configuration: CodeSeederConfig) -> None: 'timeout 15 sh -c "until docker info; do echo .; sleep 1; done"', ] configuration.python_modules = [f"seed-farmer=={__version__}"] - configuration.runtime_versions = {"nodejs": "16", "python": "3.10"} @property def PROJECT(self) -> str: diff --git a/seedfarmer/commands/_deployment_commands.py b/seedfarmer/commands/_deployment_commands.py index 1c6e1668..7a7d8997 100644 --- a/seedfarmer/commands/_deployment_commands.py +++ b/seedfarmer/commands/_deployment_commands.py @@ -22,7 +22,7 @@ import git import yaml -from git import Repo # type: ignore +from git import Repo import seedfarmer.checksum as checksum import seedfarmer.errors @@ -374,15 +374,14 @@ def _render_permissions_boundary_arn( def prime_target_accounts(deployment_manifest: DeploymentManifest) -> None: - # TODO: Investigate whether we need to validate the requested mappings against previously deployed mappings - _logger.info("Priming Accounts") with concurrent.futures.ThreadPoolExecutor(max_workers=len(deployment_manifest.target_accounts_regions)) as workers: - def _prime_accounts(args: Dict[str, Any]) -> None: + def _prime_accounts(args: Dict[str, Any]) -> List[Any]: _logger.info("Priming Acccount %s in %s", args["account_id"], args["region"]) - commands.deploy_seedkit(**args) + seedkit_stack_outputs = commands.deploy_seedkit(**args) commands.deploy_managed_policy_stack(deployment_manifest=deployment_manifest, **args) + return [args["account_id"], args["region"], seedkit_stack_outputs] params = [] for target_account_region in deployment_manifest.target_accounts_regions: @@ -401,7 +400,11 @@ def _prime_accounts(args: Dict[str, Any]) -> None: params.append(param_d) - _ = list(workers.map(_prime_accounts, params)) + output_seedkit = list(workers.map(_prime_accounts, params)) + # add these to the region mappings for reference + for out_s in output_seedkit: + deployment_manifest.populate_seedkit_metadata(account_id=out_s[0], region=out_s[1], seedkit_dict=out_s[2]) + _logger.debug(deployment_manifest.model_dump()) def tear_down_target_accounts(deployment_manifest: DeploymentManifest, retain_seedkit: bool = False) -> None: @@ -551,7 +554,7 @@ def deploy_deployment( By default False """ - deployment_manifest_wip = deployment_manifest.copy() + deployment_manifest_wip = deployment_manifest.model_copy() deployment_name = cast(str, deployment_manifest_wip.name) _logger.debug("Setting up deployment for %s", deployment_name) @@ -604,7 +607,9 @@ def deploy_deployment( deployment_manifest=deployment_manifest_wip, module=module, group_name=group.name ) - module.manifest_md5 = hashlib.md5(json.dumps(module.dict(), sort_keys=True).encode("utf-8")).hexdigest() + module.manifest_md5 = hashlib.md5( + json.dumps(module.model_dump(), sort_keys=True).encode("utf-8") + ).hexdigest() module.deployspec_md5 = hashlib.md5(open(deployspec_path, "rb").read()).hexdigest() _build_module = du.need_to_build( @@ -704,7 +709,7 @@ def apply( manifest_path = os.path.join(config.OPS_ROOT, deployment_manifest_path) with open(manifest_path) as manifest_file: deployment_manifest = DeploymentManifest(**yaml.safe_load(manifest_file)) - _logger.debug(deployment_manifest.dict()) + _logger.debug(deployment_manifest.model_dump()) # Initialize the SessionManager for the entire project session_manager = SessionManager().get_or_create( @@ -720,7 +725,9 @@ def apply( deployment_manifest._partition = partition if not dryrun: write_deployment_manifest( - cast(str, deployment_manifest.name), deployment_manifest.dict(), session=session_manager.toolchain_session + cast(str, deployment_manifest.name), + deployment_manifest.model_dump(), + session=session_manager.toolchain_session, ) for module_group in deployment_manifest.groups: diff --git a/seedfarmer/commands/_module_commands.py b/seedfarmer/commands/_module_commands.py index df52fe6e..a0585d00 100644 --- a/seedfarmer/commands/_module_commands.py +++ b/seedfarmer/commands/_module_commands.py @@ -26,6 +26,7 @@ import seedfarmer.errors from seedfarmer import config +from seedfarmer.commands._runtimes import get_runtimes from seedfarmer.models.deploy_responses import CodeSeederMetadata, ModuleDeploymentResponse, StatusType from seedfarmer.models.manifests import ModuleManifest, ModuleParameter from seedfarmer.services.session_manager import SessionManager @@ -136,6 +137,9 @@ def deploy_module( } _phases = module_manifest.deploy_spec.deploy.phases + active_codebuild_image = ( + module_manifest.codebuild_image if module_manifest.codebuild_image is not None else codebuild_image + ) try: resp_dict_str, dict_metadata = _execute_module_commands( deployment_name=deployment_name, @@ -153,9 +157,8 @@ def deploy_module( extra_env_vars=env_vars, codebuild_compute_type=module_manifest.deploy_spec.build_type, codebuild_role_name=module_role_name, - codebuild_image=module_manifest.codebuild_image - if module_manifest.codebuild_image is not None - else codebuild_image, + codebuild_image=active_codebuild_image, + runtime_versions=get_runtimes(active_codebuild_image), ) _logger.debug("CodeSeeder Metadata response is %s", dict_metadata) @@ -228,6 +231,9 @@ def destroy_module( for data_file in module_manifest.data_files } + active_codebuild_image = ( + module_manifest.codebuild_image if module_manifest.codebuild_image is not None else codebuild_image + ) try: resp_dict_str, _ = _execute_module_commands( deployment_name=deployment_name, @@ -245,9 +251,8 @@ def destroy_module( extra_env_vars=env_vars, codebuild_compute_type=module_manifest.deploy_spec.build_type, codebuild_role_name=module_role_name, - codebuild_image=module_manifest.codebuild_image - if module_manifest.codebuild_image is not None - else codebuild_image, + codebuild_image=active_codebuild_image, + runtime_versions=get_runtimes(active_codebuild_image), ) resp = ModuleDeploymentResponse( deployment=deployment_name, @@ -286,6 +291,7 @@ def _execute_module_commands( codebuild_compute_type: Optional[str] = None, codebuild_role_name: Optional[str] = None, codebuild_image: Optional[str] = None, + runtime_versions: Optional[Dict[str, str]] = None, ) -> Tuple[str, Optional[Dict[str, str]]]: session_getter: Optional[Callable[[], Session]] = None @@ -315,6 +321,7 @@ def _session_getter() -> Session: codebuild_compute_type=codebuild_compute_type, extra_files=extra_file_bundle, boto3_session=session_getter, + runtime_versions=runtime_versions, ) def _execute_module_commands( deployment_name: str, @@ -331,6 +338,7 @@ def _execute_module_commands( extra_post_build_commands: Optional[List[str]] = None, extra_env_vars: Optional[Dict[str, Any]] = None, codebuild_compute_type: Optional[str] = None, + runtime_versions: Optional[Dict[str, str]] = None, ) -> str: deploy_info = { "aws_region": os.environ.get("AWS_DEFAULT_REGION"), diff --git a/seedfarmer/commands/_parameter_commands.py b/seedfarmer/commands/_parameter_commands.py index bce8f169..bdc94443 100644 --- a/seedfarmer/commands/_parameter_commands.py +++ b/seedfarmer/commands/_parameter_commands.py @@ -57,7 +57,7 @@ def load_parameter_values( parameter_values_cache: Dict[Tuple[str, str, str], Any] = {} for parameter in parameters: if _logger.isEnabledFor(logging.DEBUG): - _logger.debug("parameter: %s", parameter.dict()) + _logger.debug("parameter: %s", parameter.model_dump()) if parameter.value is not None: _logger.debug("static parameter value: %s", parameter.value) diff --git a/seedfarmer/commands/_runtimes.py b/seedfarmer/commands/_runtimes.py new file mode 100644 index 00000000..a109f55f --- /dev/null +++ b/seedfarmer/commands/_runtimes.py @@ -0,0 +1,36 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from enum import Enum +from typing import Dict, Optional + + +class CuratedBuildImages: + class ImageEnums(Enum): + UBUNTU_STANDARD_6 = "aws/codebuild/standard:6.0" + UBUNTU_STANDARD_7 = "aws/codebuild/standard:7.0" + + class ImageRuntimes(Enum): + UBUNTU_STANDARD_6 = {"nodejs": "16", "python": "3.10", "java": "corretto17"} + UBUNTU_STANDARD_7 = {"nodejs": "18", "python": "3.11", "java": "corretto21"} + + +def get_runtimes(codebuild_image: Optional[str]) -> Optional[Dict[str, str]]: + image_vals = [cbi.value for cbi in CuratedBuildImages.ImageEnums] + if codebuild_image in image_vals: + cir_d = {cir.name: cir.value for cir in CuratedBuildImages.ImageRuntimes} + k = CuratedBuildImages.ImageEnums(codebuild_image).name + return cir_d[k] + else: + return None diff --git a/seedfarmer/commands/_stack_commands.py b/seedfarmer/commands/_stack_commands.py index db8f66bd..b6ce0cb4 100644 --- a/seedfarmer/commands/_stack_commands.py +++ b/seedfarmer/commands/_stack_commands.py @@ -435,7 +435,7 @@ def deploy_seedkit( vpc_id: Optional[str] = None, private_subnet_ids: Optional[List[str]] = None, security_group_ids: Optional[List[str]] = None, -) -> None: +) -> Dict[str, Any]: """ deploy_seedkit Accessor method to CodeSeeder to deploy the SeedKit if not deployed @@ -468,6 +468,9 @@ def deploy_seedkit( subnet_ids=private_subnet_ids, security_group_ids=security_group_ids, ) + # Go get the outputs and return them + _, _, stack_outputs = commands.seedkit_deployed(seedkit_name=config.PROJECT, session=session) + return dict(stack_outputs) def destroy_seedkit(account_id: str, region: str) -> None: diff --git a/seedfarmer/mgmt/deploy_utils.py b/seedfarmer/mgmt/deploy_utils.py index d63cfbab..5db8e88b 100644 --- a/seedfarmer/mgmt/deploy_utils.py +++ b/seedfarmer/mgmt/deploy_utils.py @@ -288,20 +288,20 @@ def prepare_ssm_for_deploy( # Remove the deployspec before writing...remove bloat as we write deployspec separately session = SessionManager().get_or_create().get_deployment_session(account_id=account_id, region_name=region) - module_manifest_wip = module_manifest.copy() + module_manifest_wip = module_manifest.model_copy() module_manifest_wip.deploy_spec = None mi.write_module_manifest( deployment=deployment_name, group=group_name, module=module_manifest.name, - data=module_manifest_wip.dict(), + data=module_manifest_wip.model_dump(), session=session, ) mi.write_deployspec( deployment=deployment_name, group=group_name, module=module_manifest.name, - data=module_manifest.deploy_spec.dict(), + data=module_manifest.deploy_spec.model_dump(), session=session, ) if module_manifest.deploy_spec else None mi.write_module_md5( @@ -343,7 +343,9 @@ def write_deployed_deployment_manifest(deployment_manifest: DeploymentManifest) for group in deployment_manifest.groups: delattr(group, "modules") session = SessionManager().get_or_create().toolchain_session - mi.write_deployed_deployment_manifest(deployment=deployment_name, data=deployment_manifest.dict(), session=session) + mi.write_deployed_deployment_manifest( + deployment=deployment_name, data=deployment_manifest.model_dump(), session=session + ) def generate_deployed_manifest( @@ -556,7 +558,7 @@ def write_group_manifest(deployment_name: str, group_manifest: ModulesManifest) The ModulesManifest object of the groups to persist """ g = group_manifest.name if group_manifest.name else "" - mi.write_group_manifest(deployment=deployment_name, group=g, data=group_manifest.dict()) + mi.write_group_manifest(deployment=deployment_name, group=g, data=group_manifest.model_dump()) def filter_deploy_destroy(apply_manifest: DeploymentManifest, module_info_index: ModuleInfoIndex) -> DeploymentManifest: @@ -580,7 +582,7 @@ def filter_deploy_destroy(apply_manifest: DeploymentManifest, module_info_index: """ deployment_name = cast(str, apply_manifest.name) - destroy_manifest = apply_manifest.copy() + destroy_manifest = apply_manifest.model_copy() delattr(destroy_manifest, "groups") destroy_group_list = _populate_groups_to_remove(deployment_name, apply_manifest.groups, module_info_index) destroy_manifest.groups = destroy_group_list @@ -673,4 +675,4 @@ def update_deployspec( d_path = mi.get_deployspec_path(module_path=module_path) with open(d_path) as deploymentspec: new_spec = DeploySpec(**yaml.safe_load(deploymentspec)) - mi.write_deployspec(deployment, group, module, new_spec.dict(), session=session) + mi.write_deployspec(deployment, group, module, new_spec.model_dump(), session=session) diff --git a/seedfarmer/mgmt/metadata_support.py b/seedfarmer/mgmt/metadata_support.py index e82ff367..03ae7436 100644 --- a/seedfarmer/mgmt/metadata_support.py +++ b/seedfarmer/mgmt/metadata_support.py @@ -41,7 +41,7 @@ def __init__(self) -> None: deploy_spec = DeploySpec(**yaml.safe_load(module_spec_file)) self.use_project_prefix = not deploy_spec.publish_generic_env_variables except Exception: - _logger.warn("Cannot read the deployspec, using project name as prefix (a non-generic module)") + _logger.warning("Cannot read the deployspec, using project name as prefix (a non-generic module)") self.use_project_prefix = True def get_ops_root_path(self) -> str: @@ -157,5 +157,5 @@ def get_parameter_value(parameter_suffix: str) -> Optional[str]: _logger.info("Getting the Env Parameter tied to %s", key) return os.getenv(key) except Exception: - _logger.warn("Error looking for %s, returning None", key) + _logger.warning("Error looking for %s, returning None", key) return None diff --git a/seedfarmer/models/_base.py b/seedfarmer/models/_base.py index 61517c71..f6d0f4fb 100644 --- a/seedfarmer/models/_base.py +++ b/seedfarmer/models/_base.py @@ -15,7 +15,7 @@ from typing import Optional, cast import humps -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict def to_camel(string: str) -> str: @@ -23,10 +23,9 @@ def to_camel(string: str) -> str: class CamelModel(BaseModel): - class Config: - alias_generator = to_camel - allow_population_by_field_name = True - underscore_attrs_are_private = True + # TODO[pydantic]: The following keys were removed: `underscore_attrs_are_private`. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-config for more information. + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) class ModuleRef(CamelModel): diff --git a/seedfarmer/models/_project_spec.py b/seedfarmer/models/_project_spec.py index f56a4b6c..33e6029e 100644 --- a/seedfarmer/models/_project_spec.py +++ b/seedfarmer/models/_project_spec.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional +from typing import Optional, Union from seedfarmer.models._base import CamelModel @@ -27,3 +27,4 @@ class ProjectSpec(CamelModel): project: str description: Optional[str] = None project_policy_path: Optional[str] = None + seedfarmer_version: Optional[Union[int, str]] = None diff --git a/seedfarmer/models/deploy_responses.py b/seedfarmer/models/deploy_responses.py index 33a4b1c9..132d48d2 100644 --- a/seedfarmer/models/deploy_responses.py +++ b/seedfarmer/models/deploy_responses.py @@ -30,11 +30,11 @@ class CodeSeederMetadata(CamelModel): _build_url: str = PrivateAttr() - aws_account_id: Optional[str] - aws_region: Optional[str] - aws_partition: Optional[str] - codebuild_build_id: Optional[str] - codebuild_log_path: Optional[str] + aws_account_id: Optional[str] = None + aws_region: Optional[str] = None + aws_partition: Optional[str] = None + codebuild_build_id: Optional[str] = None + codebuild_log_path: Optional[str] = None def __init__(self, **data: Any) -> None: super().__init__(**data) @@ -53,11 +53,11 @@ def build_url(self) -> str: class ModuleDeploymentResponse(CamelModel): deployment: str - group: Optional[str] + group: Optional[str] = None module: str status: str - codeseeder_metadata: Optional[CodeSeederMetadata] - codeseeder_output: Optional[Dict[Any, Any]] + codeseeder_metadata: Optional[CodeSeederMetadata] = None + codeseeder_output: Optional[Dict[Any, Any]] = None def __init__(self, **data: Any) -> None: super().__init__(**data) diff --git a/seedfarmer/models/manifests/_deployment_manifest.py b/seedfarmer/models/manifests/_deployment_manifest.py index 5d777362..3d444670 100644 --- a/seedfarmer/models/manifests/_deployment_manifest.py +++ b/seedfarmer/models/manifests/_deployment_manifest.py @@ -41,9 +41,9 @@ class NetworkMapping(CamelModel): This class provides network metadata """ - vpc_id: Optional[Union[str, ValueFromRef]] - private_subnet_ids: Optional[Union[List[str], ValueFromRef]] - security_group_ids: Optional[Union[List[str], ValueFromRef]] + vpc_id: Optional[Union[str, ValueFromRef]] = None + private_subnet_ids: Optional[Union[List[str], ValueFromRef]] = None + security_group_ids: Optional[Union[List[str], ValueFromRef]] = None class RegionMapping(CamelModel): @@ -57,6 +57,7 @@ class RegionMapping(CamelModel): parameters_regional: Dict[str, Any] = {} network: Optional[NetworkMapping] = None codebuild_image: Optional[str] = None + seedkit_metadata: Optional[Dict[str, Any]] = None class TargetAccountMapping(CamelModel): @@ -66,7 +67,7 @@ class TargetAccountMapping(CamelModel): """ alias: str - account_id: Union[str, ValueFromRef] + account_id: Union[int, str, ValueFromRef] default: bool = False parameters_global: Dict[str, str] = {} region_mappings: List[RegionMapping] = [] @@ -86,8 +87,8 @@ def get_region_mapping(self, region: str) -> Optional[RegionMapping]: @property def actual_account_id(self) -> str: - if isinstance(self.account_id, str): - return self.account_id + if isinstance(self.account_id, str) or isinstance(self.account_id, int): + return str(self.account_id) elif isinstance(self.account_id, ValueFromRef): if self.account_id.value_from and self.account_id.value_from.module_metadata is not None: raise seedfarmer.errors.InvalidManifestError( @@ -161,7 +162,7 @@ class DeploymentManifest(CamelModel): name_generator: Optional[NameGenerator] = None toolchain_region: str groups: List[ModulesManifest] = [] - description: Optional[str] + description: Optional[str] = None target_account_mappings: List[TargetAccountMapping] = [] force_dependency_redeploy: Optional[bool] = False _default_account: Optional[TargetAccountMapping] = PrivateAttr(default=None) @@ -284,11 +285,19 @@ def get_region_codebuild_image( # Search the region_mappings for the region, if the codebuild_image is in region for region_mapping in target_account.region_mappings: if region == region_mapping.region or (use_default_region and region_mapping.default): - return ( + image = ( region_mapping.codebuild_image if region_mapping.codebuild_image is not None else target_account.codebuild_image ) + if ( + image is None + and region_mapping.seedkit_metadata + and region_mapping.seedkit_metadata.get("CodeBuildProjectBuildImage") + ): + image = region_mapping.seedkit_metadata["CodeBuildProjectBuildImage"] + + return image else: return None @@ -324,3 +333,10 @@ def validate_and_set_module_defaults(self) -> None: def get_module(self, group: str, module: str) -> Optional[ModuleManifest]: return self._module_index.get((group, module), None) + + def populate_seedkit_metadata(self, account_id: str, region: str, seedkit_dict: Dict[str, Any]) -> None: + for target_account in self.target_account_mappings: + for region_mapping in target_account.region_mappings: + if target_account.actual_account_id == account_id and region_mapping.region == region: + region_mapping.seedkit_metadata = seedkit_dict + break diff --git a/seedfarmer/models/manifests/_module_manifest.py b/seedfarmer/models/manifests/_module_manifest.py index ac7e3fb9..afb9433d 100644 --- a/seedfarmer/models/manifests/_module_manifest.py +++ b/seedfarmer/models/manifests/_module_manifest.py @@ -71,9 +71,9 @@ class ModuleManifest(CamelModel): name: str path: str - bundle_md5: Optional[str] - manifest_md5: Optional[str] - deployspec_md5: Optional[str] + bundle_md5: Optional[str] = None + manifest_md5: Optional[str] = None + deployspec_md5: Optional[str] = None parameters: List[ModuleParameter] = [] deploy_spec: Optional[DeploySpec] = None target_account: Optional[str] = None diff --git a/setup.py b/setup.py index f7fa5f7c..08dee11b 100644 --- a/setup.py +++ b/setup.py @@ -45,12 +45,12 @@ keywords=["aws", "cdk"], python_requires=">=3.7,<3.12", install_requires=[ - "aws-codeseeder~=0.10.2", + "aws-codeseeder~=0.11.0", "cookiecutter~=2.1.0", "pyhumps~=3.5.0", - "pydantic~=1.10.0", + "pydantic~=2.5.3", "executor~=23.2", - "typing-extensions~=4.5.0", + "typing-extensions~=4.6.1", "rich~=12.4.0", "requests>=2.28,<2.32", "python-dotenv~=0.21.0", diff --git a/test/unit-test/test_cli_arg.py b/test/unit-test/test_cli_arg.py index 69352ab8..91c92db8 100644 --- a/test/unit-test/test_cli_arg.py +++ b/test/unit-test/test_cli_arg.py @@ -265,9 +265,7 @@ def test_apply_missing_deployment(): @pytest.mark.apply def test_apply_missing_deployment_group_name(): deployment_manifest = f"{_TEST_ROOT}/manifests/test-missing-deployment-group-name/deployment.yaml" - - result = _test_command(sub_command=apply, options=deployment_manifest, exit_code=1, return_result=True) - assert result.exception.args[0][0][0].exc.errors()[0]["msg"] == "none is not an allowed value" + _test_command(sub_command=apply, options=deployment_manifest, exit_code=1, return_result=True) @pytest.mark.apply diff --git a/test/unit-test/test_commands_parameters.py b/test/unit-test/test_commands_parameters.py index 0308fd13..2f409d3f 100644 --- a/test/unit-test/test_commands_parameters.py +++ b/test/unit-test/test_commands_parameters.py @@ -6,6 +6,7 @@ import pytest import yaml from moto import mock_sts +import pydantic_core import seedfarmer.commands._parameter_commands as pc import seedfarmer.errors @@ -256,9 +257,9 @@ def test_load_parameter_values_missing_param_value(session_manager, mocker): faulty_d["groups"][0]["modules"][0]["parameters"][4] = ( {"name": "test-regional-param", "value_from": {"parameterValue": "regParamMissing"}}, ) - dep = DeploymentManifest(**faulty_d) - dep.validate_and_set_module_defaults() - with pytest.raises(seedfarmer.errors.InvalidManifestError): + with pytest.raises(pydantic_core._pydantic_core.ValidationError): + dep = DeploymentManifest(**faulty_d) + dep.validate_and_set_module_defaults() pc.load_parameter_values( deployment_name="mlops", deployment_manifest=dep,