From c32a20441e1cb48f42a7dd489905aa3c8388a700 Mon Sep 17 00:00:00 2001 From: Yevhen Ivantsov Date: Mon, 23 Oct 2023 13:26:20 +1100 Subject: [PATCH] Get snapshots info from json --- .github/workflows/unit-test.yaml | 11 +- config.tfvars | 15 + dc-infrastructure.tf | 20 +- install.sh | 16 + locals.tf | 96 ++++++ modules/AWS/eks/variables.tf | 2 +- outputs.tf | 40 +++ test/dcapt-snapshots.json | 533 +++++++++++++++++++++++++++++++ test/unittest/snapshots_test.go | 136 ++++++++ variables.tf | 50 +++ 10 files changed, 905 insertions(+), 14 deletions(-) create mode 100644 test/dcapt-snapshots.json create mode 100644 test/unittest/snapshots_test.go diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/unit-test.yaml index 80ebf8bc..3f7c73da 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/unit-test.yaml @@ -20,7 +20,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - + - name: Setup Go environment uses: actions/setup-go@v3 with: @@ -48,5 +48,10 @@ jobs: role-duration-seconds: 7200 - name: Unit tests - working-directory: test/ - run: go test ./unittest/... -v \ No newline at end of file + working-directory: test/unittest + run: | + TESTS=$(go test -list . | grep -v 'ok\s\+github.com/atlassian-labs/data-center-terraform/test/unittest') + for test in ${TESTS[@]}; do + go test -run ${test} -v + rm -rf /tmp/* >/dev/null 2>&1 || true + done diff --git a/config.tfvars b/config.tfvars index 4aa54bdf..4afa3255 100644 --- a/config.tfvars +++ b/config.tfvars @@ -85,6 +85,9 @@ max_cluster_capacity = 5 # List of addtional namespaces to be created in the cluster #additional_namespaces = ["extra_namespace"] +# Path to a JSON file with EBS and RDS snapshot IDs +#snapshots_json_file_path = "test/dcapt-snapshots.json" + ################################################################################ # Osquery settings. Atlassian only! ################################################################################ @@ -215,6 +218,9 @@ jira_db_iops = 1000 # Set `null` if the snapshot does not have a default db name. jira_db_name = "jira" +# Dataset size. Used only when snapshots_json_file_path is defined. Defaults to large +# jira_dataset_size = "small" + # Database restore configuration # If you want to restore the database from a snapshot, uncomment the following line and provide the snapshot identifier. # This will restore the database from the snapshot and will not create a new database. @@ -310,6 +316,9 @@ confluence_db_iops = 1000 # Set `null` if the snapshot does not have a default db name. confluence_db_name = "confluence" +# Dataset size. Used only when snapshots_json_file_path is defined. Defaults to large +# confluence_dataset_size = "small" + # Database restore configuration # If you want to restore the database from a snapshot, uncomment the following lines and provide the snapshot identifier. # This will restore the database from the snapshot and will not create a new database. @@ -420,6 +429,9 @@ bitbucket_db_name = "bitbucket" #bitbucket_elasticsearch_storage = "" #bitbucket_elasticsearch_replicas = "" +# Dataset size. Used only when snapshots_json_file_path is defined. Defaults to large +# bitbucket_dataset_size = "small" + # Dataset Restore # Database restore configuration @@ -590,6 +602,9 @@ crowd_db_name = "crowd" # if you encounter such an issue. This will apply to Crowd pods. #crowd_termination_grace_period = 0 +# Dataset size. Used only when snapshots_json_file_path is defined. Defaults to large +# crowd_dataset_size = "small" + # Dataset Restore # Database restore configuration diff --git a/dc-infrastructure.tf b/dc-infrastructure.tf index 0ac0c22d..571cb571 100644 --- a/dc-infrastructure.tf +++ b/dc-infrastructure.tf @@ -116,7 +116,7 @@ module "jira" { db_instance_class = var.jira_db_instance_class db_iops = var.jira_db_iops db_name = var.jira_db_name - db_snapshot_id = var.jira_db_snapshot_id + db_snapshot_id = local.jira_rds_snapshot_id db_master_username = var.jira_db_master_username db_master_password = var.jira_db_master_password @@ -145,7 +145,7 @@ module "jira" { nfs_limits_cpu = var.jira_nfs_limits_cpu nfs_limits_memory = var.jira_nfs_limits_memory - shared_home_snapshot_id = var.jira_shared_home_snapshot_id + shared_home_snapshot_id = local.jira_ebs_snapshot_id # If local Helm charts path is provided, Terraform will then install using local charts and ignores remote registry local_jira_chart_path = local.local_jira_chart_path @@ -171,8 +171,8 @@ module "confluence" { db_name = var.confluence_db_name } - db_snapshot_id = var.confluence_db_snapshot_id - db_snapshot_build_number = var.confluence_db_snapshot_build_number + db_snapshot_id = local.confluence_rds_snapshot_id + db_snapshot_build_number = local.confluence_db_snapshot_build_number db_master_username = var.confluence_db_master_username db_master_password = var.confluence_db_master_password @@ -209,7 +209,7 @@ module "confluence" { nfs_limits_cpu = var.confluence_nfs_limits_cpu nfs_limits_memory = var.confluence_nfs_limits_memory - shared_home_snapshot_id = var.confluence_shared_home_snapshot_id + shared_home_snapshot_id = local.confluence_ebs_snapshot_id # If local Helm charts path is provided, Terraform will then install using local charts and ignores remote registry local_confluence_chart_path = local.local_confluence_chart_path @@ -230,7 +230,7 @@ module "bitbucket" { db_instance_class = var.bitbucket_db_instance_class db_iops = var.bitbucket_db_iops db_name = var.bitbucket_db_name - db_snapshot_id = var.bitbucket_db_snapshot_id + db_snapshot_id = local.bitbucket_rds_snapshot_id db_master_username = var.bitbucket_db_master_username db_master_password = var.bitbucket_db_master_password @@ -273,7 +273,7 @@ module "bitbucket" { elasticsearch_storage = var.bitbucket_elasticsearch_storage elasticsearch_replicas = var.bitbucket_elasticsearch_replicas - shared_home_snapshot_id = var.bitbucket_shared_home_snapshot_id + shared_home_snapshot_id = local.bitbucket_ebs_snapshot_id # If local Helm charts path is provided, Terraform will then install using local charts and ignores remote registry local_bitbucket_chart_path = local.local_bitbucket_chart_path @@ -296,8 +296,8 @@ module "crowd" { db_name = var.crowd_db_name db_master_username = var.crowd_db_master_username db_master_password = var.crowd_db_master_password - db_snapshot_id = var.crowd_db_snapshot_id - db_snapshot_build_number = var.crowd_db_snapshot_build_number + db_snapshot_id = local.crowd_rds_snapshot_id + db_snapshot_build_number = local.crowd_db_snapshot_build_number replica_count = var.crowd_replica_count installation_timeout = var.crowd_installation_timeout @@ -323,7 +323,7 @@ module "crowd" { nfs_limits_cpu = var.crowd_nfs_limits_cpu nfs_limits_memory = var.crowd_nfs_limits_memory - shared_home_snapshot_id = var.crowd_shared_home_snapshot_id + shared_home_snapshot_id = local.crowd_ebs_snapshot_id # If local Helm charts path is provided, Terraform will then install using local charts and ignores remote registry local_crowd_chart_path = local.local_crowd_chart_path diff --git a/install.sh b/install.sh index 16e755b4..7799d180 100755 --- a/install.sh +++ b/install.sh @@ -114,6 +114,10 @@ pre_flight_checks() { PRODUCT_VERSION=$(get_variable ${PRODUCT_VERSION_VAR} "${CONFIG_ABS_PATH}") MAJOR_MINOR_VERSION=$(echo "$PRODUCT_VERSION" | cut -d '.' -f1-2) EBS_SNAPSHOT_ID=$(get_variable ${SHARED_HOME_SNAPSHOT_VAR} "${CONFIG_ABS_PATH}") + SNAPSHOTS_JSON_FILE_PATH=$(get_variable 'snapshots_json_file_path' "${CONFIG_ABS_PATH}") + if [ "${SNAPSHOTS_JSON_FILE_PATH}" ]; then + EBS_SNAPSHOT_ID=$(cat ${SNAPSHOTS_JSON_FILE_PATH} | jq ".${PRODUCT}.versions[] | select(.version == \"${PRODUCT_VERSION}\") | .data[] | select(.size == \"large\" and .type == \"ebs\") | .snapshots[] | .[\"${REGION}\"]" | sed 's/"//g') + fi if [ ! -z ${EBS_SNAPSHOT_ID} ]; then log "Checking EBS snapshot ${EBS_SNAPSHOT_ID} compatibility with ${PRODUCT} version ${PRODUCT_VERSION}" EBS_SNAPSHOT_DESCRIPTION=$(aws ec2 describe-snapshots --snapshot-ids=${EBS_SNAPSHOT_ID} --region ${REGION} --query 'Snapshots[0].Description') @@ -147,6 +151,9 @@ pre_flight_checks() { fi fi RDS_SNAPSHOT_ID=$(get_variable ${RDS_SNAPSHOT_VAR} "${CONFIG_ABS_PATH}") + if [ "${SNAPSHOTS_JSON_FILE_PATH}" ]; then + RDS_SNAPSHOT_ID=$(cat ${SNAPSHOTS_JSON_FILE_PATH} | jq ".${PRODUCT}.versions[] | select(.version == \"${PRODUCT_VERSION}\") | .data[] | select(.size == \"large\" and .type == \"rds\") | .snapshots[] | .[\"${REGION}\"]" | sed 's/"//g') + fi if [ ! -z ${RDS_SNAPSHOT_ID} ]; then log "Checking RDS snapshot ${RDS_SNAPSHOT_ID} compatibility with ${PRODUCT} version ${PRODUCT_VERSION}" RDS_SNAPSHOT_VERSION=$(echo "${RDS_SNAPSHOT_ID}" | sed 's/.*dcapt-\(.*\)/\1/' | sed 's/-/./g' | cut -d '.' -f 2-) @@ -181,6 +188,15 @@ verify_configuration_file() { HAS_VALIDATION_ERR=1 fi + SNAPSHOTS_JSON_FILE_PATH=$(get_variable 'snapshots_json_file_path' "${CONFIG_ABS_PATH}") + if [ "${SNAPSHOTS_JSON_FILE_PATH}" ]; then + if [ ! -e "${SNAPSHOTS_JSON_FILE_PATH}" ]; then + log "Snapshots json file not found at ${SNAPSHOTS_JSON_FILE_PATH}" + log "Please make sure 'snapshots_json_file_path' in ${CONFIG_ABS_PATH} points to an existing valid json file" + HAS_VALIDATION_ERR=1 + fi + fi + if [ -n "${INVALID_CONTENT}" ]; then log "Configuration file '${CONFIG_ABS_PATH##*/}' is not valid." "ERROR" log "Terraform uses this file to generate customised infrastructure for '${ENVIRONMENT_NAME}' on your AWS account." diff --git a/locals.tf b/locals.tf index 68bb7bbd..75f4a2cc 100644 --- a/locals.tf +++ b/locals.tf @@ -17,4 +17,100 @@ locals { local_bamboo_chart_path = var.local_helm_charts_path != "" && var.bamboo_install_local_chart ? "${var.local_helm_charts_path}/bamboo" : "" local_agent_chart_path = var.local_helm_charts_path != "" && var.bamboo_install_local_chart ? "${var.local_helm_charts_path}/bamboo-agent" : "" local_crowd_chart_path = var.local_helm_charts_path != "" && var.crowd_install_local_chart ? "${var.local_helm_charts_path}/crowd" : "" + + # snapshots_json = var.snapshots_json_file_path != "" ? jsondecode(file(var.snapshots_json_file_path)) : jsondecode("{\"jira\": {\"versions\":[]},\"confluence\": {\"versions\":[]},\"crowd\": {\"versions\":[]},\"bitbucket\": {\"versions\":[]}}") + snapshots_json = var.snapshots_json_file_path != "" ? jsondecode(file(var.snapshots_json_file_path)) : null + + filtered_bitbucket_snapshots = local.snapshots_json != null ? flatten([ + for version in local.snapshots_json.bitbucket.versions : + [for snapshot in version.data : + merge({ version = version.version }, snapshot) + ] + ]) : [] + + bitbucket_rds_snapshot = local.snapshots_json != null ? [ + for snapshot in local.filtered_bitbucket_snapshots : + snapshot.type == "rds" && snapshot.size == var.bitbucket_dataset_size && snapshot.version == var.bitbucket_version_tag ? snapshot.snapshots[0][var.region] : "" + ] : [] + + bitbucket_ebs_snapshot = local.snapshots_json != null ? [ + for snapshot in local.filtered_bitbucket_snapshots : + snapshot.type == "ebs" && snapshot.size == var.bitbucket_dataset_size && snapshot.version == var.bitbucket_version_tag ? snapshot.snapshots[0][var.region] : "" + ] : [] + + filtered_confluence_snapshots = local.snapshots_json != null ? flatten([ + for version in local.snapshots_json.confluence.versions : + [for snapshot in version.data : + merge({ version = version.version }, snapshot) + ] + ]) : [] + + confluence_rds_snapshot = local.snapshots_json != null ? [ + for snapshot in local.filtered_confluence_snapshots : + snapshot.type == "rds" && snapshot.size == var.confluence_dataset_size && snapshot.version == var.confluence_version_tag ? snapshot.snapshots[0][var.region] : "" + ] : [] + + confluence_ebs_snapshot = local.snapshots_json != null ? [ + for snapshot in local.filtered_confluence_snapshots : + snapshot.type == "ebs" && snapshot.size == var.confluence_dataset_size && snapshot.version == var.confluence_version_tag ? snapshot.snapshots[0][var.region] : "" + ] : [] + + confluence_build_numbers = local.snapshots_json != null ? flatten([ + for version in local.snapshots_json.confluence.versions : + version.version == var.confluence_version_tag ? version.build_number : "" + ]) : [] + + filtered_crowd_snapshots = local.snapshots_json != null ? flatten([ + for version in local.snapshots_json.crowd.versions : + [for snapshot in version.data : + merge({ version = version.version }, snapshot) + ] + ]) : [] + + crowd_rds_snapshot = local.snapshots_json != null ? [ + for snapshot in local.filtered_crowd_snapshots : + snapshot.type == "rds" && snapshot.size == var.crowd_dataset_size && snapshot.version == var.crowd_version_tag ? snapshot.snapshots[0][var.region] : "" + ] : [] + + crowd_ebs_snapshot = local.snapshots_json != null ? [ + for snapshot in local.filtered_crowd_snapshots : + snapshot.type == "ebs" && snapshot.size == var.crowd_dataset_size && snapshot.version == var.crowd_version_tag ? snapshot.snapshots[0][var.region] : "" + ] : [] + + crowd_build_numbers = local.snapshots_json != null ? flatten([ + for version in local.snapshots_json.crowd.versions : + version.version == var.crowd_version_tag ? version.build_number : "" + ]) : [] + + filtered_jira_snapshots = local.snapshots_json != null ? flatten([ + for version in local.snapshots_json.jira.versions : + [for snapshot in version.data : + merge({ version = version.version }, snapshot) + ] + ]) : [] + + jira_rds_snapshot = local.snapshots_json != null ? [ + for snapshot in local.filtered_jira_snapshots : + snapshot.type == "rds" && snapshot.size == var.jira_dataset_size && snapshot.version == var.jira_version_tag ? snapshot.snapshots[0][var.region] : "" + ] : [] + + jira_ebs_snapshot = local.snapshots_json != null ? [ + for snapshot in local.filtered_jira_snapshots : + snapshot.type == "ebs" && snapshot.size == var.jira_dataset_size && snapshot.version == var.jira_version_tag ? snapshot.snapshots[0][var.region] : "" + ] : [] + + jira_rds_snapshot_id = length(compact(local.jira_rds_snapshot)) > 0 ? compact(local.jira_rds_snapshot)[0] : var.jira_db_snapshot_id != null ? var.jira_db_snapshot_id : null + jira_ebs_snapshot_id = length(compact(local.jira_ebs_snapshot)) > 0 ? compact(local.jira_ebs_snapshot)[0] : var.jira_shared_home_snapshot_id != null ? var.jira_shared_home_snapshot_id : null + + confluence_rds_snapshot_id = length(compact(local.confluence_rds_snapshot)) > 0 ? compact(local.confluence_rds_snapshot)[0] : var.confluence_db_snapshot_id != null ? var.confluence_db_snapshot_id : null + confluence_ebs_snapshot_id = length(compact(local.confluence_ebs_snapshot)) > 0 ? compact(local.confluence_ebs_snapshot)[0] : var.confluence_shared_home_snapshot_id != null ? var.confluence_shared_home_snapshot_id : null + confluence_db_snapshot_build_number = length(compact(local.confluence_build_numbers)) > 0 ? compact(local.confluence_build_numbers)[0] : var.confluence_db_snapshot_build_number != null ? var.confluence_db_snapshot_build_number : null + + crowd_rds_snapshot_id = length(compact(local.crowd_rds_snapshot)) > 0 ? compact(local.crowd_rds_snapshot)[0] : var.crowd_db_snapshot_id != null ? var.crowd_db_snapshot_id : null + crowd_ebs_snapshot_id = length(compact(local.crowd_ebs_snapshot)) > 0 ? compact(local.crowd_ebs_snapshot)[0] : var.crowd_shared_home_snapshot_id != null ? var.crowd_shared_home_snapshot_id : null + crowd_db_snapshot_build_number = length(compact(local.crowd_build_numbers)) > 0 ? compact(local.crowd_build_numbers)[0] : var.crowd_db_snapshot_build_number != null ? var.crowd_db_snapshot_build_number : null + + bitbucket_rds_snapshot_id = length(compact(local.bitbucket_rds_snapshot)) > 0 ? compact(local.bitbucket_rds_snapshot)[0] : var.bitbucket_db_snapshot_id != null ? var.bitbucket_db_snapshot_id : null + bitbucket_ebs_snapshot_id = length(compact(local.bitbucket_ebs_snapshot)) > 0 ? compact(local.bitbucket_ebs_snapshot)[0] : var.bitbucket_shared_home_snapshot_id != null ? var.bitbucket_shared_home_snapshot_id : null + } diff --git a/modules/AWS/eks/variables.tf b/modules/AWS/eks/variables.tf index 8eb7f1df..6e428272 100644 --- a/modules/AWS/eks/variables.tf +++ b/modules/AWS/eks/variables.tf @@ -115,7 +115,7 @@ variable "osquery_fleet_enrollment_host" { variable "kinesis_log_producers_role_arns" { description = "AWS kinesis log producer role" - type = object({ + type = object({ eu = string non-eu = string }) diff --git a/outputs.tf b/outputs.tf index 917777c1..1c6aebe1 100644 --- a/outputs.tf +++ b/outputs.tf @@ -111,3 +111,43 @@ output "confluence_s3_bucket" { description = "Confluence S3 bucket name" value = var.confluence_s3_attachments_storage ? "${local.cluster_name}-confluence-storage" : null } + +output "jira_rds_snapshot" { + value = local.jira_rds_snapshot_id +} + +output "jira_ebs_snapshot" { + value = local.jira_ebs_snapshot_id +} + +output "confluence_rds_snapshot" { + value = local.confluence_rds_snapshot_id +} + +output "confluence_ebs_snapshot" { + value = local.confluence_ebs_snapshot_id +} + +output "confluence_db_snapshot_build_number" { + value = local.confluence_db_snapshot_build_number +} + +output "bitbucket_rds_snapshot" { + value = local.bitbucket_rds_snapshot_id +} + +output "bitbucket_ebs_snapshot" { + value = local.bitbucket_ebs_snapshot_id +} + +output "crowd_rds_snapshot" { + value = local.crowd_rds_snapshot_id +} + +output "crowd_ebs_snapshot" { + value = local.crowd_ebs_snapshot_id +} + +output "crowd_db_snapshot_build_number" { + value = local.crowd_db_snapshot_build_number +} diff --git a/test/dcapt-snapshots.json b/test/dcapt-snapshots.json new file mode 100644 index 00000000..9a72bb13 --- /dev/null +++ b/test/dcapt-snapshots.json @@ -0,0 +1,533 @@ +{ + "jira": { + "versions": [ + { + "version": "9.4.8", + "data": [ + { + "type": "rds", + "size": "large", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-jira-9-4-8", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-jira-9-4-8", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-jira-9-4-8", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-jira-9-4-8" + } + ] + }, + { + "type": "rds", + "size": "small", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-jira-small-9-4-8", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-jira-small-9-4-8", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-jira-small-9-4-8", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-jira-small-9-4-8" + } + ] + }, + { + "type": "ebs", + "size": "large", + "snapshots": [ + { + "us-east-1": "snap-0640210f62a262aaf", + "us-east-2": "snap-0d619095feaa2eca5", + "us-west-1": "snap-00f6a0fc8ba4c4cce", + "us-west-2": "snap-0d23a05be5f527030" + } + ] + }, + { + "type": "ebs", + "size": "small", + "snapshots": [ + { + "us-east-1": "snap-05a61b57dbb4f9834", + "us-east-2": "snap-0005a8c3cc297b294", + "us-west-1": "snap-0dfb346bb01f4709a", + "us-west-2": "snap-0c17be9ae98bbd1ed" + } + ] + } + ] + }, + { + "version": "8.20.24", + "data": [ + { + "type": "rds", + "size": "large", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-jira-8-20-24", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-jira-8-20-24", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-jira-8-20-24", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-jira-8-20-24" + } + ] + }, + { + "type": "rds", + "size": "small", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-jira-small-8-20-24", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-jira-small-8-20-24", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-jira-small-8-20-24", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-jira-small-8-20-24" + } + ] + }, + { + "type": "ebs", + "size": "large", + "snapshots": [ + { + "us-east-1": "snap-029edbed07ab594e0", + "us-east-2": "snap-0b5f4473954e6d959", + "us-west-1": "snap-0ffaa992ba449a53d", + "us-west-2": "snap-01e69efc1d8943038" + } + ] + }, + { + "type": "ebs", + "size": "small", + "snapshots": [ + { + "us-east-1": "snap-083d2d8f5797f907e", + "us-east-2": "snap-0c3cb60ddc50c1136", + "us-west-1": "snap-07de609e058d28a03", + "us-west-2": "snap-01fa045458071eda5" + } + ] + } + ] + } + ] + }, + "jsm": { + "versions": [ + { + "version": "5.4.8", + "data": [ + { + "type": "rds", + "size": "large", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-jsm-5-4-8", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-jsm-5-4-8", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-jsm-5-4-8", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-jsm-5-4-8" + } + ] + }, + { + "type": "rds", + "size": "small", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-jsm-small-5-4-8", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-jsm-small-5-4-8", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-jsm-small-5-4-8", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-jsm-small-5-4-8" + } + ] + }, + { + "type": "ebs", + "size": "large", + "snapshots": [ + { + "us-east-1": "snap-0a13271b63872a2a6", + "us-east-2": "snap-0fb58e8d005edeb32", + "us-west-1": "snap-05d6aa53717fb3c6c", + "us-west-2": "snap-043842d9319f25659" + } + ] + }, + { + "type": "ebs", + "size": "small", + "snapshots": [ + { + "us-east-1": "snap-08bc969471d0ee11f", + "us-east-2": "snap-02f299ef7f1f524b2", + "us-west-1": "snap-0fa5b8bd27f66e6c3", + "us-west-2": "snap-063211f90e6d81bbd" + } + ] + } + ] + }, + { + "version": "4.20.24", + "data": [ + { + "type": "rds", + "size": "large", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-jsm-4-20-24", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-jsm-4-20-24", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-jsm-4-20-24", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-jsm-4-20-24" + } + ] + }, + { + "type": "rds", + "size": "small", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-jsm-small-4-20-24", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-jsm-small-4-20-24", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-jsm-small-4-20-24", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-jsm-small-4-20-24" + } + ] + }, + { + "type": "ebs", + "size": "large", + "snapshots": [ + { + "us-east-1": "snap-0c95405b316f28ec8", + "us-east-2": "snap-0cae5febc8127250b", + "us-west-1": "snap-081f548dda005c97e", + "us-west-2": "snap-08e23754ddc402ec4" + } + ] + }, + { + "type": "ebs", + "size": "small", + "snapshots": [ + { + "us-east-1": "snap-0d933d20b989beb7b", + "us-east-2": "snap-0971e128b8d1d2af9", + "us-west-1": "snap-0c6d8b6aa53b93e78", + "us-west-2": "snap-0734518fb7d55f7ce" + } + ] + } + ] + } + ] + }, + "confluence": { + "versions": [ + { + "version": "7.19.11", + "build_number": "8804", + "data": [ + { + "type": "rds", + "size": "large", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-confluence-7-19-11", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-confluence-7-19-11", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-confluence-7-19-11", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-confluence-7-19-11" + } + ] + }, + { + "type": "rds", + "size": "small", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-confluence-small-7-19-11", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-confluence-small-7-19-11", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-confluence-small-7-19-11", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-confluence-small-7-19-11" + } + ] + }, + { + "type": "ebs", + "size": "large", + "snapshots": [ + { + "us-east-1": "snap-03b3a8541b7466ec3", + "us-east-2": "snap-09365c581a158a979", + "us-west-1": "snap-01bc9fdb49bc6641e", + "us-west-2": "snap-061919924738ea4c3" + } + ] + }, + { + "type": "ebs", + "size": "small", + "snapshots": [ + { + "us-east-1": "snap-0dad75f94da7f317b", + "us-east-2": "snap-00ede7dca448a6243", + "us-west-1": "snap-0f72ad2146e3a19c3", + "us-west-2": "snap-09ff2c4be549518a0" + } + ] + } + ] + }, + { + "version": "7.13.18", + "build_number": "8703", + "data": [ + { + "type": "rds", + "size": "large", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-confluence-7-13-18", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-confluence-7-13-18", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-confluence-7-13-18", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-confluence-7-13-18" + } + ] + }, + { + "type": "rds", + "size": "small", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-confluence-small-7-13-18", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-confluence-small-7-13-18", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-confluence-small-7-13-18", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-confluence-small-7-13-18" + } + ] + }, + { + "type": "ebs", + "size": "large", + "snapshots": [ + { + "us-east-1": "snap-08156f8bb0099942f", + "us-east-2": "snap-04cc3d8455b1ef6e9", + "us-west-1": "snap-039e3a985cf126fc0", + "us-west-2": "snap-0ce8a0947cd581752" + } + ] + }, + { + "type": "ebs", + "size": "small", + "snapshots": [ + { + "us-east-1": "snap-01df9653b5c8f9f64", + "us-east-2": "snap-055811dae848f13ae", + "us-west-1": "snap-058e37561a1cce3e9", + "us-west-2": "snap-0bb261f0b3266d136" + } + ] + } + ] + }, + { + "version": "8.5.1", + "build_number": "9012", + "data": [ + { + "type": "rds", + "size": "large", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-confluence-8-5-1", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-confluence-8-5-1", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-confluence-8-5-1", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-confluence-8-5-1" + } + ] + }, + { + "type": "rds", + "size": "small", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-confluence-small-8-5-1", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-confluence-small-8-5-1", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-confluence-small-8-5-1", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-confluence-small-8-5-1" + } + ] + }, + { + "type": "ebs", + "size": "large", + "snapshots": [ + { + "us-east-1": "snap-099a9ce9e9c902815", + "us-east-2": "snap-074a2fdca0497b6b6", + "us-west-1": "snap-01b07710d49b113b2", + "us-west-2": "snap-031dad82fa7367921" + } + ] + }, + { + "type": "ebs", + "size": "small", + "snapshots": [ + { + "us-east-1": "snap-09d97f19261bd463e", + "us-east-2": "snap-008cc496f440198de", + "us-west-1": "snap-02b3a229b530c8a87", + "us-west-2": "snap-01d8ebf9701613c4c" + } + ] + } + ] + } + ] + }, + "bitbucket": { + "versions": [ + { + "version": "7.21.14", + "data": [ + { + "type": "rds", + "size": "large", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-bitbucket-7-21-14", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-bitbucket-7-21-14", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-bitbucket-7-21-14", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-bitbucket-7-21-14" + } + ] + }, + { + "type": "rds", + "size": "small", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-bitbucket-small-7-21-14", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-bitbucket-small-7-21-14", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-bitbucket-small-7-21-14", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-bitbucket-small-7-21-14" + } + ] + }, + { + "type": "ebs", + "size": "large", + "snapshots": [ + { + "us-east-1": "snap-01873c2840b4dd3c3", + "us-east-2": "snap-0ccb8c3d34ff171f1", + "us-west-1": "snap-0cde4bd0ed0358d0e", + "us-west-2": "snap-0f8b60b668f3bbb66" + } + ] + }, + { + "type": "ebs", + "size": "small", + "snapshots": [ + { + "us-east-1": "snap-0a869d88cb2829bc4", + "us-east-2": "snap-03893c494ba7edcf4", + "us-west-1": "snap-020733b644be01f62", + "us-west-2": "snap-07e784e26a282e18c" + } + ] + } + ] + }, + { + "version": "8.9.2", + "data": [ + { + "type": "rds", + "size": "large", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-bitbucket-8-9-2", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-bitbucket-8-9-2", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-bitbucket-8-9-2", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-bitbucket-8-9-2" + } + ] + }, + { + "type": "rds", + "size": "small", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-bitbucket-small-8-9-2", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-bitbucket-small-8-9-2", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-bitbucket-small-8-9-2", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-bitbucket-small-8-9-2" + } + ] + }, + { + "type": "ebs", + "size": "large", + "snapshots": [ + { + "us-east-1": "snap-0c5d82e754dd8f536", + "us-east-2": "snap-0d933f3d40000e877", + "us-west-1": "snap-039620171723e6e2c", + "us-west-2": "snap-01a121265b13ee758" + } + ] + }, + { + "type": "ebs", + "size": "small", + "snapshots": [ + { + "us-east-1": "snap-039acd2608c3bce3f", + "us-east-2": "snap-0fb8cd6bf387057c0", + "us-west-1": "snap-012dc94feaaa30490", + "us-west-2": "snap-0a0f1b03ec0e8bf36" + } + ] + } + ] + } + ] + }, + "crowd": { + "versions": [ + { + "version": "5.1.4", + "build_number": "1893", + "data": [ + { + "type": "rds", + "size": "large", + "snapshots": [ + { + "us-east-1": "arn:aws:rds:us-east-1:585036043680:snapshot:dcapt-crowd-5-1-4", + "us-east-2": "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-crowd-5-1-4", + "us-west-1": "arn:aws:rds:us-west-1:585036043680:snapshot:dcapt-crowd-5-1-4", + "us-west-2": "arn:aws:rds:us-west-2:585036043680:snapshot:dcapt-crowd-5-1-4" + } + ] + }, + { + "type": "ebs", + "size": "large", + "snapshots": [ + { + "us-east-1": "snap-08c6dc6abc75fe2b4", + "us-east-2": "snap-0a8e229690be9ae30", + "us-west-1": "snap-0206b022c6880fe67", + "us-west-2": "snap-07a9b523b316aeb32" + } + ] + } + ] + } + ] + } +} + diff --git a/test/unittest/snapshots_test.go b/test/unittest/snapshots_test.go new file mode 100644 index 00000000..74ee6a1a --- /dev/null +++ b/test/unittest/snapshots_test.go @@ -0,0 +1,136 @@ +package unittest + +import ( + "github.com/gruntwork-io/terratest/modules/aws" + "github.com/gruntwork-io/terratest/modules/terraform" + testStructure "github.com/gruntwork-io/terratest/modules/test-structure" + "github.com/stretchr/testify/assert" + "path/filepath" + "testing" +) + +type DCSnapshots struct { + JiraEbs string + JiraRds string + ConfluenceEbs string + ConfluenceRds string + BitbucketEbs string + BitbucketRds string + CrowdEbs string + CrowdRds string +} + +func TestSnapshotsFromJson(t *testing.T) { + t.Parallel() + + // rather than parse the ../dcapt-snapshots.json, snap ids are copied from it + dcSnapshots := DCSnapshots{ + JiraEbs: "snap-0d619095feaa2eca5", + JiraRds: "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-jira-9-4-8", + ConfluenceEbs: "snap-04cc3d8455b1ef6e9", + ConfluenceRds: "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-confluence-7-13-18", + BitbucketEbs: "snap-0ccb8c3d34ff171f1", + BitbucketRds: "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-bitbucket-7-21-14", + CrowdEbs: "snap-0a8e229690be9ae30", + CrowdRds: "arn:aws:rds:us-east-2:585036043680:snapshot:dcapt-crowd-5-1-4", + } + + vars := map[string]interface{}{ + "environment_name": "e2etests", + "snapshots_json_file_path": "test/dcapt-snapshots.json", + "products": []string{"jira", "confluence", "bitbucket", "crowd"}, + "region": "us-east-2", + "jira_version_tag": "9.4.8", + "jira_license": "license", + "jira_db_master_username": "atljira", + "jira_db_master_password": "Password1!", + "confluence_license": "license", + "confluence_version_tag": "7.13.18", + "confluence_db_master_username": "atlconfluence", + "confluence_db_master_password": "Password1!", + "bitbucket_license": "license", + "bitbucket_version_tag": "7.21.14", + "bitbucket_admin_username": "admin", + "bitbucket_admin_password": "admin", + "bitbucket_admin_display_name": "admin", + "bitbucket_admin_email_address": "admin@example.com", + "bitbucket_db_master_username": "atlbitbucket", + "bitbucket_db_master_password": "Password1!", + "crowd_license": "license", + "crowd_version_tag": "5.1.4", + "crowd_db_master_username": "atlcrowd", + "crowd_db_master_password": "Password1!", + "bamboo_license": "bamboo-license", + "bamboo_version_tag": "9.2.3", + "bamboo_agent_version_tag": "9.2.3", + "bamboo_admin_username": "admin", + "bamboo_admin_password": "admin", + "bamboo_admin_display_name": "admin", + "bamboo_admin_email_address": "admin@example.com", + "bamboo_dataset_url": "https://centaurus-datasets.s3.amazonaws.com/bamboo/dcapt-bamboo.zip", + } + + // a bit of copy-paste as we can't use GenerateTFOptions as is (we need to run terraform plan in the root of the directory) + exampleFolder := testStructure.CopyTerraformFolderToTemp(t, "../..", "") + awsRegion := aws.GetRandomStableRegion(t, nil, nil) + planFilePath := filepath.Join(exampleFolder, "plan.out") + tfOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ + TerraformDir: exampleFolder, + Vars: vars, + EnvVars: map[string]string{ + "AWS_DEFAULT_REGION": awsRegion, + }, + PlanFilePath: planFilePath, + }) + + plan, _ := terraform.InitAndPlanAndShowWithStructE(t, tfOptions) + + // assert that pre-install jobs are created (rds snapshot id is not null) + confluencePreInstallJob := plan.ResourcePlannedValuesMap["module.confluence[0].kubernetes_job.pre_install[0]"] + crowdPreInstallJob := plan.ResourcePlannedValuesMap["module.crowd[0].kubernetes_job.pre_install[0]"] + bitbucketPreInstallJob := plan.ResourcePlannedValuesMap["module.bitbucket[0].kubernetes_job.pre_install[0]"] + jiraPreInstallJob := plan.ResourcePlannedValuesMap["module.jira[0].kubernetes_job.pre_install[0]"] + + assert.NotNil(t, confluencePreInstallJob) + assert.NotNil(t, crowdPreInstallJob) + assert.NotNil(t, bitbucketPreInstallJob) + assert.NotNil(t, jiraPreInstallJob) + + // assert that snapshot IDs are in outputs, i.e. they have been correctly extracted from + // ../dcapt-snapshots.json. Additionally, Confluence and Crowd have build_numbers + assert.Equal(t, dcSnapshots.BitbucketEbs, plan.RawPlan.PlannedValues.Outputs["bitbucket_ebs_snapshot"].Value) + assert.Equal(t, dcSnapshots.BitbucketRds, plan.RawPlan.PlannedValues.Outputs["bitbucket_rds_snapshot"].Value) + + assert.Equal(t, dcSnapshots.JiraEbs, plan.RawPlan.PlannedValues.Outputs["jira_ebs_snapshot"].Value) + assert.Equal(t, dcSnapshots.JiraRds, plan.RawPlan.PlannedValues.Outputs["jira_rds_snapshot"].Value) + + assert.Equal(t, dcSnapshots.ConfluenceEbs, plan.RawPlan.PlannedValues.Outputs["confluence_ebs_snapshot"].Value) + assert.Equal(t, dcSnapshots.ConfluenceRds, plan.RawPlan.PlannedValues.Outputs["confluence_rds_snapshot"].Value) + assert.Equal(t, "8703", plan.RawPlan.PlannedValues.Outputs["confluence_db_snapshot_build_number"].Value) + + assert.Equal(t, dcSnapshots.CrowdEbs, plan.RawPlan.PlannedValues.Outputs["crowd_ebs_snapshot"].Value) + assert.Equal(t, dcSnapshots.CrowdRds, plan.RawPlan.PlannedValues.Outputs["crowd_rds_snapshot"].Value) + assert.Equal(t, "1893", plan.RawPlan.PlannedValues.Outputs["crowd_db_snapshot_build_number"].Value) + + // assert that RDS snapshots are in db_snapshot data + confluenceRdsSnapShot := plan.ResourcePlannedValuesMap["module.confluence[0].module.database.data.aws_db_snapshot.confluence_db_snapshot[0]"].AttributeValues["db_snapshot_identifier"] + bitbucketRdsSnapShot := plan.ResourcePlannedValuesMap["module.bitbucket[0].module.database.data.aws_db_snapshot.confluence_db_snapshot[0]"].AttributeValues["db_snapshot_identifier"] + crowdRdsSnapShot := plan.ResourcePlannedValuesMap["module.crowd[0].module.database.data.aws_db_snapshot.confluence_db_snapshot[0]"].AttributeValues["db_snapshot_identifier"] + jiraRdsSnapShot := plan.ResourcePlannedValuesMap["module.jira[0].module.database.data.aws_db_snapshot.confluence_db_snapshot[0]"].AttributeValues["db_snapshot_identifier"] + + assert.Equal(t, dcSnapshots.ConfluenceRds, confluenceRdsSnapShot) + assert.Equal(t, dcSnapshots.BitbucketRds, bitbucketRdsSnapShot) + assert.Equal(t, dcSnapshots.CrowdRds, crowdRdsSnapShot) + assert.Equal(t, dcSnapshots.JiraRds, jiraRdsSnapShot) + + // assert ebs snapshot is in ebs_volume aws resource + bitbucketEbsVolumeSnapshot := plan.ResourcePlannedValuesMap["module.bitbucket[0].module.nfs.aws_ebs_volume.shared_home"].AttributeValues["snapshot_id"] + jiraEbsVolumeSnapshot := plan.ResourcePlannedValuesMap["module.jira[0].module.nfs.aws_ebs_volume.shared_home"].AttributeValues["snapshot_id"] + confluenceEbsVolumeSnapshot := plan.ResourcePlannedValuesMap["module.confluence[0].module.nfs.aws_ebs_volume.shared_home"].AttributeValues["snapshot_id"] + crowdEbsVolumeSnapshot := plan.ResourcePlannedValuesMap["module.crowd[0].module.nfs.aws_ebs_volume.shared_home"].AttributeValues["snapshot_id"] + + assert.Equal(t, dcSnapshots.BitbucketEbs, bitbucketEbsVolumeSnapshot) + assert.Equal(t, dcSnapshots.JiraEbs, jiraEbsVolumeSnapshot) + assert.Equal(t, dcSnapshots.ConfluenceEbs, confluenceEbsVolumeSnapshot) + assert.Equal(t, dcSnapshots.CrowdEbs, crowdEbsVolumeSnapshot) +} diff --git a/variables.tf b/variables.tf index a30a7001..e5604e70 100644 --- a/variables.tf +++ b/variables.tf @@ -198,6 +198,16 @@ variable "monitoring_custom_values_file" { default = "" } +################################################################################ +# Snapshot Settings +################################################################################ + +variable "snapshots_json_file_path" { + description = "Absolute path to dcapt-snapshots.json" + type = string + default = "" +} + ################################################################################ # Jira Settings ################################################################################ @@ -394,6 +404,16 @@ variable "jira_install_local_chart" { type = bool } +variable "jira_dataset_size" { + description = "The size of the dataset to restore from" + type = string + default = "large" + validation { + condition = var.jira_dataset_size == null || can(regex("large|small", var.jira_dataset_size)) + error_message = "Invalid dataset size. Expected values are: small, large" + } +} + ################################################################################ # Confluence variables ################################################################################ @@ -628,6 +648,16 @@ variable "confluence_shared_home_snapshot_id" { default = null } +variable "confluence_dataset_size" { + description = "The size of the dataset to restore from" + type = string + default = "large" + validation { + condition = var.confluence_dataset_size == null || can(regex("large|small", var.confluence_dataset_size)) + error_message = "Invalid dataset size. Expected values are: small, large" + } +} + ################################################################################ # Bitbucket Variables ################################################################################ @@ -874,6 +904,16 @@ variable "bitbucket_install_local_chart" { type = bool } +variable "bitbucket_dataset_size" { + description = "The size of the dataset to restore from" + type = string + default = "large" + validation { + condition = var.bitbucket_dataset_size == null || can(regex("large|small", var.bitbucket_dataset_size)) + error_message = "Invalid dataset size. Expected values are: small, large" + } +} + ################################################################################ # Bamboo Variables ################################################################################ @@ -1312,6 +1352,16 @@ variable "crowd_license" { default = "" } +variable "crowd_dataset_size" { + description = "The size of the dataset to restore from" + type = string + default = "large" + validation { + condition = var.crowd_dataset_size == null || can(regex("large|small", var.crowd_dataset_size)) + error_message = "Invalid dataset size. Expected values are: small, large" + } +} + variable "confluence_s3_attachments_storage" { description = "Use S3 as attachment storage" type = bool