From 3f707692ed39a843c1d21966636c24190d8e7427 Mon Sep 17 00:00:00 2001 From: Dave Osment Date: Fri, 27 Jan 2017 12:04:06 +0000 Subject: [PATCH 1/8] Created new function to offsite SnapShots based upon presence of 'DestinationRegion' Tag; if the Tag is present, it must be a valid AWS Region which will trigger replication of the SnapShot to the specified Region, and then delete the SnapShot from the Source Region in any subsequent re-run after the copy has completed successfully. All Tags on the Source SnapShot will be applied to the copy. --- ebs-snapshot-offsiter.py | 114 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 ebs-snapshot-offsiter.py diff --git a/ebs-snapshot-offsiter.py b/ebs-snapshot-offsiter.py new file mode 100644 index 0000000..6805191 --- /dev/null +++ b/ebs-snapshot-offsiter.py @@ -0,0 +1,114 @@ +import boto3 +import collections +import datetime +import os + +ec = boto3.client('ec2') + +def lambda_handler(event, context): + + # Get Current Region + aws_region = os.getenv('AWS_REGION') + + # Get All SnapShots With 'Offsite' Tag Set + snapshots = ec.describe_snapshots( + Filters=[ + { 'Name': 'tag-key', 'Values': ['DestinationRegion'] }, + ] + ) + + for snapshot in snapshots['Snapshots']: + + # Reset Our Destination Region + destination_region = None + + # Obtain Tags From Source SnapShot + for tag in snapshot['Tags']: + + # Obtain Destination Region From Source Snapshot Tag + if tag['Key'] == 'DestinationRegion': + destination_region = tag['Value'] + + # Check If We Need To Do A Copy Or Not + if destination_region == aws_region: + + print "\tDestination Region %s is the same as current region (%s) - skipping copy" % ( + destination_region, + aws_region + ) + + continue + + # Construct ec2 Client For Secondary Region + secondary_ec = boto3.client('ec2', region_name=destination_region) + + # Determine If There's An Off-Site Copy Of This SnapShot + os_snapshots = secondary_ec.describe_snapshots( + Filters=[ + { 'Name': 'tag:SourceSnapshotId', 'Values': [snapshot['SnapshotId']] }, + { 'Name': 'status', 'Values': ['pending','completed'] }, + ] + ) + + # We Only Want To Delete Where Snapshot Has Copied Successfully + if len(os_snapshots['Snapshots']) >= 1: + + snapshot_states = [d['State'] for d in os_snapshots['Snapshots']] + + if 'pending' in snapshot_states: + print "\tThere is at least 1 Snapshot copy pending in %s - skipping delete & copy" % ( + destination_region + ) + + continue + + print "\t\tFound a corresponding Snapshot with Id %s in %s created from Snapshot %s" % ( + os_snapshots['Snapshots'][0]['SnapshotId'], + destination_region, + snapshot['SnapshotId'] + ) + + print "Deleting source Snapshot %s from %s" % ( + snapshot['SnapshotId'], + aws_region + ) + + ec.delete_snapshot( + DryRun=True, + SnapshotId=snapshot['SnapshotId'] + ) + + continue + + # Create Copy Of Snapshot Because One Doesn't Exist + os_snapshot = secondary_ec.copy_snapshot( + SourceRegion=aws_region, + SourceSnapshotId=snapshot['SnapshotId'], + Description=snapshot['Description'], + DestinationRegion=destination_region + ) + + # If Snapshot Copy Executed Successfully, Copy The Tags + if (os_snapshot): + + print "\t\tSnapshot copy %s created in %s of %s from %s" % ( + os_snapshot['SnapshotId'], + destination_region, + snapshot['SnapshotId'], + aws_region + ) + + # Add Tags To Off-Site SnapShot + destination_snapshot_tags = snapshot['Tags'] + [{ 'Key': 'SourceSnapshotId', 'Value': snapshot['SnapshotId'] }] + secondary_ec.create_tags( + Resources=[os_snapshot['SnapshotId']], + Tags=destination_snapshot_tags + ) + + # Add Tags To Source SnapShot + ec.create_tags( + Resources=[snapshot['SnapshotId']], + Tags=[ + { 'Key': 'OffsiteSnapshotId', 'Value': os_snapshot['SnapshotId'] }, + ] + ) \ No newline at end of file From 8f38cda6d1e4bf2584ba06b2839b4867b875d9af Mon Sep 17 00:00:00 2001 From: Dave Osment Date: Fri, 27 Jan 2017 13:14:22 +0000 Subject: [PATCH 2/8] 1. Removed DryRun from delete_snapshot 2. Excluded all source SnapShots not in yet in a 'completed' state --- ebs-snapshot-offsiter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ebs-snapshot-offsiter.py b/ebs-snapshot-offsiter.py index 6805191..676be63 100644 --- a/ebs-snapshot-offsiter.py +++ b/ebs-snapshot-offsiter.py @@ -14,6 +14,7 @@ def lambda_handler(event, context): snapshots = ec.describe_snapshots( Filters=[ { 'Name': 'tag-key', 'Values': ['DestinationRegion'] }, + { 'Name': 'status', 'Values': ['completed'] }, ] ) @@ -74,7 +75,6 @@ def lambda_handler(event, context): ) ec.delete_snapshot( - DryRun=True, SnapshotId=snapshot['SnapshotId'] ) From ad782537811c873ed06072b4a2392f077ce36fbd Mon Sep 17 00:00:00 2001 From: Dave Osment Date: Fri, 27 Jan 2017 13:33:32 +0000 Subject: [PATCH 3/8] Updated creator to get all variables from Tags. Added functionality to exclude individual Volumes attached to an Instance from SnapShots and added support for off-site tag. Configured description to use Volume Name where specified. --- ebs-snapshot-creator.py | 152 ++++++++++++++++++++++++++++------------ 1 file changed, 106 insertions(+), 46 deletions(-) diff --git a/ebs-snapshot-creator.py b/ebs-snapshot-creator.py index f5d553c..1c839fe 100644 --- a/ebs-snapshot-creator.py +++ b/ebs-snapshot-creator.py @@ -1,81 +1,141 @@ import boto3 import collections import datetime - -region = 'us-west-2' # region we're running in (should be changed to be auto-determined +import os ec = boto3.client('ec2') def lambda_handler(event, context): - reservations = ec.describe_instances( + + # Get Current Region + aws_region = os.getenv('AWS_REGION') + + # Get Retention Period From Environment Variable Or Assume Default If Not Specified + retention_days = int( + os.getenv( + 'RETENTION_DAYS', + 7 + ) + ) + + print "Setting SnapShot retention period To %s days" % (retention_days) + + # Determine Which Instances To SnapShot + instances = ec.describe_instances( Filters=[ { 'Name': 'tag:Backup', 'Values': ['Yes'] }, ] ).get( 'Reservations', [] ) + + print "Found %d instances that need backing up" % len(instances[0]['Instances']) - instances = sum( - [ - [i for i in r['Instances']] - for r in reservations - ], []) - - print "Found %d instances that need backing up" % len(instances) - + # Initialise Dictionary Objects To Store Tags In to_tag = collections.defaultdict(list) - for instance in instances: - try: - retention_days = [ - int(t.get('Value')) for t in instance['Tags'] - if t['Key'] == 'Retention'][0] - except IndexError: - retention_days = 7 + # Iterate Over Each Instance & SnapShot Volumes Not Explicitly Excluded From Backups + for instance in instances[0]['Instances']: + # List All Volumes Attached To The Instance for dev in instance['BlockDeviceMappings']: + + # Set Variable Defaults + snapshot_required = True + volume_name = None + if dev.get('Ebs', None) is None: continue vol_id = dev['Ebs']['VolumeId'] dev_name = dev['DeviceName'] - print "\tFound EBS volume %s (%s) on instance %s" % ( - vol_id, dev_name, instance['InstanceId']) - - # figure out instance name if there is one - instance_name = "" - for tag in instance['Tags']: - if tag['Key'] != 'Name': - continue - else: - instance_name = tag['Value'] - description = '%s - %s (%s)' % ( instance_name, vol_id, dev_name ) - - # trigger snapshot - snap = ec.create_snapshot( - VolumeId=vol_id, - Description=description + # Get a Volume Object Based Upon Volume ID + volume = ec.describe_volumes( + VolumeIds=[vol_id,] + ) + + vol = volume['Volumes'][0] + if 'Tags' in vol: + for tag in vol['Tags']: + + # Determine If Volume Has 'Backup' Flag Set To 'No' & Exclude From SnapShot If It Does + if tag['Key'] == 'Backup' and tag['Value'] == 'No': + snapshot_required = False + + # Determine If Volume Has a Name Specified + if tag['Key'] == 'Name': + volume_name = tag['Value'] + + # Exit This Loop If SnapShot Not Required + if snapshot_required == False: + print "\tIgnoring EBS volume %s (%s) on instance %s - 'Backup' Tag set to 'No'" % ( + vol_id, + dev_name, + instance['InstanceId'] ) - if (snap): - print "\t\tSnapshot %s created in %s of [%s]" % ( snap['SnapshotId'], region, description ) - to_tag[retention_days].append(snap['SnapshotId']) - print "\t\tRetaining snapshot %s of volume %s from instance %s (%s) for %d days" % ( - snap['SnapshotId'], - vol_id, - instance['InstanceId'], - instance_name, - retention_days, - ) + else: + print "\tFound EBS volume %s (%s) on instance %s - Proceeding with SnapShot" % ( + vol_id, + dev_name, + instance['InstanceId'] + ) + + # Determine EC2 Instance Name (If Present) + instance_name = "" + for tag in instance['Tags']: + if tag['Key'] != 'Name': + continue + else: + instance_name = tag['Value'] + + # Determine SnapShot Description (Use Volume Name If Specified) + if volume_name == None: + description = '%s - %s (%s)' % ( + instance_name, + vol_id, + dev_name + ) + else: + description = volume_name + + # Trigger SnapShot + snap = ec.create_snapshot( + VolumeId=vol_id, + Description=description + ) + + if (snap): + print "\t\tSnapshot %s created in %s of [%s]" % ( + snap['SnapshotId'], + aws_region, + description + ) + + # Tag The SnapShot To Facilitate Later Automated Deletion + to_tag[retention_days].append(snap['SnapshotId']) + + print "\t\tRetaining snapshot %s of volume %s from instance %s (%s) for %d days" % ( + snap['SnapshotId'], + vol_id, + instance['InstanceId'], + instance_name, + retention_days, + ) for retention_days in to_tag.keys(): delete_date = datetime.date.today() + datetime.timedelta(days=retention_days) delete_fmt = delete_date.strftime('%Y-%m-%d') - print "Will delete %d snapshots on %s" % (len(to_tag[retention_days]), delete_fmt) + print "Will delete %d snapshots from %s on %s" % ( + len(to_tag[retention_days]), + aws_region, + delete_fmt + ) + ec.create_tags( Resources=to_tag[retention_days], Tags=[ { 'Key': 'DeleteOn', 'Value': delete_fmt }, { 'Key': 'Type', 'Value': 'Automated' }, ] - ) + ) \ No newline at end of file From 237a22d4457350fc01f8f80982564bb1845c0a97 Mon Sep 17 00:00:00 2001 From: Dave Osment Date: Fri, 27 Jan 2017 13:55:26 +0000 Subject: [PATCH 4/8] Updated Readme, Ideas and Changelog --- CHANGELOG.md | 14 ++++++++++++++ IDEAS.md | 10 +++++----- README.md | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f757c2..3f2f3b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,20 @@ # Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## 0.0.5 2017-01-27 +### Added +- Created new function to offsite SnapShots based upon presence of 'DestinationRegion' Tag +- All Tags on the Source snapShot will be applied to the copy +- Ability to exclude individual EBS Volumes +- Linked original and copied snapshots by Tagging with the other's respective Id + +### Changed +- Automatic determination of curent AWS Region +- All variables obtained through Tags rather than hard-coded + +### Fixed +- Nothing so far + ## 0.0.4 [unreleased] ### Added - WIP: Out-of-region snapshot support diff --git a/IDEAS.md b/IDEAS.md index b98c37e..a3654ed 100644 --- a/IDEAS.md +++ b/IDEAS.md @@ -6,17 +6,17 @@ - DONE: Copying a snapshot to an additional region should be possible within the creator (hardcoded in creator) - DONE: Enabling snapshot copying out-of-region should be easily configurable in the creator script (albeit still requiring a variable parameter change) - DONE: Only copy snapshot out of region if a copy_region is defined in the creator script - - The out-of-region/copy snapshot functionality should be in its own dedicated job + - DONE: The out-of-region/copy snapshot functionality should be in its own dedicated job - because snapshots can't be copied until they're in a completed state (and this enables getting closer to that) - Job/function is easy to understand (logical point of separation) - - Copies of snapshots in the additional region should be tagged in the same manner as in-region snapshots (Automated: Yes, expiration info, etc.) - - Enabling the copying (duplication) of a snapshot out-of-region should be configurable on a per instance basis - - Out-of-region snapshots should be managed (for expiration/retention) just like in-region snapshots + - DONE: Copies of snapshots in the additional region should be tagged in the same manner as in-region snapshots (Automated: Yes, expiration info, etc.) + - DONE: Enabling the copying (duplication) of a snapshot out-of-region should be configurable on a per instance basis + - DONE: Out-of-region snapshots should be managed (for expiration/retention) just like in-region snapshots ### P2 - It should be possible to get automatically notified when the job (a Lambda function) emits an error - e.g. http://docs.aws.amazon.com/lambda/latest/dg/with-scheduledevents-example.html -- The required minimum IAM role policy should be provided +- DONE: The required minimum IAM role policy should be provided ### P3 - It should be possible to configure multiple regions to copy (duplicate) snapshots into diff --git a/README.md b/README.md index c9dc5c4..3a11899 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,18 @@ This is for managing AWS EC2 EBS volume snapshots. It consists of a snapshot cre - Ability to configure retention period on a per EC2 instance basis (applying to all volumes attached to said instance) - Ability to manually tag individual snapshots to be kept indefinitely (regardless of instance retention configuration) - Does not require a job/management instance; no resources to provision to run snapshot jobs (leverages AWS Lambda) +- Ability to snapshot all Volumes attached to a given Instance (Default), and exclude on a per-Volume basis any indivdual Volume (Through the addition of `Backup = No` Tag to Volume) +- Ability to replicate snapshot to a second AWS Region (As specified by Tag) and remove snapshot from source Region upon successful copy. Tags are replicated from source to destination snapshots + +## Tags Configuration + +- Instance Level + - `Backup` { Yes | No } + - `DestinationRegion` { us-west-1 | eu-west-1 | etc. } + - `RetentionDays` { 1..x } + +- Volume Level + - `Backup` { Yes | No } (Default if absent = 'Yes') : Overrides default to exclude a given Volume from snapshot ## Implementation Details @@ -24,11 +36,40 @@ For the moment, read these links for documentation on how to setup/use. I've ext Ideas and To Do items are currently tracked in [IDEAS](IDEAS.md). +## IAM Role + +The minimal IAM Role for these Lambda Functions is: + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + "ec2:DescribeInstances", + "ec2:DescribeVolumes", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:CopySnapshot", + "ec2:DescribeSnapshots", + "ec2:DeleteSnapshot" + ], + "Resource": "*" + } + ] +} +``` + ## Files: Each file implements a single AWS Lambda function. - ebs-snapshot-creator.py +- ebs-snapshot-offsiter.py - ebs-snapshot-manager.py ## Related: From e10201cd7bdbb6a3481f9f16e177174338e1fa75 Mon Sep 17 00:00:00 2001 From: Dave Osment Date: Fri, 27 Jan 2017 17:05:23 +0000 Subject: [PATCH 5/8] Updated mechanism through which current Account Id is acquired --- ebs-snapshot-manager.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/ebs-snapshot-manager.py b/ebs-snapshot-manager.py index c1643ca..8c56aa0 100644 --- a/ebs-snapshot-manager.py +++ b/ebs-snapshot-manager.py @@ -3,7 +3,6 @@ import datetime ec = boto3.client('ec2') -iam = boto3.client('iam') """ This function looks at *all* snapshots that have a "DeleteOn" tag containing @@ -12,21 +11,8 @@ """ def lambda_handler(event, context): - account_ids = list() - try: - """ - You can replace this try/except by filling in `account_ids` yourself. - Get your account ID with: - > import boto3 - > iam = boto3.client('iam') - > print iam.get_user()['User']['Arn'].split(':')[4] - """ - iam.get_user() - except Exception as e: - # use the exception message to get the account ID the function executes under - account_ids.append(re.search(r'(arn:aws:sts::)([0-9]+)', str(e)).groups()[1]) - - + account_ids = (boto3.client('sts').get_caller_identity()['Account']) + delete_on = datetime.date.today().strftime('%Y-%m-%d') # limit snapshots to process to ones marked for deletion on this day # AND limit snapshots to process to ones that are automated only @@ -35,7 +21,7 @@ def lambda_handler(event, context): { 'Name': 'tag:DeleteOn', 'Values': [delete_on] }, { 'Name': 'tag:Type', 'Values': ['Automated'] }, ] - snapshot_response = ec.describe_snapshots(OwnerIds=account_ids, Filters=filters) + snapshot_response = ec.describe_snapshots(OwnerIds=[account_ids], Filters=filters) for snap in snapshot_response['Snapshots']: for tag in snap['Tags']: @@ -50,4 +36,4 @@ def lambda_handler(event, context): # do nothing else else: print "Deleting snapshot %s" % snap['SnapshotId'] - ec.delete_snapshot(SnapshotId=snap['SnapshotId']) + ec.delete_snapshot(SnapshotId=snap['SnapshotId']) \ No newline at end of file From a84b0d0780bf491af772e8efb1b012805c76e137 Mon Sep 17 00:00:00 2001 From: Dave Osment Date: Fri, 27 Jan 2017 17:21:48 +0000 Subject: [PATCH 6/8] Added CFN YAML to create all three core functions in Lambda, along with required IAM Role, and CloudWatch Event Cron Trigger to fire each Function on a schedule defined within CFN paramaters --- CHANGELOG.md | 3 +- IDEAS.md | 4 +- cfn.yaml | 453 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 457 insertions(+), 3 deletions(-) create mode 100644 cfn.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f2f3b1..00dbc13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,13 @@ # Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). -## 0.0.5 2017-01-27 +## 0.0.5 - 2017-01-27 ### Added - Created new function to offsite SnapShots based upon presence of 'DestinationRegion' Tag - All Tags on the Source snapShot will be applied to the copy - Ability to exclude individual EBS Volumes - Linked original and copied snapshots by Tagging with the other's respective Id +- Added YAML CFN to create Lambda Functions from 'ebs-snapshot-creator', 'ebs-snapshot-manager' & 'ebs-snapshot-offsiter' along with IAM Roles and CloudWatch Cron triggers ### Changed - Automatic determination of curent AWS Region diff --git a/IDEAS.md b/IDEAS.md index a3654ed..1c005de 100644 --- a/IDEAS.md +++ b/IDEAS.md @@ -22,7 +22,7 @@ - It should be possible to configure multiple regions to copy (duplicate) snapshots into - It should be possible to trigger a web hook (optionally / if configured) every time the creator job runs - e.g. to use with PagerDuty to monitor if a job doesn't check-in every N days/hours/whatever -- Add JSON for IAM and/or CloudFormation and/or Terraform code and/or CLI/SH for deploying +- ADDED CFN: Add JSON for IAM and/or CloudFormation and/or Terraform code and/or CLI/SH for deploying - It should be possible to trigger snapshots of instance volumes in other regions besides the one that the creator is running in (or should it?) ## ebs-snapshot-manager.py @@ -41,7 +41,7 @@ ### P3 - Trigger a optional (if configured) web hook every time it runs (e.g. to use with PagerDuty to trigger if job doesn't check-in every N days/hours/whatever) -- Add JSON for IAM and/or CloudFormation and/or Terraform code and/or CLI/SH for deploying +- ADDED CFN: Add JSON for IAM and/or CloudFormation and/or Terraform code and/or CLI/SH for deploying ## ebs-snapshot-watcher.py diff --git a/cfn.yaml b/cfn.yaml new file mode 100644 index 0000000..1168eb3 --- /dev/null +++ b/cfn.yaml @@ -0,0 +1,453 @@ +Description: | + Creates Lambda Functions to create, offsite and age-off EBS Volume Snapshots + +Parameters: + + SnapshotSchedule: + Description: Cron expression for the schedule on which Volume Snapshots should be taken + Type: String + Default: 0 1 * * ? * + + OffsiteSchedule: + Description: Cron expression for the schedule on which Volume Snapshots should be off-sited + Type: String + Default: 0/60 * * * ? * + + DeleteSchedule: + Description: Cron expression for the schedule on which Volume Snapshots should be deleted (Aged-Off) + Type: String + Default: 0 1 * * ? * + +Outputs: {} + +Resources: + + # Create IAM Role For Lambda Functions + LambdaBackupsRole: + Type: AWS::IAM::Role + Properties: + RoleName: Lambda-Backups-Role + Path: / + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: Lambda-Backups-Policy + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - lambda:Invoke* + Resource: + - "*" + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + - ec2:DescribeInstances + - ec2:DescribeVolumes + - ec2:CreateSnapshot + - ec2:CreateTags + - ec2:CopySnapshot + - ec2:DescribeSnapshots + - ec2:DeleteSnapshot + Resource: + - "*" + + # Create CloudWatch Events To Trigger Lambda Functions + CloudWatchEventSnapshotSchedule: + Type: AWS::Events::Rule + Properties: + Name: Take-Snapshots + ScheduleExpression: !Join [ '', [ 'cron(', !Ref SnapshotSchedule, ')' ] ] + Targets: + - Arn: !GetAtt [LambdaFunctionTakeSnapshots, Arn] + Id: !Ref LambdaFunctionTakeSnapshots + + CloudWatchEventOffsiteSchedule: + Type: AWS::Events::Rule + Properties: + Name: Offsite-Snapshots + ScheduleExpression: !Join [ '', [ 'cron(', !Ref OffsiteSchedule, ')' ] ] + Targets: + - Arn: !GetAtt [LambdaFunctionOffSiteSnapshots, Arn] + Id: !Ref LambdaFunctionOffSiteSnapshots + + CloudWatchEventDeleteSchedule: + Type: AWS::Events::Rule + Properties: + Name: Delete-Snapshots + ScheduleExpression: !Join [ '', [ 'cron(', !Ref DeleteSchedule, ')' ] ] + Targets: + - Arn: !GetAtt [LambdaFunctionDeleteSnapshots, Arn] + Id: !Ref LambdaFunctionDeleteSnapshots + + + # Create Lambda Permissions To Link Rules To Functions + LambdaFunctionPermissionTakeSnapshots: + Type: AWS::Lambda::Permission + Properties: + Action: lambda:InvokeFunction + FunctionName: + Ref: LambdaFunctionTakeSnapshots + Principal: events.amazonaws.com + SourceArn: !GetAtt [CloudWatchEventSnapshotSchedule, Arn] + + LambdaFunctionPermissionOffsiteSnapshots: + Type: AWS::Lambda::Permission + Properties: + Action: lambda:InvokeFunction + FunctionName: + Ref: LambdaFunctionOffSiteSnapshots + Principal: events.amazonaws.com + SourceArn: !GetAtt [CloudWatchEventOffsiteSchedule, Arn] + + LambdaFunctionPermissionDeleteSnapshots: + Type: AWS::Lambda::Permission + Properties: + Action: lambda:InvokeFunction + FunctionName: + Ref: LambdaFunctionDeleteSnapshots + Principal: events.amazonaws.com + SourceArn: !GetAtt [CloudWatchEventDeleteSchedule, Arn] + + + # Create Lambda Functions To Create, Offsite & Delete Snapshots + LambdaFunctionOffSiteSnapshots: + Type: AWS::Lambda::Function + Properties: + FunctionName: Offsite-Snapshots + Description: Invokes EBS Snapshot Off-Siting Process + Handler: index.lambda_handler + MemorySize: 128 + Role: !GetAtt [LambdaBackupsRole, Arn] + Runtime: python2.7 + Timeout: 300 + Code: + ZipFile: + Fn::Sub: | + import boto3 + import collections + import datetime + import os + + ec = boto3.client('ec2') + + def lambda_handler(event, context): + + # Get Current Region + aws_region = os.getenv('AWS_REGION') + + # Get All SnapShots With 'Offsite' Tag Set + snapshots = ec.describe_snapshots( + Filters=[ + { 'Name': 'tag-key', 'Values': ['DestinationRegion'] }, + { 'Name': 'status', 'Values': ['completed'] }, + ] + ) + + for snapshot in snapshots['Snapshots']: + + # Reset Our Destination Region + destination_region = None + + # Obtain Tags From Source SnapShot + for tag in snapshot['Tags']: + + # Obtain Destination Region From Source Snapshot Tag + if tag['Key'] == 'DestinationRegion': + destination_region = tag['Value'] + + # Check If We Need To Do A Copy Or Not + if destination_region == aws_region: + + print "\tDestination Region %s is the same as current region (%s) - skipping copy" % ( + destination_region, + aws_region + ) + + continue + + # Construct ec2 Client For Secondary Region + secondary_ec = boto3.client('ec2', region_name=destination_region) + + # Determine If There's An Off-Site Copy Of This SnapShot + os_snapshots = secondary_ec.describe_snapshots( + Filters=[ + { 'Name': 'tag:SourceSnapshotId', 'Values': [snapshot['SnapshotId']] }, + { 'Name': 'status', 'Values': ['pending','completed'] }, + ] + ) + + # We Only Want To Delete Where Snapshot Has Copied Successfully + if len(os_snapshots['Snapshots']) >= 1: + + snapshot_states = [d['State'] for d in os_snapshots['Snapshots']] + + if 'pending' in snapshot_states: + print "\tThere is at least 1 Snapshot copy pending in %s - skipping delete & copy" % ( + destination_region + ) + + continue + + print "\t\tFound a corresponding Snapshot with Id %s in %s created from Snapshot %s" % ( + os_snapshots['Snapshots'][0]['SnapshotId'], + destination_region, + snapshot['SnapshotId'] + ) + + print "Deleting source Snapshot %s from %s" % ( + snapshot['SnapshotId'], + aws_region + ) + + ec.delete_snapshot( + SnapshotId=snapshot['SnapshotId'] + ) + + continue + + # Create Copy Of Snapshot Because One Doesn't Exist + os_snapshot = secondary_ec.copy_snapshot( + SourceRegion=aws_region, + SourceSnapshotId=snapshot['SnapshotId'], + Description=snapshot['Description'], + DestinationRegion=destination_region + ) + + # If Snapshot Copy Executed Successfully, Copy The Tags + if (os_snapshot): + + print "\t\tSnapshot copy %s created in %s of %s from %s" % ( + os_snapshot['SnapshotId'], + destination_region, + snapshot['SnapshotId'], + aws_region + ) + + # Add Tags To Off-Site SnapShot + destination_snapshot_tags = snapshot['Tags'] + [{ 'Key': 'SourceSnapshotId', 'Value': snapshot['SnapshotId'] }] + secondary_ec.create_tags( + Resources=[os_snapshot['SnapshotId']], + Tags=destination_snapshot_tags + ) + + # Add Tags To Source SnapShot + ec.create_tags( + Resources=[snapshot['SnapshotId']], + Tags=[ + { 'Key': 'OffsiteSnapshotId', 'Value': os_snapshot['SnapshotId'] }, + ] + ) + + LambdaFunctionTakeSnapshots: + Type: AWS::Lambda::Function + Properties: + FunctionName: Create-Snapshots + Description: Invokes EBS Snapshot Creation Process + Handler: index.lambda_handler + MemorySize: 128 + Role: !GetAtt [LambdaBackupsRole, Arn] + Runtime: python2.7 + Timeout: 300 + Code: + ZipFile: + Fn::Sub: | + import boto3 + import collections + import datetime + import os + + ec = boto3.client('ec2') + + def lambda_handler(event, context): + + # Get Current Region + aws_region = os.getenv('AWS_REGION') + + # Determine Which Instances To SnapShot + instances = ec.describe_instances( + Filters=[ + { 'Name': 'tag:Backup', 'Values': ['Yes'] }, + ] + ).get( + 'Reservations', [] + )[0]['Instances'] + + print "Found %d instances that need backing up" % len(instances) + + # Iterate Over Each Instance & SnapShot Volumes Not Explicitly Excluded From Backups + for instance in instances: + + # Determine Retention Period Based Upon Tags + retention_days = 7 + destination_region = None + instance_name = "" + for tag in instance['Tags']: + if tag['Key'] == 'RetentionDays' and tag['Value'] > 0: + retention_days = int(tag['Value']) + + if tag['Key'] == 'DestinationRegion' and len(tag['Value']) > 0: + destination_region = tag['Value'] + + if tag['Key'] == 'Name' and len(tag['Value']) > 0: + instance_name = tag['Value'] + + print "Setting SnapShot retention period To %s days" % (retention_days) + + # Determine When We're Going To Delete This SnapShot + delete_date = datetime.date.today() + datetime.timedelta(days=retention_days) + delete_fmt = delete_date.strftime('%Y-%m-%d') + + # Set Default SnapShot Tags + snapshot_tags = [ + { 'Key': 'DeleteOn', 'Value': delete_fmt }, + { 'Key': 'Type', 'Value': 'Automated' }, + ] + + # If We Want To Offsite This SnapShot, Set The Appropriate Tag + if destination_region != None: + snapshot_tags = snapshot_tags + [{ 'Key': 'DestinationRegion', 'Value': destination_region }] + + # List All Volumes Attached To The Instance + for dev in instance['BlockDeviceMappings']: + + # Set Variable Defaults + snapshot_required = True + volume_name = None + + if dev.get('Ebs', None) is None: + continue + vol_id = dev['Ebs']['VolumeId'] + dev_name = dev['DeviceName'] + + # Get a Volume Object Based Upon Volume ID + volume = ec.describe_volumes( + VolumeIds=[vol_id,] + )['Volumes'][0] + + # Set Default SnapShot Description + description = '%s - %s (%s)' % ( + instance_name, + vol_id, + dev_name + ) + + if 'Tags' in volume: + for tag in volume['Tags']: + + # Determine If Volume Has 'Backup' Flag Set To 'No' & Exclude From SnapShot If It Does + if tag['Key'] == 'Backup' and tag['Value'] == 'No': + snapshot_required = False + + # Override Default Description With Volume Name If One Specified + if tag['Key'] == 'Name': + description = tag['Value'] + + + # We Don't Want To SnapShot Any Volume Explictly Excluded + if snapshot_required == False: + print "\tIgnoring EBS volume %s (%s) on instance %s - 'Backup' Tag set to 'No'" % ( + vol_id, + dev_name, + instance['InstanceId'] + ) + + continue + + + print "\tFound EBS volume %s (%s) on instance %s - Proceeding with SnapShot" % ( + vol_id, + dev_name, + instance['InstanceId'] + ) + + # Take SnapShot Of Volume + snap = ec.create_snapshot( + VolumeId=vol_id, + Description=description + ) + + if not (snap): + print "\t\tSnapShot operation failed!" + continue + + print "\t\tSnapshot %s created in %s of [%s]" % ( + snap['SnapshotId'], + aws_region, + description + ) + + print "\t\tRetaining snapshot %s of volume %s from instance %s (%s) for %d days" % ( + snap['SnapshotId'], + vol_id, + instance['InstanceId'], + instance_name, + retention_days, + ) + + # Tag The SnapShot To Facilitate Later Automated Deletion & Offsiting + ec.create_tags( + Resources=[snap['SnapshotId']], + Tags=snapshot_tags + ) + + LambdaFunctionDeleteSnapshots: + Type: AWS::Lambda::Function + Properties: + FunctionName: Delete-Snapshots + Description: Invokes EBS Snapshot Deletion (Age-Off) Process + Handler: index.lambda_handler + MemorySize: 128 + Role: !GetAtt [LambdaBackupsRole, Arn] + Runtime: python2.7 + Timeout: 300 + Code: + ZipFile: + Fn::Sub: | + import boto3 + import re + import datetime + + ec = boto3.client('ec2') + + """ + This function looks at *all* snapshots that have a "DeleteOn" tag containing + the current day formatted as YYYY-MM-DD. This function should be run at least + daily. + """ + + def lambda_handler(event, context): + account_ids = (boto3.client('sts').get_caller_identity()['Account']) + + delete_on = datetime.date.today().strftime('%Y-%m-%d') + # limit snapshots to process to ones marked for deletion on this day + # AND limit snapshots to process to ones that are automated only + # AND exclude automated snapshots marked for permanent retention + filters = [ + { 'Name': 'tag:DeleteOn', 'Values': [delete_on] }, + { 'Name': 'tag:Type', 'Values': ['Automated'] }, + ] + snapshot_response = ec.describe_snapshots(OwnerIds=[account_ids], Filters=filters) + + for snap in snapshot_response['Snapshots']: + for tag in snap['Tags']: + if tag['Key'] != 'KeepForever': + skipping_this_one = False + continue + else: + skipping_this_one = True + + if skipping_this_one == True: + print "Skipping snapshot %s (marked KeepForever)" % snap['SnapshotId'] + # do nothing else + else: + print "Deleting snapshot %s" % snap['SnapshotId'] + ec.delete_snapshot(SnapshotId=snap['SnapshotId']) \ No newline at end of file From 9b8a9274ff97db3c12a1ccc7a6b3163a155889f6 Mon Sep 17 00:00:00 2001 From: Dave Osment Date: Fri, 27 Jan 2017 18:08:01 +0000 Subject: [PATCH 7/8] Updated ebs-snapshot-creator with correct version of file --- ebs-snapshot-creator.py | 255 ++++++++++++++++++++-------------------- 1 file changed, 126 insertions(+), 129 deletions(-) diff --git a/ebs-snapshot-creator.py b/ebs-snapshot-creator.py index 1c839fe..247759d 100644 --- a/ebs-snapshot-creator.py +++ b/ebs-snapshot-creator.py @@ -6,136 +6,133 @@ ec = boto3.client('ec2') def lambda_handler(event, context): - - # Get Current Region - aws_region = os.getenv('AWS_REGION') - - # Get Retention Period From Environment Variable Or Assume Default If Not Specified - retention_days = int( - os.getenv( - 'RETENTION_DAYS', - 7 - ) - ) - + + # Get Current Region + aws_region = os.getenv('AWS_REGION') + + # Determine Which Instances To SnapShot + instances = ec.describe_instances( + Filters=[ + { 'Name': 'tag:Backup', 'Values': ['Yes'] }, + ] + ).get( + 'Reservations', [] + )[0]['Instances'] + + print "Found %d instances that need backing up" % len(instances) + + # Iterate Over Each Instance & SnapShot Volumes Not Explicitly Excluded From Backups + for instance in instances: + + # Determine Retention Period Based Upon Tags + retention_days = 7 + destination_region = None + instance_name = "" + for tag in instance['Tags']: + if tag['Key'] == 'RetentionDays' and tag['Value'] > 0: + retention_days = int(tag['Value']) + + if tag['Key'] == 'DestinationRegion' and len(tag['Value']) > 0: + destination_region = tag['Value'] + + if tag['Key'] == 'Name' and len(tag['Value']) > 0: + instance_name = tag['Value'] + print "Setting SnapShot retention period To %s days" % (retention_days) - - # Determine Which Instances To SnapShot - instances = ec.describe_instances( - Filters=[ - { 'Name': 'tag:Backup', 'Values': ['Yes'] }, - ] - ).get( - 'Reservations', [] - ) - - print "Found %d instances that need backing up" % len(instances[0]['Instances']) - - # Initialise Dictionary Objects To Store Tags In - to_tag = collections.defaultdict(list) - - # Iterate Over Each Instance & SnapShot Volumes Not Explicitly Excluded From Backups - for instance in instances[0]['Instances']: - - # List All Volumes Attached To The Instance - for dev in instance['BlockDeviceMappings']: - - # Set Variable Defaults - snapshot_required = True - volume_name = None - - if dev.get('Ebs', None) is None: - continue - vol_id = dev['Ebs']['VolumeId'] - dev_name = dev['DeviceName'] - - # Get a Volume Object Based Upon Volume ID - volume = ec.describe_volumes( - VolumeIds=[vol_id,] - ) - - vol = volume['Volumes'][0] - if 'Tags' in vol: - for tag in vol['Tags']: - - # Determine If Volume Has 'Backup' Flag Set To 'No' & Exclude From SnapShot If It Does - if tag['Key'] == 'Backup' and tag['Value'] == 'No': - snapshot_required = False - - # Determine If Volume Has a Name Specified - if tag['Key'] == 'Name': - volume_name = tag['Value'] - - # Exit This Loop If SnapShot Not Required - if snapshot_required == False: - print "\tIgnoring EBS volume %s (%s) on instance %s - 'Backup' Tag set to 'No'" % ( - vol_id, - dev_name, - instance['InstanceId'] - ) + + # Determine When We're Going To Delete This SnapShot + delete_date = datetime.date.today() + datetime.timedelta(days=retention_days) + delete_fmt = delete_date.strftime('%Y-%m-%d') + + # Set Default SnapShot Tags + snapshot_tags = [ + { 'Key': 'DeleteOn', 'Value': delete_fmt }, + { 'Key': 'Type', 'Value': 'Automated' }, + ] + + # If We Want To Offsite This SnapShot, Set The Appropriate Tag + if destination_region != None: + snapshot_tags = snapshot_tags + [{ 'Key': 'DestinationRegion', 'Value': destination_region }] + + # List All Volumes Attached To The Instance + for dev in instance['BlockDeviceMappings']: + + # Set Variable Defaults + snapshot_required = True + volume_name = None + + if dev.get('Ebs', None) is None: + continue + vol_id = dev['Ebs']['VolumeId'] + dev_name = dev['DeviceName'] + + # Get a Volume Object Based Upon Volume ID + volume = ec.describe_volumes( + VolumeIds=[vol_id,] + )['Volumes'][0] + + # Set Default SnapShot Description + description = '%s - %s (%s)' % ( + instance_name, + vol_id, + dev_name + ) + + if 'Tags' in volume: + for tag in volume['Tags']: + + # Determine If Volume Has 'Backup' Flag Set To 'No' & Exclude From SnapShot If It Does + if tag['Key'] == 'Backup' and tag['Value'] == 'No': + snapshot_required = False + + # Override Default Description With Volume Name If One Specified + if tag['Key'] == 'Name': + description = tag['Value'] + - else: - print "\tFound EBS volume %s (%s) on instance %s - Proceeding with SnapShot" % ( - vol_id, - dev_name, - instance['InstanceId'] - ) - - # Determine EC2 Instance Name (If Present) - instance_name = "" - for tag in instance['Tags']: - if tag['Key'] != 'Name': - continue - else: - instance_name = tag['Value'] - - # Determine SnapShot Description (Use Volume Name If Specified) - if volume_name == None: - description = '%s - %s (%s)' % ( - instance_name, - vol_id, - dev_name - ) - else: - description = volume_name - - # Trigger SnapShot - snap = ec.create_snapshot( - VolumeId=vol_id, - Description=description - ) - - if (snap): - print "\t\tSnapshot %s created in %s of [%s]" % ( - snap['SnapshotId'], - aws_region, - description - ) - - # Tag The SnapShot To Facilitate Later Automated Deletion - to_tag[retention_days].append(snap['SnapshotId']) - - print "\t\tRetaining snapshot %s of volume %s from instance %s (%s) for %d days" % ( - snap['SnapshotId'], - vol_id, - instance['InstanceId'], - instance_name, - retention_days, - ) - - for retention_days in to_tag.keys(): - delete_date = datetime.date.today() + datetime.timedelta(days=retention_days) - delete_fmt = delete_date.strftime('%Y-%m-%d') - print "Will delete %d snapshots from %s on %s" % ( - len(to_tag[retention_days]), - aws_region, - delete_fmt + # We Don't Want To SnapShot Any Volume Explictly Excluded + if snapshot_required == False: + print "\tIgnoring EBS volume %s (%s) on instance %s - 'Backup' Tag set to 'No'" % ( + vol_id, + dev_name, + instance['InstanceId'] ) + + continue + + + print "\tFound EBS volume %s (%s) on instance %s - Proceeding with SnapShot" % ( + vol_id, + dev_name, + instance['InstanceId'] + ) - ec.create_tags( - Resources=to_tag[retention_days], - Tags=[ - { 'Key': 'DeleteOn', 'Value': delete_fmt }, - { 'Key': 'Type', 'Value': 'Automated' }, - ] - ) \ No newline at end of file + # Take SnapShot Of Volume + snap = ec.create_snapshot( + VolumeId=vol_id, + Description=description + ) + + if not (snap): + print "\t\tSnapShot operation failed!" + continue + + print "\t\tSnapshot %s created in %s of [%s]" % ( + snap['SnapshotId'], + aws_region, + description + ) + + print "\t\tRetaining snapshot %s of volume %s from instance %s (%s) for %d days" % ( + snap['SnapshotId'], + vol_id, + instance['InstanceId'], + instance_name, + retention_days, + ) + + # Tag The SnapShot To Facilitate Later Automated Deletion & Offsiting + ec.create_tags( + Resources=[snap['SnapshotId']], + Tags=snapshot_tags + ) \ No newline at end of file From f7963827c9f5cce1a906a74279edcf7f880e26d2 Mon Sep 17 00:00:00 2001 From: Dave Osment Date: Mon, 30 Jan 2017 11:45:19 +0000 Subject: [PATCH 8/8] Corrected incorrect assignment of zero-index Instance to 'instances' variable, which prevented Snapshot-ing of > 1 Instance --- cfn.yaml | 5 ++++- ebs-snapshot-creator.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cfn.yaml b/cfn.yaml index 1168eb3..5db7e45 100644 --- a/cfn.yaml +++ b/cfn.yaml @@ -279,13 +279,16 @@ Resources: ] ).get( 'Reservations', [] - )[0]['Instances'] + ) print "Found %d instances that need backing up" % len(instances) # Iterate Over Each Instance & SnapShot Volumes Not Explicitly Excluded From Backups for instance in instances: + # Get Instance Object + instance = instance['Instances'][0] + # Determine Retention Period Based Upon Tags retention_days = 7 destination_region = None diff --git a/ebs-snapshot-creator.py b/ebs-snapshot-creator.py index 247759d..6c7b956 100644 --- a/ebs-snapshot-creator.py +++ b/ebs-snapshot-creator.py @@ -17,13 +17,16 @@ def lambda_handler(event, context): ] ).get( 'Reservations', [] - )[0]['Instances'] + ) print "Found %d instances that need backing up" % len(instances) # Iterate Over Each Instance & SnapShot Volumes Not Explicitly Excluded From Backups for instance in instances: + # Get Instance Object + instance = instance['Instances'][0] + # Determine Retention Period Based Upon Tags retention_days = 7 destination_region = None