diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f757c2..00dbc13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,21 @@ # Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## 0.0.5 - 2017-01-27 +### Added +- Created new function to offsite SnapShots based upon presence of 'DestinationRegion' Tag +- All Tags on the Source snapShot will be applied to the copy +- Ability to exclude individual EBS Volumes +- Linked original and copied snapshots by Tagging with the other's respective Id +- Added YAML CFN to create Lambda Functions from 'ebs-snapshot-creator', 'ebs-snapshot-manager' & 'ebs-snapshot-offsiter' along with IAM Roles and CloudWatch Cron triggers + +### Changed +- Automatic determination of curent AWS Region +- All variables obtained through Tags rather than hard-coded + +### Fixed +- Nothing so far + ## 0.0.4 [unreleased] ### Added - WIP: Out-of-region snapshot support diff --git a/IDEAS.md b/IDEAS.md index b98c37e..1c005de 100644 --- a/IDEAS.md +++ b/IDEAS.md @@ -6,23 +6,23 @@ - DONE: Copying a snapshot to an additional region should be possible within the creator (hardcoded in creator) - DONE: Enabling snapshot copying out-of-region should be easily configurable in the creator script (albeit still requiring a variable parameter change) - DONE: Only copy snapshot out of region if a copy_region is defined in the creator script - - The out-of-region/copy snapshot functionality should be in its own dedicated job + - DONE: The out-of-region/copy snapshot functionality should be in its own dedicated job - because snapshots can't be copied until they're in a completed state (and this enables getting closer to that) - Job/function is easy to understand (logical point of separation) - - Copies of snapshots in the additional region should be tagged in the same manner as in-region snapshots (Automated: Yes, expiration info, etc.) - - Enabling the copying (duplication) of a snapshot out-of-region should be configurable on a per instance basis - - Out-of-region snapshots should be managed (for expiration/retention) just like in-region snapshots + - DONE: Copies of snapshots in the additional region should be tagged in the same manner as in-region snapshots (Automated: Yes, expiration info, etc.) + - DONE: Enabling the copying (duplication) of a snapshot out-of-region should be configurable on a per instance basis + - DONE: Out-of-region snapshots should be managed (for expiration/retention) just like in-region snapshots ### P2 - It should be possible to get automatically notified when the job (a Lambda function) emits an error - e.g. http://docs.aws.amazon.com/lambda/latest/dg/with-scheduledevents-example.html -- The required minimum IAM role policy should be provided +- DONE: The required minimum IAM role policy should be provided ### P3 - It should be possible to configure multiple regions to copy (duplicate) snapshots into - It should be possible to trigger a web hook (optionally / if configured) every time the creator job runs - e.g. to use with PagerDuty to monitor if a job doesn't check-in every N days/hours/whatever -- Add JSON for IAM and/or CloudFormation and/or Terraform code and/or CLI/SH for deploying +- ADDED CFN: Add JSON for IAM and/or CloudFormation and/or Terraform code and/or CLI/SH for deploying - It should be possible to trigger snapshots of instance volumes in other regions besides the one that the creator is running in (or should it?) ## ebs-snapshot-manager.py @@ -41,7 +41,7 @@ ### P3 - Trigger a optional (if configured) web hook every time it runs (e.g. to use with PagerDuty to trigger if job doesn't check-in every N days/hours/whatever) -- Add JSON for IAM and/or CloudFormation and/or Terraform code and/or CLI/SH for deploying +- ADDED CFN: Add JSON for IAM and/or CloudFormation and/or Terraform code and/or CLI/SH for deploying ## ebs-snapshot-watcher.py diff --git a/README.md b/README.md index c9dc5c4..3a11899 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,18 @@ This is for managing AWS EC2 EBS volume snapshots. It consists of a snapshot cre - Ability to configure retention period on a per EC2 instance basis (applying to all volumes attached to said instance) - Ability to manually tag individual snapshots to be kept indefinitely (regardless of instance retention configuration) - Does not require a job/management instance; no resources to provision to run snapshot jobs (leverages AWS Lambda) +- Ability to snapshot all Volumes attached to a given Instance (Default), and exclude on a per-Volume basis any indivdual Volume (Through the addition of `Backup = No` Tag to Volume) +- Ability to replicate snapshot to a second AWS Region (As specified by Tag) and remove snapshot from source Region upon successful copy. Tags are replicated from source to destination snapshots + +## Tags Configuration + +- Instance Level + - `Backup` { Yes | No } + - `DestinationRegion` { us-west-1 | eu-west-1 | etc. } + - `RetentionDays` { 1..x } + +- Volume Level + - `Backup` { Yes | No } (Default if absent = 'Yes') : Overrides default to exclude a given Volume from snapshot ## Implementation Details @@ -24,11 +36,40 @@ For the moment, read these links for documentation on how to setup/use. I've ext Ideas and To Do items are currently tracked in [IDEAS](IDEAS.md). +## IAM Role + +The minimal IAM Role for these Lambda Functions is: + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + "ec2:DescribeInstances", + "ec2:DescribeVolumes", + "ec2:CreateSnapshot", + "ec2:CreateTags", + "ec2:CopySnapshot", + "ec2:DescribeSnapshots", + "ec2:DeleteSnapshot" + ], + "Resource": "*" + } + ] +} +``` + ## Files: Each file implements a single AWS Lambda function. - ebs-snapshot-creator.py +- ebs-snapshot-offsiter.py - ebs-snapshot-manager.py ## Related: diff --git a/cfn.yaml b/cfn.yaml new file mode 100644 index 0000000..5db7e45 --- /dev/null +++ b/cfn.yaml @@ -0,0 +1,456 @@ +Description: | + Creates Lambda Functions to create, offsite and age-off EBS Volume Snapshots + +Parameters: + + SnapshotSchedule: + Description: Cron expression for the schedule on which Volume Snapshots should be taken + Type: String + Default: 0 1 * * ? * + + OffsiteSchedule: + Description: Cron expression for the schedule on which Volume Snapshots should be off-sited + Type: String + Default: 0/60 * * * ? * + + DeleteSchedule: + Description: Cron expression for the schedule on which Volume Snapshots should be deleted (Aged-Off) + Type: String + Default: 0 1 * * ? * + +Outputs: {} + +Resources: + + # Create IAM Role For Lambda Functions + LambdaBackupsRole: + Type: AWS::IAM::Role + Properties: + RoleName: Lambda-Backups-Role + Path: / + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: Lambda-Backups-Policy + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - lambda:Invoke* + Resource: + - "*" + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + - ec2:DescribeInstances + - ec2:DescribeVolumes + - ec2:CreateSnapshot + - ec2:CreateTags + - ec2:CopySnapshot + - ec2:DescribeSnapshots + - ec2:DeleteSnapshot + Resource: + - "*" + + # Create CloudWatch Events To Trigger Lambda Functions + CloudWatchEventSnapshotSchedule: + Type: AWS::Events::Rule + Properties: + Name: Take-Snapshots + ScheduleExpression: !Join [ '', [ 'cron(', !Ref SnapshotSchedule, ')' ] ] + Targets: + - Arn: !GetAtt [LambdaFunctionTakeSnapshots, Arn] + Id: !Ref LambdaFunctionTakeSnapshots + + CloudWatchEventOffsiteSchedule: + Type: AWS::Events::Rule + Properties: + Name: Offsite-Snapshots + ScheduleExpression: !Join [ '', [ 'cron(', !Ref OffsiteSchedule, ')' ] ] + Targets: + - Arn: !GetAtt [LambdaFunctionOffSiteSnapshots, Arn] + Id: !Ref LambdaFunctionOffSiteSnapshots + + CloudWatchEventDeleteSchedule: + Type: AWS::Events::Rule + Properties: + Name: Delete-Snapshots + ScheduleExpression: !Join [ '', [ 'cron(', !Ref DeleteSchedule, ')' ] ] + Targets: + - Arn: !GetAtt [LambdaFunctionDeleteSnapshots, Arn] + Id: !Ref LambdaFunctionDeleteSnapshots + + + # Create Lambda Permissions To Link Rules To Functions + LambdaFunctionPermissionTakeSnapshots: + Type: AWS::Lambda::Permission + Properties: + Action: lambda:InvokeFunction + FunctionName: + Ref: LambdaFunctionTakeSnapshots + Principal: events.amazonaws.com + SourceArn: !GetAtt [CloudWatchEventSnapshotSchedule, Arn] + + LambdaFunctionPermissionOffsiteSnapshots: + Type: AWS::Lambda::Permission + Properties: + Action: lambda:InvokeFunction + FunctionName: + Ref: LambdaFunctionOffSiteSnapshots + Principal: events.amazonaws.com + SourceArn: !GetAtt [CloudWatchEventOffsiteSchedule, Arn] + + LambdaFunctionPermissionDeleteSnapshots: + Type: AWS::Lambda::Permission + Properties: + Action: lambda:InvokeFunction + FunctionName: + Ref: LambdaFunctionDeleteSnapshots + Principal: events.amazonaws.com + SourceArn: !GetAtt [CloudWatchEventDeleteSchedule, Arn] + + + # Create Lambda Functions To Create, Offsite & Delete Snapshots + LambdaFunctionOffSiteSnapshots: + Type: AWS::Lambda::Function + Properties: + FunctionName: Offsite-Snapshots + Description: Invokes EBS Snapshot Off-Siting Process + Handler: index.lambda_handler + MemorySize: 128 + Role: !GetAtt [LambdaBackupsRole, Arn] + Runtime: python2.7 + Timeout: 300 + Code: + ZipFile: + Fn::Sub: | + import boto3 + import collections + import datetime + import os + + ec = boto3.client('ec2') + + def lambda_handler(event, context): + + # Get Current Region + aws_region = os.getenv('AWS_REGION') + + # Get All SnapShots With 'Offsite' Tag Set + snapshots = ec.describe_snapshots( + Filters=[ + { 'Name': 'tag-key', 'Values': ['DestinationRegion'] }, + { 'Name': 'status', 'Values': ['completed'] }, + ] + ) + + for snapshot in snapshots['Snapshots']: + + # Reset Our Destination Region + destination_region = None + + # Obtain Tags From Source SnapShot + for tag in snapshot['Tags']: + + # Obtain Destination Region From Source Snapshot Tag + if tag['Key'] == 'DestinationRegion': + destination_region = tag['Value'] + + # Check If We Need To Do A Copy Or Not + if destination_region == aws_region: + + print "\tDestination Region %s is the same as current region (%s) - skipping copy" % ( + destination_region, + aws_region + ) + + continue + + # Construct ec2 Client For Secondary Region + secondary_ec = boto3.client('ec2', region_name=destination_region) + + # Determine If There's An Off-Site Copy Of This SnapShot + os_snapshots = secondary_ec.describe_snapshots( + Filters=[ + { 'Name': 'tag:SourceSnapshotId', 'Values': [snapshot['SnapshotId']] }, + { 'Name': 'status', 'Values': ['pending','completed'] }, + ] + ) + + # We Only Want To Delete Where Snapshot Has Copied Successfully + if len(os_snapshots['Snapshots']) >= 1: + + snapshot_states = [d['State'] for d in os_snapshots['Snapshots']] + + if 'pending' in snapshot_states: + print "\tThere is at least 1 Snapshot copy pending in %s - skipping delete & copy" % ( + destination_region + ) + + continue + + print "\t\tFound a corresponding Snapshot with Id %s in %s created from Snapshot %s" % ( + os_snapshots['Snapshots'][0]['SnapshotId'], + destination_region, + snapshot['SnapshotId'] + ) + + print "Deleting source Snapshot %s from %s" % ( + snapshot['SnapshotId'], + aws_region + ) + + ec.delete_snapshot( + SnapshotId=snapshot['SnapshotId'] + ) + + continue + + # Create Copy Of Snapshot Because One Doesn't Exist + os_snapshot = secondary_ec.copy_snapshot( + SourceRegion=aws_region, + SourceSnapshotId=snapshot['SnapshotId'], + Description=snapshot['Description'], + DestinationRegion=destination_region + ) + + # If Snapshot Copy Executed Successfully, Copy The Tags + if (os_snapshot): + + print "\t\tSnapshot copy %s created in %s of %s from %s" % ( + os_snapshot['SnapshotId'], + destination_region, + snapshot['SnapshotId'], + aws_region + ) + + # Add Tags To Off-Site SnapShot + destination_snapshot_tags = snapshot['Tags'] + [{ 'Key': 'SourceSnapshotId', 'Value': snapshot['SnapshotId'] }] + secondary_ec.create_tags( + Resources=[os_snapshot['SnapshotId']], + Tags=destination_snapshot_tags + ) + + # Add Tags To Source SnapShot + ec.create_tags( + Resources=[snapshot['SnapshotId']], + Tags=[ + { 'Key': 'OffsiteSnapshotId', 'Value': os_snapshot['SnapshotId'] }, + ] + ) + + LambdaFunctionTakeSnapshots: + Type: AWS::Lambda::Function + Properties: + FunctionName: Create-Snapshots + Description: Invokes EBS Snapshot Creation Process + Handler: index.lambda_handler + MemorySize: 128 + Role: !GetAtt [LambdaBackupsRole, Arn] + Runtime: python2.7 + Timeout: 300 + Code: + ZipFile: + Fn::Sub: | + import boto3 + import collections + import datetime + import os + + ec = boto3.client('ec2') + + def lambda_handler(event, context): + + # Get Current Region + aws_region = os.getenv('AWS_REGION') + + # Determine Which Instances To SnapShot + instances = ec.describe_instances( + Filters=[ + { 'Name': 'tag:Backup', 'Values': ['Yes'] }, + ] + ).get( + 'Reservations', [] + ) + + print "Found %d instances that need backing up" % len(instances) + + # Iterate Over Each Instance & SnapShot Volumes Not Explicitly Excluded From Backups + for instance in instances: + + # Get Instance Object + instance = instance['Instances'][0] + + # Determine Retention Period Based Upon Tags + retention_days = 7 + destination_region = None + instance_name = "" + for tag in instance['Tags']: + if tag['Key'] == 'RetentionDays' and tag['Value'] > 0: + retention_days = int(tag['Value']) + + if tag['Key'] == 'DestinationRegion' and len(tag['Value']) > 0: + destination_region = tag['Value'] + + if tag['Key'] == 'Name' and len(tag['Value']) > 0: + instance_name = tag['Value'] + + print "Setting SnapShot retention period To %s days" % (retention_days) + + # Determine When We're Going To Delete This SnapShot + delete_date = datetime.date.today() + datetime.timedelta(days=retention_days) + delete_fmt = delete_date.strftime('%Y-%m-%d') + + # Set Default SnapShot Tags + snapshot_tags = [ + { 'Key': 'DeleteOn', 'Value': delete_fmt }, + { 'Key': 'Type', 'Value': 'Automated' }, + ] + + # If We Want To Offsite This SnapShot, Set The Appropriate Tag + if destination_region != None: + snapshot_tags = snapshot_tags + [{ 'Key': 'DestinationRegion', 'Value': destination_region }] + + # List All Volumes Attached To The Instance + for dev in instance['BlockDeviceMappings']: + + # Set Variable Defaults + snapshot_required = True + volume_name = None + + if dev.get('Ebs', None) is None: + continue + vol_id = dev['Ebs']['VolumeId'] + dev_name = dev['DeviceName'] + + # Get a Volume Object Based Upon Volume ID + volume = ec.describe_volumes( + VolumeIds=[vol_id,] + )['Volumes'][0] + + # Set Default SnapShot Description + description = '%s - %s (%s)' % ( + instance_name, + vol_id, + dev_name + ) + + if 'Tags' in volume: + for tag in volume['Tags']: + + # Determine If Volume Has 'Backup' Flag Set To 'No' & Exclude From SnapShot If It Does + if tag['Key'] == 'Backup' and tag['Value'] == 'No': + snapshot_required = False + + # Override Default Description With Volume Name If One Specified + if tag['Key'] == 'Name': + description = tag['Value'] + + + # We Don't Want To SnapShot Any Volume Explictly Excluded + if snapshot_required == False: + print "\tIgnoring EBS volume %s (%s) on instance %s - 'Backup' Tag set to 'No'" % ( + vol_id, + dev_name, + instance['InstanceId'] + ) + + continue + + + print "\tFound EBS volume %s (%s) on instance %s - Proceeding with SnapShot" % ( + vol_id, + dev_name, + instance['InstanceId'] + ) + + # Take SnapShot Of Volume + snap = ec.create_snapshot( + VolumeId=vol_id, + Description=description + ) + + if not (snap): + print "\t\tSnapShot operation failed!" + continue + + print "\t\tSnapshot %s created in %s of [%s]" % ( + snap['SnapshotId'], + aws_region, + description + ) + + print "\t\tRetaining snapshot %s of volume %s from instance %s (%s) for %d days" % ( + snap['SnapshotId'], + vol_id, + instance['InstanceId'], + instance_name, + retention_days, + ) + + # Tag The SnapShot To Facilitate Later Automated Deletion & Offsiting + ec.create_tags( + Resources=[snap['SnapshotId']], + Tags=snapshot_tags + ) + + LambdaFunctionDeleteSnapshots: + Type: AWS::Lambda::Function + Properties: + FunctionName: Delete-Snapshots + Description: Invokes EBS Snapshot Deletion (Age-Off) Process + Handler: index.lambda_handler + MemorySize: 128 + Role: !GetAtt [LambdaBackupsRole, Arn] + Runtime: python2.7 + Timeout: 300 + Code: + ZipFile: + Fn::Sub: | + import boto3 + import re + import datetime + + ec = boto3.client('ec2') + + """ + This function looks at *all* snapshots that have a "DeleteOn" tag containing + the current day formatted as YYYY-MM-DD. This function should be run at least + daily. + """ + + def lambda_handler(event, context): + account_ids = (boto3.client('sts').get_caller_identity()['Account']) + + delete_on = datetime.date.today().strftime('%Y-%m-%d') + # limit snapshots to process to ones marked for deletion on this day + # AND limit snapshots to process to ones that are automated only + # AND exclude automated snapshots marked for permanent retention + filters = [ + { 'Name': 'tag:DeleteOn', 'Values': [delete_on] }, + { 'Name': 'tag:Type', 'Values': ['Automated'] }, + ] + snapshot_response = ec.describe_snapshots(OwnerIds=[account_ids], Filters=filters) + + for snap in snapshot_response['Snapshots']: + for tag in snap['Tags']: + if tag['Key'] != 'KeepForever': + skipping_this_one = False + continue + else: + skipping_this_one = True + + if skipping_this_one == True: + print "Skipping snapshot %s (marked KeepForever)" % snap['SnapshotId'] + # do nothing else + else: + print "Deleting snapshot %s" % snap['SnapshotId'] + ec.delete_snapshot(SnapshotId=snap['SnapshotId']) \ No newline at end of file diff --git a/ebs-snapshot-creator.py b/ebs-snapshot-creator.py index f5d553c..6c7b956 100644 --- a/ebs-snapshot-creator.py +++ b/ebs-snapshot-creator.py @@ -1,81 +1,141 @@ import boto3 import collections import datetime - -region = 'us-west-2' # region we're running in (should be changed to be auto-determined +import os ec = boto3.client('ec2') def lambda_handler(event, context): - reservations = ec.describe_instances( - Filters=[ - { 'Name': 'tag:Backup', 'Values': ['Yes'] }, - ] - ).get( - 'Reservations', [] - ) - - instances = sum( - [ - [i for i in r['Instances']] - for r in reservations - ], []) - - print "Found %d instances that need backing up" % len(instances) - - to_tag = collections.defaultdict(list) - - for instance in instances: - try: - retention_days = [ - int(t.get('Value')) for t in instance['Tags'] - if t['Key'] == 'Retention'][0] - except IndexError: - retention_days = 7 - - for dev in instance['BlockDeviceMappings']: - if dev.get('Ebs', None) is None: - continue - vol_id = dev['Ebs']['VolumeId'] - dev_name = dev['DeviceName'] - print "\tFound EBS volume %s (%s) on instance %s" % ( - vol_id, dev_name, instance['InstanceId']) - - # figure out instance name if there is one - instance_name = "" - for tag in instance['Tags']: - if tag['Key'] != 'Name': - continue - else: - instance_name = tag['Value'] - - description = '%s - %s (%s)' % ( instance_name, vol_id, dev_name ) + + # Get Current Region + aws_region = os.getenv('AWS_REGION') + + # Determine Which Instances To SnapShot + instances = ec.describe_instances( + Filters=[ + { 'Name': 'tag:Backup', 'Values': ['Yes'] }, + ] + ).get( + 'Reservations', [] + ) + + print "Found %d instances that need backing up" % len(instances) + + # Iterate Over Each Instance & SnapShot Volumes Not Explicitly Excluded From Backups + for instance in instances: + + # Get Instance Object + instance = instance['Instances'][0] + + # Determine Retention Period Based Upon Tags + retention_days = 7 + destination_region = None + instance_name = "" + for tag in instance['Tags']: + if tag['Key'] == 'RetentionDays' and tag['Value'] > 0: + retention_days = int(tag['Value']) + + if tag['Key'] == 'DestinationRegion' and len(tag['Value']) > 0: + destination_region = tag['Value'] + + if tag['Key'] == 'Name' and len(tag['Value']) > 0: + instance_name = tag['Value'] + + print "Setting SnapShot retention period To %s days" % (retention_days) + + # Determine When We're Going To Delete This SnapShot + delete_date = datetime.date.today() + datetime.timedelta(days=retention_days) + delete_fmt = delete_date.strftime('%Y-%m-%d') + + # Set Default SnapShot Tags + snapshot_tags = [ + { 'Key': 'DeleteOn', 'Value': delete_fmt }, + { 'Key': 'Type', 'Value': 'Automated' }, + ] + + # If We Want To Offsite This SnapShot, Set The Appropriate Tag + if destination_region != None: + snapshot_tags = snapshot_tags + [{ 'Key': 'DestinationRegion', 'Value': destination_region }] + + # List All Volumes Attached To The Instance + for dev in instance['BlockDeviceMappings']: + + # Set Variable Defaults + snapshot_required = True + volume_name = None + + if dev.get('Ebs', None) is None: + continue + vol_id = dev['Ebs']['VolumeId'] + dev_name = dev['DeviceName'] + + # Get a Volume Object Based Upon Volume ID + volume = ec.describe_volumes( + VolumeIds=[vol_id,] + )['Volumes'][0] + + # Set Default SnapShot Description + description = '%s - %s (%s)' % ( + instance_name, + vol_id, + dev_name + ) + + if 'Tags' in volume: + for tag in volume['Tags']: + + # Determine If Volume Has 'Backup' Flag Set To 'No' & Exclude From SnapShot If It Does + if tag['Key'] == 'Backup' and tag['Value'] == 'No': + snapshot_required = False + + # Override Default Description With Volume Name If One Specified + if tag['Key'] == 'Name': + description = tag['Value'] - # trigger snapshot - snap = ec.create_snapshot( - VolumeId=vol_id, - Description=description - ) - if (snap): - print "\t\tSnapshot %s created in %s of [%s]" % ( snap['SnapshotId'], region, description ) - to_tag[retention_days].append(snap['SnapshotId']) - print "\t\tRetaining snapshot %s of volume %s from instance %s (%s) for %d days" % ( - snap['SnapshotId'], - vol_id, - instance['InstanceId'], - instance_name, - retention_days, - ) - - for retention_days in to_tag.keys(): - delete_date = datetime.date.today() + datetime.timedelta(days=retention_days) - delete_fmt = delete_date.strftime('%Y-%m-%d') - print "Will delete %d snapshots on %s" % (len(to_tag[retention_days]), delete_fmt) - ec.create_tags( - Resources=to_tag[retention_days], - Tags=[ - { 'Key': 'DeleteOn', 'Value': delete_fmt }, - { 'Key': 'Type', 'Value': 'Automated' }, - ] + # We Don't Want To SnapShot Any Volume Explictly Excluded + if snapshot_required == False: + print "\tIgnoring EBS volume %s (%s) on instance %s - 'Backup' Tag set to 'No'" % ( + vol_id, + dev_name, + instance['InstanceId'] ) + + continue + + + print "\tFound EBS volume %s (%s) on instance %s - Proceeding with SnapShot" % ( + vol_id, + dev_name, + instance['InstanceId'] + ) + + # Take SnapShot Of Volume + snap = ec.create_snapshot( + VolumeId=vol_id, + Description=description + ) + + if not (snap): + print "\t\tSnapShot operation failed!" + continue + + print "\t\tSnapshot %s created in %s of [%s]" % ( + snap['SnapshotId'], + aws_region, + description + ) + + print "\t\tRetaining snapshot %s of volume %s from instance %s (%s) for %d days" % ( + snap['SnapshotId'], + vol_id, + instance['InstanceId'], + instance_name, + retention_days, + ) + + # Tag The SnapShot To Facilitate Later Automated Deletion & Offsiting + ec.create_tags( + Resources=[snap['SnapshotId']], + Tags=snapshot_tags + ) \ No newline at end of file diff --git a/ebs-snapshot-manager.py b/ebs-snapshot-manager.py index c1643ca..8c56aa0 100644 --- a/ebs-snapshot-manager.py +++ b/ebs-snapshot-manager.py @@ -3,7 +3,6 @@ import datetime ec = boto3.client('ec2') -iam = boto3.client('iam') """ This function looks at *all* snapshots that have a "DeleteOn" tag containing @@ -12,21 +11,8 @@ """ def lambda_handler(event, context): - account_ids = list() - try: - """ - You can replace this try/except by filling in `account_ids` yourself. - Get your account ID with: - > import boto3 - > iam = boto3.client('iam') - > print iam.get_user()['User']['Arn'].split(':')[4] - """ - iam.get_user() - except Exception as e: - # use the exception message to get the account ID the function executes under - account_ids.append(re.search(r'(arn:aws:sts::)([0-9]+)', str(e)).groups()[1]) - - + account_ids = (boto3.client('sts').get_caller_identity()['Account']) + delete_on = datetime.date.today().strftime('%Y-%m-%d') # limit snapshots to process to ones marked for deletion on this day # AND limit snapshots to process to ones that are automated only @@ -35,7 +21,7 @@ def lambda_handler(event, context): { 'Name': 'tag:DeleteOn', 'Values': [delete_on] }, { 'Name': 'tag:Type', 'Values': ['Automated'] }, ] - snapshot_response = ec.describe_snapshots(OwnerIds=account_ids, Filters=filters) + snapshot_response = ec.describe_snapshots(OwnerIds=[account_ids], Filters=filters) for snap in snapshot_response['Snapshots']: for tag in snap['Tags']: @@ -50,4 +36,4 @@ def lambda_handler(event, context): # do nothing else else: print "Deleting snapshot %s" % snap['SnapshotId'] - ec.delete_snapshot(SnapshotId=snap['SnapshotId']) + ec.delete_snapshot(SnapshotId=snap['SnapshotId']) \ No newline at end of file diff --git a/ebs-snapshot-offsiter.py b/ebs-snapshot-offsiter.py new file mode 100644 index 0000000..676be63 --- /dev/null +++ b/ebs-snapshot-offsiter.py @@ -0,0 +1,114 @@ +import boto3 +import collections +import datetime +import os + +ec = boto3.client('ec2') + +def lambda_handler(event, context): + + # Get Current Region + aws_region = os.getenv('AWS_REGION') + + # Get All SnapShots With 'Offsite' Tag Set + snapshots = ec.describe_snapshots( + Filters=[ + { 'Name': 'tag-key', 'Values': ['DestinationRegion'] }, + { 'Name': 'status', 'Values': ['completed'] }, + ] + ) + + for snapshot in snapshots['Snapshots']: + + # Reset Our Destination Region + destination_region = None + + # Obtain Tags From Source SnapShot + for tag in snapshot['Tags']: + + # Obtain Destination Region From Source Snapshot Tag + if tag['Key'] == 'DestinationRegion': + destination_region = tag['Value'] + + # Check If We Need To Do A Copy Or Not + if destination_region == aws_region: + + print "\tDestination Region %s is the same as current region (%s) - skipping copy" % ( + destination_region, + aws_region + ) + + continue + + # Construct ec2 Client For Secondary Region + secondary_ec = boto3.client('ec2', region_name=destination_region) + + # Determine If There's An Off-Site Copy Of This SnapShot + os_snapshots = secondary_ec.describe_snapshots( + Filters=[ + { 'Name': 'tag:SourceSnapshotId', 'Values': [snapshot['SnapshotId']] }, + { 'Name': 'status', 'Values': ['pending','completed'] }, + ] + ) + + # We Only Want To Delete Where Snapshot Has Copied Successfully + if len(os_snapshots['Snapshots']) >= 1: + + snapshot_states = [d['State'] for d in os_snapshots['Snapshots']] + + if 'pending' in snapshot_states: + print "\tThere is at least 1 Snapshot copy pending in %s - skipping delete & copy" % ( + destination_region + ) + + continue + + print "\t\tFound a corresponding Snapshot with Id %s in %s created from Snapshot %s" % ( + os_snapshots['Snapshots'][0]['SnapshotId'], + destination_region, + snapshot['SnapshotId'] + ) + + print "Deleting source Snapshot %s from %s" % ( + snapshot['SnapshotId'], + aws_region + ) + + ec.delete_snapshot( + SnapshotId=snapshot['SnapshotId'] + ) + + continue + + # Create Copy Of Snapshot Because One Doesn't Exist + os_snapshot = secondary_ec.copy_snapshot( + SourceRegion=aws_region, + SourceSnapshotId=snapshot['SnapshotId'], + Description=snapshot['Description'], + DestinationRegion=destination_region + ) + + # If Snapshot Copy Executed Successfully, Copy The Tags + if (os_snapshot): + + print "\t\tSnapshot copy %s created in %s of %s from %s" % ( + os_snapshot['SnapshotId'], + destination_region, + snapshot['SnapshotId'], + aws_region + ) + + # Add Tags To Off-Site SnapShot + destination_snapshot_tags = snapshot['Tags'] + [{ 'Key': 'SourceSnapshotId', 'Value': snapshot['SnapshotId'] }] + secondary_ec.create_tags( + Resources=[os_snapshot['SnapshotId']], + Tags=destination_snapshot_tags + ) + + # Add Tags To Source SnapShot + ec.create_tags( + Resources=[snapshot['SnapshotId']], + Tags=[ + { 'Key': 'OffsiteSnapshotId', 'Value': os_snapshot['SnapshotId'] }, + ] + ) \ No newline at end of file