Initial commit

README.md

+tf_mod_aws_lambda_ebs_snapshots
+==============
+A Terraform module for creating a Lambda function and associated IAM role for creating and managing EBS snapshots and rotation.
+The module makes heavy use of tagging, and required that EBS volumes and/or EC2 instances are creating with specific
+tags based upon the defaults listed below or a custom set of tags.
+
+Input Variables
+---------------
+
+- `min_retention_count` - Minimum number of EBS snapshots to retain. Defaults to 1.
+- `default_retention_days` - Default number of days to retain EBS snapshots for. Defaults to 7 days.
+- `snapshot_tag` - Tag to use to mark EBS volumes for snapshotting. Defaults to 'MakeSnapshot'.
+- `retention_tag` - Tag to use to mark EBS volume for retention. Defaults to 'Retention'.
+- `delete_tag` - Tag to use to allow Lamba to automatically delete old EBS snapshots. Defaults to 'AutoDelete'.
+- `cloudwatch_schedule` - Schedule for Cloudwatch event to invoke Lambda EBS snapshot function. Defaults to `cron(0 1 ? * * *)` which is every day at 01:00.
+
+Outputs
+-------
+
+None.
+
+Usage
+-----
+
+You can use this module in your Terraform template with the following steps. All variables have defaults set so setting any variables is optional.
+
+1.) Adding a module resource to your template, e.g. `main.tf`
+
+```
+variable "min_retention_count" {}
+variable "default_retention_days" {}
+variable "snapshot_tag" {}
+variable "retention_tag" {}
+variable "delete_tag" {}
+variable "cloudwatch_schedule" {}
+
+#############################################################################################################
+# Lambda EBS Snapshots
+#############################################################################################################
+module "lambda_ebs_snapshots" {
+  source = "git::https://git.steamhaus.co.uk/steamhaus/tf_mod_aws_lambda_ebs_snapshots"
+  
+  min_retention_count    = "${var.min_retention_count}"
+  default_retention_days = "${var.default_retention_days}"
+  snapshot_tag           = "${var.snapshot_tag}"
+  retention_tag          = "${var.retention_tag}"
+  delete_tag             = "${var.delete_tag}"
+  cloudwatch_schedule    = "${var.cloudwatch_schedule}"
+}
+```

lambda_ebs_snapshots.py

+import os
+import boto3
+import datetime
+import logging
+
+"""
+This function lists all instances that have the `MakeSnapshot` tag and performs
+a snapshot of all EBS volumes attached to it. It also tags the snapshots for
+deletion after a number of retention days.
+"""
+
+MIN_RETENTION_COUNT = os.environ['MIN_RETENTION_COUNT']
+RETENTION_DAYS_DEFAULT = os.environ['DEFAULT_RETENTION_DAYS']
+SNAPSHOT_TAG = os.environ['SNAPSHOT_TAG']
+RETENTION_TAG = os.environ['RETENTION_TAG']
+DELETE_TAG = os.environ['DELETE_TAG']
+
+ec2 = boto3.resource('ec2')
+
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+def lambda_handler(event, context):
+    logger.info('AWS snapshot backups started for event [%s]', event)
+    instances = ec2.instances.filter(Filters=[
+        {'Name': 'tag-key', 'Values': [SNAPSHOT_TAG]},
+        {'Name': 'instance-state-name', 'Values': ['running']},
+    ])
+    for instance in instances:
+        instance_tags = tags_dict(instance.tags)
+        make_snapshot = str(instance_tags.get(SNAPSHOT_TAG)).lower()
+        if make_snapshot in ('false', '0', 'none'):
+            logger.info('Skip instance [%s]: [%s] tag value is false',
+                instance.id, SNAPSHOT_TAG)
+            continue
+        instance_name = instance_tags.get('Name', instance.id)
+        logger.info('Instance [%s]: [%s]', instance.id, instance_name)
+        retention_days = int(instance_tags.get(RETENTION_TAG, RETENTION_DAYS_DEFAULT))
+        for volume in instance.volumes.all():
+            volume_tags = tags_dict(volume.tags)
+            make_snapshot = str(volume_tags.get(SNAPSHOT_TAG, True)).lower()
+            if make_snapshot in ('false', '0', 'none'):
+                logger.info('\tSkip volume [%s]: [%s] tag value is false',
+                    volume.id, SNAPSHOT_TAG)
+                continue
+            logger.info('\tVolume [%s]', volume.id)
+            snapshot_date = datetime.datetime.utcnow().isoformat().rpartition('.')[0]
+            description = '{}.{}.{}'.format(instance_name, volume.id, snapshot_date)
+            new_snapshot = volume.create_snapshot(Description=description)
+            new_snapshot.create_tags(Tags=[{'Key': DELETE_TAG, 'Value': 'true'}])
+            logger.info('\t\tSnapshot created with description [%s]', description)
+            volume_retention_days = int(volume_tags.get(RETENTION_TAG, retention_days))
+            snapshots = volume.snapshots.filter(Filters=[
+                {'Name': 'tag-key', 'Values': [DELETE_TAG]},
+            ])
+            snapshots = list(snapshots)
+            if len(snapshots) <= MIN_RETENTION_COUNT:
+                logger.info('\t\tSkip deletion of snapshots: count [%d] <= minimum retention count [%d]',
+                    len(snapshots), MIN_RETENTION_COUNT)
+                continue
+            for snapshot in snapshots:
+                snapshot_tags = tags_dict(snapshot.tags)
+                auto_delete = str(snapshot_tags.get(DELETE_TAG)).lower()
+                if auto_delete != 'true':
+                    logger.info('\t\tSkip deletion of snapshot [%s]: [%s] tag value is not ["true"]',
+                        snapshot.id, DELETE_TAG)
+                    continue
+                tz = snapshot.start_time.tzinfo
+                snapshot_retention = datetime.timedelta(days=volume_retention_days)
+                is_expired = (datetime.datetime.now(tz) - snapshot.start_time) > snapshot_retention
+                if not is_expired:
+                    logger.info('\t\tSkip deletion of snapshot [%s]: still within retention period of [%d] days',
+                        snapshot.id, volume_retention_days)
+                    continue
+                logger.info('\t\tDelete snapshot [%s]: [%s]', snapshot.id, snapshot.description)
+                snapshot.delete()
+    logger.info('AWS snapshot backups completed')
+    return True
+
+
+def tags_dict(tags):
+        '''Flatten a list of tags dicts into a single dict'''
+        return {t['Key']: t['Value'] for t in tags} if tags else {}

main.tf

+#############################################################################################################
+# Variables
+#############################################################################################################
+variable "min_retention_count" {
+  description = "Minimum number of EBS snapshots to retain"
+  default     = "1"
+}
+
+variable "default_retention_days" {
+  description = "Default number of days to retain EBS snapshots for"
+  default     = "7"
+}
+
+variable "snapshot_tag" {
+  description = "Tag to use to mark EBS volumes for snapshotting"
+  default     = "MakeSnapshot"
+}
+
+variable "retention_tag" {
+  description = "Tag to use to mark EBS volume for retention"
+  default     = "Retention"
+}
+
+variable "delete_tag" {
+  description = "Tag to use to allow Lamba to automatically delete old EBS snapshots"
+  default     = "AutoDelete"
+}
+
+variable "cloudwatch_schedule" {
+  description = "Schedule for Cloudwatch event to invoke Lambda EBS snapshot function"
+  default     = "cron(0 1 ? * * *)"
+}
+
+#############################################################################################################
+# IAM
+#############################################################################################################
+data "aws_iam_policy_document" "main" {
+  statement {
+    effect = "Allow"
+    actions = [
+      "logs:*",
+    ]
+    resources = [
+      "arn:aws:logs:*:*:*",
+    ]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "ec2:Describe*",
+    ]
+    resources = [
+      "*",
+    ]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "ec2:CreateSnapshot",
+      "ec2:DeleteSnapshot",
+      "ec2:CreateTags",
+      "ec2:ModifySnapshotAttribute",
+      "ec2:ResetSnapshotAttribute",
+    ]
+    resources = [
+      "*",
+    ]
+  }
+}
+
+resource "aws_iam_role" "main" {
+  name               = "lambda_ebs_snapshots"
+  assume_role_policy = "${data.aws_iam_policy_document.main.json}"
+}
+
+#############################################################################################################
+# Lambda
+#############################################################################################################
+resource "aws_lambda_function" "main" {
+  filename         = "${path.module}/lambda_ebs_snapshots.zip"
+  function_name    = "lambda_ebs_snapshots"
+  role             = "${aws_iam_role.main.arn}"
+  handler          = "lambda_function.lambda_handler"
+  timeout          = "300"
+  runtime          = "python2.7"
+  source_code_hash = "${base64sha256(file("${path.module}/lambda_ebs_snapshots.zip"))}"
+
+  environment {
+    variables = {
+      MIN_RETENTION_COUNT    = "${var.min_retention_count}"
+      DEFAULT_RETENTION_DAYS = "${var.default_retention_days}"
+      SNAPSHOT_TAG           = "${var.snapshot_tag}"
+      RETENTION_TAG          = "${var.retention_tag}"
+      DELETE_TAG             = "${var.delete_tag}"
+      CLOUDWATCH_SCHEDULE    = "${var.cloudwatch_schedule}"
+    }
+  }
+}
+
+resource "aws_lambda_permission" "main" {
+  statement_id  = "AllowExecutionFromCloudWatch"
+  action        = "lambda:InvokeFunction"
+  function_name = "${aws_lambda_function.main.arn}"
+  principal     = "events.amazonaws.com"
+}
+
+#############################################################################################################
+# CloudWatch
+#############################################################################################################
+resource "aws_cloudwatch_event_rule" "main" {
+  name                = "lambda_ebs_snapshots_nightly"
+  description         = "Run EBS snapshot function each morning"
+  schedule_expression = "${var.cloudwatch_schedule}"
+}
+
+resource "aws_cloudwatch_event_target" "main" {
+  rule = "${aws_cloudwatch_event_rule.main.name}"
+  arn  = "${aws_lambda_function.main.arn}"
+}