Adding a cleanup function

once/delete-served-files/handler.py

@@ -0,0 +1,56 @@
+import os
+import io
+import json
+import logging
+from typing import Dict
+
+import boto3
+from boto3.dynamodb.conditions import Key
+
+
+def is_debug_enabled() -> bool:
+    value = os.getenv('DEBUG', 'false').lower()
+    if value in ['false', '0']:
+        return False
+    else:
+        return bool(value)
+
+
+DEBUG = is_debug_enabled()
+FILES_BUCKET = os.getenv('FILES_BUCKET')
+FILES_TABLE_NAME = os.getenv('FILES_TABLE_NAME')
+
+
+log = logging.getLogger()
+if DEBUG:
+    log.setLevel(logging.DEBUG)
+else:
+    log.setLevel(logging.INFO)
+
+
+def on_event(event, context):
+    log.info(f'Event received: {event}')
+    log.info(f'Context is: {context}')
+    log.debug(f'Debug mode is {DEBUG}')
+    log.debug(f'Files bucket is "{FILES_BUCKET}"')
+
+    dynamodb = boto3.client('dynamodb')
+    response = dynamodb.scan(
+        TableName=FILES_TABLE_NAME,
+        Select='ALL_ATTRIBUTES',
+        FilterExpression='deleted = :deleted',
+        ExpressionAttributeValues={
+            ':deleted': {'BOOL': True}
+        })
+
+    s3 = boto3.client('s3')
+    for item in response['Items']:
+        object_name = item['object_name']['S']
+        log.info(f'Deleting file {object_name}')
+        try:
+            s3.delete_object(Bucket=FILES_BUCKET, Key=object_name)
+        except:
+            log.exception('Could not delete file {object_name}')
+
+        response = dynamodb.delete_item(TableName=FILES_TABLE_NAME, Key={'id': item['id']})
+        log.debug(f'dynamodb delete item: {response}')

once/once_stack.py

@@ -4,6 +4,8 @@ from aws_cdk import(
     core,
     aws_apigatewayv2 as apigw,
     aws_dynamodb as dynamodb,
+    aws_events as events,
+    aws_events_targets as targets,
     aws_lambda as lambda_,
     aws_s3 as s3)
 
@@ -31,6 +33,8 @@ class OnceStack(core.Stack):
 
         self.api = apigw.HttpApi(self, 'once-api', api_name='once-api')
 
+        core.CfnOutput(self, 'api-url', value=self.api.url)
+
         self.get_upload_ticket_function = lambda_.Function(self, 'get-upload-ticket-function',
             function_name='once-get-upload-ticket',
             runtime=lambda_.Runtime.PYTHON_3_7,
@@ -73,4 +77,20 @@ class OnceStack(core.Stack):
             methods=[apigw.HttpMethod.GET],
             integration=download_and_delete_integration)
 
-        core.CfnOutput(self, 'api-url', value=self.api.url)
+        self.cleanup_function = lambda_.Function(self, 'delete-served-files-function',
+            function_name='once-delete-served-files',
+            runtime=lambda_.Runtime.PYTHON_3_7,
+            code=lambda_.Code.from_asset(os.path.join(BASE_PATH, 'delete-served-files')),
+            handler='handler.on_event',
+            description='Deletes files from S3 once they have been marked as deleted in DynamoDB',
+            environment={
+                'FILES_BUCKET': self.files_bucket.bucket_name,
+                'FILES_TABLE_NAME': self.files_table.table_name
+            })
+
+        self.files_bucket.grant_delete(self.cleanup_function)
+        self.files_table.grant_read_write_data(self.cleanup_function)
+
+        events.Rule(self, "once-delete-served-files-rule",
+            schedule=events.Schedule.rate(core.Duration.hours(24)),
+            targets=[targets.LambdaFunction(self.cleanup_function)])

requirements.txt

@@ -2,5 +2,7 @@
 aws_cdk.core
 aws_cdk.aws_apigatewayv2
 aws_cdk.aws_dynamodb
+aws_cdk.aws_events
+aws_cdk.aws_events_targets
 aws_cdk.aws_lambda
 aws_cdk.aws_s3