diff --git a/once/delete-served-files/handler.py b/once/delete-served-files/handler.py new file mode 100644 index 0000000..6aead43 --- /dev/null +++ b/once/delete-served-files/handler.py @@ -0,0 +1,56 @@ +import os +import io +import json +import logging +from typing import Dict + +import boto3 +from boto3.dynamodb.conditions import Key + + +def is_debug_enabled() -> bool: + value = os.getenv('DEBUG', 'false').lower() + if value in ['false', '0']: + return False + else: + return bool(value) + + +DEBUG = is_debug_enabled() +FILES_BUCKET = os.getenv('FILES_BUCKET') +FILES_TABLE_NAME = os.getenv('FILES_TABLE_NAME') + + +log = logging.getLogger() +if DEBUG: + log.setLevel(logging.DEBUG) +else: + log.setLevel(logging.INFO) + + +def on_event(event, context): + log.info(f'Event received: {event}') + log.info(f'Context is: {context}') + log.debug(f'Debug mode is {DEBUG}') + log.debug(f'Files bucket is "{FILES_BUCKET}"') + + dynamodb = boto3.client('dynamodb') + response = dynamodb.scan( + TableName=FILES_TABLE_NAME, + Select='ALL_ATTRIBUTES', + FilterExpression='deleted = :deleted', + ExpressionAttributeValues={ + ':deleted': {'BOOL': True} + }) + + s3 = boto3.client('s3') + for item in response['Items']: + object_name = item['object_name']['S'] + log.info(f'Deleting file {object_name}') + try: + s3.delete_object(Bucket=FILES_BUCKET, Key=object_name) + except: + log.exception('Could not delete file {object_name}') + + response = dynamodb.delete_item(TableName=FILES_TABLE_NAME, Key={'id': item['id']}) + log.debug(f'dynamodb delete item: {response}') diff --git a/once/once_stack.py b/once/once_stack.py index dd374ad..3ad70ae 100644 --- a/once/once_stack.py +++ b/once/once_stack.py @@ -4,6 +4,8 @@ from aws_cdk import( core, aws_apigatewayv2 as apigw, aws_dynamodb as dynamodb, + aws_events as events, + aws_events_targets as targets, aws_lambda as lambda_, aws_s3 as s3) @@ -31,6 +33,8 @@ class OnceStack(core.Stack): self.api = apigw.HttpApi(self, 'once-api', api_name='once-api') + core.CfnOutput(self, 'api-url', value=self.api.url) + self.get_upload_ticket_function = lambda_.Function(self, 'get-upload-ticket-function', function_name='once-get-upload-ticket', runtime=lambda_.Runtime.PYTHON_3_7, @@ -73,4 +77,20 @@ class OnceStack(core.Stack): methods=[apigw.HttpMethod.GET], integration=download_and_delete_integration) - core.CfnOutput(self, 'api-url', value=self.api.url) + self.cleanup_function = lambda_.Function(self, 'delete-served-files-function', + function_name='once-delete-served-files', + runtime=lambda_.Runtime.PYTHON_3_7, + code=lambda_.Code.from_asset(os.path.join(BASE_PATH, 'delete-served-files')), + handler='handler.on_event', + description='Deletes files from S3 once they have been marked as deleted in DynamoDB', + environment={ + 'FILES_BUCKET': self.files_bucket.bucket_name, + 'FILES_TABLE_NAME': self.files_table.table_name + }) + + self.files_bucket.grant_delete(self.cleanup_function) + self.files_table.grant_read_write_data(self.cleanup_function) + + events.Rule(self, "once-delete-served-files-rule", + schedule=events.Schedule.rate(core.Duration.hours(24)), + targets=[targets.LambdaFunction(self.cleanup_function)]) diff --git a/requirements.txt b/requirements.txt index 079a913..acb7ca8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,7 @@ aws_cdk.core aws_cdk.aws_apigatewayv2 aws_cdk.aws_dynamodb +aws_cdk.aws_events +aws_cdk.aws_events_targets aws_cdk.aws_lambda aws_cdk.aws_s3