From a2ce68cbf2fd9788263fa9a46eec75957b87c075 Mon Sep 17 00:00:00 2001 From: Domenico Testa Date: Mon, 8 Jun 2020 20:14:56 +0200 Subject: [PATCH] Adding more features. - Implementing support for custom domains - Blocking common link previewers to "unwind" the links - Setting a log retention policy for the lambda functions - Updating to latest CDK version (1.44) --- README.md | 25 ++++++-- app.py | 17 +++++- once/download-and-delete/handler.py | 27 ++++++++- once/get-upload-ticket/handler.py | 4 +- once/once_stack.py | 91 ++++++++++++++++++++++++++--- requirements.txt | 4 ++ setup.py | 2 +- 7 files changed, 149 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index b6b4fda..63c723d 100644 --- a/README.md +++ b/README.md @@ -21,13 +21,26 @@ With CDK I could create the following resources: I will use API Gateway to expose the lambda functions as an HTTP API. - HERE BE DIAGRAM! +## TODO - mkdir once - cd once - cdk init app +[+] Publish it to a custom domain name: DONE +[+] Set logs retention policy +[+] Deploy custom domain as a nested stack +[+] Mask link preview depending on the user agent -Then it should be easy to start organizing the project layout. -One single stack, one folder for each lambda function. +- Add a robust authentication method +- Add progressbar to client + +- Package application as a click app + + + +- Write a proper README with instructions +- Record a demo +- write tests with pytest + +- publish the source code +- write a blog post +- add a link to the blog post in the README \ No newline at end of file diff --git a/app.py b/app.py index f5dd07e..e3f8e5a 100644 --- a/app.py +++ b/app.py @@ -1,11 +1,22 @@ #!/usr/bin/env python3 -from aws_cdk import core +import os +from aws_cdk import ( + core, + aws_route53 as route53) -from once.once_stack import OnceStack +from once.once_stack import OnceStack, CustomDomainStack + +USE_CUSTOM_DOMAIN = True +DOMAIN_NAME = os.getenv('DOMAIN_NAME') +HOSTED_ZONE_NAME = os.getenv('HOSTED_ZONE_NAME') +HOSTED_ZONE_ID = os.getenv('HOSTED_ZONE_ID') app = core.App() -OnceStack(app, "once") +once = OnceStack(app, 'once', + custom_domain=DOMAIN_NAME, + hosted_zone_id=HOSTED_ZONE_ID, + hosted_zone_name=HOSTED_ZONE_NAME) app.synth() diff --git a/once/download-and-delete/handler.py b/once/download-and-delete/handler.py index 6d52fed..086e597 100644 --- a/once/download-and-delete/handler.py +++ b/once/download-and-delete/handler.py @@ -2,6 +2,7 @@ import os import io import json import logging +import re import boto3 @@ -18,6 +19,20 @@ DEBUG = is_debug_enabled() FILES_BUCKET = os.getenv('FILES_BUCKET') FILES_TABLE_NAME = os.getenv('FILES_TABLE_NAME') PRESIGNED_URL_EXPIRES_IN = int(os.getenv('PRESIGNED_URL_EXPIRES_IN', 20)) +MASKED_USER_AGENTS = os.getenv('MASKED_USER_AGENTS', ','.join([ + '^Facebook.*', + '^Google.*', + '^Instagram.*', + '^LinkedIn.*', + '^Outlook.*', + '^Reddit.*', + '^Slack.*', + '^Skype.*', + '^SnapChat.*', + '^Telegram.*', + '^Twitter.*', + '^WhatsApp.*' +])).split(',') log = logging.getLogger() @@ -49,6 +64,17 @@ def on_event(event, context): log.info(error_message) return {'statusCode': 404, 'body': error_message} + # Some rich clients try to get a preview of any link pasted + # into text controls. + user_agent = event['headers'].get('user-agent', '') + is_masked_agent = any([re.match(agent, user_agent) for agent in MASKED_USER_AGENTS]) + if is_masked_agent: + log.info('Serving possible link preview. Download prevented.') + return { + 'statusCode': 200, + 'headers': {} + } + s3 = boto3.client('s3') download_url = s3.generate_presigned_url( 'get_object', @@ -69,4 +95,3 @@ def on_event(event, context): 'Location': download_url } } - diff --git a/once/get-upload-ticket/handler.py b/once/get-upload-ticket/handler.py index 7bb3498..ea59750 100644 --- a/once/get-upload-ticket/handler.py +++ b/once/get-upload-ticket/handler.py @@ -66,9 +66,7 @@ def on_event(event, context): response = {} try: if filename is None: - raise BadRequestError( - 'Please provide a valid value for the `filename_prefix` ' - 'query parameter') + raise BadRequestError('Provide a valid value for the `f` query parameter') domain = string.ascii_uppercase + string.ascii_lowercase + string.digits entry_id = ''.join(random.choice(domain) for _ in range(6)) diff --git a/once/once_stack.py b/once/once_stack.py index 3ad70ae..7f05f0a 100644 --- a/once/once_stack.py +++ b/once/once_stack.py @@ -1,22 +1,85 @@ import os +from typing import Optional +import jsii from aws_cdk import( core, aws_apigatewayv2 as apigw, + aws_certificatemanager as certmgr, + aws_cloudformation as cfn, aws_dynamodb as dynamodb, aws_events as events, aws_events_targets as targets, aws_lambda as lambda_, + aws_logs as logs, + aws_route53 as route53, + aws_route53_targets as route53_targets, aws_s3 as s3) from .utils import make_python_zip_bundle BASE_PATH = os.path.dirname(os.path.abspath(__file__)) +LOG_RETENTION = getattr(logs.RetentionDays, os.getenv('LOG_RETENTION', 'TWO_WEEKS')) + + +@jsii.implements(route53.IAliasRecordTarget) +class ApiGatewayV2Domain(object): + def __init__(self, domain_name: apigw.CfnDomainName): + self.domain_name = domain_name + + @jsii.member(jsii_name='bind') + def bind(self, _record: route53.IRecordSet) -> route53.AliasRecordTargetConfig: + return { + 'dnsName': self.domain_name.get_att('RegionalDomainName').to_string(), + 'hostedZoneId': self.domain_name.get_att('RegionalHostedZoneId').to_string() + } + +class CustomDomainStack(cfn.NestedStack): + def __init__(self, scope: core.Construct, id: str, + hosted_zone_id: str, + hosted_zone_name: str, + domain_name: str, + api: apigw.HttpApi): + super().__init__(scope, id) + + hosted_zone = route53.HostedZone.from_hosted_zone_attributes(self, id='dns-hosted-zone', + hosted_zone_id=hosted_zone_id, + zone_name=hosted_zone_name) + + certificate = certmgr.DnsValidatedCertificate(self, 'tls-certificate', + domain_name=domain_name, + hosted_zone=hosted_zone, + validation_method=certmgr.ValidationMethod.DNS) + + custom_domain = apigw.CfnDomainName(self, 'custom-domain', + domain_name=domain_name, + domain_name_configurations=[ + apigw.CfnDomainName.DomainNameConfigurationProperty( + certificate_arn=certificate.certificate_arn)]) + + custom_domain.node.add_dependency(api) + custom_domain.node.add_dependency(certificate) + + api_mapping = apigw.CfnApiMapping(self, 'custom-domain-mapping', + api_id=api.http_api_id, + domain_name=domain_name, + stage='$default') + + api_mapping.node.add_dependency(custom_domain) + + route53.ARecord(self, 'custom-domain-record', + target=route53.RecordTarget.from_alias(ApiGatewayV2Domain(custom_domain)), + zone=hosted_zone, + record_name=domain_name) class OnceStack(core.Stack): - def __init__(self, scope: core.Construct, id: str, **kwargs) -> None: + def __init__(self, scope: core.Construct, id: str, + custom_domain: Optional[str] = None, + hosted_zone_id: Optional[str] = None, + hosted_zone_name: Optional[str] = None, + **kwargs) -> None: super().__init__(scope, id, **kwargs) self.files_bucket = s3.Bucket(self, 'files-bucket', @@ -33,16 +96,21 @@ class OnceStack(core.Stack): self.api = apigw.HttpApi(self, 'once-api', api_name='once-api') - core.CfnOutput(self, 'api-url', value=self.api.url) + api_url = self.api.url + if custom_domain is not None: + api_url = f'https://{custom_domain}/' + + core.CfnOutput(self, 'api-url', value=api_url) self.get_upload_ticket_function = lambda_.Function(self, 'get-upload-ticket-function', function_name='once-get-upload-ticket', + description='Returns a pre-signed request to share a file', runtime=lambda_.Runtime.PYTHON_3_7, code=make_python_zip_bundle(os.path.join(BASE_PATH, 'get-upload-ticket')), handler='handler.on_event', - description='Returns a pre-signed request to share a file', + log_retention=LOG_RETENTION, environment={ - 'APP_URL': self.api.url, + 'APP_URL': api_url, 'FILES_TABLE_NAME': self.files_table.table_name, 'FILES_BUCKET': self.files_bucket.bucket_name }) @@ -52,10 +120,11 @@ class OnceStack(core.Stack): self.download_and_delete_function = lambda_.Function(self, 'download-and-delete-function', function_name='once-download-and-delete', + description='Serves a file from S3 and deletes it as soon as it has been successfully transferred', runtime=lambda_.Runtime.PYTHON_3_7, code=lambda_.Code.from_asset(os.path.join(BASE_PATH, 'download-and-delete')), handler='handler.on_event', - description='Serves a file from S3 and deletes it as soon as it has been successfully transferred.', + log_retention=LOG_RETENTION, environment={ 'FILES_BUCKET': self.files_bucket.bucket_name, 'FILES_TABLE_NAME': self.files_table.table_name @@ -79,10 +148,11 @@ class OnceStack(core.Stack): self.cleanup_function = lambda_.Function(self, 'delete-served-files-function', function_name='once-delete-served-files', + description='Deletes files from S3 once they have been marked as deleted in DynamoDB', runtime=lambda_.Runtime.PYTHON_3_7, code=lambda_.Code.from_asset(os.path.join(BASE_PATH, 'delete-served-files')), handler='handler.on_event', - description='Deletes files from S3 once they have been marked as deleted in DynamoDB', + log_retention=LOG_RETENTION, environment={ 'FILES_BUCKET': self.files_bucket.bucket_name, 'FILES_TABLE_NAME': self.files_table.table_name @@ -91,6 +161,13 @@ class OnceStack(core.Stack): self.files_bucket.grant_delete(self.cleanup_function) self.files_table.grant_read_write_data(self.cleanup_function) - events.Rule(self, "once-delete-served-files-rule", + events.Rule(self, 'once-delete-served-files-rule', schedule=events.Schedule.rate(core.Duration.hours(24)), targets=[targets.LambdaFunction(self.cleanup_function)]) + + if custom_domain is not None: + self.custom_domain_stack = CustomDomainStack(self, 'custom-domain', + api=self.api, + domain_name=custom_domain, + hosted_zone_id=hosted_zone_id, + hosted_zone_name=hosted_zone_name) diff --git a/requirements.txt b/requirements.txt index acb7ca8..7972894 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,12 @@ -e . aws_cdk.core aws_cdk.aws_apigatewayv2 +aws_cdk.aws_certificatemanager +aws_cdk.aws_cloudformation aws_cdk.aws_dynamodb aws_cdk.aws_events aws_cdk.aws_events_targets aws_cdk.aws_lambda +aws_cdk.aws_logs +aws_cdk.aws_route53 aws_cdk.aws_s3 diff --git a/setup.py b/setup.py index 0f350e1..79fd7d1 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ setuptools.setup( long_description=long_description, long_description_content_type="text/markdown", - author="author", + author="Domenico Testa", package_dir={"": "once"}, packages=setuptools.find_packages(where="once"),