Media Transcode Pipeline

AWS architecture template · S3 upload triggers a Lambda dispatcher that enqueues SQS jobs consumed by auto-scaled Fargate workers. Renditions are served via CloudFront.

Media Transcode Pipeline is a free AWS architecture template built with S3, Auto Scaling, CloudFront, CloudWatch, ECS, Fargate, IAM, Lambda, Route 53 and SQS. Open it in TheArchitect to edit the diagram, estimate monthly cost, and export the CloudFormation or Terraform that deploys it.

Open in TheArchitect All templates

Media Transcode Pipeline architecture diagram

AWS services in this template

S3×2
Auto Scaling
CloudFront
CloudWatch
ECS
Fargate
IAM
Lambda
Route 53
SQS

Deployment notes

Needs input

Configure S3 ObjectCreated notification to invoke the dispatcher Lambda.
Set SQS queue URL as environment variable on the Fargate task.
Create ACM certificate for custom CloudFront domain.

Generated CloudFormation — 13 resources

Logical ID	Resource type
UploadBuckets3	AWS::S3::Bucket
JobDispatcherlambda	AWS::Lambda::Function
TranscodeQueuesqs	AWS::SQS::Queue
TranscodeClusterecs	AWS::ECS::Cluster
TranscodeWorkersfargate	AWS::ECS::TaskDefinition
QueueDepthScalingautoscaling	AWS::AutoScaling::AutoScalingGroup
RenditionsBuckets3	AWS::S3::Bucket
VideoCDNcloudfront	AWS::CloudFront::Distribution
videoexamplecomroute53	AWS::Route53::HostedZone
TaskRoleiam	AWS::IAM::Role
BacklogAlarmcloudwatch	AWS::CloudWatch::Alarm
JobDispatcherlambdaUploadBuckets3Permission	AWS::Lambda::Permission
JobDispatcherlambdaTranscodeQueuesqsEventSourceMapping	AWS::Lambda::EventSourceMapping

View the full CloudFormation template

{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Description": "Generated by TheArchitect — https://www.thecloudarchitect.dev",
  "Metadata": {
    "TheArchitect": {
      "Edges": [
        {
          "From": "UploadBuckets3",
          "To": "JobDispatcherlambda",
          "Label": "ObjectCreated"
        },
        {
          "From": "JobDispatcherlambda",
          "To": "TranscodeQueuesqs",
          "Label": "enqueue job"
        },
        {
          "From": "TranscodeQueuesqs",
          "To": "TranscodeWorkersfargate",
          "Label": "long-poll consume"
        },
        {
          "From": "RenditionsBuckets3",
          "To": "VideoCDNcloudfront",
          "Label": "OAC origin"
        },
        {
          "From": "TaskRoleiam",
          "To": "TranscodeWorkersfargate",
          "Label": "task role"
        }
      ]
    }
  },
  "Resources": {
    "UploadBuckets3": {
      "Type": "AWS::S3::Bucket",
      "Properties": {
        "BucketName": "media-upload-bucket",
        "VersioningConfiguration": {
          "Status": "Suspended"
        },
        "PublicAccessBlockConfiguration": {
          "BlockPublicAcls": true,
          "BlockPublicPolicy": true,
          "IgnorePublicAcls": true,
          "RestrictPublicBuckets": true
        },
        "NotificationConfiguration": {
          "LambdaConfigurations": [
            {
              "Event": "s3:ObjectCreated:*",
              "Function": {
                "Fn::GetAtt": [
                  "JobDispatcherlambda",
                  "Arn"
                ]
              }
            }
          ]
        }
      },
      "DependsOn": [
        "JobDispatcherlambdaUploadBuckets3Permission"
      ]
    },
    "JobDispatcherlambda": {
      "Type": "AWS::Lambda::Function",
      "Properties": {
        "FunctionName": "media-job-dispatcher",
        "Runtime": "nodejs20.x",
        "Handler": "dispatch.handler",
        "MemorySize": 256,
        "Timeout": 30,
        "Description": "Triggered by S3 ObjectCreated event. Validates file type and enqueues a transcode job in SQS."
      }
    },
    "TranscodeQueuesqs": {
      "Type": "AWS::SQS::Queue",
      "Properties": {
        "QueueName": "media-transcode-queue",
        "VisibilityTimeout": 3600,
        "MessageRetentionPeriod": 86400,
        "ReceiveMessageWaitTimeSeconds": 20
      }
    },
    "TranscodeClusterecs": {
      "Type": "AWS::ECS::Cluster",
      "Properties": {
        "ClusterName": "media-transcode-cluster",
        "ClusterSettings": [
          {
            "Name": "containerInsights",
            "Value": "enabled"
          }
        ]
      }
    },
    "TranscodeWorkersfargate": {
      "Type": "AWS::ECS::TaskDefinition",
      "DependsOn": [
        "TaskRoleiam"
      ],
      "Properties": {
        "Family": "media-transcode-worker",
        "Cpu": "4096",
        "Memory": "8192",
        "NetworkMode": "awsvpc",
        "RequiresCompatibilities": [
          "FARGATE"
        ],
        "ExecutionRoleArn": {
          "Fn::GetAtt": [
            "TaskRoleiam",
            "Arn"
          ]
        },
        "TaskRoleArn": {
          "Fn::GetAtt": [
            "TaskRoleiam",
            "Arn"
          ]
        }
      }
    },
    "QueueDepthScalingautoscaling": {
      "Type": "AWS::AutoScaling::AutoScalingGroup",
      "Properties": {
        "AutoScalingGroupName": "transcode-queue-scaling",
        "MinSize": "0",
        "MaxSize": "20",
        "DesiredCapacity": "0"
      }
    },
    "RenditionsBuckets3": {
      "Type": "AWS::S3::Bucket",
      "Properties": {
        "BucketName": "media-renditions-bucket",
        "VersioningConfiguration": {
          "Status": "Suspended"
        },
        "PublicAccessBlockConfiguration": {
          "BlockPublicAcls": true,
          "BlockPublicPolicy": true,
          "IgnorePublicAcls": true,
          "RestrictPublicBuckets": true
        }
      }
    },
    "VideoCDNcloudfront": {
      "Type": "AWS::CloudFront::Distribution",
      "Properties": {
        "DistributionConfig": {
          "Enabled": true,
          "Comment": "Video CDN for media renditions",
          "PriceClass": "PriceClass_All",
          "DefaultCacheBehavior": {
            "TargetOriginId": "S3Origin",
            "ViewerProtocolPolicy": "redirect-to-https",
            "Compress": true,
            "ForwardedValues": {
              "QueryString": false,
              "Cookies": {
                "Forward": "none"
              }
            }
          },
          "ViewerCertificate": {
            "CloudFrontDefaultCertificate": true
          },
          "Origins": [
            {
              "DomainName": {
                "Fn::GetAtt": [
                  "RenditionsBuckets3",
                  "RegionalDomainName"
                ]
              },
              "Id": "S3Origin",
              "S3OriginConfig": {
                "OriginAccessIdentity": ""
              }
            }
          ]
        }
      }
    },
    "videoexamplecomroute53": {
      "Type": "AWS::Route53::HostedZone",
      "Properties": {
        "HostedZoneConfig": {
          "Comment": "Video delivery hostname pointing to CloudFront distribution"
        }
      }
    },
    "TaskRoleiam": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "RoleName": "media-task-role",
        "Description": "Fargate task role — SQS consumer + S3 read/write + CloudWatch Logs",
        "AssumeRolePolicyDocument": {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Effect": "Allow",
              "Principal": {
                "Service": "ecs-tasks.amazonaws.com"
              },
              "Action": "sts:AssumeRole"
            }
          ]
        },
        "ManagedPolicyArns": [
          "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"
        ]
      }
    },
    "BacklogAlarmcloudwatch": {
      "Type": "AWS::CloudWatch::Alarm",
      "Properties": {
        "AlarmName": "media-transcode-backlog",
        "AlarmDescription": "SQS visible messages > 100 — scale up Fargate workers to drain queue",
        "Namespace": "AWS/SQS",
        "MetricName": "ApproximateNumberOfMessagesVisible",
        "Statistic": "Average",
        "Period": 60,
        "EvaluationPeriods": 2,
        "Threshold": 100,
        "ComparisonOperator": "GreaterThanThreshold",
        "TreatMissingData": "notBreaching"
      }
    },
    "JobDispatcherlambdaUploadBuckets3Permission": {
      "Type": "AWS::Lambda::Permission",
      "DependsOn": [
        "JobDispatcherlambda"
      ],
      "Properties": {
        "Action": "lambda:InvokeFunction",
        "FunctionName": {
          "Ref": "JobDispatcherlambda"
        },
        "Principal": "s3.amazonaws.com",
        "SourceAccount": {
          "Ref": "AWS::AccountId"
        },
        "SourceArn": {
          "Fn::Sub": "arn:${AWS::Partition}:s3:::media-upload-bucket"
        }
      }
    },
    "JobDispatcherlambdaTranscodeQueuesqsEventSourceMapping": {
      "Type": "AWS::Lambda::EventSourceMapping",
      "DependsOn": [
        "JobDispatcherlambda",
        "TranscodeQueuesqs"
      ],
      "Properties": {
        "EventSourceArn": {
          "Fn::GetAtt": [
            "TranscodeQueuesqs",
            "Arn"
          ]
        },
        "FunctionName": {
          "Ref": "JobDispatcherlambda"
        },
        "Enabled": true,
        "BatchSize": 10
      }
    }
  }
}

Generated by TheArchitect from the diagram above. Review values such as names, CIDR ranges and instance sizes before deploying to your account.

Media Transcode Pipeline

AWS services in this template

Deployment notes

Generated CloudFormation — 13 resources

Related templates