Centralised Log Analytics

AWS architecture template · CloudWatch subscription filters stream application logs through Kinesis into a Lambda transformer that writes partitioned Parquet to S3 for Athena investigation.

Centralised Log Analytics is a free AWS architecture template built with CloudWatch, Athena, CloudTrail, IAM, Kinesis, Lambda, S3 and SNS. Open it in TheArchitect to edit the diagram, estimate monthly cost, and export the CloudFormation or Terraform that deploys it.

Open in TheArchitect All templates

Centralised Log Analytics architecture diagram

AWS services in this template

CloudWatch×2
Athena
CloudTrail
IAM
Kinesis
Lambda
S3
SNS

Deployment notes

Needs input

Add CloudWatch Logs subscription filters pointing to the Kinesis stream.
Grant CloudWatch Logs permission to put records to the Kinesis stream.

Generated CloudFormation — 12 resources

Logical ID	Resource type
AppLogGroupscloudwatch	AWS::CloudWatch::Alarm
LogStreamkinesis	AWS::Kinesis::Stream
LogTransformerlambda	AWS::Lambda::Function
LogLakes3	AWS::S3::Bucket
InvestigationQueriesathena	AWS::Athena::WorkGroup
APIAuditTrailcloudtrail	AWS::CloudTrail::Trail
ErrorSpikeAlertssns	AWS::SNS::Topic
TransformerRoleiam	AWS::IAM::Role
PipelineHealthAlarmcloudwatch	AWS::CloudWatch::Alarm
LogTransformerlambdaErrorSpikeAlertssnsPermission	AWS::Lambda::Permission
ErrorSpikeAlertssnsLogTransformerlambdaSubscription	AWS::SNS::Subscription
LogTransformerlambdaLogLakes3Permission	AWS::Lambda::Permission

View the full CloudFormation template

{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Description": "Generated by TheArchitect — https://www.thecloudarchitect.dev",
  "Metadata": {
    "TheArchitect": {
      "Edges": [
        {
          "From": "LogStreamkinesis",
          "To": "LogTransformerlambda",
          "Label": "batch 60s"
        },
        {
          "From": "LogTransformerlambda",
          "To": "LogLakes3",
          "Label": "parquet partitions"
        },
        {
          "From": "LogTransformerlambda",
          "To": "ErrorSpikeAlertssns",
          "Label": "ERROR spike"
        },
        {
          "From": "APIAuditTrailcloudtrail",
          "To": "LogLakes3",
          "Label": "audit log delivery"
        },
        {
          "From": "LogLakes3",
          "To": "InvestigationQueriesathena",
          "Label": "query source"
        },
        {
          "From": "TransformerRoleiam",
          "To": "LogTransformerlambda",
          "Label": "execution role"
        },
        {
          "From": "LogTransformerlambda",
          "To": "PipelineHealthAlarmcloudwatch",
          "Label": "errors metric"
        }
      ]
    }
  },
  "Resources": {
    "AppLogGroupscloudwatch": {
      "Type": "AWS::CloudWatch::Alarm",
      "Properties": {
        "AlarmName": "app-log-groups-monitor",
        "AlarmDescription": "Application log groups with subscription filter forwarding to Kinesis stream",
        "Namespace": "AWS/Logs",
        "MetricName": "IncomingLogEvents",
        "Statistic": "Sum",
        "Period": 300,
        "EvaluationPeriods": 1,
        "Threshold": 10000,
        "ComparisonOperator": "GreaterThanThreshold",
        "TreatMissingData": "notBreaching"
      }
    },
    "LogStreamkinesis": {
      "Type": "AWS::Kinesis::Stream",
      "Properties": {
        "Name": "application-log-stream",
        "StreamModeDetails": {
          "StreamMode": "PROVISIONED"
        },
        "ShardCount": 2,
        "RetentionPeriodHours": 24
      }
    },
    "LogTransformerlambda": {
      "Type": "AWS::Lambda::Function",
      "DependsOn": [
        "TransformerRoleiam"
      ],
      "Properties": {
        "FunctionName": "log-transformer",
        "Runtime": "python3.12",
        "Handler": "transform.handler",
        "MemorySize": 1024,
        "Timeout": 300,
        "Description": "Decompresses CloudWatch log batches, filters ERROR lines, converts to Parquet and writes to S3 partitioned by year/month/day. Publishes to SNS on ERROR spike.",
        "Role": {
          "Fn::GetAtt": [
            "TransformerRoleiam",
            "Arn"
          ]
        }
      }
    },
    "LogLakes3": {
      "Type": "AWS::S3::Bucket",
      "Properties": {
        "BucketName": "central-log-lake",
        "VersioningConfiguration": {
          "Status": "Suspended"
        },
        "PublicAccessBlockConfiguration": {
          "BlockPublicAcls": true,
          "BlockPublicPolicy": true,
          "IgnorePublicAcls": true,
          "RestrictPublicBuckets": true
        },
        "NotificationConfiguration": {
          "LambdaConfigurations": [
            {
              "Event": "s3:ObjectCreated:*",
              "Function": {
                "Fn::GetAtt": [
                  "LogTransformerlambda",
                  "Arn"
                ]
              }
            }
          ]
        }
      },
      "DependsOn": [
        "LogTransformerlambdaLogLakes3Permission"
      ]
    },
    "InvestigationQueriesathena": {
      "Type": "AWS::Athena::WorkGroup",
      "Properties": {
        "Name": "log-investigation",
        "Description": "Ad-hoc log investigation queries on partitioned log lake",
        "State": "ENABLED",
        "WorkGroupConfiguration": {
          "EnforceWorkGroupConfiguration": true,
          "PublishCloudWatchMetricsEnabled": true,
          "BytesScannedCutoffPerQuery": 5368709120,
          "ResultConfiguration": {
            "OutputLocation": "s3://central-log-lake/athena-results/"
          }
        }
      }
    },
    "APIAuditTrailcloudtrail": {
      "Type": "AWS::CloudTrail::Trail",
      "Properties": {
        "TrailName": "api-audit-trail",
        "S3BucketName": "central-log-lake",
        "IsLogging": true,
        "IsMultiRegionTrail": true,
        "EnableLogFileValidation": true,
        "IncludeGlobalServiceEvents": true,
        "S3KeyPrefix": "cloudtrail/"
      }
    },
    "ErrorSpikeAlertssns": {
      "Type": "AWS::SNS::Topic",
      "Properties": {
        "TopicName": "log-error-spike-alerts",
        "DisplayName": "Log Error Spike Alerts"
      }
    },
    "TransformerRoleiam": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "RoleName": "log-transformer-role",
        "Description": "Lambda execution role — Kinesis read, S3 write, SNS publish",
        "AssumeRolePolicyDocument": {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Effect": "Allow",
              "Principal": {
                "Service": "lambda.amazonaws.com"
              },
              "Action": "sts:AssumeRole"
            }
          ]
        },
        "ManagedPolicyArns": [
          "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
        ]
      }
    },
    "PipelineHealthAlarmcloudwatch": {
      "Type": "AWS::CloudWatch::Alarm",
      "Properties": {
        "AlarmName": "log-pipeline-transformer-errors",
        "AlarmDescription": "Log transformer Lambda errors > 5 — pipeline may be dropping log data",
        "Namespace": "AWS/Lambda",
        "MetricName": "Errors",
        "Statistic": "Sum",
        "Period": 300,
        "EvaluationPeriods": 1,
        "Threshold": 5,
        "ComparisonOperator": "GreaterThanThreshold",
        "TreatMissingData": "notBreaching"
      }
    },
    "LogTransformerlambdaErrorSpikeAlertssnsPermission": {
      "Type": "AWS::Lambda::Permission",
      "DependsOn": [
        "LogTransformerlambda",
        "ErrorSpikeAlertssns"
      ],
      "Properties": {
        "Action": "lambda:InvokeFunction",
        "FunctionName": {
          "Ref": "LogTransformerlambda"
        },
        "Principal": "sns.amazonaws.com",
        "SourceArn": {
          "Ref": "ErrorSpikeAlertssns"
        }
      }
    },
    "ErrorSpikeAlertssnsLogTransformerlambdaSubscription": {
      "Type": "AWS::SNS::Subscription",
      "DependsOn": [
        "LogTransformerlambdaErrorSpikeAlertssnsPermission"
      ],
      "Properties": {
        "TopicArn": {
          "Ref": "ErrorSpikeAlertssns"
        },
        "Protocol": "lambda",
        "Endpoint": {
          "Fn::GetAtt": [
            "LogTransformerlambda",
            "Arn"
          ]
        }
      }
    },
    "LogTransformerlambdaLogLakes3Permission": {
      "Type": "AWS::Lambda::Permission",
      "DependsOn": [
        "LogTransformerlambda"
      ],
      "Properties": {
        "Action": "lambda:InvokeFunction",
        "FunctionName": {
          "Ref": "LogTransformerlambda"
        },
        "Principal": "s3.amazonaws.com",
        "SourceAccount": {
          "Ref": "AWS::AccountId"
        },
        "SourceArn": {
          "Fn::Sub": "arn:${AWS::Partition}:s3:::central-log-lake"
        }
      }
    }
  }
}

Generated by TheArchitect from the diagram above. Review values such as names, CIDR ranges and instance sizes before deploying to your account.

Centralised Log Analytics

AWS services in this template

Deployment notes

Generated CloudFormation — 12 resources

Related templates