centralized-logs.yaml

AWSTemplateFormatVersion: "2010-09-09"

Parameters:
  LogsLifeCycle:
    Type: Number
    Description: Number in days to retain logs
    Default: 30

  ApplicationAccount:
    Type: String
    Description: Application account id
    Default: ""

Conditions:
  HasApplicationAccount: !Not [!Equals [!Ref "ApplicationAccount", ""]]

Resources:
  LogBucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketName: !Sub "logs-${AWS::AccountId}"
      AccessControl: Private
      PublicAccessBlockConfiguration:
        BlockPublicAcls: true
        BlockPublicPolicy: true
        IgnorePublicAcls: true
        RestrictPublicBuckets: true
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: AES256
      LifecycleConfiguration:
        Rules:
          - Status: Enabled
            ExpirationInDays: !Ref LogsLifeCycle

  GlueRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub "GlueCrawler-to-S3-${AWS::AccountId}"
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - glue.amazonaws.com
            Action:
              - "sts:AssumeRole"
      Policies:
        - PolicyName: gluecrawler-s3-policy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - s3:*
                Resource:
                  - !GetAtt LogBucket.Arn
                  - !Sub "${LogBucket.Arn}/*"
              - Effect: Allow
                Action:
                  - glue:*
                  - logs:*
                Resource:
                  - "*"

  KinesisRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub "KinesisFirehose-to-S3-${AWS::AccountId}"
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - firehose.amazonaws.com
            Action:
              - "sts:AssumeRole"
      Policies:
        - PolicyName: kinesisfirehose-s3-policy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - s3:AbortMultipartUpload
                  - s3:GetBucketLocation
                  - s3:GetObject
                  - s3:ListBucket
                  - s3:ListBucketMultipartUploads
                  - s3:PutObject
                Resource:
                  - !GetAtt LogBucket.Arn
                  - !Sub "${LogBucket.Arn}/*"
              - Effect: Allow
                Action:
                  - kinesis:DescribeStream
                  - kinesis:GetShardIterator
                  - kinesis:GetRecords
                  - kinesis:ListShards
                  - kms:Decrypt
                  - kms:GenerateDataKey
                  - logs:PutLogEvents
                  - lambda:InvokeFunction
                  - lambda:GetFunctionConfiguration
                Resource:
                  - "*"

  GlueDatabase:
    Type: AWS::Glue::Database
    Properties:
      CatalogId: !Ref AWS::AccountId
      DatabaseInput:
        Name: !Sub "logdb-${AWS::AccountId}"

  GlueCrawler:
    Type: AWS::Glue::Crawler
    Properties:
      Name: !Sub "crawler-${AWS::AccountId}"
      Role: !GetAtt GlueRole.Arn
      Schedule:
        ScheduleExpression: cron(0 12 * * ? *)
      DatabaseName: !Ref GlueDatabase
      Targets:
        S3Targets:
          - Path: !Sub "${LogBucket}"

  KinesisFirehose:
    Type: AWS::KinesisFirehose::DeliveryStream
    Properties:
      DeliveryStreamName: !Sub "kinesis-firehose-${AWS::AccountId}"
      DeliveryStreamType: DirectPut
      S3DestinationConfiguration:
        BucketARN: !GetAtt LogBucket.Arn
        Prefix: "logging/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/"
        ErrorOutputPrefix: "ecs-error/!{firehose:random-string}/!{firehose:error-output-type}/!{timestamp:yyyy/MM/dd}/"
        RoleARN: !GetAtt KinesisRole.Arn
        BufferingHints:
          IntervalInSeconds: 60
          SizeInMBs: 5

  ApplicationAccountRole:
    Condition: HasApplicationAccount
    Type: AWS::IAM::Role
    Properties:
      RoleName: !Sub "Application-Assume-${AWS::AccountId}"
      AssumeRolePolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Principal:
              AWS:
                - !Ref ApplicationAccount
            Action:
              - "sts:AssumeRole"
      Policies:
        - PolicyName: firehose-put-policy
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action:
                  - "firehose:PutRecord"
                  - "firehose:PutRecordBatch"
                Resource:
                  - !GetAtt KinesisFirehose.Arn

Outputs:
  BucketName:
    Value: !Ref LogBucket
  KinesisFirehoseECSName:
    Value: !Ref KinesisFirehose
  KinesisFirehoseECSArn:
    Value: !GetAtt KinesisFirehose.Arn
  ApplicationAccountRoleArn:
    Condition: HasApplicationAccount
    Value: !GetAtt ApplicationAccountRole.Arn