大量のデータを処理するための最適な選択: Kinesis Data Streams と Firehose の比較

Resources:
  MyKinesisDataStream:
    Type: AWS::Kinesis::Stream
    Properties:
      Name: my-kinesis-data-stream
      RetentionPeriodHours: 24
      ShardCount: 1

  MyKinesisDataProducer:
    Type: AWS::Lambda::Function
    Properties:
      Code:
        S3Bucket: my-lambda-code-bucket
        S3Key: my-lambda-code.zip
      Description: "My Kinesis Data Producer"
      FunctionName: "my-kinesis-data-producer"
      Handler: "lambda_function.lambda_handler"
      MemorySize: 128
      Role: "arn:aws:iam::123456789012:role/MyLambdaRole"
      Runtime: "python3.7"
      Timeout: 60
      Environment:
        Variables:
          STREAM_NAME: !Ref MyKinesisDataStream
          REGION: !Ref AWS::Region

  MyKinesisDataProducerPermission:
    Type: AWS::Lambda::Permission
    Properties:
      Action: "lambda:InvokeFunction"
      FunctionName: !Ref MyKinesisDataProducer
      Principal: "kinesis.amazonaws.com"
      SourceArn: !GetAtt MyKinesisDataStream.Arn

  MyKinesisDataConsumer:
    Type: AWS::Lambda::Function
    Properties:
      Code:
        S3Bucket: my-lambda-code-bucket
        S3Key: my-lambda-code.zip
      Description: "My Kinesis Data Consumer"
      FunctionName: "my-kinesis-data-consumer"
      Handler: "lambda_function.lambda_handler"
      MemorySize: 128
      Role: "arn:aws:iam::123456789012:role/MyLambdaRole"
      Runtime: "python3.7"
      Timeout: 60
      Environment:
        Variables:
          STREAM_NAME: !Ref MyKinesisDataStream
          REGION: !Ref AWS::Region

  MyKinesisDataConsumerPermission:
    Type: AWS::Lambda::Permission
    Properties:
      Action: "lambda:InvokeFunction"
      FunctionName: !Ref MyKinesisDataConsumer
      Principal: "kinesis.amazonaws.com"
      SourceArn: !GetAtt MyKinesisDataStream.Arn

  MyKinesisDataConsumerSubscription:
    Type: AWS::Kinesis::StreamConsumer
    Properties:
      ConsumerName: "my-kinesis-data-consumer"
      StreamARN: !GetAtt MyKinesisDataStream.Arn
      RoleARN: "arn:aws:iam::123456789012:role/MyConsumerRole"
      StartingPosition: "TRIM_HORIZON"

Resources:
  MyDeliveryStream:
    Type: AWS::KinesisFirehose::DeliveryStream
    Properties:
      DeliveryStreamName: my-delivery-stream
      S3DestinationConfiguration:
        BucketARN: arn:aws:s3:::my-s3-bucket
        RoleARN: arn:aws:iam::123456789012:role/my-firehose-role
        Prefix: my-data/
        BufferingHints:
          IntervalInSeconds: 60
          SizeInMBs: 1
        CompressionFormat: UNCOMPRESSED
        CloudWatchLoggingOptions:
          Enabled: true
          LogGroupName: my-delivery-stream-logs
          LogStreamName: my-delivery-stream
      ExtendedS3DestinationConfiguration:
        RoleARN: arn:aws:iam::123456789012:role/my-firehose-role
        BucketARN: arn:aws:s3:::my-s3-bucket
        Prefix: my-data/
        ErrorOutputPrefix: my-data/errors/
        BufferingHints:
          IntervalInSeconds: 60
          SizeInMBs: 1
        CompressionFormat: UNCOMPRESSED
        CloudWatchLoggingOptions:
          Enabled: true
          LogGroupName: my-delivery-stream-logs
          LogStreamName: my-delivery-stream
        ProcessingConfiguration:
          Enabled: true
          Processors:
            - Type: Lambda
              Parameters:
                - ParameterName: "LambdaArn"
                  ParameterValue: "arn:aws:lambda:us-east-1:123456789012:function:my-lambda-function"
        S3BackupMode: Disabled