AWS ECS(Fargate)で動作しているAPPのログをNewRelic、CloudWatchLogs、S3の3箇所に流す方法についてのメモ書きです。
NewRelicだけにログを流すのであればNewRelic専用のfluentbitイメージを利用すれば良いのですが、複数箇所にログを流すとなると、fluentbitをAWSサービスに対応したイメージに変更し、fluent-bit.confにログを流したいサービスを記述する必要があります。（他に上手い方法があるかもしれませんが）
構成としては次のようなイメージです。
ECS ─> awsfirelens ─> fluentbit ─> CloudWatchLogs
└───> Kinesis Data Firehose ─> S3
└───> Kinesis Data Firehose ─> NewRelic
※ できればKinesis Data Firehoseを経由せずfluentbit─> NewRelicとしたかったが、NewRelic専用のfluentbitイメージのソースのfluent-bit.confをいじってみても、AWSに接続するためのモジュールがないせいか上手くいかなかった。
手順
NewRelic以外へのサービスにログを流す方法はこちらの記事にまとめています。
【ECS】fluent bitでAPPのログをCloudWatchとS3へ同時に流す
NewRelicにログを流すには、上記記事の中で用意しているfluentbitの設定ファイルextra.conf(fluent-bit.confに設定を追記するファイル)に次のような設定を加えます。
# extra.conf
[OUTPUT]
Name cloudwatch
Match **
region ap-northeast-1
log_group_name app-logs
log_stream_prefix app-log-
auto_create_group true
[OUTPUT]
Name firehose
Match **
region ap-northeast-1
delivery_stream S3-Delivery-Stream-app-logs
# 下記を追加
[OUTPUT]
Name firehose
Match **
region ap-northeast-1
delivery_stream NewRelic-Delivery-Stream
次にNewRelic用のKinesis Data Firehoseを用意します。
こちらは下記公式手順を参考に設定します。
Amazon Kinesis Data Firehoseから直接New Relicにデータを取り込む
配信ストリームの名前はNewRelic-Delivery-Stream （extra.confで設定した名前）を設定します。
これで、APPを動作させればログが取得できているはずです。
terraform
terraformでNewRlic用のfilrehoseを実装すると以下のようになる。
# NewRlic用のfilrehose
resource "aws_kinesis_firehose_delivery_stream" "newrelic_stream" {
destination = "http_endpoint"
name = "NewRelic-Delivery-Stream"
http_endpoint_configuration {
access_key = data.terraform_remote_state.log.outputs.ssm_params_name_newrelic_license_key_value # パラメータストアからNewRelicのライセンスキーを渡す
buffering_interval = 60
buffering_size = 5
name = "New Relic"
retry_duration = 60
role_arn = module.iam_role_kinesisi-firehose-service.iam_role_arn
url = "https://aws-api.newrelic.com/firehose/v1"
cloudwatch_logging_options {
enabled = true
log_group_name = "/aws/kinesisfirehose/NewRelic-Delivery-Stream"
log_stream_name = "DestinationDelivery"
}
processing_configuration {
enabled = false
}
request_configuration {
content_encoding = "GZIP"
}
}
s3_configuration {
bucket_arn = xxxx # 任意のs3バケットのarn
compression_format = "UNCOMPRESSED"
error_output_prefix = null
kms_key_arn = null
prefix = null
role_arn = module.iam_role_kinesisi-firehose-service.iam_role_arn
cloudwatch_logging_options {
enabled = true
log_group_name = "/aws/kinesisfirehose/NewRelic-Delivery-Stream"
log_stream_name = "BackupDelivery"
}
}
server_side_encryption {
enabled = false
key_arn = null
key_type = "AWS_OWNED_CMK"
}
}
# iam_role_kinesisi-firehose-service.iam_role_arn
module "iam_role_kinesisi-firehose-service" {
source = "../../../../modules/iam/sts_assume_role"
name = "KinesisFirehoseServiceRole"
attach_policy_arn_map = {
firehose_stream_policy = aws_iam_policy.firehose_stream_policy.arn
}
principal_service_names = [
"logs.ap-northeast-1.amazonaws.com",
"firehose.amazonaws.com"
]
}
# modules/iam/sts_assume_role
data "aws_iam_policy_document" "this" {
statement {
actions = ["sts:AssumeRole"]
effect = var.effect
principals {
identifiers = var.principal_service_names
type = "Service"
}
}
}
resource "aws_iam_role" "this" {
name = var.name
assume_role_policy = data.aws_iam_policy_document.this.json
}
resource "aws_iam_role_policy_attachment" "this" {
for_each = var.attach_policy_arn_map
role = aws_iam_role.this.id
policy_arn = each.value
}
# aws_iam_policy.firehose_stream_policy.arn
resource "aws_iam_policy" "firehose_stream_policy" {
name = "FirehoseStreamPolicy"
policy = data.aws_iam_policy_document.firehose_stream_policy.json
}
data "aws_iam_policy_document" "firehose_stream_policy" {
statement {
actions = [
"glue:GetTable",
"glue:GetTableVersion",
"glue:GetTableVersions"
]
effect = "Allow"
resources = [
"arn:aws:glue:ap-northeast-1:${var.aws_account_id}:catalog",
"arn:aws:glue:ap-northeast-1:${var.aws_account_id}:database/%FIREHOSE_POLICY_TEMPLATE_PLACEHOLDER%",
"arn:aws:glue:ap-northeast-1:${var.aws_account_id}:table/%FIREHOSE_POLICY_TEMPLATE_PLACEHOLDER%/%FIREHOSE_POLICY_TEMPLATE_PLACEHOLDER%"
]
}
statement {
actions = [
"s3:AbortMultipartUpload",
"s3:GetBucketLocation",
"s3:GetObject",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:PutObject"
]
effect = "Allow"
resources = [
"arn:aws:s3:::newrelic-firehose-log-test",
"arn:aws:s3:::newrelic-firehose-log-test/*"
]
}
statement {
actions = [
"lambda:InvokeFunction",
"lambda:GetFunctionConfiguration"
]
effect = "Allow"
resources = [
"arn:aws:lambda:ap-northeast-1:${var.aws_account_id}:function:%FIREHOSE_POLICY_TEMPLATE_PLACEHOLDER%"
]
}
statement {
actions = [
"kms:GenerateDataKey",
"kms:Decrypt"
]
effect = "Allow"
resources = [
"arn:aws:kms:ap-northeast-1:${var.aws_account_id}:key/%FIREHOSE_POLICY_TEMPLATE_PLACEHOLDER%"
]
condition {
test = "StringEquals"
variable = "kms:ViaService"
values = ["s3.ap-northeast-1.amazonaws.com"]
}
condition {
test = "StringLike"
variable = "kms:EncryptionContext:aws:s3:arn"
values = [
"arn:aws:s3:::%FIREHOSE_POLICY_TEMPLATE_PLACEHOLDER%/*",
"arn:aws:s3:::%FIREHOSE_POLICY_TEMPLATE_PLACEHOLDER%"]
}
}
statement {
actions = [
"logs:PutLogEvents"
]
effect = "Allow"
resources = [
"arn:aws:logs:ap-northeast-1:${var.aws_account_id}:log-group:/aws/kinesisfirehose/NewRelic-Delivery-Stream-from-firehose:log-stream:*",
"arn:aws:logs:ap-northeast-1:${var.aws_account_id}:log-group:%FIREHOSE_POLICY_TEMPLATE_PLACEHOLDER%:log-stream:*"
]
}
statement {
actions = [
"kinesis:DescribeStream",
"kinesis:GetShardIterator",
"kinesis:GetRecords",
"kinesis:ListShards"
]
effect = "Allow"
resources = [
"arn:aws:kinesis:ap-northeast-1:${var.aws_account_id}:stream/%FIREHOSE_POLICY_TEMPLATE_PLACEHOLDER%"
]
}
statement {
actions = [
"kms:Decrypt"
]
effect = "Allow"
resources = [
"arn:aws:kms:ap-northeast-1:${var.aws_account_id}:key/%FIREHOSE_POLICY_TEMPLATE_PLACEHOLDER%"
]
condition {
test = "StringEquals"
variable = "kms:ViaService"
values = ["kinesis.ap-northeast-1.amazonaws.com"]
}
condition {
test = "StringLike"
variable = "kms:EncryptionContext:aws:kinesis:arn"
values = [
"arn:aws:kinesis:ap-northeast-1:${var.aws_account_id}:stream/%FIREHOSE_POLICY_TEMPLATE_PLACEHOLDER%"]
}
}
}
