Feature/s3 logs sqs (#175)

* manage logs sqs queue

* update changelog

* add tags to sns topics and sqs queues

Co-authored-by: Raj Poluri <rpoluri@expediagroup.com>

Author: rpoluri

CHANGELOG.md

@@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [6.5.2] - 2020-09-08
+### Changed
+- Enable SQS events on managed logs bucket.
+
 ## [6.5.1] - 2020-09-02
 ### Changed
 - [Issue 165](https://github.com/ExpediaGroup/apiary-data-lake/issues/173) Configure metastore IAM roles using apiary bucket prefix.

s3-other.tf

@@ -88,6 +88,16 @@ resource "aws_s3_bucket_public_access_block" "apiary_managed_logs_bucket" {
   ignore_public_acls  = true
 }
 
+resource "aws_s3_bucket_notification" "apiary_managed_logs_bucket" {
+  count  = local.enable_apiary_s3_log_management ? 1 : 0
+  bucket = aws_s3_bucket.apiary_managed_logs_bucket[0].bucket
+
+  queue {
+    queue_arn = aws_sqs_queue.apiary_managed_logs_queue[0].arn
+    events    = ["s3:ObjectCreated:*", "s3:ObjectRemoved:*"]
+  }
+}
+
 resource "aws_s3_bucket" "apiary_access_logs_hive" {
   count  = local.enable_apiary_s3_log_management ? 1 : 0
   bucket = local.apiary_s3_hive_logs_bucket

sns.tf

@@ -6,11 +6,13 @@
 
 resource "aws_sns_topic" "apiary_ops_sns" {
   name = "${local.instance_alias}-operational-events"
+  tags = var.apiary_tags
 }
 
 resource "aws_sns_topic" "apiary_metadata_events" {
   count = var.enable_metadata_events ? 1 : 0
   name  = "${local.instance_alias}-metadata-events"
+  tags  = var.apiary_tags
 
   policy = length(var.apiary_customer_accounts) == 0 ? null : <<POLICY
 {
@@ -33,6 +35,8 @@ resource "aws_sns_topic" "apiary_data_events" {
   } : {}
   name = "${local.instance_alias}-${each.value["resource_suffix"]}-data-events"
 
+  tags = var.apiary_tags
+
   policy = <<POLICY
 {
     "Version":"2012-10-17",
@@ -52,6 +56,7 @@ POLICY
 resource "aws_sqs_queue" "apiary_data_event_queue" {
   count = local.create_sqs_data_event_queue ? 1 : 0
   name  = "${local.instance_alias}-data-event-queue"
+  tags  = var.apiary_tags
 
   policy = <<POLICY
 {
@@ -71,3 +76,25 @@ resource "aws_sqs_queue" "apiary_data_event_queue" {
 POLICY
 }
 
+resource "aws_sqs_queue" "apiary_managed_logs_queue" {
+  count = local.enable_apiary_s3_log_management ? 1 : 0
+  name  = "${local.instance_alias}-s3-logs-queue"
+  tags  = var.apiary_tags
+
+  policy = <<POLICY
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "s3.amazonaws.com" },
+      "Action": "sqs:SendMessage",
+      "Resource": "arn:aws:sqs:*:*:${local.instance_alias}-s3-logs-queue",
+      "Condition":{
+          "ArnEquals":{"aws:SourceArn":"arn:aws:s3:::${local.apiary_s3_logs_bucket}"}
+      }
+    }
+  ]
+}
+POLICY
+}