Skip to content

Commit 693e75c

Browse files
rpoluriRaj Poluri
rpoluri
and
Raj Poluri
authored
Feature/hms iam wildcard (#174)
* use wildcard to configure metastore iam roles * fix * arn fix * fix master user secret count * fix templates * fix allow-grant path * k8s-secret fix * fix init container commands * update changelog * remove mysql_commands template variable * remove init container image and use hms docker for init container also * fix Co-authored-by: Raj Poluri <rpoluri@expediagroup.com>
1 parent 01aaf9e commit 693e75c

11 files changed

Lines changed: 47 additions & 218 deletions

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,11 @@ All notable changes to this project will be documented in this file.
33

44
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
55

6+
## [6.5.1] - 2020-09-02
7+
### Changed
8+
- [Issue 165](https://github.com/ExpediaGroup/apiary-data-lake/issues/173) Configure metastore IAM roles using apiary bucket prefix.
9+
- Fix init container deployment.
10+
611
## [6.5.0] - 2020-08-31
712
### Changed
813
- [Issue 165](https://github.com/ExpediaGroup/apiary-data-lake/issues/165) Use init containers instead of `mysql` commands to initialize mysql users.

‎VARIABLES.md‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,6 @@
5252
| hms_rw_heapsize | Heapsize for the read/write Hive Metastore. Valid values: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html | string | - | yes |
5353
| iam_name_root | Name to identify Hive Metastore IAM roles. | string | `hms` | no |
5454
| ingress_cidr | Generally allowed ingress CIDR list. | list | - | yes |
55-
| init_container_image | Docker image for running HMS init container. Required if `external_database_host` is unset. | string | `` | no |
56-
| init_container_version | Docker image version for running HMS init container. Required if `external_database_host` is unset. | string | `` | no |
5755
| instance_name | Apiary instance name to identify resources in multi-instance deployments. | string | `` | no |
5856
| k8s_docker_registry_secret| Docker Registry authentication K8s secret name. | string | `` | no |
5957
| kiam_arn | Kiam server IAM role ARN. | string | `` | no |

‎db.tf‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -92,21 +92,21 @@ resource "aws_rds_cluster_instance" "apiary_cluster_instance" {
9292
# In order to avoid resource collision when deleting & immediately recreating SecretsManager secrets in AWS, we set a random suffix on the name of the secret.
9393
# This allows us to avoid the issue of AWS's imposed 7 day recovery window.
9494
resource "random_string" "secret_name_suffix" {
95-
count = "${var.external_database_host == "" ? var.db_instance_count : 0}"
95+
count = var.external_database_host == "" ? 1 : 0
9696
length = 8
9797
special = false
9898
}
9999

100100
resource "aws_secretsmanager_secret" "apiary_mysql_master_credentials" {
101-
count = "${var.external_database_host == "" ? var.db_instance_count : 0}"
101+
count = var.external_database_host == "" ? 1 : 0
102102
name = "${local.instance_alias}_db_master_user_${random_string.secret_name_suffix[0].result}"
103103
tags = var.apiary_tags
104104
recovery_window_in_days = 0
105105
}
106106

107107
resource "aws_secretsmanager_secret_version" "apiary_mysql_master_credentials" {
108-
count = "${var.external_database_host == "" ? var.db_instance_count : 0}"
109-
secret_id = aws_secretsmanager_secret.apiary_mysql_master_credentials[0].id
108+
count = var.external_database_host == "" ? 1 : 0
109+
secret_id = aws_secretsmanager_secret.apiary_mysql_master_credentials[0].id
110110
secret_string = jsonencode(
111111
map(
112112
"username", var.db_master_username,

‎iam-policy-s3-buckets.tf‎

Lines changed: 4 additions & 154 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ resource "aws_iam_role_policy" "s3_data_for_hms_readwrite" {
3030
"s3:PutObjectVersionTagging"
3131
],
3232
"Resource": [
33-
"${join("\",\"", formatlist("arn:aws:s3:::%s", local.schemas_info[*]["data_bucket"]))}",
34-
"${join("\",\"", formatlist("arn:aws:s3:::%s/*", local.schemas_info[*]["data_bucket"]))}"
33+
"arn:aws:s3:::${local.apiary_bucket_prefix}-*",
34+
"arn:aws:s3:::${local.apiary_bucket_prefix}-*/*"
3535
]
3636
}
3737
]
@@ -55,8 +55,8 @@ resource "aws_iam_role_policy" "s3_data_for_hms_readonly" {
5555
"s3:List*"
5656
],
5757
"Resource": [
58-
"${join("\",\"", formatlist("arn:aws:s3:::%s", local.schemas_info[*]["data_bucket"]))}",
59-
"${join("\",\"", formatlist("arn:aws:s3:::%s/*", local.schemas_info[*]["data_bucket"]))}"
58+
"arn:aws:s3:::${local.apiary_bucket_prefix}-*",
59+
"arn:aws:s3:::${local.apiary_bucket_prefix}-*/*"
6060
]
6161
}
6262
]
@@ -123,153 +123,3 @@ resource "aws_iam_role_policy" "external_s3_data_for_hms_readonly" {
123123
}
124124
EOF
125125
}
126-
127-
resource "aws_iam_role_policy" "s3_inventory_for_hms_readwrite" {
128-
count = var.s3_enable_inventory ? 1 : 0
129-
name = "s3-inventory"
130-
role = "${aws_iam_role.apiary_hms_readwrite.id}"
131-
132-
policy = <<EOF
133-
{
134-
"Version": "2012-10-17",
135-
"Statement": [
136-
{
137-
"Effect": "Allow",
138-
"Action": [
139-
"s3:Get*",
140-
"s3:List*"
141-
],
142-
"Resource": [
143-
"${format("arn:aws:s3:::%s", local.s3_inventory_bucket)}",
144-
"${format("arn:aws:s3:::%s/*", local.s3_inventory_bucket)}"
145-
]
146-
}
147-
]
148-
}
149-
EOF
150-
}
151-
152-
resource "aws_iam_role_policy" "s3_inventory_for_hms_readonly" {
153-
count = var.s3_enable_inventory ? 1 : 0
154-
name = "s3-inventory"
155-
role = "${aws_iam_role.apiary_hms_readonly.id}"
156-
157-
policy = <<EOF
158-
{
159-
"Version": "2012-10-17",
160-
"Statement": [
161-
{
162-
"Effect": "Allow",
163-
"Action": [
164-
"s3:Get*",
165-
"s3:List*"
166-
],
167-
"Resource": [
168-
"${format("arn:aws:s3:::%s", local.s3_inventory_bucket)}",
169-
"${format("arn:aws:s3:::%s/*", local.s3_inventory_bucket)}"
170-
]
171-
}
172-
]
173-
}
174-
EOF
175-
}
176-
177-
resource "aws_iam_role_policy" "s3_access_logs_for_hms_readwrite" {
178-
count = local.enable_apiary_s3_log_management ? 1 : 0
179-
name = "s3-access-logs"
180-
role = "${aws_iam_role.apiary_hms_readwrite.id}"
181-
182-
policy = <<EOF
183-
{
184-
"Version": "2012-10-17",
185-
"Statement": [
186-
{
187-
"Effect": "Allow",
188-
"Action": [
189-
"s3:Get*",
190-
"s3:List*"
191-
],
192-
"Resource": [
193-
"${format("arn:aws:s3:::%s", local.apiary_s3_hive_logs_bucket)}",
194-
"${format("arn:aws:s3:::%s/*", local.apiary_s3_hive_logs_bucket)}"
195-
]
196-
}
197-
]
198-
}
199-
EOF
200-
}
201-
202-
resource "aws_iam_role_policy" "s3_access_logs_for_hms_readonly" {
203-
count = local.enable_apiary_s3_log_management ? 1 : 0
204-
name = "s3-access-logs"
205-
role = "${aws_iam_role.apiary_hms_readonly.id}"
206-
207-
policy = <<EOF
208-
{
209-
"Version": "2012-10-17",
210-
"Statement": [
211-
{
212-
"Effect": "Allow",
213-
"Action": [
214-
"s3:Get*",
215-
"s3:List*"
216-
],
217-
"Resource": [
218-
"${format("arn:aws:s3:::%s", local.apiary_s3_hive_logs_bucket)}",
219-
"${format("arn:aws:s3:::%s/*", local.apiary_s3_hive_logs_bucket)}"
220-
]
221-
}
222-
]
223-
}
224-
EOF
225-
}
226-
227-
resource "aws_iam_role_policy" "system_for_hms_readwrite" {
228-
229-
name = "system"
230-
role = "${aws_iam_role.apiary_hms_readwrite.id}"
231-
232-
policy = <<EOF
233-
{
234-
"Version": "2012-10-17",
235-
"Statement": [
236-
{
237-
"Effect": "Allow",
238-
"Action": [
239-
"s3:Get*",
240-
"s3:List*"
241-
],
242-
"Resource": [
243-
"${format("arn:aws:s3:::%s", local.apiary_system_bucket)}",
244-
"${format("arn:aws:s3:::%s/*", local.apiary_system_bucket)}"
245-
]
246-
}
247-
]
248-
}
249-
EOF
250-
}
251-
252-
resource "aws_iam_role_policy" "system_for_hms_readonly" {
253-
254-
name = "system"
255-
role = "${aws_iam_role.apiary_hms_readonly.id}"
256-
257-
policy = <<EOF
258-
{
259-
"Version": "2012-10-17",
260-
"Statement": [
261-
{
262-
"Effect": "Allow",
263-
"Action": [
264-
"s3:Get*",
265-
"s3:List*"
266-
],
267-
"Resource": [
268-
"${format("arn:aws:s3:::%s", local.apiary_system_bucket)}",
269-
"${format("arn:aws:s3:::%s/*", local.apiary_system_bucket)}"
270-
]
271-
}
272-
]
273-
}
274-
EOF
275-
}

‎k8s-readonly.tf‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,25 +39,25 @@ resource "kubernetes_deployment" "apiary_hms_readonly" {
3939
spec {
4040
dynamic "init_container" {
4141
for_each = var.external_database_host == "" ? ["enabled"] : []
42-
42+
4343
content {
44-
image = "${var.init_container_image}:${var.init_container_version}"
44+
image = "${var.hms_docker_image}:${var.hms_docker_version}"
4545
name = "${local.hms_alias}-sql-init-readonly"
46-
47-
command = ["sh allow-grant.sh"]
46+
47+
command = ["sh", "/allow-grant.sh"]
4848

4949
env {
50-
name = "MYSQL_HOST"
50+
name = "MYSQL_HOST"
5151
value = var.external_database_host == "" ? join("", aws_rds_cluster.apiary_cluster.*.endpoint) : var.external_database_host
5252
}
5353

5454
env {
55-
name = "MYSQL_DB"
55+
name = "MYSQL_DB"
5656
value = var.apiary_database_name
5757
}
5858

5959
env {
60-
name = "MYSQL_PERMISSIONS"
60+
name = "MYSQL_PERMISSIONS"
6161
value = "SELECT"
6262
}
6363

@@ -66,7 +66,7 @@ resource "kubernetes_deployment" "apiary_hms_readonly" {
6666
value_from {
6767
secret_key_ref {
6868
name = kubernetes_secret.hms_secrets[0].metadata[0].name
69-
key = "master_creds"
69+
key = "master_creds"
7070
}
7171
}
7272
}
@@ -76,7 +76,7 @@ resource "kubernetes_deployment" "apiary_hms_readonly" {
7676
value_from {
7777
secret_key_ref {
7878
name = kubernetes_secret.hms_secrets[0].metadata[0].name
79-
key = "ro_creds"
79+
key = "ro_creds"
8080
}
8181
}
8282
}

‎k8s-readwrite.tf‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -37,26 +37,26 @@ resource "kubernetes_deployment" "apiary_hms_readwrite" {
3737
}
3838

3939
spec {
40-
dynamic "init_container" {
41-
for_each = var.external_database_host == "" ? ["enabled"] : []
42-
content {
43-
image = "${var.init_container_image}:${var.init_container_version}"
40+
dynamic "init_container" {
41+
for_each = var.external_database_host == "" ? ["enabled"] : []
42+
content {
43+
image = "${var.hms_docker_image}:${var.hms_docker_version}"
4444
name = "${local.hms_alias}-sql-init-readwrite"
45-
46-
command = ["sh allow-grant.sh"]
45+
46+
command = ["sh", "/allow-grant.sh"]
4747

4848
env {
49-
name = "MYSQL_HOST"
49+
name = "MYSQL_HOST"
5050
value = var.external_database_host == "" ? join("", aws_rds_cluster.apiary_cluster.*.endpoint) : var.external_database_host
5151
}
5252

5353
env {
54-
name = "MYSQL_DB"
54+
name = "MYSQL_DB"
5555
value = var.apiary_database_name
5656
}
5757

5858
env {
59-
name = "MYSQL_PERMISSIONS"
59+
name = "MYSQL_PERMISSIONS"
6060
value = "ALL"
6161
}
6262

@@ -65,7 +65,7 @@ resource "kubernetes_deployment" "apiary_hms_readwrite" {
6565
value_from {
6666
secret_key_ref {
6767
name = kubernetes_secret.hms_secrets[0].metadata[0].name
68-
key = "master_creds"
68+
key = "master_creds"
6969
}
7070
}
7171
}
@@ -75,7 +75,7 @@ resource "kubernetes_deployment" "apiary_hms_readwrite" {
7575
value_from {
7676
secret_key_ref {
7777
name = kubernetes_secret.hms_secrets[0].metadata[0].name
78-
key = "rw_creds"
78+
key = "rw_creds"
7979
}
8080
}
8181
}

‎k8s-secrets.tf‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,7 @@ resource "kubernetes_secret" "hms_secrets" {
77

88
data = {
99
master_creds = aws_secretsmanager_secret_version.apiary_mysql_master_credentials[0].secret_string
10-
ro_creds = data.aws_secretsmanager_secret_version.db_ro_user.secret_string
11-
rw_creds = data.aws_secretsmanager_secret_version.db_rw_user.secret_string
10+
ro_creds = data.aws_secretsmanager_secret_version.db_ro_user.secret_string
11+
rw_creds = data.aws_secretsmanager_secret_version.db_rw_user.secret_string
1212
}
13-
14-
type = "kubernetes.io/basic-auth"
1513
}

‎templates.tf‎

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -52,16 +52,13 @@ data "template_file" "hms_readwrite" {
5252

5353
s3_enable_inventory = var.s3_enable_inventory ? "1" : ""
5454
# If user sets "apiary_log_bucket", then they are doing their own access logs mgmt, and not using Apiary's log mgmt.
55-
s3_enable_logs = local.enable_apiary_s3_log_management ? "1" : ""
55+
s3_enable_logs = local.enable_apiary_s3_log_management ? "1" : ""
5656

5757
# Template vars for init container
5858
init_container_enabled = var.external_database_host == "" ? true : false
59-
init_container_image = "${var.init_container_image}"
60-
init_container_version = "${var.init_container_version}"
61-
mysql_permissions = "ALL"
62-
mysql_master_cred_arn = aws_secretsmanager_secret.apiary_mysql_master_credentials[0].arn
63-
mysql_user_cred_arn = data.aws_secretsmanager_secret.db_rw_user.arn
64-
mysql_commands = "sh allow-grant.sh"
59+
mysql_permissions = "ALL"
60+
mysql_master_cred_arn = aws_secretsmanager_secret.apiary_mysql_master_credentials[0].arn
61+
mysql_user_cred_arn = data.aws_secretsmanager_secret.db_rw_user.arn
6562
}
6663
}
6764

@@ -101,11 +98,9 @@ data "template_file" "hms_readonly" {
10198

10299
# Template vars for init container
103100
init_container_enabled = var.external_database_host == "" ? true : false
104-
init_container_image = "${var.init_container_image}"
105-
mysql_permissions = "SELECT"
106-
mysql_write_db = "${var.external_database_host == "" ? join("", aws_rds_cluster.apiary_cluster.*.endpoint) : var.external_database_host}"
107-
mysql_master_cred_arn = aws_secretsmanager_secret.apiary_mysql_master_credentials[0].arn
108-
mysql_user_cred_arn = data.aws_secretsmanager_secret.db_ro_user.arn
109-
mysql_commands = "sh allow-grant.sh"
101+
mysql_permissions = "SELECT"
102+
mysql_write_db = "${var.external_database_host == "" ? join("", aws_rds_cluster.apiary_cluster.*.endpoint) : var.external_database_host}"
103+
mysql_master_cred_arn = aws_secretsmanager_secret.apiary_mysql_master_credentials[0].arn
104+
mysql_user_cred_arn = data.aws_secretsmanager_secret.db_ro_user.arn
110105
}
111106
}

‎templates/apiary-hms-readonly.json‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
{
44
"name": "mysql-setup",
55
"essential": false,
6-
"image": "${var.init_container_image}:${var.init_container_version}",
6+
"image": "${hms_docker_image}:${hms_docker_version}",
77
${docker_auth}
88
"logConfiguration": {
99
"logDriver": "awslogs",
@@ -37,9 +37,7 @@
3737
"name": "MYSQL_USER_CREDS"
3838
}
3939
],
40-
"entryPoint": [ "/bin/sh", "-c" ],
41-
"workingDirectory": "/init",
42-
"command": ["${mysql_commands}"]
40+
"command": ["sh", "/allow-grant.sh"]
4341
},
4442
%{ endif }
4543
{

0 commit comments

Comments
 (0)