Skip to content

Commit b5eb803

Browse files
jeffreyavengeneral-kroll-4-life
jeffreyaven
authored and
general-kroll-4-life
committed
dbx examples
Summary: - Added databricks examples.
1 parent 2c88afa commit b5eb803

25 files changed

+794
-29
lines changed

‎.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,13 @@ stackql-azure-cloud-shell.sh
44
stackql-google-cloud-shell.sh
55
stackql
66
/.stackql
7-
.env
7+
**/.env
88
.pypirc
99
stack/
1010
oss-activity-monitor/
1111
testcreds/
1212
*.log
13+
venv/
1314

1415
# Byte-compiled / optimized / DLL files
1516
__pycache__/

‎.vscode/settings.json

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"files.associations": {
3+
"*.iql": "sql"
4+
}
5+
}

‎README.md

+6-1
Original file line numberDiff line numberDiff line change
@@ -241,8 +241,13 @@ stackql-deploy --help
241241
242242
To get started with **stackql-deploy**, install it locally using pip:
243243
244-
```
244+
```bash
245+
python3 -m venv venv
246+
source venv/bin/activate
245247
pip install -e .
248+
# ...
249+
deactivate
250+
rm -rf venv/
246251
```
247252
248253
### To Remove the Locally Installed Package

‎examples/databricks/all-purpose-cluster/README.md

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# `stackql-deploy` starter project for `azure`
2+
3+
```bash
4+
stackql-deploy test \
5+
examples/databricks/all-purpose-cluster prd \
6+
-e AWS_REGION=ap-southeast-2 \
7+
-e DATABRICKS_ACCOUNT_ID=ebfcc5a9-9d49-4c93-b651-b3ee6cf1c9ce
8+
```
9+
10+
```bash
11+
stackql-deploy build \
12+
examples/databricks/all-purpose-cluster prd \
13+
-e AWS_REGION=ap-southeast-2 \
14+
-e DATABRICKS_ACCOUNT_ID=ebfcc5a9-9d49-4c93-b651-b3ee6cf1c9ce
15+
```

‎examples/databricks/all-purpose-cluster/resources/aws_s3_bucket.iql

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*+ exists, retries=3, retry_delay=5 */
2+
SELECT COUNT(*) as count
3+
FROM aws.s3.buckets
4+
WHERE region = '{{ region }}'
5+
AND data__Identifier = '{{ bucket_name }}'
6+
7+
/*+ create */
8+
INSERT INTO aws.s3.buckets (
9+
BucketName,
10+
OwnershipControls,
11+
AccessControl,
12+
BucketEncryption,
13+
PublicAccessBlockConfiguration,
14+
VersioningConfiguration,
15+
Tags,
16+
region
17+
)
18+
SELECT
19+
'{{ bucket_name }}',
20+
'{{ ownership_controls }}',
21+
'{{ access_control }}',
22+
'{{ bucket_encryption }}',
23+
'{{ public_access_block_configuration }}',
24+
'{{ versioning_configuration }}',
25+
'{{ global_tags }}',
26+
'{{ region }}'
27+
28+
/*+ update */
29+
update aws.s3.buckets
30+
set data__PatchDocument = string('{{ {
31+
"OwnershipControls": ownership_controls,
32+
"AccessControl": access_control,
33+
"BucketEncryption": bucket_encryption,
34+
"PublicAccessBlockConfiguration": public_access_block_configuration,
35+
"VersioningConfiguration": versioning_configuration,
36+
"Tags": global_tags
37+
} | generate_patch_document }}')
38+
WHERE
39+
region = '{{ region }}'
40+
AND data__Identifier = '{{ bucket_name }}'
41+
42+
/*+ statecheck, retries=5, retry_delay=5 */
43+
SELECT COUNT(*) as count FROM (
44+
SELECT
45+
JSON_EQUAL(ownership_controls, '{{ ownership_controls }}') as test_ownership_controls,
46+
JSON_EQUAL(bucket_encryption, '{{ bucket_encryption }}') as test_encryption,
47+
JSON_EQUAL(public_access_block_configuration, '{{ public_access_block_configuration }}') as test_public_access_block_configuration,
48+
JSON_EQUAL(versioning_configuration, '{{ versioning_configuration }}') as test_versioning_configuration
49+
FROM aws.s3.buckets
50+
WHERE region = '{{ region }}'
51+
AND data__Identifier = '{{ bucket_name }}'
52+
AND access_control = '{{ access_control }}')t
53+
WHERE test_ownership_controls = 1
54+
AND test_encryption = 1
55+
AND test_public_access_block_configuration = 1
56+
AND test_versioning_configuration = 1
57+
58+
/*+ exports, retries=3, retry_delay=5 */
59+
SELECT
60+
arn as bucket_arn
61+
FROM aws.s3.buckets
62+
WHERE region = '{{ region }}'
63+
AND data__Identifier = '{{ bucket_name }}'

‎examples/databricks/all-purpose-cluster/resources/aws_s3_bucket_policy.iql

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*+ exists */
2+
SELECT COUNT(*) as count
3+
FROM aws.s3.bucket_policies
4+
WHERE region = '{{ region }}'
5+
AND bucket = '{{ transfer_bucket_name }}';
6+
7+
/*+ create */
8+
INSERT INTO aws.s3.bucket_policies (
9+
Bucket,
10+
PolicyDocument,
11+
region
12+
)
13+
SELECT
14+
'{{ transfer_bucket_name }}',
15+
'{{ policy_document }}',
16+
'{{ region }}'
17+
18+
/*+ update */
19+
update aws.s3.bucket_policies
20+
set data__PatchDocument = string('{{ {
21+
"PolicyDocument": policy_document
22+
} | generate_patch_document }}')
23+
WHERE
24+
region = '{{ region }}'
25+
AND data__Identifier = '{{ transfer_bucket_name }}';
26+
27+
/*+ statecheck, retries=5, retry_delay=5 */
28+
SELECT COUNT(*) as count FROM (
29+
SELECT
30+
JSON_EQUAL(policy_document, '{{ policy_document }}') as test_policy_document
31+
FROM aws.s3.bucket_policies
32+
WHERE region = '{{ region }}'
33+
AND data__Identifier = '{{ transfer_bucket_name }}')t
34+
WHERE test_policy_document = 1;

‎examples/databricks/all-purpose-cluster/resources/aws_vpc.iql

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*+ exists, retries=3, retry_delay=5 */
2+
SELECT COUNT(*) as count FROM
3+
(
4+
SELECT vpc_id
5+
FROM aws.ec2.vpcs
6+
WHERE region = '{{ region }}'
7+
AND cidr_block = '{{ cidr_block }}'
8+
) t
9+
10+
/*+ create */
11+
INSERT INTO aws.ec2.vpcs (
12+
CidrBlock,
13+
Tags,
14+
EnableDnsSupport,
15+
EnableDnsHostnames,
16+
region
17+
)
18+
SELECT
19+
'{{ cidr_block }}',
20+
'{{ tags }}',
21+
true,
22+
true,
23+
'{{ region }}'
24+
25+
/*+ statecheck, retries=3, retry_delay=5 */
26+
SELECT COUNT(*) as count FROM
27+
(
28+
SELECT vpc_id,
29+
json_group_object(tag_key, tag_value) as tags
30+
FROM aws.ec2.vpc_tags
31+
WHERE region = '{{ region }}'
32+
AND cidr_block = '{{ cidr_block }}'
33+
GROUP BY vpc_id
34+
HAVING json_extract(tags, '$.Provisioner') = 'stackql'
35+
AND json_extract(tags, '$.StackName') = '{{ stack_name }}'
36+
AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}'
37+
) t
38+
39+
/*+ exports, retries=3, retry_delay=5 */
40+
SELECT vpc_id FROM
41+
(
42+
SELECT vpc_id,
43+
json_group_object(tag_key, tag_value) as tags
44+
FROM aws.ec2.vpc_tags
45+
WHERE region = '{{ region }}'
46+
AND cidr_block = '{{ cidr_block }}'
47+
GROUP BY vpc_id
48+
HAVING json_extract(tags, '$.Provisioner') = 'stackql'
49+
AND json_extract(tags, '$.StackName') = '{{ stack_name }}'
50+
AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}'
51+
) t

‎examples/databricks/all-purpose-cluster/resources/aws_vpc_inet_gateway.iql

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*+ create */
2+
INSERT INTO aws.ec2.internet_gateways (
3+
Tags,
4+
region
5+
)
6+
SELECT
7+
'{{ tags }}',
8+
'{{ region }}';
9+
10+
/*+ statecheck, retries=5, retry_delay=5 */
11+
SELECT COUNT(*) as count FROM
12+
(
13+
SELECT internet_gateway_id,
14+
json_group_object(tag_key, tag_value) as tags
15+
FROM aws.ec2.internet_gateway_tags
16+
WHERE region = '{{ region }}'
17+
GROUP BY internet_gateway_id
18+
HAVING json_extract(tags, '$.Provisioner') = 'stackql'
19+
AND json_extract(tags, '$.StackName') = '{{ stack_name }}'
20+
AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}'
21+
) t;
22+
23+
/*+ exports, retries=3, retry_delay=5 */
24+
SELECT internet_gateway_id FROM
25+
(
26+
SELECT internet_gateway_id,
27+
json_group_object(tag_key, tag_value) as tags
28+
FROM aws.ec2.internet_gateway_tags
29+
WHERE region = '{{ region }}'
30+
GROUP BY internet_gateway_id
31+
HAVING json_extract(tags, '$.Provisioner') = 'stackql'
32+
AND json_extract(tags, '$.StackName') = '{{ stack_name }}'
33+
AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}'
34+
) t;
35+
36+
/*+ delete */
37+
DELETE FROM aws.ec2.internet_gateways
38+
WHERE data__Identifier = '{{ internet_gateway_id }}'
39+
AND region = '{{ region }}';

‎examples/databricks/all-purpose-cluster/resources/aws_vpc_inet_gw_attachment.iql

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*+ create */
2+
INSERT INTO aws.ec2.vpc_gateway_attachments (
3+
InternetGatewayId,
4+
VpcId,
5+
region
6+
)
7+
SELECT
8+
'{{ internet_gateway_id }}',
9+
'{{ vpc_id }}',
10+
'{{ region }}';
11+
12+
/*+ statecheck, retries=5, retry_delay=5 */
13+
SELECT COUNT(*) as count FROM
14+
(
15+
SELECT
16+
attachment_type,
17+
vpc_id
18+
FROM aws.ec2.vpc_gateway_attachments
19+
WHERE region = '{{ region }}'
20+
AND internet_gateway_id = '{{ internet_gateway_id }}'
21+
AND vpc_id = '{{ vpc_id }}'
22+
) t;
23+
24+
/*+ delete */
25+
DELETE FROM aws.ec2.vpc_gateway_attachments
26+
WHERE data__Identifier = 'IGW|{{ vpc_id }}'
27+
AND region = '{{ region }}';

‎examples/databricks/all-purpose-cluster/resources/aws_vpc_inet_route.iql

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*+ create */
2+
INSERT INTO aws.ec2.routes (
3+
DestinationCidrBlock,
4+
GatewayId,
5+
RouteTableId,
6+
region
7+
)
8+
SELECT
9+
'0.0.0.0/0',
10+
'{{ internet_gateway_id }}',
11+
'{{ route_table_id }}',
12+
'{{ region }}';
13+
14+
/*+ statecheck, retries=5, retry_delay=5 */
15+
SELECT COUNT(*) as count FROM
16+
(
17+
SELECT data__Identifier
18+
FROM aws.ec2.routes
19+
WHERE region = '{{ region }}'
20+
AND data__Identifier = '{{ route_table_id }}|0.0.0.0/0'
21+
) t;
22+
23+
/*+ exports, retries=3, retry_delay=5 */
24+
SELECT data__Identifier as inet_route_indentifer
25+
FROM aws.ec2.routes
26+
WHERE region = '{{ region }}'
27+
AND data__Identifier = '{{ route_table_id }}|0.0.0.0/0';
28+
29+
/*+ delete */
30+
DELETE FROM aws.ec2.routes
31+
WHERE data__Identifier = '{{ inet_route_indentifer }}'
32+
AND region = '{{ region }}';

‎examples/databricks/all-purpose-cluster/resources/aws_vpc_route_table.iql

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
/*+ command */
2+
update aws.ec2.route_tables
3+
set data__PatchDocument = string('{{ {
4+
"Tags": tags
5+
} | generate_patch_document }}')
6+
WHERE region = '{{ region }}'
7+
AND data__Identifier = '{{ route_table_id }}';

‎examples/databricks/all-purpose-cluster/resources/aws_vpc_security_group.iql

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*+ exists, retries=3, retry_delay=5 */
2+
SELECT COUNT(*) as count FROM
3+
(
4+
SELECT group_id
5+
FROM aws.ec2.security_groups
6+
WHERE region = '{{ region }}'
7+
AND vpc_id = '{{ vpc_id }}'
8+
AND group_name = '{{ group_name }}'
9+
) t;
10+
11+
/*+ create */
12+
INSERT INTO aws.ec2.security_groups (
13+
GroupName,
14+
GroupDescription,
15+
VpcId,
16+
SecurityGroupIngress,
17+
SecurityGroupEgress,
18+
Tags,
19+
region
20+
)
21+
SELECT
22+
'{{ group_name }}',
23+
'{{ group_description }}',
24+
'{{ vpc_id }}',
25+
'{{ security_group_ingress }}',
26+
'{{ security_group_egress }}',
27+
'{{ tags }}',
28+
'{{ region }}';
29+
30+
/*+ statecheck, retries=5, retry_delay=5 */
31+
SELECT COUNT(*) as count FROM
32+
(
33+
SELECT
34+
JSON_EQUAL(security_group_ingress, '{{ security_group_ingress }}') as ingress_test,
35+
JSON_EQUAL(security_group_egress, '{{ security_group_egress }}') as egress_test
36+
FROM aws.ec2.security_groups
37+
WHERE region = '{{ region }}'
38+
AND vpc_id = '{{ vpc_id }}'
39+
AND group_name = '{{ group_name }}'
40+
AND group_description = '{{ group_description }}'
41+
AND ingress_test = 1
42+
AND egress_test = 1
43+
) t;
44+
45+
/*+ exports, retries=3, retry_delay=5 */
46+
SELECT group_id as 'security_group_id' FROM
47+
(
48+
SELECT group_id
49+
FROM aws.ec2.security_groups
50+
WHERE region = '{{ region }}'
51+
AND vpc_id = '{{ vpc_id }}'
52+
AND group_name = '{{ group_name }}'
53+
) t;
54+
55+
/*+ delete */
56+
DELETE FROM aws.ec2.security_groups
57+
WHERE data__Identifier = '{{ security_group_id }}'
58+
AND region = '{{ region }}';

0 commit comments

Comments
 (0)