Skip to content

Commit 3aee5f2

Browse files
akolenskyakolensky
authored
Merge pull request #32 from Barts-Life-Science/benedict-dev
Data Push - ADF Connection
2 parents 37bea52 + 61197ed commit 3aee5f2

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+3917
-3
lines changed

‎.github/workflows/deploy_tre.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@ name: Deploy Azure TRE
44
# It also runs on a schedule, serving as the nightly build
55

66
on: # yamllint disable-line rule:truthy
7-
# schedule:
8-
# # midnight every day https://crontab.guru/#0_0_*_*_*
9-
# - cron: "0 0 * * *"
7+
schedule:
8+
# midnight every day https://crontab.guru/#0_0_*_*_*
9+
- cron: "0 0 * * *"
1010
push:
1111
branches: [main]
1212
workflow_dispatch:

‎templates/shared_services/adf/.dockerignore

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# See https://docs.docker.com/engine/reference/builder/#dockerignore-file
2+
# Put files here that you don't want copied into your bundle's invocation image
3+
.gitignore
4+
**/.terraform/*
5+
**/*_backend.tf
6+
Dockerfile.tmpl
7+
terraform/import_state.sh
8+
terraform/remove_state.sh

‎templates/shared_services/adf/Dockerfile.tmpl

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# syntax=docker/dockerfile-upstream:1.4.0
2+
FROM --platform=linux/amd64 debian:bullseye-slim
3+
4+
# PORTER_INIT
5+
6+
# PORTER_MIXINS
7+
8+
# Use the BUNDLE_DIR build argument to copy files into the bundle
9+
COPY --link . ${BUNDLE_DIR}//

‎templates/shared_services/adf/parameters.json

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
{
2+
"schemaType": "ParameterSet",
3+
"schemaVersion": "1.0.1",
4+
"namespace": "",
5+
"name": "tre-shared-service-adf",
6+
"parameters": [
7+
{
8+
"name": "tre_id",
9+
"source": {
10+
"env": "TRE_ID"
11+
}
12+
},
13+
{
14+
"name": "id",
15+
"source": {
16+
"env": "ID"
17+
}
18+
},
19+
{
20+
"name": "tfstate_container_name",
21+
"source": {
22+
"env": "TERRAFORM_STATE_CONTAINER_NAME"
23+
}
24+
},
25+
{
26+
"name": "tfstate_resource_group_name",
27+
"source": {
28+
"env": "MGMT_RESOURCE_GROUP_NAME"
29+
}
30+
},
31+
{
32+
"name": "tfstate_storage_account_name",
33+
"source": {
34+
"env": "MGMT_STORAGE_ACCOUNT_NAME"
35+
}
36+
},
37+
{
38+
"name": "arm_environment",
39+
"source": {
40+
"env": "ARM_ENVIRONMENT"
41+
}
42+
}
43+
]
44+
}

‎templates/shared_services/adf/porter.yaml

+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
---
2+
schemaVersion: 1.0.0
3+
name: tre-shared-service-adf
4+
version: 0.0.2
5+
description: "An Azure Data Factory workspace service"
6+
registry: azuretre
7+
dockerfile: Dockerfile.tmpl
8+
9+
credentials:
10+
- name: azure_tenant_id
11+
env: ARM_TENANT_ID
12+
- name: azure_subscription_id
13+
env: ARM_SUBSCRIPTION_ID
14+
- name: azure_client_id
15+
env: ARM_CLIENT_ID
16+
- name: azure_client_secret
17+
env: ARM_CLIENT_SECRET
18+
parameters:
19+
- name: tre_id
20+
type: string
21+
22+
# the following are added automatically by the resource processor
23+
- name: id
24+
type: string
25+
description: "Resource ID"
26+
env: id
27+
- name: tfstate_resource_group_name
28+
type: string
29+
description: "Resource group containing the Terraform state storage account"
30+
- name: tfstate_storage_account_name
31+
type: string
32+
description: "The name of the Terraform state storage account"
33+
- name: tfstate_container_name
34+
env: tfstate_container_name
35+
type: string
36+
default: "tfstate"
37+
description: "The name of the Terraform state storage container"
38+
- name: arm_use_msi
39+
env: ARM_USE_MSI
40+
type: boolean
41+
default: false
42+
- name: arm_environment
43+
env: ARM_ENVIRONMENT
44+
type: string
45+
default: "public"
46+
47+
mixins:
48+
- exec
49+
- terraform:
50+
clientVersion: 1.3.6
51+
52+
install:
53+
- terraform:
54+
description: "Deploy ADF shared service"
55+
vars:
56+
tre_id: ${ bundle.parameters.tre_id }
57+
tre_resource_id: ${ bundle.parameters.id }
58+
backendConfig:
59+
resource_group_name: ${ bundle.parameters.tfstate_resource_group_name }
60+
storage_account_name: ${ bundle.parameters.tfstate_storage_account_name }
61+
container_name: ${ bundle.parameters.tfstate_container_name }
62+
key: tre-shared-service-adf-${ bundle.parameters.id }
63+
64+
upgrade:
65+
- exec:
66+
description: "Upgrade shared service"
67+
command: echo
68+
arguments:
69+
- "This shared service does not implement upgrade action"
70+
71+
uninstall:
72+
- terraform:
73+
description: "Tear down adf shared service"
74+
vars:
75+
tre_id: ${ bundle.parameters.tre_id }
76+
tre_resource_id: ${ bundle.parameters.id }
77+
backendConfig:
78+
resource_group_name: ${ bundle.parameters.tfstate_resource_group_name }
79+
storage_account_name: ${ bundle.parameters.tfstate_storage_account_name }
80+
container_name: ${ bundle.parameters.tfstate_container_name }
81+
key: tre-shared-service-adf-${ bundle.parameters.id }

‎templates/shared_services/adf/template_schema.json

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"$schema": "http://json-schema.org/draft-07/schema",
3+
"$id": "https://github.com/Barts-Life-Science/AzureTRE/templates/workspace_services/sql/template_schema.json",
4+
"type": "object",
5+
"title": "ADF Shared Service",
6+
"description": "Deploys an ADF and an integration runtime on a managed VNET to the TRE's core",
7+
"required": [],
8+
"properties": {}
9+
}

‎templates/shared_services/adf/terraform/.terraform.lock.hcl

+42
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎templates/shared_services/adf/terraform/adf.tf

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# This script is to create the adf that is used for the data transfer
2+
# The private endpoint will need approving from the storage account
3+
# Create the data factory
4+
data "azurerm_resource_group" "rg" {
5+
name = local.core_resource_group_name
6+
}
7+
8+
resource "azurerm_data_factory" "adf_core" {
9+
name = "adf-${var.tre_id}"
10+
location = data.azurerm_resource_group.rg.location
11+
resource_group_name = data.azurerm_resource_group.rg.name
12+
managed_virtual_network_enabled = true
13+
tags = local.tre_shared_service_tags
14+
}
15+
16+
# Create a managed integration runtime
17+
resource "azurerm_data_factory_integration_runtime_azure" "adf_ir" {
18+
name = "adf-ir-${var.tre_id}"
19+
data_factory_id = azurerm_data_factory.adf_core.id
20+
location = data.azurerm_resource_group.rg.location
21+
virtual_network_enabled = true
22+
}
23+
24+

‎templates/shared_services/adf/terraform/locals.tf

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
locals {
2+
core_vnet = "vnet-${var.tre_id}"
3+
core_resource_group_name = "rg-${var.tre_id}"
4+
keyvault_name = "kv-${var.tre_id}"
5+
tre_shared_service_tags = {
6+
tre_id = var.tre_id
7+
tre_shared_service_id = var.tre_resource_id
8+
}
9+
}

‎templates/shared_services/adf/terraform/main.tf

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Azure Provider source and version being used
2+
terraform {
3+
required_providers {
4+
azurerm = {
5+
source = "hashicorp/azurerm"
6+
version = "=3.23.0"
7+
}
8+
random = {
9+
source = "hashicorp/random"
10+
version = "=3.4.3"
11+
}
12+
}
13+
14+
backend "azurerm" {}
15+
}
16+
17+
provider "azurerm" {
18+
features {
19+
key_vault {
20+
# Don't purge on destroy (this would fail due to purge protection being enabled on keyvault)
21+
purge_soft_delete_on_destroy = false
22+
purge_soft_deleted_secrets_on_destroy = false
23+
purge_soft_deleted_certificates_on_destroy = false
24+
purge_soft_deleted_keys_on_destroy = false
25+
# When recreating an environment, recover any previously soft deleted secrets - set to true by default
26+
recover_soft_deleted_key_vaults = true
27+
recover_soft_deleted_secrets = true
28+
recover_soft_deleted_certificates = true
29+
recover_soft_deleted_keys = true
30+
}
31+
}
32+
}

‎templates/shared_services/adf/terraform/variables.tf

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
variable "tre_id" {
2+
type = string
3+
}
4+
5+
variable "tre_resource_id" {
6+
type = string
7+
description = "Resource ID"
8+
}

‎templates/workspaces/base-adf/.dockerignore

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Local .terraform directories
2+
**/.terraform/*
3+
4+
# TF backend files
5+
**/*_backend.tf
6+
7+
Dockerfile.tmpl
8+
terraform/deploy.sh
9+
terraform/destroy.sh

‎templates/workspaces/base-adf/.env.sample

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
ARM_CLIENT_ID="__CHANGE_ME__"
2+
ARM_CLIENT_SECRET="__CHANGE_ME__"
3+
ARM_TENANT_ID="__CHANGE_ME__"
4+
ARM_SUBSCRIPTION_ID="__CHANGE_ME__"
5+
AUTH_TENANT_ID="__CHANGE_ME__"
6+
7+
# These are passed in if Terraform will create the Workspace AAD Application
8+
REGISTER_AAD_APPLICATION=true
9+
CREATE_AAD_GROUPS=true
10+
AUTH_CLIENT_ID="__CHANGE_ME__"
11+
AUTH_CLIENT_SECRET="__CHANGE_ME__"
12+
WORKSPACE_OWNER_OBJECT_ID="__CHANGE_ME__"
13+
14+
# These are passed in if you register the Workspace AAD Application before hand
15+
# REGISTER_AAD_APPLICATION=false
16+
# CLIENT_ID="__CHANGE_ME__"
17+
# CLIENT_SECRET="__CHANGE_ME__"
18+
# WORKSPACE_OWNER_OBJECT_ID=""
19+
20+
# Used by Porter, aka TRE_RESOURCE_ID
21+
ID="MadeUp123"
22+
SP_ID=""
23+
SCOPE_ID="api://ws_0001"
24+
APP_ROLE_ID_WORKSPACE_OWNER=""
25+
APP_ROLE_ID_WORKSPACE_RESEARCHER=""
26+
APP_ROLE_ID_WORKSPACE_AIRLOCK_MANAGER=""
27+
# Complex types are base 64 encoded by resource processor
28+
ADDRESS_SPACES="WyIxMC4xLjEwLjAvMjQiXQ=="
29+
SHARED_STORAGE_QUOTA=50
30+
ENABLE_LOCAL_DEBUGGING=true
31+
32+
AAD_REDIRECT_URIS="W10="
33+
WORKSPACE_APP_SERVICE_PLAN_SKU=S1

‎templates/workspaces/base-adf/Dockerfile.tmpl

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# syntax=docker/dockerfile-upstream:1.4.0
2+
FROM --platform=linux/amd64 debian:bullseye-slim
3+
4+
# PORTER_INIT
5+
6+
RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
7+
8+
# Git is required for terraform_azurerm_environment_configuration
9+
RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
10+
apt-get update && apt-get install -y git jq --no-install-recommends
11+
12+
# PORTER_MIXINS
13+
14+
# Use the BUNDLE_DIR build argument to copy files into the bundle
15+
COPY --link . ${BUNDLE_DIR}/

0 commit comments

Comments
 (0)