diff --git a/airflow/config/webserver_config.py b/airflow/config/webserver_config.py deleted file mode 100644 index 1f3e63d7..00000000 --- a/airflow/config/webserver_config.py +++ /dev/null @@ -1,3 +0,0 @@ -# Issue 404: DISABLE AIRRLOW AUTHENTICATION (https://github.com/unity-sds/unity-sps/issues/404) -# Disable airflow authentication, https://airflow.apache.org/docs/apache-airflow-providers-fab/stable/auth-manager/webserver-authentication.html -AUTH_ROLE_PUBLIC = "Admin" diff --git a/airflow/dags/cwl_dag_modular.py b/airflow/dags/cwl_dag_modular.py index 990b0bab..6e4d5db1 100644 --- a/airflow/dags/cwl_dag_modular.py +++ b/airflow/dags/cwl_dag_modular.py @@ -25,13 +25,11 @@ from airflow.utils.trigger_rule import TriggerRule from kubernetes.client import models as k8s from unity_sps_utils import ( - CS_SHARED_SERVICES_ACCOUNT_ID, - CS_SHARED_SERVICES_ACCOUNT_REGION, DEFAULT_LOG_LEVEL, - DS_COGNITO_CLIENT_ID, DS_S3_BUCKET_PARAM, EC2_TYPES, LOG_LEVEL_TYPE, + MDPS_CLIENT_ID, NODE_POOL_DEFAULT, NODE_POOL_HIGH_WORKLOAD, POD_LABEL, @@ -175,16 +173,9 @@ def select_stage_in(ti, stac_json, unity_stac_auth_type): """Retrieve stage in arguments based on authentication type parameter.""" stage_in_args = {"stac_json": stac_json, "stac_auth_type": "NONE"} if unity_stac_auth_type: - shared_services_account = SSM_CLIENT.get_parameter( - Name=CS_SHARED_SERVICES_ACCOUNT_ID, WithDecryption=True - )["Parameter"]["Value"] - shared_services_region = SSM_CLIENT.get_parameter( - Name=CS_SHARED_SERVICES_ACCOUNT_REGION, WithDecryption=True - )["Parameter"]["Value"] - unity_client_id = SSM_CLIENT.get_parameter( - Name=f"arn:aws:ssm:{shared_services_region}:{shared_services_account}:parameter{DS_COGNITO_CLIENT_ID}", - WithDecryption=True, - )["Parameter"]["Value"] + unity_client_id = SSM_CLIENT.get_parameter(Name=MDPS_CLIENT_ID, WithDecryption=True)["Parameter"][ + "Value" + ] stage_in_args["unity_client_id"] = unity_client_id stage_in_args["stac_auth_type"] = "UNITY" diff --git a/airflow/dags/run_ogc_process.py b/airflow/dags/run_ogc_process.py index cc54f19d..46b8e521 100644 --- a/airflow/dags/run_ogc_process.py +++ b/airflow/dags/run_ogc_process.py @@ -19,13 +19,13 @@ from kubernetes.client import models as k8s from unity_sps_utils import POD_LABEL, POD_NAMESPACE, get_affinity -PROCESSES_ENDPOINT = "https://api.dit.maap-project.org/api/ogc/processes" +API_HOST = "https://api.dit.maap-project.org/api/" def fetch_ogc_processes(): """Fetch available processes from the OGC API and create mapping.""" try: - response = requests.get(PROCESSES_ENDPOINT, timeout=30) + response = requests.get(API_HOST + "ogc/processes", timeout=30) response.raise_for_status() processes_data = response.json() @@ -65,7 +65,8 @@ def fetch_ogc_processes(): # Constants K8S_SECRET_NAME = "sps-app-credentials" -DOCKER_IMAGE = "jplmdps/ogc-job-runner:latest" +# This docker image is generated by the files in docker/run_ogc_process +DOCKER_IMAGE = "jplmdps/ogc-job-runner:v1.0.0" PROCESS_MAPPING, DROPDOWN_OPTIONS = fetch_ogc_processes() # SPS-specific secrets @@ -155,7 +156,7 @@ def _build_submit_env_vars(self): return [ k8s.V1EnvVar( name="SUBMIT_JOB_URL", - value="https://api.dit.maap-project.org/api/ogc/processes/{process_id}/execution", + value=API_HOST + "ogc/processes/{process_id}/execution", ), k8s.V1EnvVar(name="PROCESS_ID", value=str(numerical_process_id)), k8s.V1EnvVar(name="JOB_INPUTS", value=self.job_inputs or "{}"), @@ -168,7 +169,7 @@ def _build_monitor_env_vars(self): return [ k8s.V1EnvVar( name="MONITOR_JOB_URL", - value="https://api.dit.maap-project.org/api/ogc/jobs/{job_id}", + value=API_HOST + "ogc/jobs/{job_id}", ), k8s.V1EnvVar(name="JOB_ID", value=self.job_id), k8s.V1EnvVar(name="SUBMIT_JOB", value="false"), diff --git a/airflow/docker/run_ogc_process/run_ogc_process_entrypoint.sh b/airflow/docker/run_ogc_process/run_ogc_process_entrypoint.sh index 590e239e..bba284d1 100644 --- a/airflow/docker/run_ogc_process/run_ogc_process_entrypoint.sh +++ b/airflow/docker/run_ogc_process/run_ogc_process_entrypoint.sh @@ -19,7 +19,7 @@ if [ "$SUBMIT_JOB" = "true" ] || [ "$SUBMIT_JOB" = "True" ]; then --data "${SUBMIT_JOB_ARGUMENTS}") echo "API Response: $response" - job_id=$(echo "$response" | jq -r .id) + job_id=$(echo "$response" | jq -r .jobID) if [ "$job_id" = "null" ] || [ -z "$job_id" ]; then echo "Failed to get jobID from response." diff --git a/airflow/helm/values.tmpl.yaml b/airflow/helm/values.tmpl.yaml index 2d6abe31..7accb7ec 100644 --- a/airflow/helm/values.tmpl.yaml +++ b/airflow/helm/values.tmpl.yaml @@ -165,10 +165,6 @@ webserverSecretKeySecretName: ${webserver_secret_name} webserver: replicas: 3 - # Issue 404: DISABLE AIRRLOW AUTHENTICATION (https://github.com/unity-sds/unity-sps/issues/404) - webserverConfig: |- - ${webserver_config} - startupProbe: timeoutSeconds: 20 failureThreshold: 60 # Number of tries before giving up (10 minutes with periodSeconds of 10) diff --git a/airflow/helm/values_high_load.tmpl.yaml b/airflow/helm/values_high_load.tmpl.yaml index 041d86cb..2b62148a 100644 --- a/airflow/helm/values_high_load.tmpl.yaml +++ b/airflow/helm/values_high_load.tmpl.yaml @@ -165,10 +165,6 @@ webserverSecretKeySecretName: ${webserver_secret_name} webserver: replicas: 3 - # Issue 404: DISABLE AIRRLOW AUTHENTICATION (https://github.com/unity-sds/unity-sps/issues/404) - webserverConfig: |- - ${webserver_config} - startupProbe: timeoutSeconds: 20 failureThreshold: 60 # Number of tries before giving up (10 minutes with periodSeconds of 10) diff --git a/airflow/plugins/unity_sps_utils.py b/airflow/plugins/unity_sps_utils.py index 97cebce9..b36e46d3 100644 --- a/airflow/plugins/unity_sps_utils.py +++ b/airflow/plugins/unity_sps_utils.py @@ -20,8 +20,8 @@ CS_SHARED_SERVICES_ACCOUNT_ID = "/unity/shared-services/aws/account" CS_SHARED_SERVICES_ACCOUNT_REGION = "/unity/shared-services/aws/account/region" -DS_COGNITO_CLIENT_ID = "/unity/shared-services/dapa/client-id" -DS_S3_BUCKET_PARAM = f"/unity/unity/{os.environ['AIRFLOW_VAR_UNITY_VENUE']}/ds/datastore-bucket" +MDPS_CLIENT_ID = "/sps/processing/workflows/unity_client_id" +DS_S3_BUCKET_PARAM = f"/smce/mdps/{os.environ['AIRFLOW_VAR_UNITY_VENUE']}/ds/datastore-bucket" DEFAULT_LOG_LEVEL = "INFO" LOG_LEVEL_TYPE = {"DEBUG": 10, "INFO": 20, "WARNING": 30, "ERROR": 40, "CRITICAL": 50} diff --git a/pyproject.toml b/pyproject.toml index 6ecd8a69..96c8fb1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "unity-sps" -version = "3.1.0" +version = "3.1.1" authors = [ { name = "Luca Cinquini", email = "luca.cinquini@jpl.nasa.gov" }, { name = "Grace Llewellyn", email = "grace.llewellyn@jpl.nasa.gov" }, diff --git a/terraform-unity/.terraform.lock.hcl b/terraform-unity/.terraform.lock.hcl index 8f9f50ae..61004c96 100644 --- a/terraform-unity/.terraform.lock.hcl +++ b/terraform-unity/.terraform.lock.hcl @@ -89,22 +89,22 @@ provider "registry.terraform.io/hashicorp/kubernetes" { } provider "registry.terraform.io/hashicorp/local" { - version = "2.5.3" + version = "2.6.1" constraints = ">= 2.5.1" hashes = [ - "h1:MCzg+hs1/ZQ32u56VzJMWP9ONRQPAAqAjuHuzbyshvI=", - "zh:284d4b5b572eacd456e605e94372f740f6de27b71b4e1fd49b63745d8ecd4927", - "zh:40d9dfc9c549e406b5aab73c023aa485633c1b6b730c933d7bcc2fa67fd1ae6e", - "zh:6243509bb208656eb9dc17d3c525c89acdd27f08def427a0dce22d5db90a4c8b", + "h1:DbiR/D2CPigzCGweYIyJH0N0x04oyI5xiZ9wSW/s3kQ=", + "zh:10050d08f416de42a857e4b6f76809aae63ea4ec6f5c852a126a915dede814b4", + "zh:2df2a3ebe9830d4759c59b51702e209fe053f47453cb4688f43c063bac8746b7", + "zh:2e759568bcc38c86ca0e43701d34cf29945736fdc8e429c5b287ddc2703c7b18", + "zh:6a62a34e48500ab4aea778e355e162ebde03260b7a9eb9edc7e534c84fbca4c6", + "zh:74373728ba32a1d5450a3a88ac45624579e32755b086cd4e51e88d9aca240ef6", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:885d85869f927853b6fe330e235cd03c337ac3b933b0d9ae827ec32fa1fdcdbf", - "zh:bab66af51039bdfcccf85b25fe562cbba2f54f6b3812202f4873ade834ec201d", - "zh:c505ff1bf9442a889ac7dca3ac05a8ee6f852e0118dd9a61796a2f6ff4837f09", - "zh:d36c0b5770841ddb6eaf0499ba3de48e5d4fc99f4829b6ab66b0fab59b1aaf4f", - "zh:ddb6a407c7f3ec63efb4dad5f948b54f7f4434ee1a2607a49680d494b1776fe1", - "zh:e0dafdd4500bec23d3ff221e3a9b60621c5273e5df867bc59ef6b7e41f5c91f6", - "zh:ece8742fd2882a8fc9d6efd20e2590010d43db386b920b2a9c220cfecc18de47", - "zh:f4c6b3eb8f39105004cf720e202f04f57e3578441cfb76ca27611139bc116a82", + "zh:8dddae588971a996f622e7589cd8b9da7834c744ac12bfb59c97fa77ded95255", + "zh:946f82f66353bb97aefa8d95c4ca86db227f9b7c50b82415289ac47e4e74d08d", + "zh:e9a5c09e6f35e510acf15b666fd0b34a30164cecdcd81ce7cda0f4b2dade8d91", + "zh:eafe5b873ef42b32feb2f969c38ff8652507e695620cbaf03b9db714bee52249", + "zh:ec146289fa27650c9d433bb5c7847379180c0b7a323b1b94e6e7ad5d2a7dbe71", + "zh:fc882c35ce05631d76c0973b35adde26980778fc81d9da81a2fade2b9d73423b", ] } diff --git a/terraform-unity/README.md b/terraform-unity/README.md index 467dd794..f12d37d8 100644 --- a/terraform-unity/README.md +++ b/terraform-unity/README.md @@ -81,13 +81,14 @@ terraform-docs tfvars hcl . --output-file "terraform.tfvars" | [karpenter\_node\_classes](#input\_karpenter\_node\_classes) | Configuration for karpenter\_node\_classes | 
map(object({
    volume_size = string
  }))
| 
{
  "airflow-kubernetes-pod-operator-high-workload": {
    "volume_size": "300Gi"
  },
  "default": {
    "volume_size": "30Gi"
  }
}
| no | | [karpenter\_node\_pools](#input\_karpenter\_node\_pools) | Configuration for Karpenter node pools | 
map(object({
    requirements : list(object({
      key : string
      operator : string
      values : list(string)
    }))
    nodeClassRef : string
    limits : object({
      cpu : string
      memory : string
    })
    disruption : object({
      consolidationPolicy : string
      consolidateAfter : string
    })
  }))
| 
{
  "airflow-celery-workers": {
    "disruption": {
      "consolidateAfter": "1m",
      "consolidationPolicy": "WhenEmpty"
    },
    "limits": {
      "cpu": "80",
      "memory": "320Gi"
    },
    "nodeClassRef": "default",
    "requirements": [
      {
        "key": "karpenter.k8s.aws/instance-family",
        "operator": "In",
        "values": [
          "m7i",
          "m6i",
          "m5",
          "t3",
          "c7i",
          "c6i",
          "c6id",
          "c5",
          "r7i",
          "r6i",
          "r5",
          "m5ad"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Gt",
        "values": [
          "1"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Lt",
        "values": [
          "17"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Gt",
        "values": [
          "4095"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Lt",
        "values": [
          "65537"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-hypervisor",
        "operator": "In",
        "values": [
          "nitro"
        ]
      }
    ]
  },
  "airflow-core-components": {
    "disruption": {
      "consolidateAfter": "1m",
      "consolidationPolicy": "WhenEmpty"
    },
    "limits": {
      "cpu": "40",
      "memory": "160Gi"
    },
    "nodeClassRef": "default",
    "requirements": [
      {
        "key": "karpenter.k8s.aws/instance-family",
        "operator": "In",
        "values": [
          "m7i",
          "m6i",
          "m5",
          "t3",
          "c7i",
          "c6i",
          "c6id",
          "c5",
          "r7i",
          "r6i",
          "r5",
          "m5ad"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Gt",
        "values": [
          "1"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Lt",
        "values": [
          "17"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Gt",
        "values": [
          "4095"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Lt",
        "values": [
          "65537"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-hypervisor",
        "operator": "In",
        "values": [
          "nitro"
        ]
      }
    ]
  },
  "airflow-kubernetes-pod-operator": {
    "disruption": {
      "consolidateAfter": "1m",
      "consolidationPolicy": "WhenEmpty"
    },
    "limits": {
      "cpu": "6400",
      "memory": "12800Gi"
    },
    "nodeClassRef": "default",
    "requirements": [
      {
        "key": "karpenter.k8s.aws/instance-family",
        "operator": "In",
        "values": [
          "m7i",
          "m6i",
          "m5",
          "m5ad",
          "t3",
          "c7i",
          "c6i",
          "c6id",
          "c5",
          "r7i",
          "r6i",
          "r5"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Gt",
        "values": [
          "0"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Lt",
        "values": [
          "17"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Gt",
        "values": [
          "511"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Lt",
        "values": [
          "65537"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-hypervisor",
        "operator": "In",
        "values": [
          "nitro"
        ]
      }
    ]
  },
  "airflow-kubernetes-pod-operator-high-workload": {
    "disruption": {
      "consolidateAfter": "1m",
      "consolidationPolicy": "WhenEmpty"
    },
    "limits": {
      "cpu": "6400",
      "memory": "12800Gi"
    },
    "nodeClassRef": "airflow-kubernetes-pod-operator-high-workload",
    "requirements": [
      {
        "key": "karpenter.k8s.aws/instance-family",
        "operator": "In",
        "values": [
          "m7i",
          "m6i",
          "m5",
          "m5ad",
          "t3",
          "c7i",
          "c6i",
          "c6id",
          "c5",
          "r7i",
          "r6i",
          "r5"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Gt",
        "values": [
          "0"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Lt",
        "values": [
          "65"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Gt",
        "values": [
          "511"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Lt",
        "values": [
          "262145"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-hypervisor",
        "operator": "In",
        "values": [
          "nitro"
        ]
      }
    ]
  }
}
| no | | [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | The path to the kubeconfig file for the Kubernetes cluster. | `string` | n/a | yes | -| [mcp\_ami\_owner\_id](#input\_mcp\_ami\_owner\_id) | The owner ID of the MCP AMIs | `string` | `"794625662971"` | no | +| [smce\_ami\_owner\_id](#input\_smce\_ami\_owner\_id) | The owner ID of the SMCE AMIs | `string` | `""` | no | +| [aws\_ami\_owner\_id](#input\_aws\_ami\_owner\_id) | The owner ID of the AWS AMI we are using for EKS optimized AMI | `string` | `"602401143452"` | no | | [ogc\_processes\_docker\_images](#input\_ogc\_processes\_docker\_images) | Docker images for the associated OGC Processes API services. | 
object({
    ogc_processes_api = object({
      name = string
      tag  = string
    })
    git_sync = object({
      name = string
      tag  = string
    })
    redis = object({
      name = string
      tag  = string
    })
  })
| 
{
  "git_sync": {
    "name": "registry.k8s.io/git-sync/git-sync",
    "tag": "v4.2.4"
  },
  "ogc_processes_api": {
    "name": "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api",
    "tag": "2.1.0"
  },
  "redis": {
    "name": "redis",
    "tag": "7.4.0"
  }
}
| no | | [project](#input\_project) | The project or mission deploying Unity SPS. | `string` | `"unity"` | no | | [release](#input\_release) | The software release version. | `string` | `"25.3"` | no | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed. | `string` | `"sps"` | no | | [tags](#input\_tags) | Tags for the deployment (unused) | `map(string)` | 
{
  "empty": ""
}
| no | -| [venue](#input\_venue) | The MCP venue in which the resources will be deployed. | `string` | n/a | yes | +| [venue](#input\_venue) | The SMCE venue in which the resources will be deployed. | `string` | n/a | yes | ## Outputs @@ -99,7 +100,7 @@ terraform-docs tfvars hcl . --output-file "terraform.tfvars" ## Deploy the Cluster -### Deploying in into Different MCP Venues +### Deploying in into Different SMCE Venues ### Deploying into Different EKS Clusters @@ -231,13 +232,14 @@ terraform apply -no-color 2>&1 | tee apply_output.txt | [karpenter\_node\_classes](#input\_karpenter\_node\_classes) | Configuration for karpenter\_node\_classes | 
map(object({
    volume_size = string
  }))
| 
{
  "airflow-kubernetes-pod-operator-high-workload": {
    "volume_size": "300Gi"
  },
  "default": {
    "volume_size": "30Gi"
  }
}
| no | | [karpenter\_node\_pools](#input\_karpenter\_node\_pools) | Configuration for Karpenter node pools | 
map(object({
    requirements : list(object({
      key : string
      operator : string
      values : list(string)
    }))
    nodeClassRef : string
    limits : object({
      cpu : string
      memory : string
    })
    disruption : object({
      consolidationPolicy : string
      consolidateAfter : string
    })
  }))
| 
{
  "airflow-celery-workers": {
    "disruption": {
      "consolidateAfter": "1m",
      "consolidationPolicy": "WhenEmpty"
    },
    "limits": {
      "cpu": "80",
      "memory": "320Gi"
    },
    "nodeClassRef": "default",
    "requirements": [
      {
        "key": "karpenter.k8s.aws/instance-family",
        "operator": "In",
        "values": [
          "m7i",
          "m6i",
          "m5",
          "t3",
          "c7i",
          "c6i",
          "c6id",
          "c5",
          "r7i",
          "r6i",
          "r5",
          "m5ad"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Gt",
        "values": [
          "1"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Lt",
        "values": [
          "17"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Gt",
        "values": [
          "4095"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Lt",
        "values": [
          "65537"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-hypervisor",
        "operator": "In",
        "values": [
          "nitro"
        ]
      }
    ]
  },
  "airflow-core-components": {
    "disruption": {
      "consolidateAfter": "1m",
      "consolidationPolicy": "WhenEmpty"
    },
    "limits": {
      "cpu": "40",
      "memory": "160Gi"
    },
    "nodeClassRef": "default",
    "requirements": [
      {
        "key": "karpenter.k8s.aws/instance-family",
        "operator": "In",
        "values": [
          "m7i",
          "m6i",
          "m5",
          "t3",
          "c7i",
          "c6i",
          "c6id",
          "c5",
          "r7i",
          "r6i",
          "r5",
          "m5ad"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Gt",
        "values": [
          "1"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Lt",
        "values": [
          "17"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Gt",
        "values": [
          "4095"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Lt",
        "values": [
          "65537"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-hypervisor",
        "operator": "In",
        "values": [
          "nitro"
        ]
      }
    ]
  },
  "airflow-kubernetes-pod-operator": {
    "disruption": {
      "consolidateAfter": "1m",
      "consolidationPolicy": "WhenEmpty"
    },
    "limits": {
      "cpu": "6400",
      "memory": "12800Gi"
    },
    "nodeClassRef": "default",
    "requirements": [
      {
        "key": "karpenter.k8s.aws/instance-family",
        "operator": "In",
        "values": [
          "m7i",
          "m6i",
          "m5",
          "m5ad",
          "t3",
          "c7i",
          "c6i",
          "c6id",
          "c5",
          "r7i",
          "r6i",
          "r5"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Gt",
        "values": [
          "0"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Lt",
        "values": [
          "17"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Gt",
        "values": [
          "511"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Lt",
        "values": [
          "65537"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-hypervisor",
        "operator": "In",
        "values": [
          "nitro"
        ]
      }
    ]
  },
  "airflow-kubernetes-pod-operator-high-workload": {
    "disruption": {
      "consolidateAfter": "1m",
      "consolidationPolicy": "WhenEmpty"
    },
    "limits": {
      "cpu": "6400",
      "memory": "12800Gi"
    },
    "nodeClassRef": "airflow-kubernetes-pod-operator-high-workload",
    "requirements": [
      {
        "key": "karpenter.k8s.aws/instance-family",
        "operator": "In",
        "values": [
          "m7i",
          "m6i",
          "m5",
          "m5ad",
          "t3",
          "c7i",
          "c6i",
          "c6id",
          "c5",
          "r7i",
          "r6i",
          "r5"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Gt",
        "values": [
          "0"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "Lt",
        "values": [
          "65"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Gt",
        "values": [
          "511"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Lt",
        "values": [
          "262145"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-hypervisor",
        "operator": "In",
        "values": [
          "nitro"
        ]
      }
    ]
  }
}
| no | | [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | The path to the kubeconfig file for the Kubernetes cluster. | `string` | n/a | yes | -| [mcp\_ami\_owner\_id](#input\_mcp\_ami\_owner\_id) | The owner ID of the MCP AMIs | `string` | `"794625662971"` | no | +| [smce\_ami\_owner\_id](#input\_smce\_ami\_owner\_id) | The owner ID of the SMCE AMIs | `string` | `""` | no | +| [aws\_ami\_owner\_id](#input\_aws\_ami\_owner\_id) | The owner ID of the AWS AMI we are using for EKS optimized AMI | `string` | `"602401143452"` | no | | [ogc\_processes\_docker\_images](#input\_ogc\_processes\_docker\_images) | Docker images for the associated OGC Processes API services. | 
object({
    ogc_processes_api = object({
      name = string
      tag  = string
    })
    git_sync = object({
      name = string
      tag  = string
    })
    redis = object({
      name = string
      tag  = string
    })
  })
| 
{
  "git_sync": {
    "name": "registry.k8s.io/git-sync/git-sync",
    "tag": "v4.2.4"
  },
  "ogc_processes_api": {
    "name": "ghcr.io/unity-sds/unity-sps-ogc-processes-api/unity-sps-ogc-processes-api",
    "tag": "2.0.0"
  },
  "redis": {
    "name": "redis",
    "tag": "7.4.0"
  }
}
| no | | [project](#input\_project) | The project or mission deploying Unity SPS. | `string` | `"unity"` | no | | [release](#input\_release) | The software release version. | `string` | `"25.2"` | no | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed. | `string` | `"sps"` | no | | [tags](#input\_tags) | Tags for the deployment (unused) | `map(string)` | 
{
  "empty": ""
}
| no | -| [venue](#input\_venue) | The MCP venue in which the resources will be deployed. | `string` | n/a | yes | +| [venue](#input\_venue) | The SMCE venue in which the resources will be deployed. | `string` | n/a | yes | ## Outputs diff --git a/terraform-unity/main.tf b/terraform-unity/main.tf index fb4bdb9f..ce780675 100644 --- a/terraform-unity/main.tf +++ b/terraform-unity/main.tf @@ -1,6 +1,6 @@ terraform { backend "s3" { - bucket = "unity-unity-dev-bucket" + bucket = "smce-mdps-dev-bucket" workspace_key_prefix = "sps/tfstates" key = "terraform.tfstate" region = "us-west-2" @@ -82,7 +82,8 @@ module "unity-sps-karpenter-node-config" { service_area = var.service_area release = var.release kubeconfig_filepath = var.kubeconfig_filepath - mcp_ami_owner_id = var.mcp_ami_owner_id + smce_ami_owner_id = var.smce_ami_owner_id + aws_ami_owner_id = var.aws_ami_owner_id karpenter_node_classes = var.karpenter_node_classes karpenter_node_pools = var.karpenter_node_pools } diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/README.md b/terraform-unity/modules/terraform-unity-sps-airflow/README.md index 0a40c6c7..bd845014 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/README.md +++ b/terraform-unity/modules/terraform-unity-sps-airflow/README.md @@ -93,7 +93,6 @@ No modules. | [aws_ssm_parameter.shared_services_domain](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/ssm_parameter) | data source | | [aws_ssm_parameter.shared_services_region](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/ssm_parameter) | data source | | [aws_ssm_parameter.subnet_ids](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/ssm_parameter) | data source | -| [aws_ssm_parameter.venue_proxy_baseurl](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/ssm_parameter) | data source | | [aws_vpc.cluster_vpc](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/vpc) | data source | | [kubernetes_namespace.service_area](https://registry.terraform.io/providers/hashicorp/kubernetes/2.32.0/docs/data-sources/namespace) | data source | | [kubernetes_service.airflow_ingress_internal](https://registry.terraform.io/providers/hashicorp/kubernetes/2.32.0/docs/data-sources/service) | data source | @@ -116,7 +115,7 @@ No modules. | [project](#input\_project) | The project or mission deploying Unity SPS | `string` | n/a | yes | | [release](#input\_release) | The software release version. | `string` | n/a | yes | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | n/a | yes | -| [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | +| [venue](#input\_venue) | The SMCE venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | ## Outputs diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/data.tf b/terraform-unity/modules/terraform-unity-sps-airflow/data.tf index 0e4f7724..a4bec754 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/data.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/data.tf @@ -58,27 +58,6 @@ data "aws_ssm_parameter" "shared_services_region" { name = "/unity/shared-services/aws/account/region" } -data "aws_ssm_parameter" "shared_services_domain" { - name = "arn:aws:ssm:${data.aws_ssm_parameter.shared_services_region.value}:${data.aws_ssm_parameter.shared_services_account.value}:parameter/unity/shared-services/domain" -} - -data "aws_ssm_parameter" "venue_proxy_baseurl" { - name = "/unity/${var.project}/${var.venue}/management/httpd/loadbalancer-url" -} - -data "aws_api_gateway_rest_api" "rest_api" { - name = "unity-${var.project}-${var.venue}-rest-api-gateway" -} - -data "aws_api_gateway_authorizers" "unity_cs_common_authorizers_list" { - rest_api_id = data.aws_api_gateway_rest_api.rest_api.id -} - -data "aws_api_gateway_authorizer" "unity_cs_common_authorizer" { - rest_api_id = data.aws_api_gateway_rest_api.rest_api.id - authorizer_id = data.aws_api_gateway_authorizers.unity_cs_common_authorizers_list.ids[0] -} - data "aws_lb" "airflow_k8s_lb" { tags = { Venue = var.venue diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf b/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf index 84bd5a05..87343dd0 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf @@ -1,6 +1,7 @@ locals { resource_name_prefix = join("-", compact([var.project, var.venue, var.service_area, "%s"])) + s3_bucket_name_prefix = join("-", compact([var.project, var.venue, var.service_area, "%s", "smce"])) common_tags = { Name = "" Venue = var.venue diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf index fc5c0df9..2cf4b6da 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf @@ -115,10 +115,10 @@ resource "kubernetes_role_binding" "airflow_pod_creator_binding" { } resource "aws_s3_bucket" "airflow_logs" { - bucket = format(local.resource_name_prefix, "airflowlogs") + bucket = format(local.s3_bucket_name_prefix, "airflowlogs") force_destroy = true tags = merge(local.common_tags, { - Name = format(local.resource_name_prefix, "airflowlogs") + Name = format(local.s3_bucket_name_prefix, "airflowlogs") Component = "airflow" Stack = "airflow" }) @@ -136,8 +136,8 @@ resource "aws_s3_bucket_policy" "airflow_logs_s3_policy" { "Action" : "s3:*", "Effect" : "Deny", "Resource" : [ - format("%s%s", "arn:aws:s3:::", format(local.resource_name_prefix, "airflowlogs")), - format("%s%s/%s", "arn:aws:s3:::", format(local.resource_name_prefix, "airflowlogs"), "*") + format("%s%s", "arn:aws:s3:::", format(local.s3_bucket_name_prefix, "airflowlogs")), + format("%s%s/%s", "arn:aws:s3:::", format(local.s3_bucket_name_prefix, "airflowlogs"), "*") ], "Condition" : { "Bool" : { @@ -208,7 +208,7 @@ resource "aws_iam_role" "airflow_worker_role" { ] } ) - permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/mcp-tenantOperator-AMI-APIG" + permissions_boundary = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/zsmce-tenantOperator-AMI-APIG" } resource "aws_iam_role_policy_attachment" "airflow_worker_policy_attachment" { @@ -413,8 +413,6 @@ resource "helm_release" "airflow" { unity_cluster_name = data.aws_eks_cluster.cluster.name karpenter_node_pools = join(",", var.karpenter_node_pools) cwl_dag_ecr_uri = "${data.aws_caller_identity.current.account_id}.dkr.ecr.us-west-2.amazonaws.com" - # Issue 404: DISABLE AIRRLOW AUTHENTICATION (https://github.com/unity-sds/unity-sps/issues/404) - webserver_config = indent(4, file("${path.module}/../../../airflow/config/webserver_config.py")) }) ] set_sensitive { @@ -520,13 +518,14 @@ resource "aws_vpc_security_group_ingress_rule" "airflow_ingress_sg_proxy_rule" { } #tfsec:ignore:AVD-AWS-0107 -resource "aws_vpc_security_group_ingress_rule" "airflow_api_ingress_sg_proxy_rule" { +resource "aws_vpc_security_group_ingress_rule" "airflow_jpl_ingress_rule" { + for_each = toset(["128.149.0.0/16", "137.78.0.0/16", "137.79.0.0/16"]) security_group_id = aws_security_group.airflow_ingress_sg_internal.id - description = "SecurityGroup ingress rule for api-gateway (temporary)" + description = "SecurityGroup ingress rule for JPL-local addresses" ip_protocol = "tcp" from_port = local.load_balancer_port to_port = local.load_balancer_port - cidr_ipv4 = "0.0.0.0/0" + cidr_ipv4 = each.key } resource "kubernetes_service" "airflow_ingress_internal" { @@ -534,10 +533,10 @@ resource "kubernetes_service" "airflow_ingress_internal" { name = "airflow-ingress-internal" namespace = data.kubernetes_namespace.service_area.metadata[0].name annotations = { - "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internal" + "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" "service.beta.kubernetes.io/aws-load-balancer-type" = "external" "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "ip" - "service.beta.kubernetes.io/aws-load-balancer-subnets" = join(",", jsondecode(data.aws_ssm_parameter.subnet_ids.value)["private"]) + "service.beta.kubernetes.io/aws-load-balancer-subnets" = join(",", jsondecode(data.aws_ssm_parameter.subnet_ids.value)["public"]) "service.beta.kubernetes.io/aws-load-balancer-healthcheck-path" = "/health" "service.beta.kubernetes.io/aws-load-balancer-attributes" = "load_balancing.cross_zone.enabled=true" "service.beta.kubernetes.io/aws-load-balancer-security-groups" = aws_security_group.airflow_ingress_sg_internal.id @@ -577,92 +576,94 @@ resource "time_sleep" "wait_for_airflow_lb" { create_duration = "180s" } -resource "aws_api_gateway_vpc_link" "rest_api_sps_vpc_link" { - name = "ogc-nlb-vpc-link-${var.project}-${var.venue}" - description = "ogc-nlb-vpc-link-${var.project}-${var.venue}" - target_arns = [data.aws_lb.airflow_k8s_lb.arn] - depends_on = [time_sleep.wait_for_airflow_lb] -} - -resource "aws_api_gateway_resource" "rest_api_resource_sps_path" { - rest_api_id = data.aws_api_gateway_rest_api.rest_api.id - parent_id = data.aws_api_gateway_rest_api.rest_api.root_resource_id - path_part = "sps" -} - -resource "aws_api_gateway_resource" "rest_api_resource_airflow_api_path" { - rest_api_id = data.aws_api_gateway_rest_api.rest_api.id - parent_id = aws_api_gateway_resource.rest_api_resource_sps_path.id - path_part = "api" -} - -resource "aws_api_gateway_resource" "rest_api_resource_airflow_proxy_path" { - rest_api_id = data.aws_api_gateway_rest_api.rest_api.id - parent_id = aws_api_gateway_resource.rest_api_resource_airflow_api_path.id - path_part = "{proxy+}" -} - -resource "aws_api_gateway_method" "rest_api_method_for_airflow_proxy_method" { - rest_api_id = data.aws_api_gateway_rest_api.rest_api.id - resource_id = aws_api_gateway_resource.rest_api_resource_airflow_proxy_path.id - http_method = "ANY" - authorization = "CUSTOM" - authorizer_id = data.aws_api_gateway_authorizer.unity_cs_common_authorizer.id - request_parameters = { "method.request.path.proxy" = true } -} - -resource "aws_api_gateway_integration" "rest_api_integration_for_airflow_api" { - rest_api_id = data.aws_api_gateway_rest_api.rest_api.id - resource_id = aws_api_gateway_resource.rest_api_resource_airflow_proxy_path.id - http_method = aws_api_gateway_method.rest_api_method_for_airflow_proxy_method.http_method - type = "HTTP_PROXY" - uri = format("%s://%s:%s%s", "http", data.kubernetes_service.airflow_ingress_internal.status[0].load_balancer[0].ingress[0].hostname, local.load_balancer_port, "/api/{proxy}") - integration_http_method = "ANY" - passthrough_behavior = "WHEN_NO_MATCH" - connection_type = "VPC_LINK" - connection_id = aws_api_gateway_vpc_link.rest_api_sps_vpc_link.id - - # this integration includes a sneaky injected airflow auth header - # when cognito groups are integrated with airflow this will need to be amended - request_parameters = { - "integration.request.path.proxy" = "method.request.path.proxy" - "integration.request.header.Authorization" = "'Basic ${base64encode("${var.airflow_webserver_username}:${var.airflow_webserver_password}")}'" - } - tls_config { # the k8s ingress backends aren't set up with TLS - insecure_skip_verification = true - } - - depends_on = [aws_api_gateway_vpc_link.rest_api_sps_vpc_link, aws_api_gateway_method.rest_api_method_for_airflow_proxy_method] -} - -resource "aws_api_gateway_method_response" "response_200" { - rest_api_id = data.aws_api_gateway_rest_api.rest_api.id - resource_id = aws_api_gateway_resource.rest_api_resource_airflow_proxy_path.id - http_method = aws_api_gateway_method.rest_api_method_for_airflow_proxy_method.http_method - status_code = "200" - - depends_on = [aws_api_gateway_method.rest_api_method_for_airflow_proxy_method] -} - -resource "time_sleep" "wait_for_gateway_integration" { - # need to make sure both the proxy method and integration have time to settle before deploying - depends_on = [aws_api_gateway_integration.rest_api_integration_for_airflow_api] - create_duration = "60s" -} - -# API Gateway deployment -resource "aws_api_gateway_deployment" "airflow-api-gateway-deployment" { - rest_api_id = data.aws_api_gateway_rest_api.rest_api.id - stage_name = var.venue - # stage_name = "default" - depends_on = [time_sleep.wait_for_gateway_integration, aws_api_gateway_method_response.response_200] -} +# API Gateway resources commented out - using LoadBalancer instead +# resource "aws_api_gateway_vpc_link" "rest_api_sps_vpc_link" { +# name = "ogc-nlb-vpc-link-${var.project}-${var.venue}" +# description = "ogc-nlb-vpc-link-${var.project}-${var.venue}" +# target_arns = [data.aws_lb.airflow_k8s_lb.arn] +# depends_on = [time_sleep.wait_for_airflow_lb] +# } + +# resource "aws_api_gateway_resource" "rest_api_resource_sps_path" { +# rest_api_id = data.aws_api_gateway_rest_api.rest_api.id +# parent_id = data.aws_api_gateway_rest_api.rest_api.root_resource_id +# path_part = "sps" +# } + +# resource "aws_api_gateway_resource" "rest_api_resource_airflow_api_path" { +# rest_api_id = data.aws_api_gateway_rest_api.rest_api.id +# parent_id = aws_api_gateway_resource.rest_api_resource_sps_path.id +# path_part = "api" +# } + +# resource "aws_api_gateway_resource" "rest_api_resource_airflow_proxy_path" { +# rest_api_id = data.aws_api_gateway_rest_api.rest_api.id +# parent_id = aws_api_gateway_resource.rest_api_resource_airflow_api_path.id +# path_part = "{proxy+}" +# } + +# resource "aws_api_gateway_method" "rest_api_method_for_airflow_proxy_method" { +# rest_api_id = data.aws_api_gateway_rest_api.rest_api.id +# resource_id = aws_api_gateway_resource.rest_api_resource_airflow_proxy_path.id +# http_method = "ANY" +# authorization = "CUSTOM" +# authorizer_id = data.aws_api_gateway_authorizer.unity_cs_common_authorizer.id +# request_parameters = { "method.request.path.proxy" = true } +# } + +# resource "aws_api_gateway_integration" "rest_api_integration_for_airflow_api" { +# rest_api_id = data.aws_api_gateway_rest_api.rest_api.id +# resource_id = aws_api_gateway_resource.rest_api_resource_airflow_proxy_path.id +# http_method = aws_api_gateway_method.rest_api_method_for_airflow_proxy_method.http_method +# type = "HTTP_PROXY" +# uri = format("%s://%s:%s%s", "http", data.kubernetes_service.airflow_ingress_internal.status[0].load_balancer[0].ingress[0].hostname, local.load_balancer_port, "/api/{proxy}") +# integration_http_method = "ANY" +# passthrough_behavior = "WHEN_NO_MATCH" +# connection_type = "VPC_LINK" +# connection_id = aws_api_gateway_vpc_link.rest_api_sps_vpc_link.id + +# # this integration includes a sneaky injected airflow auth header +# # when cognito groups are integrated with airflow this will need to be amended +# request_parameters = { +# "integration.request.path.proxy" = "method.request.path.proxy" +# "integration.request.header.Authorization" = "'Basic ${base64encode("${var.airflow_webserver_username}:${var.airflow_webserver_password}")}'" +# } +# tls_config { # the k8s ingress backends aren't set up with TLS +# insecure_skip_verification = true +# } + +# depends_on = [aws_api_gateway_vpc_link.rest_api_sps_vpc_link, aws_api_gateway_method.rest_api_method_for_airflow_proxy_method] +# } + +# resource "aws_api_gateway_method_response" "response_200" { +# rest_api_id = data.aws_api_gateway_rest_api.rest_api.id +# resource_id = aws_api_gateway_resource.rest_api_resource_airflow_proxy_path.id +# http_method = aws_api_gateway_method.rest_api_method_for_airflow_proxy_method.http_method +# status_code = "200" + +# depends_on = [aws_api_gateway_method.rest_api_method_for_airflow_proxy_method] +# } + +# resource "time_sleep" "wait_for_gateway_integration" { +# # need to make sure both the proxy method and integration have time to settle before deploying +# depends_on = [aws_api_gateway_integration.rest_api_integration_for_airflow_api] +# create_duration = "60s" +# } + +# # API Gateway deployment +# resource "aws_api_gateway_deployment" "airflow-api-gateway-deployment" { +# rest_api_id = data.aws_api_gateway_rest_api.rest_api.id +# stage_name = var.venue +# # stage_name = "default" +# depends_on = [time_sleep.wait_for_gateway_integration, aws_api_gateway_method_response.response_200] +# } resource "aws_ssm_parameter" "airflow_ui_url" { name = format("/%s", join("/", compact(["", var.project, var.venue, var.service_area, "processing", "airflow", "ui_url"]))) description = "The URL of the Airflow UI." type = "String" - value = "https://www.${data.aws_ssm_parameter.shared_services_domain.value}:4443/${var.project}/${var.venue}/sps/" + # Updated to use LoadBalancer instead of shared services domain + value = "http://${data.kubernetes_service.airflow_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:${local.load_balancer_port}/" tags = merge(local.common_tags, { Name = format(local.resource_name_prefix, "endpoints-airflow_ui") Component = "SSM" @@ -675,14 +676,15 @@ resource "aws_ssm_parameter" "airflow_ui_health_check_endpoint" { name = format("/%s", join("/", compact(["", "unity", var.project, var.venue, "component", "airflow-ui"]))) description = "The URL of the Airflow UI." type = "String" + # Updated to use LoadBalancer instead of shared services domain value = jsonencode({ "componentCategory" : "processing" "componentName" : "Airflow UI" "componentType" : "ui" "description" : "The primary GUI for the Science Processing System (SPS) to run and monitor jobs at scale." - "healthCheckUrl" : "https://www.${data.aws_ssm_parameter.shared_services_domain.value}:4443/${var.project}/${var.venue}/sps/health" + "healthCheckUrl" : "http://${data.kubernetes_service.airflow_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:${local.load_balancer_port}/health" "isPortalIntegrated" : false - "landingPageUrl" : "https://www.${data.aws_ssm_parameter.shared_services_domain.value}:4443/${var.project}/${var.venue}/sps/" + "landingPageUrl" : "http://${data.kubernetes_service.airflow_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:${local.load_balancer_port}/" }) tags = merge(local.common_tags, { Name = format(local.resource_name_prefix, "health-check-endpoints-airflow_ui") @@ -699,7 +701,8 @@ resource "aws_ssm_parameter" "airflow_api_url" { name = format("/%s", join("/", compact(["", var.project, var.venue, var.service_area, "processing", "airflow", "api_url"]))) description = "The URL of the Airflow REST API." type = "String" - value = "${aws_api_gateway_deployment.airflow-api-gateway-deployment.invoke_url}/sps/api/v1" + # Updated to use LoadBalancer instead of API Gateway + value = "http://${data.kubernetes_service.airflow_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:${local.load_balancer_port}/api/v1" tags = merge(local.common_tags, { Name = format(local.resource_name_prefix, "endpoints-airflow_api") Component = "SSM" @@ -712,14 +715,15 @@ resource "aws_ssm_parameter" "airflow_api_health_check_endpoint" { name = format("/%s", join("/", compact(["", "unity", var.project, var.venue, "component", "airflow-api"]))) description = "The URL of the Airflow REST API." type = "String" + # Updated to use LoadBalancer instead of API Gateway value = jsonencode({ "componentCategory" : "processing" "componentName" : "Airflow API" "componentType" : "api" "description" : "The direct API for the job management system underlying the SPS (Airflow). Typically the OGC Processes API should be used instead, because it will abstract out a particular job engine." - "healthCheckUrl" : "${aws_api_gateway_deployment.airflow-api-gateway-deployment.invoke_url}/sps/api/v1/health" + "healthCheckUrl" : "http://${data.kubernetes_service.airflow_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:${local.load_balancer_port}/api/v1/health" "isPortalIntegrated" : false - "landingPageUrl" : "${aws_api_gateway_deployment.airflow-api-gateway-deployment.invoke_url}/sps/api/v1" + "landingPageUrl" : "http://${data.kubernetes_service.airflow_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:${local.load_balancer_port}/api/v1" }) tags = merge(local.common_tags, { Name = format(local.resource_name_prefix, "health-check-endpoints-airflow_api") diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/outputs.tf b/terraform-unity/modules/terraform-unity-sps-airflow/outputs.tf index f7ed98c0..e3106502 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/outputs.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/outputs.tf @@ -13,13 +13,13 @@ output "airflow_urls" { } output "airflow_venue_urls" { - description = "URLs for the various Airflow endpoints at venue-proxy level." + description = "URLs for the various Airflow endpoints (LoadBalancer direct access)." value = { "ui" = { - "url" = nonsensitive(replace(data.aws_ssm_parameter.venue_proxy_baseurl.value, "management/ui", "sps/")) + "url" = nonsensitive(aws_ssm_parameter.airflow_ui_url.value) } "rest_api" = { - "url" = nonsensitive(replace(data.aws_ssm_parameter.venue_proxy_baseurl.value, "management/ui", "sps/api/v1")) + "url" = nonsensitive(aws_ssm_parameter.airflow_api_url.value) } } } diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/variables.tf b/terraform-unity/modules/terraform-unity-sps-airflow/variables.tf index 04287efa..a9351c40 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/variables.tf @@ -4,7 +4,7 @@ variable "project" { } variable "venue" { - description = "The MCP venue in which the cluster will be deployed (dev, test, prod)" + description = "The SMCE venue in which the cluster will be deployed (dev, test, prod)" type = string } diff --git a/terraform-unity/modules/terraform-unity-sps-database/README.md b/terraform-unity/modules/terraform-unity-sps-database/README.md index ede88a7a..5e8efecd 100644 --- a/terraform-unity/modules/terraform-unity-sps-database/README.md +++ b/terraform-unity/modules/terraform-unity-sps-database/README.md @@ -47,7 +47,7 @@ No modules. | [project](#input\_project) | The project or mission deploying Unity SPS | `string` | n/a | yes | | [release](#input\_release) | The software release version. | `string` | n/a | yes | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | n/a | yes | -| [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | +| [venue](#input\_venue) | The SMCE venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | ## Outputs diff --git a/terraform-unity/modules/terraform-unity-sps-database/variables.tf b/terraform-unity/modules/terraform-unity-sps-database/variables.tf index e99fb7d0..289c0bf4 100644 --- a/terraform-unity/modules/terraform-unity-sps-database/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-database/variables.tf @@ -4,7 +4,7 @@ variable "project" { } variable "venue" { - description = "The MCP venue in which the cluster will be deployed (dev, test, prod)" + description = "The SMCE venue in which the cluster will be deployed (dev, test, prod)" type = string } diff --git a/terraform-unity/modules/terraform-unity-sps-efs/README.md b/terraform-unity/modules/terraform-unity-sps-efs/README.md index 2d6c9acc..6b2e48a6 100644 --- a/terraform-unity/modules/terraform-unity-sps-efs/README.md +++ b/terraform-unity/modules/terraform-unity-sps-efs/README.md @@ -31,7 +31,7 @@ No modules. | [project](#input\_project) | The project or mission deploying Unity SPS | `string` | n/a | yes | | [release](#input\_release) | The software release version. | `string` | n/a | yes | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | n/a | yes | -| [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | +| [venue](#input\_venue) | The SMCE venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | ## Outputs diff --git a/terraform-unity/modules/terraform-unity-sps-efs/variables.tf b/terraform-unity/modules/terraform-unity-sps-efs/variables.tf index 54189e70..7cc1b8c6 100644 --- a/terraform-unity/modules/terraform-unity-sps-efs/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-efs/variables.tf @@ -4,7 +4,7 @@ variable "project" { } variable "venue" { - description = "The MCP venue in which the cluster will be deployed (dev, test, prod)" + description = "The SMCE venue in which the cluster will be deployed (dev, test, prod)" type = string } diff --git a/terraform-unity/modules/terraform-unity-sps-eks/.terraform.lock.hcl b/terraform-unity/modules/terraform-unity-sps-eks/.terraform.lock.hcl index 1d66f0c6..eb483bdf 100644 --- a/terraform-unity/modules/terraform-unity-sps-eks/.terraform.lock.hcl +++ b/terraform-unity/modules/terraform-unity-sps-eks/.terraform.lock.hcl @@ -26,63 +26,60 @@ provider "registry.terraform.io/hashicorp/aws" { } provider "registry.terraform.io/hashicorp/cloudinit" { - version = "2.3.5" + version = "2.3.7" constraints = ">= 2.0.0" hashes = [ - "h1:HCoabXm6NQwCivl1q24+l9VUufc2mFqNeulsQBA9iFg=", - "h1:Sf1Lt21oTADbzsnlU38ylpkl8YXP0Beznjcy5F/Yx64=", - "zh:17c20574de8eb925b0091c9b6a4d859e9d6e399cd890b44cfbc028f4f312ac7a", - "zh:348664d9a900f7baf7b091cf94d657e4c968b240d31d9e162086724e6afc19d5", - "zh:5a876a468ffabff0299f8348e719cb704daf81a4867f8c6892f3c3c4add2c755", - "zh:6ef97ee4c8c6a69a3d36746ba5c857cf4f4d78f32aa3d0e1ce68f2ece6a5dba5", + "h1:M9TpQxKAE/hyOwytdX9MUNZw30HoD/OXqYIug5fkqH8=", + "zh:06f1c54e919425c3139f8aeb8fcf9bceca7e560d48c9f0c1e3bb0a8ad9d9da1e", + "zh:0e1e4cf6fd98b019e764c28586a386dc136129fef50af8c7165a067e7e4a31d5", + "zh:1871f4337c7c57287d4d67396f633d224b8938708b772abfc664d1f80bd67edd", + "zh:2b9269d91b742a71b2248439d5e9824f0447e6d261bfb86a8a88528609b136d1", + "zh:3d8ae039af21426072c66d6a59a467d51f2d9189b8198616888c1b7fc42addc7", + "zh:3ef4e2db5bcf3e2d915921adced43929214e0946a6fb11793085d9a48995ae01", + "zh:42ae54381147437c83cbb8790cc68935d71b6357728a154109d3220b1beb4dc9", + "zh:4496b362605ae4cbc9ef7995d102351e2fe311897586ffc7a4a262ccca0c782a", + "zh:652a2401257a12706d32842f66dac05a735693abcb3e6517d6b5e2573729ba13", + "zh:7406c30806f5979eaed5f50c548eced2ea18ea121e01801d2f0d4d87a04f6a14", + "zh:7848429fd5a5bcf35f6fee8487df0fb64b09ec071330f3ff240c0343fe2a5224", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:8283e5a785e3c518a440f6ac6e7cc4fc07fe266bf34974246f4e2ef05762feda", - "zh:a44eb5077950168b571b7eb65491246c00f45409110f0f172cc3a7605f19dba9", - "zh:aa0806cbff72b49c1b389c0b8e6904586e5259c08dabb7cb5040418568146530", - "zh:bec4613c3beaad9a7be7ca99cdb2852073f782355b272892e6ee97a22856aec1", - "zh:d7fe368577b6c8d1ae44c751ed42246754c10305c7f001cc0109833e95aa107d", - "zh:df2409fc6a364b1f0a0f8a9cd8a86e61e80307996979ce3790243c4ce88f2915", - "zh:ed3c263396ff1f4d29639cc43339b655235acf4d06296a7c120a80e4e0fd6409", ] } provider "registry.terraform.io/hashicorp/helm" { - version = "2.15.0" + version = "3.1.1" hashes = [ - "h1:VymvscRkDy0+zN2uKpKYY6njXPY8JROARuaL3VPsEos=", - "h1:WfjJptfaDzC4XCht262FFizAMX8fvRDZWtqUmuLcg88=", - "zh:18b94c7c83c30ad166722a61a412e3de6a67935772960e79aaa24c15f8ea0d0f", - "zh:4f07c929a71e8169f7471b7600bfcca36dfb295787e975e82ac0455a3ab68b47", - "zh:776b804a14c3c4ae6075b12176f81c1f1987214ee1cae4a542599389591cde11", - "zh:7c11e3adbe9bd26e88484dcdbd28c473ce3a5c58950a3e3c4f0a2caee225b845", - "zh:821e1a53415df0ae4ed523f098360d367a95d6ce3872ba841f22adfdd2f97664", - "zh:94c06e483f75a11c3f139c41b3f64b51a96d1d1485e7d1fd3c0f795e2e750945", - "zh:aa2040de0b8150ef40222a965445ec40e3df2997ffde1fb062ab4c226689115e", - "zh:ad73eebeffe20228656567963477d034b9ed3d1bd2075c1c81150def4927d810", - "zh:b77450a36807f3ad1d3ae736d1d165a94fa26f476504a280e9fb2ccb89f648d0", - "zh:d2ebd3c34c50c92106ce2df25d5598f47127dc7c60172b9e2fe56ac73dc863a8", - "zh:e565995e2614df5ddde75a743a674129288fb91669596a7b0b2580fa7ed49979", + "h1:47CqNwkxctJtL/N/JuEj+8QMg8mRNI/NWeKO5/ydfZU=", + "zh:1a6d5ce931708aec29d1f3d9e360c2a0c35ba5a54d03eeaff0ce3ca597cd0275", + "zh:3411919ba2a5941801e677f0fea08bdd0ae22ba3c9ce3309f55554699e06524a", + "zh:81b36138b8f2320dc7f877b50f9e38f4bc614affe68de885d322629dd0d16a29", + "zh:95a2a0a497a6082ee06f95b38bd0f0d6924a65722892a856cfd914c0d117f104", + "zh:9d3e78c2d1bb46508b972210ad706dd8c8b106f8b206ecf096cd211c54f46990", + "zh:a79139abf687387a6efdbbb04289a0a8e7eaca2bd91cdc0ce68ea4f3286c2c34", + "zh:aaa8784be125fbd50c48d84d6e171d3fb6ef84a221dbc5165c067ce05faab4c8", + "zh:afecd301f469975c9d8f350cc482fe656e082b6ab0f677d1a816c3c615837cc1", + "zh:c54c22b18d48ff9053d899d178d9ffef7d9d19785d9bf310a07d648b7aac075b", + "zh:db2eefd55aea48e73384a555c72bac3f7d428e24147bedb64e1a039398e5b903", + "zh:ee61666a233533fd2be971091cecc01650561f1585783c381b6f6e8a390198a4", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } provider "registry.terraform.io/hashicorp/kubernetes" { - version = "2.32.0" + version = "3.0.1" constraints = ">= 2.10.0" hashes = [ - "h1:3j4XBR5UWQA7xXaiEnzZp0bHbcwOhWetHYKTWIrUTI0=", - "h1:HqeU0sZBh+2loFYqPMFx7jJamNUPEykyqJ9+CkMCYE0=", - "zh:0e715d7fb13a8ad569a5fdc937b488590633f6942e986196fdb17cd7b8f7720e", - "zh:495fc23acfe508ed981e60af9a3758218b0967993065e10a297fdbc210874974", - "zh:4b930a8619910ef528bc90dae739cb4236b9b76ce41367281e3bc3cf586101c7", - "zh:5344405fde7b1febf0734052052268ee24e7220818155702907d9ece1c0697c7", - "zh:92ee11e8c23bbac3536df7b124456407f35c6c2468bc0dbab15c3fc9f414bd0e", - "zh:a45488fe8d5bb59c49380f398da5d109a4ac02ebc10824567dabb87f6102fda8", - "zh:a4a0b57cf719a4c91f642436882b7bea24d659c08a5b6f4214ce4fe6a0204caa", - "zh:b7a27a6d11ba956a2d7b0f7389a46ec857ebe46ae3aeee537250e66cac15bf03", - "zh:bf94ce389028b686bfa70a90f536e81bb776c5c20ab70138bbe5c3d0a04c4253", - "zh:d965b2608da0212e26a65a0b3f33c5baae46cbe839196be15d93f70061516908", - "zh:f441fc793d03057a17af8bdca8b26d54916645bc5c148f54e22a54ed39089e83", + "h1:P0c8knzZnouTNFIRij8IS7+pqd0OKaFDYX0j4GRsiqo=", + "zh:02d55b0b2238fd17ffa12d5464593864e80f402b90b31f6e1bd02249b9727281", + "zh:20b93a51bfeed82682b3c12f09bac3031f5bdb4977c47c97a042e4df4fb2f9ba", + "zh:6e14486ecfaee38c09ccf33d4fdaf791409f90795c1b66e026c226fad8bc03c7", + "zh:8d0656ff422df94575668e32c310980193fccb1c28117e5c78dd2d4050a760a6", + "zh:9795119b30ec0c1baa99a79abace56ac850b6e6fbce60e7f6067792f6eb4b5f4", + "zh:b388c87acc40f6bd9620f4e23f01f3c7b41d9b88a68d5255dec0a72f0bdec249", + "zh:b59abd0a980649c2f97f172392f080eaeb18e486b603f83bf95f5d93aeccc090", + "zh:ba6e3060fddf4a022087d8f09e38aa0001c705f21170c2ded3d1c26c12f70d97", + "zh:c12626d044b1d5501cf95ca78cbe507c13ad1dd9f12d4736df66eb8e5f336eb8", + "zh:c55203240d50f4cdeb3df1e1760630d677679f5b1a6ffd9eba23662a4ad05119", + "zh:ea206a5a32d6e0d6e32f1849ad703da9a28355d9c516282a8458b5cf1502b2a1", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } @@ -109,43 +106,41 @@ provider "registry.terraform.io/hashicorp/null" { } provider "registry.terraform.io/hashicorp/time" { - version = "0.12.1" + version = "0.13.1" constraints = ">= 0.9.0" hashes = [ - "h1:6BhxSYBJdBBKyuqatOGkuPKVenfx6UmLdiI13Pb3his=", - "h1:JzYsPugN8Fb7C4NlfLoFu7BBPuRVT2/fCOdCaxshveI=", - "zh:090023137df8effe8804e81c65f636dadf8f9d35b79c3afff282d39367ba44b2", - "zh:26f1e458358ba55f6558613f1427dcfa6ae2be5119b722d0b3adb27cd001efea", - "zh:272ccc73a03384b72b964918c7afeb22c2e6be22460d92b150aaf28f29a7d511", - "zh:438b8c74f5ed62fe921bd1078abe628a6675e44912933100ea4fa26863e340e9", + "h1:ZT5ppCNIModqk3iOkVt5my8b8yBHmDpl663JtXAIRqM=", + "zh:02cb9aab1002f0f2a94a4f85acec8893297dc75915f7404c165983f720a54b74", + "zh:04429b2b31a492d19e5ecf999b116d396dac0b24bba0d0fb19ecaefe193fdb8f", + "zh:26f8e51bb7c275c404ba6028c1b530312066009194db721a8427a7bc5cdbc83a", + "zh:772ff8dbdbef968651ab3ae76d04afd355c32f8a868d03244db3f8496e462690", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:85c8bd8eefc4afc33445de2ee7fbf33a7807bc34eb3734b8eefa4e98e4cddf38", - "zh:98bbe309c9ff5b2352de6a047e0ec6c7e3764b4ed3dfd370839c4be2fbfff869", - "zh:9c7bf8c56da1b124e0e2f3210a1915e778bab2be924481af684695b52672891e", - "zh:d2200f7f6ab8ecb8373cda796b864ad4867f5c255cff9d3b032f666e4c78f625", - "zh:d8c7926feaddfdc08d5ebb41b03445166df8c125417b28d64712dccd9feef136", - "zh:e2412a192fc340c61b373d6c20c9d805d7d3dee6c720c34db23c2a8ff0abd71b", - "zh:e6ac6bba391afe728a099df344dbd6481425b06d61697522017b8f7a59957d44", + "zh:898db5d2b6bd6ca5457dccb52eedbc7c5b1a71e4a4658381bcbb38cedbbda328", + "zh:8de913bf09a3fa7bedc29fec18c47c571d0c7a3d0644322c46f3aa648cf30cd8", + "zh:9402102c86a87bdfe7e501ffbb9c685c32bbcefcfcf897fd7d53df414c36877b", + "zh:b18b9bb1726bb8cfbefc0a29cf3657c82578001f514bcf4c079839b6776c47f0", + "zh:b9d31fdc4faecb909d7c5ce41d2479dd0536862a963df434be4b16e8e4edc94d", + "zh:c951e9f39cca3446c060bd63933ebb89cedde9523904813973fbc3d11863ba75", + "zh:e5b773c0d07e962291be0e9b413c7a22c044b8c7b58c76e8aa91d1659990dfb5", ] } provider "registry.terraform.io/hashicorp/tls" { - version = "4.0.6" + version = "4.1.0" constraints = ">= 3.0.0" hashes = [ - "h1:dYSb3V94K5dDMtrBRLPzBpkMTPn+3cXZ/kIJdtFL+2M=", - "h1:n3M50qfWfRSpQV9Pwcvuse03pEizqrmYEryxKky4so4=", - "zh:10de0d8af02f2e578101688fd334da3849f56ea91b0d9bd5b1f7a243417fdda8", - "zh:37fc01f8b2bc9d5b055dc3e78bfd1beb7c42cfb776a4c81106e19c8911366297", - "zh:4578ca03d1dd0b7f572d96bd03f744be24c726bfd282173d54b100fd221608bb", - "zh:6c475491d1250050765a91a493ef330adc24689e8837a0f07da5a0e1269e11c1", - "zh:81bde94d53cdababa5b376bbc6947668be4c45ab655de7aa2e8e4736dfd52509", - "zh:abdce260840b7b050c4e401d4f75c7a199fafe58a8b213947a258f75ac18b3e8", - "zh:b754cebfc5184873840f16a642a7c9ef78c34dc246a8ae29e056c79939963c7a", - "zh:c928b66086078f9917aef0eec15982f2e337914c5c4dbc31dd4741403db7eb18", - "zh:cded27bee5f24de6f2ee0cfd1df46a7f88e84aaffc2ecbf3ff7094160f193d50", - "zh:d65eb3867e8f69aaf1b8bb53bd637c99c6b649ba3db16ded50fa9a01076d1a27", - "zh:ecb0c8b528c7a619fa71852bb3fb5c151d47576c5aab2bf3af4db52588722eeb", + "h1:zEv9tY1KR5vaLSyp2lkrucNJ+Vq3c+sTFK9GyQGLtFs=", + "zh:14c35d89307988c835a7f8e26f1b83ce771e5f9b41e407f86a644c0152089ac2", + "zh:2fb9fe7a8b5afdbd3e903acb6776ef1be3f2e587fb236a8c60f11a9fa165faa8", + "zh:35808142ef850c0c60dd93dc06b95c747720ed2c40c89031781165f0c2baa2fc", + "zh:35b5dc95bc75f0b3b9c5ce54d4d7600c1ebc96fbb8dfca174536e8bf103c8cdc", + "zh:38aa27c6a6c98f1712aa5cc30011884dc4b128b4073a4a27883374bfa3ec9fac", + "zh:51fb247e3a2e88f0047cb97bb9df7c228254a3b3021c5534e4563b4007e6f882", + "zh:62b981ce491e38d892ba6364d1d0cdaadcee37cc218590e07b310b1dfa34be2d", + "zh:bc8e47efc611924a79f947ce072a9ad698f311d4a60d0b4dfff6758c912b7298", + "zh:c149508bd131765d1bc085c75a870abb314ff5a6d7f5ac1035a8892d686b6297", + "zh:d38d40783503d278b63858978d40e07ac48123a2925e1a6b47e62179c046f87a", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fb07f708e3316615f6d218cec198504984c0ce7000b9f1eebff7516e384f4b54", ] } diff --git a/terraform-unity/modules/terraform-unity-sps-eks/README.md b/terraform-unity/modules/terraform-unity-sps-eks/README.md index 8e32f7f3..115b3c8c 100644 --- a/terraform-unity/modules/terraform-unity-sps-eks/README.md +++ b/terraform-unity/modules/terraform-unity-sps-eks/README.md @@ -35,15 +35,15 @@ | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [cluster\_version](#input\_cluster\_version) | The EKS cluster version (must be supported by the cs-infra module). | `string` | `"1.31"` | no | +| [cluster\_version](#input\_cluster\_version) | The EKS cluster version (must be supported by the cs-infra module). | `string` | `"1.33"` | no | | [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | `""` | no | | [installprefix](#input\_installprefix) | The install prefix for the service area (unused) | `string` | `""` | no | -| [nodegroups](#input\_nodegroups) | A map of node group configurations | 
map(object({
    create_iam_role            = optional(bool)
    iam_role_arn               = optional(string)
    ami_id                     = optional(string)
    min_size                   = optional(number)
    max_size                   = optional(number)
    desired_size               = optional(number)
    instance_types             = optional(list(string))
    capacity_type              = optional(string)
    enable_bootstrap_user_data = optional(bool)
    metadata_options           = optional(map(any))
    block_device_mappings = optional(map(object({
      device_name = string
      ebs = object({
        volume_size           = number
        volume_type           = string
        encrypted             = bool
        delete_on_termination = bool
      })
    })))
  }))
| 
{
  "defaultGroup": {
    "block_device_mappings": {
      "xvda": {
        "device_name": "/dev/xvda",
        "ebs": {
          "delete_on_termination": true,
          "encrypted": true,
          "volume_size": 100,
          "volume_type": "gp2"
        }
      }
    },
    "desired_size": 1,
    "instance_types": [
      "m5.2xlarge"
    ],
    "max_size": 1,
    "metadata_options": {
      "http_endpoint": "enabled",
      "http_put_response_hop_limit": 3
    },
    "min_size": 1
  }
}
| no | +| [nodegroups](#input\_nodegroups) | A map of node group configurations | 
map(object({
    create_iam_role            = optional(bool)
    iam_role_arn               = optional(string)
    ami_id                     = optional(string)
    min_size                   = optional(number)
    max_size                   = optional(number)
    desired_size               = optional(number)
    instance_types             = optional(list(string))
    capacity_type              = optional(string)
    metadata_options           = optional(map(any))
    block_device_mappings = optional(map(object({
      device_name = string
      ebs = object({
        volume_size           = number
        volume_type           = string
        encrypted             = bool
        delete_on_termination = bool
      })
    })))
  }))
| 
{
  "defaultGroup": {
    "block_device_mappings": {
      "xvda": {
        "device_name": "/dev/xvda",
        "ebs": {
          "delete_on_termination": true,
          "encrypted": true,
          "volume_size": 100,
          "volume_type": "gp2"
        }
      }
    },
    "desired_size": 1,
    "instance_types": [
      "m5.2xlarge"
    ],
    "max_size": 1,
    "metadata_options": {
      "http_endpoint": "enabled",
      "http_put_response_hop_limit": 3
    },
    "min_size": 1
  }
}
| no | | [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `"unity"` | no | | [release](#input\_release) | The software release version. | `string` | `"25.3"` | no | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no | | [tags](#input\_tags) | Tags for the deployment (unused) | `map(string)` | 
{
  "empty": ""
}
| no | -| [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | +| [venue](#input\_venue) | The SMCE venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | ## Outputs diff --git a/terraform-unity/modules/terraform-unity-sps-eks/main.tf b/terraform-unity/modules/terraform-unity-sps-eks/main.tf index 227e2ab8..cb5a0573 100644 --- a/terraform-unity/modules/terraform-unity-sps-eks/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-eks/main.tf @@ -1,6 +1,6 @@ terraform { backend "s3" { - bucket = "unity-unity-dev-bucket" + bucket = "smce-mdps-dev-bucket" workspace_key_prefix = "sps/tfstates" key = "terraform.tfstate" region = "us-west-2" @@ -9,13 +9,13 @@ terraform { } module "unity-eks" { - source = "git::https://github.com/unity-sds/unity-cs-infra.git//terraform-unity-eks_module?ref=unity-sps-2.5.0" + source = "git::https://github.com/unity-sds/unity-cs-infra.git//terraform-unity-eks_module?ref=main" deployment_name = local.cluster_name project = var.project venue = var.venue nodegroups = var.nodegroups aws_auth_roles = [{ - rolearn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/mcp-tenantOperator" + rolearn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/smce_deployment" username = "admin" groups = ["system:masters"] }] diff --git a/terraform-unity/modules/terraform-unity-sps-eks/variables.tf b/terraform-unity/modules/terraform-unity-sps-eks/variables.tf index dffa6d1d..05c4f592 100644 --- a/terraform-unity/modules/terraform-unity-sps-eks/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-eks/variables.tf @@ -5,7 +5,7 @@ variable "project" { } variable "venue" { - description = "The MCP venue in which the cluster will be deployed (dev, test, prod)" + description = "The SMCE venue in which the cluster will be deployed (dev, test, prod)" type = string } @@ -18,7 +18,7 @@ variable "service_area" { variable "cluster_version" { description = "The EKS cluster version (must be supported by the cs-infra module)." type = string - default = "1.31" + default = "1.33" } variable "release" { @@ -59,7 +59,6 @@ variable "nodegroups" { desired_size = optional(number) instance_types = optional(list(string)) capacity_type = optional(string) - enable_bootstrap_user_data = optional(bool) metadata_options = optional(map(any)) block_device_mappings = optional(map(object({ device_name = string diff --git a/terraform-unity/modules/terraform-unity-sps-initiators/README.md b/terraform-unity/modules/terraform-unity-sps-initiators/README.md index 7cceb880..fa44ecae 100644 --- a/terraform-unity/modules/terraform-unity-sps-initiators/README.md +++ b/terraform-unity/modules/terraform-unity-sps-initiators/README.md @@ -43,7 +43,7 @@ | [project](#input\_project) | The project or mission deploying Unity SPS | `string` | n/a | yes | | [release](#input\_release) | The software release version. | `string` | n/a | yes | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | n/a | yes | -| [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | +| [venue](#input\_venue) | The SMCE venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | ## Outputs diff --git a/terraform-unity/modules/terraform-unity-sps-initiators/locals.tf b/terraform-unity/modules/terraform-unity-sps-initiators/locals.tf index e61981e2..468722a7 100644 --- a/terraform-unity/modules/terraform-unity-sps-initiators/locals.tf +++ b/terraform-unity/modules/terraform-unity-sps-initiators/locals.tf @@ -1,6 +1,7 @@ locals { resource_name_prefix = join("-", compact([var.project, var.venue, var.service_area, "%s"])) + s3_bucket_name_prefix = join("-", compact([var.project, var.venue, var.service_area, "%s", "smce"])) common_tags = { Name = "" Venue = var.venue diff --git a/terraform-unity/modules/terraform-unity-sps-initiators/main.tf b/terraform-unity/modules/terraform-unity-sps-initiators/main.tf index 0e9f6e8e..663bbe18 100644 --- a/terraform-unity/modules/terraform-unity-sps-initiators/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-initiators/main.tf @@ -1,28 +1,28 @@ resource "aws_s3_bucket" "inbound_staging_location" { - bucket = format(local.resource_name_prefix, "isl") + bucket = format(local.s3_bucket_name_prefix, "isl") force_destroy = true tags = merge(local.common_tags, { - Name = format(local.resource_name_prefix, "S3-ISL") + Name = format(local.s3_bucket_name_prefix, "S3-ISL") Component = "S3" Stack = "S3" }) } resource "aws_s3_bucket" "code" { - bucket = format(local.resource_name_prefix, "code") + bucket = format(local.s3_bucket_name_prefix, "code") force_destroy = true tags = merge(local.common_tags, { - Name = format(local.resource_name_prefix, "S3-code") + Name = format(local.s3_bucket_name_prefix, "S3-code") Component = "S3" Stack = "S3" }) } resource "aws_s3_bucket" "config" { - bucket = format(local.resource_name_prefix, "config") + bucket = format(local.s3_bucket_name_prefix, "config") force_destroy = true tags = merge(local.common_tags, { - Name = format(local.resource_name_prefix, "S3-config") + Name = format(local.s3_bucket_name_prefix, "S3-config") Component = "S3" Stack = "S3" }) @@ -34,7 +34,7 @@ resource "aws_s3_bucket_policy" "ssl_s3_policy" { "code", "config" ]) - bucket = format(local.resource_name_prefix, each.key) + bucket = format(local.s3_bucket_name_prefix, each.key) policy = jsonencode( { "Id" : "ExamplePolicy", @@ -45,8 +45,8 @@ resource "aws_s3_bucket_policy" "ssl_s3_policy" { "Action" : "s3:*", "Effect" : "Deny", "Resource" : [ - format("%s%s", "arn:aws:s3:::", format(local.resource_name_prefix, each.key)), - format("%s%s/%s", "arn:aws:s3:::", format(local.resource_name_prefix, each.key), "*") + format("%s%s", "arn:aws:s3:::", format(local.s3_bucket_name_prefix, each.key)), + format("%s%s/%s", "arn:aws:s3:::", format(local.s3_bucket_name_prefix, each.key), "*") ], "Condition" : { "Bool" : { @@ -78,7 +78,7 @@ resource "aws_s3_object" "router_config" { } module "unity_initiator" { - source = "git::https://github.com/unity-sds/unity-initiator.git//terraform-unity/initiator?ref=unity-sps-2.2.0" + source = "git::https://github.com/unity-sds/unity-initiator.git//terraform-unity/initiator?ref=smce" code_bucket = aws_s3_bucket.code.id project = var.project router_config = "s3://${aws_s3_bucket.config.id}/${aws_s3_object.router_config.key}" @@ -91,7 +91,7 @@ resource "aws_s3_object" "isl_stacam_rawdp_folder" { } module "s3_bucket_notification" { - source = "git::https://github.com/unity-sds/unity-initiator.git//terraform-unity/triggers/s3-bucket-notification?ref=unity-sps-2.2.0" + source = "git::https://github.com/unity-sds/unity-initiator.git//terraform-unity/triggers/s3-bucket-notification?ref=smce" initiator_topic_arn = module.unity_initiator.initiator_topic_arn isl_bucket = aws_s3_bucket.inbound_staging_location.id isl_bucket_prefix = "STACAM/RawDP/" diff --git a/terraform-unity/modules/terraform-unity-sps-initiators/variables.tf b/terraform-unity/modules/terraform-unity-sps-initiators/variables.tf index f9701f9d..c7df21cc 100644 --- a/terraform-unity/modules/terraform-unity-sps-initiators/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-initiators/variables.tf @@ -4,7 +4,7 @@ variable "project" { } variable "venue" { - description = "The MCP venue in which the cluster will be deployed (dev, test, prod)" + description = "The SMCE venue in which the cluster will be deployed (dev, test, prod)" type = string } diff --git a/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/README.md b/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/README.md index 112105c8..a252fdb6 100644 --- a/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/README.md +++ b/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/README.md @@ -27,10 +27,10 @@ No modules. | [kubernetes_manifest.karpenter_node_classes](https://registry.terraform.io/providers/hashicorp/kubernetes/2.32.0/docs/resources/manifest) | resource | | [kubernetes_manifest.karpenter_node_pools](https://registry.terraform.io/providers/hashicorp/kubernetes/2.32.0/docs/resources/manifest) | resource | | [null_resource.remove_node_class_finalizers](https://registry.terraform.io/providers/hashicorp/null/3.2.3/docs/resources/resource) | resource | -| [aws_ami.al2_eks_optimized](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/ami) | data source | +| [aws_ami.al2023_eks_optimized](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/ami) | data source | | [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/eks_cluster) | data source | | [aws_iam_role.cluster_iam_role](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/iam_role) | data source | -| [aws_ssm_parameter.al2_eks_optimized_ami](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/ssm_parameter) | data source | +| [aws_ssm_parameter.al2023_eks_optimized_ami](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/ssm_parameter) | data source | | [aws_ssm_parameter.subnet_ids](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/ssm_parameter) | data source | ## Inputs @@ -40,11 +40,11 @@ No modules. | [karpenter\_node\_classes](#input\_karpenter\_node\_classes) | n/a | 
map(object({
    volume_size = string
  }))
| n/a | yes | | [karpenter\_node\_pools](#input\_karpenter\_node\_pools) | Configuration for Karpenter node pools | 
map(object({
    requirements : list(object({
      key : string
      operator : string
      values : list(string)
    }))
    nodeClassRef : string
    limits : object({
      cpu : string
      memory : string
    })
    disruption : object({
      consolidationPolicy : string
      consolidateAfter : string
    })
  }))
| n/a | yes | | [kubeconfig\_filepath](#input\_kubeconfig\_filepath) | The path to the kubeconfig file for the Kubernetes cluster. | `string` | n/a | yes | -| [mcp\_ami\_owner\_id](#input\_mcp\_ami\_owner\_id) | The ID of the MCP AMIs | `string` | n/a | yes | +| [smce\_ami\_owner\_id](#input\_smce\_ami\_owner\_id) | The ID of the SMCE AMIs | `string` | n/a | yes | | [project](#input\_project) | The project or mission deploying Unity SPS | `string` | n/a | yes | | [release](#input\_release) | The software release version. | `string` | n/a | yes | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | n/a | yes | -| [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | +| [venue](#input\_venue) | The SMCE venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | ## Outputs diff --git a/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/data.tf b/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/data.tf index fa1a7c35..72959f21 100644 --- a/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/data.tf +++ b/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/data.tf @@ -10,14 +10,14 @@ data "aws_ssm_parameter" "subnet_ids" { name = "/unity/account/network/subnet_list" } -data "aws_ssm_parameter" "al2_eks_optimized_ami" { - name = "/mcp/amis/aml2-eks-${replace(data.aws_eks_cluster.cluster.version, ".", "-")}" +data "aws_ssm_parameter" "al2023_eks_optimized_ami" { + name = "/smce/amis/aml2023-eks-${replace(data.aws_eks_cluster.cluster.version, ".", "-")}" } -data "aws_ami" "al2_eks_optimized" { +data "aws_ami" "al2023_eks_optimized" { filter { name = "image-id" - values = [data.aws_ssm_parameter.al2_eks_optimized_ami.value] + values = [data.aws_ssm_parameter.al2023_eks_optimized_ami.value] } - owners = [var.mcp_ami_owner_id] + owners = [var.aws_ami_owner_id] } diff --git a/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/main.tf b/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/main.tf index 79ff3778..88c70db8 100644 --- a/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/main.tf @@ -8,9 +8,9 @@ resource "kubernetes_manifest" "karpenter_node_classes" { name = each.key } spec = { - amiFamily = "AL2" + amiFamily = "AL2023" amiSelectorTerms = [{ - id = data.aws_ami.al2_eks_optimized.image_id + id = data.aws_ami.al2023_eks_optimized.image_id }] userData = file("${path.module}/node-user-data.sh") role = data.aws_iam_role.cluster_iam_role.name @@ -23,7 +23,7 @@ resource "kubernetes_manifest" "karpenter_node_classes" { "Name" = "${data.aws_eks_cluster.cluster.name}-node" } }] - blockDeviceMappings = [for bd in tolist(data.aws_ami.al2_eks_optimized.block_device_mappings) : { + blockDeviceMappings = [for bd in tolist(data.aws_ami.al2023_eks_optimized.block_device_mappings) : { deviceName = bd.device_name ebs = { volumeSize = each.value.volume_size diff --git a/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/node-user-data.sh b/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/node-user-data.sh index e68fbc39..a94c01f0 100644 --- a/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/node-user-data.sh +++ b/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/node-user-data.sh @@ -30,8 +30,8 @@ mkdir -p $MOUNT_LOCATION # # Install NVMe CLI, Software RAID Utility # -yum update -y -yum install nvme-cli mdadm -y +dnf update -y +dnf install nvme-cli mdadm -y # # Get a list of instance-store NVMe drives. If none found, do not fail. diff --git a/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/variables.tf b/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/variables.tf index e8f983d9..867114d0 100644 --- a/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-karpenter-node-config/variables.tf @@ -4,7 +4,7 @@ variable "project" { } variable "venue" { - description = "The MCP venue in which the cluster will be deployed (dev, test, prod)" + description = "The SMCE venue in which the cluster will be deployed (dev, test, prod)" type = string } @@ -23,8 +23,13 @@ variable "kubeconfig_filepath" { type = string } -variable "mcp_ami_owner_id" { - description = "The ID of the MCP AMIs" +variable "smce_ami_owner_id" { + description = "The ID of the SMCE AMIs" + type = string +} + +variable "aws_ami_owner_id" { + description = "The ID of the AWS AMIs we use for EKS optimized AMIs" type = string } diff --git a/terraform-unity/modules/terraform-unity-sps-karpenter/README.md b/terraform-unity/modules/terraform-unity-sps-karpenter/README.md index 68b09a14..7fe700c1 100644 --- a/terraform-unity/modules/terraform-unity-sps-karpenter/README.md +++ b/terraform-unity/modules/terraform-unity-sps-karpenter/README.md @@ -42,7 +42,7 @@ | [release](#input\_release) | The software release version. | `string` | `"25.3"` | no | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no | | [tags](#input\_tags) | Tags for the deployment (unused) | `map(string)` | 
{
  "empty": ""
}
| no | -| [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | +| [venue](#input\_venue) | The SMCE venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | ## Outputs diff --git a/terraform-unity/modules/terraform-unity-sps-karpenter/main.tf b/terraform-unity/modules/terraform-unity-sps-karpenter/main.tf index d11e581f..4d489798 100644 --- a/terraform-unity/modules/terraform-unity-sps-karpenter/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-karpenter/main.tf @@ -1,6 +1,6 @@ terraform { backend "s3" { - bucket = "unity-unity-dev-bucket" + bucket = "smce-mdps-dev-bucket" workspace_key_prefix = "sps/tfstates" key = "terraform.tfstate" region = "us-west-2" @@ -18,7 +18,7 @@ module "karpenter" { iam_role_use_name_prefix = false create_node_iam_role = false node_iam_role_arn = data.aws_iam_role.cluster_iam_role.arn - iam_role_permissions_boundary_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/mcp-tenantOperator-AMI-APIG" + iam_role_permissions_boundary_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/zsmce-tenantOperator-AMI-APIG" enable_irsa = true irsa_oidc_provider_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.oidc_provider_url}" # Since the nodegroup role will already have an access entry diff --git a/terraform-unity/modules/terraform-unity-sps-karpenter/variables.tf b/terraform-unity/modules/terraform-unity-sps-karpenter/variables.tf index 730cf602..24d2ec6b 100644 --- a/terraform-unity/modules/terraform-unity-sps-karpenter/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-karpenter/variables.tf @@ -5,7 +5,7 @@ variable "project" { } variable "venue" { - description = "The MCP venue in which the cluster will be deployed (dev, test, prod)" + description = "The SMCE venue in which the cluster will be deployed (dev, test, prod)" type = string } diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/README.md b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/README.md index 09f5ab83..2ea756cd 100644 --- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/README.md +++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/README.md @@ -60,7 +60,6 @@ No modules. | [aws_ssm_parameter.shared_services_domain](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/ssm_parameter) | data source | | [aws_ssm_parameter.shared_services_region](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/ssm_parameter) | data source | | [aws_ssm_parameter.subnet_ids](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/ssm_parameter) | data source | -| [aws_ssm_parameter.venue_proxy_baseurl](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/ssm_parameter) | data source | | [aws_vpc.cluster_vpc](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/vpc) | data source | | [kubernetes_namespace.service_area](https://registry.terraform.io/providers/hashicorp/kubernetes/2.32.0/docs/data-sources/namespace) | data source | | [kubernetes_persistent_volume_claim.airflow_deployed_dags](https://registry.terraform.io/providers/hashicorp/kubernetes/2.32.0/docs/data-sources/persistent_volume_claim) | data source | @@ -82,7 +81,7 @@ No modules. | [project](#input\_project) | The project or mission deploying Unity SPS | `string` | n/a | yes | | [release](#input\_release) | The software release version. | `string` | n/a | yes | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | n/a | yes | -| [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | +| [venue](#input\_venue) | The SMCE venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | ## Outputs diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/data.tf b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/data.tf index 45f482d7..5bb04943 100644 --- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/data.tf +++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/data.tf @@ -58,37 +58,6 @@ data "aws_ssm_parameter" "shared_services_region" { name = "/unity/shared-services/aws/account/region" } -data "aws_ssm_parameter" "shared_services_domain" { - name = "arn:aws:ssm:${data.aws_ssm_parameter.shared_services_region.value}:${data.aws_ssm_parameter.shared_services_account.value}:parameter/unity/shared-services/domain" -} - -data "aws_ssm_parameter" "venue_proxy_baseurl" { - name = "/unity/${var.project}/${var.venue}/management/httpd/loadbalancer-url" -} - -data "aws_api_gateway_rest_api" "rest_api" { - name = "unity-${var.project}-${var.venue}-rest-api-gateway" -} - -data "aws_api_gateway_authorizers" "unity_cs_common_authorizers_list" { - rest_api_id = data.aws_api_gateway_rest_api.rest_api.id -} - -data "aws_api_gateway_authorizer" "unity_cs_common_authorizer" { - rest_api_id = data.aws_api_gateway_rest_api.rest_api.id - authorizer_id = data.aws_api_gateway_authorizers.unity_cs_common_authorizers_list.ids[0] -} - -data "aws_lb" "ogc_k8s_lb" { - tags = { - Venue = var.venue - Proj = var.project - Name = format(local.resource_name_prefix, "OgcLB") - Stack = "ogc" - } - depends_on = [kubernetes_service.ogc_processes_api_ingress_internal] -} - data "aws_lambda_functions" "lambda_check_all" {} data "aws_security_groups" "venue_proxy_sg" { diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf index d17e7842..8912389c 100644 --- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf @@ -375,89 +375,12 @@ resource "time_sleep" "wait_for_ogc_lb" { create_duration = "180s" } -resource "aws_api_gateway_vpc_link" "rest_api_ogc_vpc_link" { - name = "sps-nlb-vpc-link-${var.project}-${var.venue}" - description = "sps-nlb-vpc-link-${var.project}-${var.venue}" - target_arns = [data.aws_lb.ogc_k8s_lb.arn] - depends_on = [time_sleep.wait_for_ogc_lb] -} - -resource "aws_api_gateway_resource" "rest_api_resource_management_path" { - rest_api_id = data.aws_api_gateway_rest_api.rest_api.id - parent_id = data.aws_api_gateway_rest_api.rest_api.root_resource_id - path_part = "ogc" -} - -resource "aws_api_gateway_resource" "rest_api_resource_ogc_api_path" { - rest_api_id = data.aws_api_gateway_rest_api.rest_api.id - parent_id = aws_api_gateway_resource.rest_api_resource_management_path.id - path_part = "api" -} - -resource "aws_api_gateway_resource" "rest_api_resource_ogc_proxy_path" { - rest_api_id = data.aws_api_gateway_rest_api.rest_api.id - parent_id = aws_api_gateway_resource.rest_api_resource_ogc_api_path.id - path_part = "{proxy+}" -} - -resource "aws_api_gateway_method" "rest_api_method_for_ogc_proxy_method" { - rest_api_id = data.aws_api_gateway_rest_api.rest_api.id - resource_id = aws_api_gateway_resource.rest_api_resource_ogc_proxy_path.id - http_method = "ANY" - authorization = "CUSTOM" - authorizer_id = data.aws_api_gateway_authorizer.unity_cs_common_authorizer.id - request_parameters = { "method.request.path.proxy" = true } -} - -resource "aws_api_gateway_integration" "rest_api_integration_for_ogc_api" { - rest_api_id = data.aws_api_gateway_rest_api.rest_api.id - resource_id = aws_api_gateway_resource.rest_api_resource_ogc_proxy_path.id - http_method = aws_api_gateway_method.rest_api_method_for_ogc_proxy_method.http_method - type = "HTTP_PROXY" - uri = format("%s://%s:%s%s", "http", data.kubernetes_service.ogc_processes_api_ingress_internal.status[0].load_balancer[0].ingress[0].hostname, local.load_balancer_port, "/{proxy}") - integration_http_method = "ANY" - passthrough_behavior = "WHEN_NO_MATCH" - connection_type = "VPC_LINK" - connection_id = aws_api_gateway_vpc_link.rest_api_ogc_vpc_link.id - - request_parameters = { - "integration.request.path.proxy" = "method.request.path.proxy" - } - tls_config { # the k8s ingress backends aren't set up with TLS - insecure_skip_verification = true - } - - depends_on = [aws_api_gateway_vpc_link.rest_api_ogc_vpc_link, aws_api_gateway_method.rest_api_method_for_ogc_proxy_method] -} - -resource "aws_api_gateway_method_response" "response_200" { - rest_api_id = data.aws_api_gateway_rest_api.rest_api.id - resource_id = aws_api_gateway_resource.rest_api_resource_ogc_proxy_path.id - http_method = aws_api_gateway_method.rest_api_method_for_ogc_proxy_method.http_method - status_code = "200" - - depends_on = [aws_api_gateway_method.rest_api_method_for_ogc_proxy_method] -} - -resource "time_sleep" "wait_for_gateway_integration" { - # need to make sure both the proxy method and integration have time to settle before deploying - depends_on = [aws_api_gateway_integration.rest_api_integration_for_ogc_api, aws_api_gateway_method.rest_api_method_for_ogc_proxy_method] - create_duration = "180s" -} - -# API Gateway deployment -resource "aws_api_gateway_deployment" "ogc-api-gateway-deployment" { - rest_api_id = data.aws_api_gateway_rest_api.rest_api.id - stage_name = var.venue - # stage_name = "default" - depends_on = [time_sleep.wait_for_gateway_integration, aws_api_gateway_method_response.response_200] -} - resource "aws_ssm_parameter" "ogc_processes_ui_url" { name = format("/%s", join("/", compact(["", var.project, var.venue, var.service_area, "processing", "ogc_processes", "ui_url"]))) description = "The URL of the OGC Proccesses API Docs UI." type = "String" - value = "https://www.${data.aws_ssm_parameter.shared_services_domain.value}:4443/${var.project}/${var.venue}/ogc/redoc" + # Updated to use LoadBalancer instead of shared services domain + value = "http://${data.kubernetes_service.ogc_processes_api_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:${local.load_balancer_port}/redoc" tags = merge(local.common_tags, { Name = format(local.resource_name_prefix, "endpoints-ogc_processes_ui") Component = "SSM" @@ -470,7 +393,8 @@ resource "aws_ssm_parameter" "ogc_processes_api_url" { name = format("/%s", join("/", compact(["", var.project, var.venue, var.service_area, "processing", "ogc_processes", "api_url"]))) description = "The URL of the OGC Processes REST API." type = "String" - value = "${aws_api_gateway_deployment.ogc-api-gateway-deployment.invoke_url}/ogc/api/" + # Updated to use LoadBalancer instead of API Gateway + value = "http://${data.kubernetes_service.ogc_processes_api_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:${local.load_balancer_port}/" tags = merge(local.common_tags, { Name = format(local.resource_name_prefix, "endpoints-ogc_processes_api") Component = "SSM" @@ -483,14 +407,15 @@ resource "aws_ssm_parameter" "ogc_processes_api_health_check_endpoint" { name = format("/%s", join("/", compact(["", "unity", var.project, var.venue, "component", "ogc-api"]))) description = "The URL of the OGC Processes REST API." type = "String" + # Updated to use LoadBalancer instead of shared services domain value = jsonencode({ "componentCategory" : "processing" "componentName" : "OGC API" "componentType" : "api" "description" : "A standards-compliant programming interface for Application deployment, job execution and job tracking. May be used to execute jobs in batches." - "healthCheckUrl" : "https://www.${data.aws_ssm_parameter.shared_services_domain.value}:4443/${var.project}/${var.venue}/ogc/health" + "healthCheckUrl" : "http://${data.kubernetes_service.ogc_processes_api_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:${local.load_balancer_port}/health" "isPortalIntegrated" : false - "landingPageUrl" : "https://www.${data.aws_ssm_parameter.shared_services_domain.value}:4443/${var.project}/${var.venue}/ogc/" + "landingPageUrl" : "http://${data.kubernetes_service.ogc_processes_api_ingress_internal.status[0].load_balancer[0].ingress[0].hostname}:${local.load_balancer_port}/" }) tags = merge(local.common_tags, { Name = format(local.resource_name_prefix, "health-check-endpoints-ogc_processes_api") @@ -538,33 +463,3 @@ resource "aws_lambda_invocation" "unity_proxy_lambda_invocation" { ])) } } - -resource "null_resource" "check_ogc_api_status" { - provisioner "local-exec" { - command = "./check_ogc_api_status.sh" - working_dir = "${path.module}/../../../utils" - environment = { - OGC_PROCESSES_API = nonsensitive(aws_ssm_parameter.ogc_processes_api_url.value) - TOKEN_URL = "https://cognito-idp.${local.region}.amazonaws.com" - UNITY_CLIENTID = nonsensitive(data.aws_ssm_parameter.unity_client_id.value) - UNITY_PASSWORD = nonsensitive(data.aws_ssm_parameter.unity_password.value) - UNITY_USERNAME = nonsensitive(data.aws_ssm_parameter.unity_username.value) - } - } - depends_on = [aws_api_gateway_deployment.ogc-api-gateway-deployment, aws_ssm_parameter.ogc_processes_api_url] -} - -resource "null_resource" "register_ogc_processes" { - provisioner "local-exec" { - command = "./post_deployment_terraform.sh" - working_dir = "${path.module}/../../../utils" - environment = { - OGC_PROCESSES_API = nonsensitive(aws_ssm_parameter.ogc_processes_api_url.value) - TOKEN_URL = "https://cognito-idp.${local.region}.amazonaws.com" - UNITY_CLIENTID = nonsensitive(data.aws_ssm_parameter.unity_client_id.value) - UNITY_PASSWORD = nonsensitive(data.aws_ssm_parameter.unity_password.value) - UNITY_USERNAME = nonsensitive(data.aws_ssm_parameter.unity_username.value) - } - } - depends_on = [null_resource.check_ogc_api_status] -} diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/outputs.tf b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/outputs.tf index e4e892d0..b3283cf6 100644 --- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/outputs.tf +++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/outputs.tf @@ -13,13 +13,13 @@ output "ogc_processes_urls" { } output "ogc_processes_venue_urls" { - description = "URLs for the various OGC Processes endpoints at venue-proxy level." + description = "URLs for the various OGC Processes endpoints (LoadBalancer direct access)." value = { "ui" = { - "url" = nonsensitive(replace(data.aws_ssm_parameter.venue_proxy_baseurl.value, "management/ui", "ogc/redoc")) + "url" = nonsensitive(aws_ssm_parameter.ogc_processes_ui_url.value) } "rest_api" = { - "url" = nonsensitive(replace(data.aws_ssm_parameter.venue_proxy_baseurl.value, "management/ui", "ogc/")) + "url" = nonsensitive(aws_ssm_parameter.ogc_processes_api_url.value) } } } diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/variables.tf b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/variables.tf index 30fdaf23..7509c8ba 100644 --- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/variables.tf @@ -4,7 +4,7 @@ variable "project" { } variable "venue" { - description = "The MCP venue in which the cluster will be deployed (dev, test, prod)" + description = "The SMCE venue in which the cluster will be deployed (dev, test, prod)" type = string } diff --git a/terraform-unity/modules/terraform-unity-sps-s3/README.md b/terraform-unity/modules/terraform-unity-sps-s3/README.md index b441fda0..f50213ac 100644 --- a/terraform-unity/modules/terraform-unity-sps-s3/README.md +++ b/terraform-unity/modules/terraform-unity-sps-s3/README.md @@ -43,7 +43,7 @@ No modules. | [project](#input\_project) | The project or mission deploying Unity SPS | `string` | n/a | yes | | [release](#input\_release) | The software release version | `string` | n/a | yes | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | n/a | yes | -| [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | +| [venue](#input\_venue) | The SMCE venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes | ## Outputs diff --git a/terraform-unity/modules/terraform-unity-sps-s3/locals.tf b/terraform-unity/modules/terraform-unity-sps-s3/locals.tf index e61981e2..468722a7 100644 --- a/terraform-unity/modules/terraform-unity-sps-s3/locals.tf +++ b/terraform-unity/modules/terraform-unity-sps-s3/locals.tf @@ -1,6 +1,7 @@ locals { resource_name_prefix = join("-", compact([var.project, var.venue, var.service_area, "%s"])) + s3_bucket_name_prefix = join("-", compact([var.project, var.venue, var.service_area, "%s", "smce"])) common_tags = { Name = "" Venue = var.venue diff --git a/terraform-unity/modules/terraform-unity-sps-s3/main.tf b/terraform-unity/modules/terraform-unity-sps-s3/main.tf index 1b6a54f1..fdf1f77e 100644 --- a/terraform-unity/modules/terraform-unity-sps-s3/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-s3/main.tf @@ -1,9 +1,9 @@ # Create an S3 bucket to use as Kubernetes Persistent Volume resource "aws_s3_bucket" "s3_pv_bucket" { - bucket = format(local.resource_name_prefix, "s3-pv") + bucket = format(local.s3_bucket_name_prefix, "s3-pv") force_destroy = true tags = merge(local.common_tags, { - Name = format(local.resource_name_prefix, "s3-pv") + Name = format(local.s3_bucket_name_prefix, "s3-pv") Component = "airflow" Stack = "airflow" }) @@ -21,8 +21,8 @@ resource "aws_s3_bucket_policy" "s3_pv_bucket_policy" { "Action" : "s3:*", "Effect" : "Deny", "Resource" : [ - format("%s%s", "arn:aws:s3:::", format(local.resource_name_prefix, "s3-pv")), - format("%s%s/%s", "arn:aws:s3:::", format(local.resource_name_prefix, "s3-pv"), "*") + format("%s%s", "arn:aws:s3:::", format(local.s3_bucket_name_prefix, "s3-pv")), + format("%s%s/%s", "arn:aws:s3:::", format(local.s3_bucket_name_prefix, "s3-pv"), "*") ], "Condition" : { "Bool" : { diff --git a/terraform-unity/modules/terraform-unity-sps-s3/variables.tf b/terraform-unity/modules/terraform-unity-sps-s3/variables.tf index 19913ba8..3ba64cfa 100644 --- a/terraform-unity/modules/terraform-unity-sps-s3/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-s3/variables.tf @@ -4,7 +4,7 @@ variable "project" { } variable "venue" { - description = "The MCP venue in which the cluster will be deployed (dev, test, prod)" + description = "The SMCE venue in which the cluster will be deployed (dev, test, prod)" type = string } diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf index a88d425e..4e6bbe48 100644 --- a/terraform-unity/variables.tf +++ b/terraform-unity/variables.tf @@ -5,7 +5,7 @@ variable "project" { } variable "venue" { - description = "The MCP venue in which the resources will be deployed." + description = "The SMCE venue in which the resources will be deployed." type = string validation { condition = can(regex("^(dev|test|prod|ops|sbg-dev|int)$", var.venue)) @@ -52,12 +52,12 @@ variable "helm_charts" { airflow = { repository = "https://airflow.apache.org" chart = "airflow" - version = "1.18.0" + version = "1.15.0" }, keda = { repository = "https://kedacore.github.io/charts" chart = "keda" - version = "v2.17.2" + version = "v2.15.1" } } } @@ -79,7 +79,7 @@ variable "airflow_docker_images" { default = { airflow = { name = "ghcr.io/unity-sds/unity-sps/sps-airflow" - tag = "3.1.0" + tag = "3.1.1" } } } @@ -116,10 +116,17 @@ variable "ogc_processes_docker_images" { } } -variable "mcp_ami_owner_id" { - description = "The owner ID of the MCP AMIs" +# If we eventually have AMIs from our SMCE account, we need to get our account ID from an SSM parameter +variable "smce_ami_owner_id" { + description = "The owner ID of the SMCE AMIs" type = string - default = "794625662971" + default = "" +} + +variable "aws_ami_owner_id" { + description = "The owner ID of the AWS EKS optimized AMIs (this is a AWS official owner ID)" + type = string + default = "602401143452" } variable "karpenter_node_classes" {