453 smce deployment

airflow/dags/cwl_dag_modular.py

@@ -25,13 +25,11 @@
 from airflow.utils.trigger_rule import TriggerRule
 from kubernetes.client import models as k8s
 from unity_sps_utils import (
-    CS_SHARED_SERVICES_ACCOUNT_ID,
-    CS_SHARED_SERVICES_ACCOUNT_REGION,
     DEFAULT_LOG_LEVEL,
-    DS_COGNITO_CLIENT_ID,
     DS_S3_BUCKET_PARAM,
     EC2_TYPES,
     LOG_LEVEL_TYPE,
+    MDPS_CLIENT_ID,
     NODE_POOL_DEFAULT,
     NODE_POOL_HIGH_WORKLOAD,
     POD_LABEL,
@@ -175,16 +173,9 @@ def select_stage_in(ti, stac_json, unity_stac_auth_type):
     """Retrieve stage in arguments based on authentication type parameter."""
     stage_in_args = {"stac_json": stac_json, "stac_auth_type": "NONE"}
     if unity_stac_auth_type:
-        shared_services_account = SSM_CLIENT.get_parameter(
-            Name=CS_SHARED_SERVICES_ACCOUNT_ID, WithDecryption=True
-        )["Parameter"]["Value"]
-        shared_services_region = SSM_CLIENT.get_parameter(
-            Name=CS_SHARED_SERVICES_ACCOUNT_REGION, WithDecryption=True
-        )["Parameter"]["Value"]
-        unity_client_id = SSM_CLIENT.get_parameter(
-            Name=f"arn:aws:ssm:{shared_services_region}:{shared_services_account}:parameter{DS_COGNITO_CLIENT_ID}",
-            WithDecryption=True,
-        )["Parameter"]["Value"]
+        unity_client_id = SSM_CLIENT.get_parameter(Name=MDPS_CLIENT_ID, WithDecryption=True)["Parameter"][
+            "Value"
+        ]
         stage_in_args["unity_client_id"] = unity_client_id
         stage_in_args["stac_auth_type"] = "UNITY"
 

airflow/dags/run_ogc_process.py

@@ -19,13 +19,13 @@
 from kubernetes.client import models as k8s
 from unity_sps_utils import POD_LABEL, POD_NAMESPACE, get_affinity
 
-PROCESSES_ENDPOINT = "https://api.dit.maap-project.org/api/ogc/processes"
+API_HOST = "https://api.dit.maap-project.org/api/"
 
 
 def fetch_ogc_processes():
     """Fetch available processes from the OGC API and create mapping."""
     try:
-        response = requests.get(PROCESSES_ENDPOINT, timeout=30)
+        response = requests.get(API_HOST + "ogc/processes", timeout=30)
         response.raise_for_status()
 
         processes_data = response.json()
@@ -65,7 +65,8 @@ def fetch_ogc_processes():
 
 # Constants
 K8S_SECRET_NAME = "sps-app-credentials"
-DOCKER_IMAGE = "jplmdps/ogc-job-runner:latest"
+# This docker image is generated by the files in docker/run_ogc_process
+DOCKER_IMAGE = "jplmdps/ogc-job-runner:v1.0.0"
 PROCESS_MAPPING, DROPDOWN_OPTIONS = fetch_ogc_processes()
 
 # SPS-specific secrets
@@ -155,7 +156,7 @@ def _build_submit_env_vars(self):
         return [
             k8s.V1EnvVar(
                 name="SUBMIT_JOB_URL",
-                value="https://api.dit.maap-project.org/api/ogc/processes/{process_id}/execution",
+                value=API_HOST + "ogc/processes/{process_id}/execution",
             ),
             k8s.V1EnvVar(name="PROCESS_ID", value=str(numerical_process_id)),
             k8s.V1EnvVar(name="JOB_INPUTS", value=self.job_inputs or "{}"),
@@ -168,7 +169,7 @@ def _build_monitor_env_vars(self):
         return [
             k8s.V1EnvVar(
                 name="MONITOR_JOB_URL",
-                value="https://api.dit.maap-project.org/api/ogc/jobs/{job_id}",
+                value=API_HOST + "ogc/jobs/{job_id}",
             ),
             k8s.V1EnvVar(name="JOB_ID", value=self.job_id),
             k8s.V1EnvVar(name="SUBMIT_JOB", value="false"),

airflow/docker/run_ogc_process/run_ogc_process_entrypoint.sh

@@ -19,7 +19,7 @@ if [ "$SUBMIT_JOB" = "true" ] || [ "$SUBMIT_JOB" = "True" ]; then
     --data "${SUBMIT_JOB_ARGUMENTS}")
 
     echo "API Response: $response"
-    job_id=$(echo "$response" | jq -r .id)
+    job_id=$(echo "$response" | jq -r .jobID)
 
     if [ "$job_id" = "null" ] || [ -z "$job_id" ]; then
         echo "Failed to get jobID from response."

airflow/helm/values.tmpl.yaml

@@ -165,10 +165,6 @@ webserverSecretKeySecretName: ${webserver_secret_name}
 webserver:
   replicas: 3
 
-  # Issue 404: DISABLE AIRRLOW AUTHENTICATION (https://github.com/unity-sds/unity-sps/issues/404)
-  webserverConfig: |-
-    ${webserver_config}
-
   startupProbe:
     timeoutSeconds: 20
     failureThreshold: 60 # Number of tries before giving up (10 minutes with periodSeconds of 10)

airflow/helm/values_high_load.tmpl.yaml

@@ -165,10 +165,6 @@ webserverSecretKeySecretName: ${webserver_secret_name}
 webserver:
   replicas: 3
 
-  # Issue 404: DISABLE AIRRLOW AUTHENTICATION (https://github.com/unity-sds/unity-sps/issues/404)
-  webserverConfig: |-
-    ${webserver_config}
-
   startupProbe:
     timeoutSeconds: 20
     failureThreshold: 60 # Number of tries before giving up (10 minutes with periodSeconds of 10)

airflow/plugins/unity_sps_utils.py

@@ -20,8 +20,8 @@
 
 CS_SHARED_SERVICES_ACCOUNT_ID = "/unity/shared-services/aws/account"
 CS_SHARED_SERVICES_ACCOUNT_REGION = "/unity/shared-services/aws/account/region"
-DS_COGNITO_CLIENT_ID = "/unity/shared-services/dapa/client-id"
-DS_S3_BUCKET_PARAM = f"/unity/unity/{os.environ['AIRFLOW_VAR_UNITY_VENUE']}/ds/datastore-bucket"
+MDPS_CLIENT_ID = "/sps/processing/workflows/unity_client_id"
+DS_S3_BUCKET_PARAM = f"/smce/mdps/{os.environ['AIRFLOW_VAR_UNITY_VENUE']}/ds/datastore-bucket"
 
 DEFAULT_LOG_LEVEL = "INFO"
 LOG_LEVEL_TYPE = {"DEBUG": 10, "INFO": 20, "WARNING": 30, "ERROR": 40, "CRITICAL": 50}

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "unity-sps"
-version = "3.1.0"
+version = "3.1.1"
 authors = [
     { name = "Luca Cinquini", email = "[email protected]" },
     { name = "Grace Llewellyn", email = "[email protected]" },

terraform-unity/main.tf

@@ -1,6 +1,6 @@
 terraform {
   backend "s3" {
-    bucket               = "unity-unity-dev-bucket"
+    bucket               = "smce-mdps-dev-bucket"
     workspace_key_prefix = "sps/tfstates"
     key                  = "terraform.tfstate"
     region               = "us-west-2"
@@ -82,7 +82,8 @@ module "unity-sps-karpenter-node-config" {
   service_area           = var.service_area
   release                = var.release
   kubeconfig_filepath    = var.kubeconfig_filepath
-  mcp_ami_owner_id       = var.mcp_ami_owner_id
+  smce_ami_owner_id      = var.smce_ami_owner_id
+  aws_ami_owner_id       = var.aws_ami_owner_id
   karpenter_node_classes = var.karpenter_node_classes
   karpenter_node_pools   = var.karpenter_node_pools
 }

terraform-unity/modules/terraform-unity-sps-airflow/README.md

@@ -93,7 +93,6 @@ No modules.
 | [aws_ssm_parameter.shared_services_domain](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/ssm_parameter) | data source |
 | [aws_ssm_parameter.shared_services_region](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/ssm_parameter) | data source |
 | [aws_ssm_parameter.subnet_ids](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/ssm_parameter) | data source |
-| [aws_ssm_parameter.venue_proxy_baseurl](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/ssm_parameter) | data source |
 | [aws_vpc.cluster_vpc](https://registry.terraform.io/providers/hashicorp/aws/5.67.0/docs/data-sources/vpc) | data source |
 | [kubernetes_namespace.service_area](https://registry.terraform.io/providers/hashicorp/kubernetes/2.32.0/docs/data-sources/namespace) | data source |
 | [kubernetes_service.airflow_ingress_internal](https://registry.terraform.io/providers/hashicorp/kubernetes/2.32.0/docs/data-sources/service) | data source |
@@ -116,7 +115,7 @@ No modules.
 | <a name="input_project"></a> [project](#input\_project) | The project or mission deploying Unity SPS | `string` | n/a | yes |
 | <a name="input_release"></a> [release](#input\_release) | The software release version. | `string` | n/a | yes |
 | <a name="input_service_area"></a> [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | n/a | yes |
-| <a name="input_venue"></a> [venue](#input\_venue) | The MCP venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes |
+| <a name="input_venue"></a> [venue](#input\_venue) | The SMCE venue in which the cluster will be deployed (dev, test, prod) | `string` | n/a | yes |
 
 ## Outputs
 

terraform-unity/modules/terraform-unity-sps-airflow/data.tf

@@ -58,27 +58,6 @@ data "aws_ssm_parameter" "shared_services_region" {
   name = "/unity/shared-services/aws/account/region"
 }
 
-data "aws_ssm_parameter" "shared_services_domain" {
-  name = "arn:aws:ssm:${data.aws_ssm_parameter.shared_services_region.value}:${data.aws_ssm_parameter.shared_services_account.value}:parameter/unity/shared-services/domain"
-}
-
-data "aws_ssm_parameter" "venue_proxy_baseurl" {
-  name = "/unity/${var.project}/${var.venue}/management/httpd/loadbalancer-url"
-}
-
-data "aws_api_gateway_rest_api" "rest_api" {
-  name = "unity-${var.project}-${var.venue}-rest-api-gateway"
-}
-
-data "aws_api_gateway_authorizers" "unity_cs_common_authorizers_list" {
-  rest_api_id = data.aws_api_gateway_rest_api.rest_api.id
-}
-
-data "aws_api_gateway_authorizer" "unity_cs_common_authorizer" {
-  rest_api_id   = data.aws_api_gateway_rest_api.rest_api.id
-  authorizer_id = data.aws_api_gateway_authorizers.unity_cs_common_authorizers_list.ids[0]
-}
-
 data "aws_lb" "airflow_k8s_lb" {
   tags = {
     Venue = var.venue

terraform-unity/modules/terraform-unity-sps-airflow/locals.tf

@@ -1,6 +1,7 @@
 
 locals {
   resource_name_prefix = join("-", compact([var.project, var.venue, var.service_area, "%s"]))
+  s3_bucket_name_prefix = join("-", compact([var.project, var.venue, var.service_area, "%s", "smce"]))
   common_tags = {
     Name        = ""
     Venue       = var.venue