diff --git a/.gitignore b/.gitignore index 6b8cf2b..a384d8b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,29 @@ venv node_modules .ruff_cache .coverage -htmlcov/ \ No newline at end of file +htmlcov/ + +# Terraform +.terraform/ +*.tfstate +*.tfstate.* +crash.log +crash.log.* +*.tfvars +*.tfvars.json +override.tf +override.tf.json +_override.tf +_override.tf.json +.terraformrc +terraform.rc +tfplan +infra/dcp/backend.tf +.terraform.lock.hcl + +# Configuration +infra/dcp/.env + +*.pyc +.venv/ +uv.lock diff --git a/infra/dcp/README.md b/infra/dcp/README.md new file mode 100644 index 0000000..e9fa14e --- /dev/null +++ b/infra/dcp/README.md @@ -0,0 +1,64 @@ +# Data Commons Platform (DCP) Infrastructure + +This directory contains the Terraform configuration to deploy the Data Commons Platform on Google Cloud Platform (GCP). + +## Prerequisites +* **GCP Project**: A GCP project with billing enabled. +* **Terraform**: Terraform installed locally (>= 1.0.0). +* **gcloud CLI**: GCP CLI installed and authenticated. + +## Setup + +1. **Configure Local Variables**: + Copy the example variable file and fill in your project details. + ```bash + cp terraform.tfvars.example terraform.tfvars + ``` + Edit `terraform.tfvars` with your `project_id` and other preferred settings. + +2. **Run Setup Script**: + The `setup.sh` script creates a GCS bucket for your Terraform state and initializes the backend. + ```bash + ./setup.sh + ``` + +## Deployment + +1. **Initialize**: + Initialize Terraform (if not already done by setup.sh). + ```bash + terraform init + ``` + +2. **Plan**: + Review the changes Terraform will make. + ```bash + terraform plan + ``` + +3. **Apply**: + Provision the infrastructure. + ```bash + terraform apply + ``` + +4. **Teardown**: + Destroy all resources. + ```bash + terraform destroy + ``` + +## Architecture + +This setup uses an **Orchestrator Pattern**: +- `infra/dcp/main.tf`: The root entrypoint that calls modules. +- `infra/dcp/modules/dcp/`: The new Data Commons Platform stack (Cloud Run + Spanner). +- `infra/dcp/modules/cdc/`: The legacy Custom Data Commons stack (Cloud Run + MySQL + Redis). + +Each module is independent and can be toggled via the root variables in `terraform.tfvars`. + +## Troubleshooting +* **Deletion Errors**: If you get a "cannot destroy... deletion_protection" error, ensure `deletion_protection = false` in your `terraform.tfvars`, run `terraform apply`, and then try `terraform destroy` again. Alternatively, use the helper command: + ```bash + make force-destroy + ``` diff --git a/infra/dcp/main.tf b/infra/dcp/main.tf new file mode 100644 index 0000000..8d86e82 --- /dev/null +++ b/infra/dcp/main.tf @@ -0,0 +1,124 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = ">= 5.0" + } + null = { + source = "hashicorp/null" + version = ">= 3.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} + +# Enable required APIs for both stacks +resource "google_project_service" "apis" { + for_each = toset(concat([ + "run.googleapis.com", + "iam.googleapis.com", + "sqladmin.googleapis.com", + "redis.googleapis.com", + "secretmanager.googleapis.com", + "vpcaccess.googleapis.com", + "artifactregistry.googleapis.com", + "compute.googleapis.com" + ], var.enable_dcp ? ["spanner.googleapis.com"] : [])) + + service = each.key + disable_on_destroy = false +} + +# --- Data Commons Platform (DCP) Stack --- +module "dcp" { + source = "./modules/dcp" + count = var.enable_dcp ? 1 : 0 + + project_id = var.project_id + namespace = var.namespace + region = var.region + image_base = var.dcp_image_base + image_tag = var.dcp_image_tag + service_name = var.dcp_service_name + service_account_name = var.dcp_service_account_name + create_spanner_instance = var.dcp_create_spanner_instance + create_spanner_db = var.dcp_create_spanner_db + spanner_instance_id = var.dcp_spanner_instance_id + spanner_database_id = var.dcp_spanner_database_id + spanner_processing_units = var.dcp_spanner_processing_units + service_cpu = var.dcp_service_cpu + service_memory = var.dcp_service_memory + service_min_instances = var.dcp_service_min_instances + service_max_instances = var.dcp_service_max_instances + service_concurrency = var.dcp_service_concurrency + service_timeout_seconds = var.dcp_service_timeout_seconds + deletion_protection = var.deletion_protection + + depends_on = [google_project_service.apis] +} + +# --- Custom Data Commons (CDC) Legacy Stack --- +module "cdc" { + source = "./modules/cdc" + count = var.enable_cdc ? 1 : 0 + + project_id = var.project_id + namespace = var.namespace + dc_api_key = var.cdc_dc_api_key + maps_api_key = var.cdc_maps_api_key + disable_google_maps = var.cdc_disable_google_maps + region = var.region + google_analytics_tag_id = var.cdc_google_analytics_tag_id + gcs_data_bucket_name = var.cdc_gcs_data_bucket_name + gcs_data_bucket_input_folder = var.cdc_gcs_data_bucket_input_folder + gcs_data_bucket_output_folder = var.cdc_gcs_data_bucket_output_folder + gcs_data_bucket_location = var.cdc_gcs_data_bucket_location + mysql_instance_name = var.cdc_mysql_instance_name + mysql_database_name = var.cdc_mysql_database_name + mysql_database_version = var.cdc_mysql_database_version + mysql_cpu_count = var.cdc_mysql_cpu_count + mysql_memory_size_mb = var.cdc_mysql_memory_size_mb + mysql_storage_size_gb = var.cdc_mysql_storage_size_gb + mysql_user = var.cdc_mysql_user + mysql_deletion_protection = var.deletion_protection + dc_web_service_image = var.cdc_web_service_image + dc_web_service_min_instance_count = var.cdc_web_service_min_instance_count + dc_web_service_max_instance_count = var.cdc_web_service_max_instance_count + dc_web_service_cpu = var.cdc_web_service_cpu + dc_web_service_memory = var.cdc_web_service_memory + make_dc_web_service_public = var.cdc_make_dc_web_service_public + dc_data_job_image = var.cdc_data_job_image + dc_data_job_cpu = var.cdc_data_job_cpu + dc_data_job_memory = var.cdc_data_job_memory + dc_data_job_timeout = var.cdc_data_job_timeout + dc_search_scope = var.cdc_search_scope + enable_mcp = var.cdc_enable_mcp + vpc_network_name = var.cdc_vpc_network_name + vpc_network_subnet_name = var.cdc_vpc_network_subnet_name + enable_redis = var.cdc_enable_redis + redis_instance_name = var.cdc_redis_instance_name + redis_memory_size_gb = var.cdc_redis_memory_size_gb + redis_tier = var.cdc_redis_tier + redis_location_id = var.cdc_redis_location_id + redis_alternative_location_id = var.cdc_redis_alternative_location_id + redis_replica_count = var.cdc_redis_replica_count + vpc_connector_cidr = var.cdc_vpc_connector_cidr + use_spanner = var.enable_dcp + spanner_instance_id = var.enable_dcp ? module.dcp[0].spanner_instance_id : "" + spanner_database_id = var.enable_dcp ? module.dcp[0].spanner_database_id : "" + deletion_protection = var.deletion_protection + + depends_on = [google_project_service.apis] +} + +# Ensure Spanner instance ID is provided when not creating a new one +check "spanner_instance_id_provided" { + assert { + condition = !var.enable_dcp || var.dcp_create_spanner_instance || var.dcp_spanner_instance_id != "" + error_message = "dcp_spanner_instance_id must be provided when reusing an existing instance (dcp_create_spanner_instance = false)." + } +} diff --git a/infra/dcp/modules/cdc/locals.tf b/infra/dcp/modules/cdc/locals.tf new file mode 100644 index 0000000..35079e2 --- /dev/null +++ b/infra/dcp/modules/cdc/locals.tf @@ -0,0 +1,109 @@ +# Local variable definitions + +locals { + # Data Commons Data Bucket + name_prefix = var.namespace != "" ? "${var.namespace}-" : "" + gcs_data_bucket_name = var.gcs_data_bucket_name != "" ? var.gcs_data_bucket_name : "${local.name_prefix}datacommons-data-${var.project_id}" + + # Use var.maps_api_key if set, otherwise use generated Maps API key + maps_api_key = var.maps_api_key != null ? var.maps_api_key : google_apikeys_key.maps_api_key[0].key_string + + # Data Commons API hostname + dc_api_hostname = "api.datacommons.org" + + # Data Commons API protocol + dc_api_protocol = "https" + + # Data Commons API root URL + dc_api_root = "${local.dc_api_protocol}://${local.dc_api_hostname}" + + # Optionally-configured Redis instance + redis_instance = var.enable_redis ? google_redis_instance.redis_instance[0] : null + + + # Shared environment variables used by the Data Commons web service and the Data + # Commons data loading job + cloud_run_shared_env_variables = [ + { + name = "USE_CLOUDSQL" + value = var.use_spanner ? "false" : "true" + }, + { + name = "CLOUDSQL_INSTANCE" + value = var.use_spanner ? "" : google_sql_database_instance.mysql_instance[0].connection_name + }, + { + name = "DB_NAME" + value = var.mysql_database_name + }, + { + name = "DB_USER" + value = var.mysql_user + }, + { + name = "DB_HOST" + value = "" + }, + { + name = "DB_PORT" + value = "3306" + }, + { + name = "OUTPUT_DIR" + value = "gs://${local.gcs_data_bucket_name}/${var.gcs_data_bucket_output_folder}" + }, + { + name = "FORCE_RESTART" + value = "${timestamp()}" + }, + { + name = "REDIS_HOST" + value = try(local.redis_instance.host, "") + }, + { + name = "REDIS_PORT" + value = try(local.redis_instance.port, "") + }, + { + name = "GCP_SPANNER_INSTANCE_ID" + value = var.spanner_instance_id + }, + { + name = "GCP_SPANNER_DATABASE_NAME" + value = var.spanner_database_id + } + ] + + # Shared environment variables containing secret refs used by the Data Commons + # web service and the Data Commons data loading job + cloud_run_shared_env_variable_secrets = concat([ + { + name = "DC_API_KEY" + value_source = { + secret_key_ref = { + secret = google_secret_manager_secret.dc_api_key.secret_id + version = "latest" + } + } + }, + { + name = "MAPS_API_KEY" + value_source = { + secret_key_ref = { + secret = var.disable_google_maps ? "" : google_secret_manager_secret.maps_api_key[0].secret_id + version = "latest" + } + } + } + ], var.use_spanner ? [] : [ + { + name = "DB_PASS" + value_source = { + secret_key_ref = { + secret = google_secret_manager_secret.mysql_password_secret[0].id + version = "latest" + } + } + } + ]) +} diff --git a/infra/dcp/modules/cdc/main.tf b/infra/dcp/modules/cdc/main.tf new file mode 100644 index 0000000..05b72e9 --- /dev/null +++ b/infra/dcp/modules/cdc/main.tf @@ -0,0 +1,342 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Custom Data Commons terraform resources + +# Reference the default VPC network +data "google_compute_network" "default" { + name = var.vpc_network_name +} + +# Reference the default VPC network subnet +data "google_compute_subnetwork" "default" { + name = var.vpc_network_subnet_name + region = var.region +} + +# Enable required Google Cloud APIs +resource "google_project_service" "required_apis" { + for_each = toset([ + "run.googleapis.com", + "sqladmin.googleapis.com", + "compute.googleapis.com", + "redis.googleapis.com", + "secretmanager.googleapis.com", + "vpcaccess.googleapis.com", + "artifactregistry.googleapis.com", + "iam.googleapis.com" + ]) + project = var.project_id + service = each.value + disable_on_destroy = false +} + +# Cloud SQL instance for Data Commons +resource "google_sql_database_instance" "mysql_instance" { + count = var.use_spanner ? 0 : 1 + name = "${local.name_prefix}${var.mysql_instance_name}" + database_version = var.mysql_database_version + region = var.region + + settings { + tier = "db-custom-${var.mysql_cpu_count}-${var.mysql_memory_size_mb}" + ip_configuration { + ipv4_enabled = true + } + backup_configuration { + enabled = true + } + } + + deletion_protection = var.deletion_protection + depends_on = [google_project_service.required_apis] +} + +# MySQL Database +resource "google_sql_database" "mysql_db" { + count = var.use_spanner ? 0 : 1 + name = var.mysql_database_name + instance = google_sql_database_instance.mysql_instance[0].name +} + +# Generate a random password for the MySQL user +resource "random_password" "mysql_password" { + length = 16 + special = true +} + +# Store MySQL password in Secret Manager +resource "google_secret_manager_secret" "mysql_password_secret" { + count = var.use_spanner ? 0 : 1 + secret_id = "${local.name_prefix}mysql-password" + + replication { + auto {} + } + depends_on = [google_project_service.required_apis] +} + +resource "google_secret_manager_secret_version" "mysql_password_secret_version" { + count = var.use_spanner ? 0 : 1 + secret = google_secret_manager_secret.mysql_password_secret[0].id + secret_data = random_password.mysql_password.result +} + +# MySQL User +resource "google_sql_user" "mysql_user" { + count = var.use_spanner ? 0 : 1 + name = var.mysql_user + instance = google_sql_database_instance.mysql_instance[0].name + password = random_password.mysql_password.result +} + +# Optional Redis instance +resource "google_redis_instance" "redis_instance" { + count = var.enable_redis ? 1 : 0 + name = "${local.name_prefix}${var.redis_instance_name}" + memory_size_gb = var.redis_memory_size_gb + tier = var.redis_tier + region = var.region + location_id = var.redis_location_id + alternative_location_id = var.redis_alternative_location_id + redis_version = "REDIS_6_X" + display_name = "Data Commons Redis Instance" + reserved_ip_range = null + replica_count = var.redis_replica_count + authorized_network = data.google_compute_network.default.id + connect_mode = "DIRECT_PEERING" + depends_on = [google_project_service.required_apis] +} + +# VPC Access Connector for private connections +resource "google_vpc_access_connector" "connector" { + name = "${local.name_prefix}vpc-conn" + region = var.region + network = data.google_compute_network.default.name + ip_cidr_range = var.vpc_connector_cidr + min_instances = 2 + max_instances = 10 + depends_on = [google_project_service.required_apis] +} + +# GCS Bucket for data storage +resource "google_storage_bucket" "data_bucket" { + name = local.gcs_data_bucket_name + location = var.gcs_data_bucket_location + force_destroy = true + + uniform_bucket_level_access = true + depends_on = [google_project_service.required_apis] +} + +# Maps API Key +resource "google_apikeys_key" "maps_api_key" { + count = var.maps_api_key == null && !var.disable_google_maps ? 1 : 0 + name = "${local.name_prefix}maps-key" + display_name = "Maps API Key for ${var.namespace != "" ? var.namespace : "Data Commons"}" + project = var.project_id + + restrictions { + api_targets { + service = "maps-backend.googleapis.com" + } + } + depends_on = [google_project_service.required_apis] +} + +# Cloud Run job for data management +resource "google_cloud_run_v2_job" "dc_data_job" { + name = "${local.name_prefix}datacommons-data-job" + location = var.region + deletion_protection = var.deletion_protection + + template { + template { + containers { + image = var.dc_data_job_image + resources { + limits = { + cpu = var.dc_data_job_cpu + memory = var.dc_data_job_memory + } + } + dynamic "env" { + for_each = local.cloud_run_shared_env_variables + content { + name = env.value.name + value = env.value.value + } + } + + dynamic "env" { + for_each = local.cloud_run_shared_env_variable_secrets + content { + name = env.value.name + value_source { + secret_key_ref { + secret = env.value.value_source.secret_key_ref.secret + version = env.value.value_source.secret_key_ref.version + } + } + } + } + + env { + name = "GCS_BUCKET" + value = google_storage_bucket.data_bucket.name + } + env { + name = "GCS_INPUT_FOLDER" + value = var.gcs_data_bucket_input_folder + } + env { + name = "GCS_OUTPUT_FOLDER" + value = var.gcs_data_bucket_output_folder + } + env { + name = "INPUT_DIR" + value = "gs://${google_storage_bucket.data_bucket.name}/${var.gcs_data_bucket_input_folder}" + } + } + vpc_access { + connector = google_vpc_access_connector.connector.id + egress = "PRIVATE_RANGES_ONLY" + } + max_retries = 0 + timeout = var.dc_data_job_timeout + service_account = google_service_account.datacommons_service_account.email + } + } + depends_on = [google_project_service.required_apis] +} + +# Run the db init job on terraform apply to create tables +resource "null_resource" "run_db_init" { + depends_on = [ + google_cloud_run_v2_job.dc_data_job + ] + + triggers = { + # Run once per deployment or when the job image changes + job_image = var.dc_data_job_image + } + + provisioner "local-exec" { + command = </dev/null; then + echo "Creating bucket gs://${BUCKET_NAME} in ${PROJECT_ID}..." + gcloud storage buckets create "gs://${BUCKET_NAME}" --project="${PROJECT_ID}" --location=us --uniform-bucket-level-access + + echo "Enabling versioning on gs://${BUCKET_NAME}..." + gcloud storage buckets update "gs://${BUCKET_NAME}" --versioning +else + echo "Bucket gs://${BUCKET_NAME} already exists." +fi + +# Generate the backend.tf file dynamically +echo "Generating backend.tf..." +cat < backend.tf +terraform { + backend "gcs" { + bucket = "${BUCKET_NAME}" + prefix = "terraform/state" + } +} +EOF + +echo "✅ backend.tf created successfully!" +echo "Now running terraform init..." +terraform init + +echo "" +echo "Setup complete! You can now run 'terraform apply' to deploy your infrastructure." diff --git a/infra/dcp/terraform.tfvars.example b/infra/dcp/terraform.tfvars.example new file mode 100644 index 0000000..529069f --- /dev/null +++ b/infra/dcp/terraform.tfvars.example @@ -0,0 +1,23 @@ +# --- Shared Global Variables --- +project_id = "your-project-id" +namespace = "your-namespace" +region = "us-central1" + +# --- Stack Toggles --- +enable_dcp = true +enable_cdc = false + +# --- DCP Stack Variables --- +# Only set to true if you need to create a dedicated Spanner Instance (Expensive!) +# Otherwise, provide an existing instance ID in dcp_spanner_instance_id +dcp_create_spanner_instance = false +dcp_spanner_instance_id = "your-existing-spanner-instance" + +# The Docker image tag to deploy. +# We recommend you run ./update_image_tag.sh to fetch the latest commit SHA. +# Using "latest" is discouraged as it prevents Terraform from detecting image updates. +dcp_image_tag = "latest" + +# --- CDC Stack Variables (Legacy) --- +# cdc_dc_api_key = "your-dc-api-key" +# cdc_maps_api_key = "your-maps-api-key" \ No newline at end of file diff --git a/infra/dcp/update_image_tag.sh b/infra/dcp/update_image_tag.sh new file mode 100755 index 0000000..2f4fc19 --- /dev/null +++ b/infra/dcp/update_image_tag.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# update_image_tag.sh +# Updates the dcp_image_tag in terraform.tfvars with the latest version from Artifact Registry. + +set -e + +# Configuration +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +IMAGE_BASE="us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-platform" +TFVARS_FILE="${SCRIPT_DIR}/terraform.tfvars" + +echo "🔍 Preparing to update image tag..." + +# 1. Extract Details +# Parse Project, Repo, and Package from IMAGE_BASE +# Assumes format: LOCATION-docker.pkg.dev/PROJECT/REPO/PACKAGE +REG_PROJECT=$(echo "$IMAGE_BASE" | cut -d'/' -f2) +REG_REPO=$(echo "$IMAGE_BASE" | cut -d'/' -f3) +REG_PACKAGE=$(echo "$IMAGE_BASE" | cut -d'/' -f4) +REG_LOCATION=$(echo "$IMAGE_BASE" | cut -d'-' -f1) + +if [ -z "$REG_PROJECT" ] || [ -z "$REG_REPO" ] || [ -z "$REG_PACKAGE" ]; then + echo "❌ Error: Could not parse registry details from $IMAGE_BASE" + exit 1 +fi + +# 2. Determine the tag +if [ "$1" ]; then + TAG="$1" + echo "📍 Using manually specified tag: $TAG" +else + echo "🌐 Fetching latest tag from Artifact Registry ($REG_PROJECT)..." + + # Logic: + # 1. Find which image version is currently tagged as 'latest' + # 2. Find all tags associated with that version (e.g. the commit SHA) + # 3. Pick the one that ISN'T 'latest' + + VERSION_ID=$(gcloud artifacts tags list \ + --project="$REG_PROJECT" \ + --location="$REG_LOCATION" \ + --repository="$REG_REPO" \ + --package="$REG_PACKAGE" \ + --filter="name:latest" \ + --format="value(version)" \ + --limit=1 2>/dev/null || true) + + if [ -z "$VERSION_ID" ]; then + echo "❌ Error: Could not find version with tag 'latest' in $IMAGE_BASE" + exit 1 + fi + + echo "✅ Newest image version (tagged as 'latest'): $VERSION_ID" + + # Get all tags for this specific version ID + ALL_TAGS=$(gcloud artifacts tags list \ + --project="$REG_PROJECT" \ + --location="$REG_LOCATION" \ + --repository="$REG_REPO" \ + --package="$REG_PACKAGE" \ + --filter="version:$VERSION_ID" \ + --format="value(TAG)") + + if [ -z "$ALL_TAGS" ]; then + echo "⚠️ No specific tags found for this version. Falling back to 'latest'." + TAG="latest" + else + # Pick the first tag that is NOT exactly 'latest' + TAG=$(echo "$ALL_TAGS" | tr ' ' '\n' | grep -v "^latest$" | head -n 1) + + # Fallback to latest if no other tag exists + if [ -z "$TAG" ]; then + TAG="latest" + fi + echo "✅ Detected alternative tag: $TAG" + fi +fi + +# 3. Update terraform.tfvars +echo "📝 Updating $TFVARS_FILE..." + +# Check if dcp_image_tag already exists +if grep -q "^dcp_image_tag" "$TFVARS_FILE"; then + # Use sed to replace the line. (Special syntax for macOS) + sed -i '' "s/^dcp_image_tag.*/dcp_image_tag = \"$TAG\"/" "$TFVARS_FILE" +else + # Append to end of file if it doesn't exist + echo "" >> "$TFVARS_FILE" + echo "dcp_image_tag = \"$TAG\"" >> "$TFVARS_FILE" +fi + +echo "✨ Success! image tag is now set to \"$TAG\" in $TFVARS_FILE." +echo "You can now run 'terraform plan' to see the changes." diff --git a/infra/dcp/variables.tf b/infra/dcp/variables.tf new file mode 100644 index 0000000..47cdf71 --- /dev/null +++ b/infra/dcp/variables.tf @@ -0,0 +1,349 @@ +# --- Shared Global Variables --- +variable "project_id" { + description = "GCP Project ID" + type = string +} + +variable "region" { + description = "GCP Region" + type = string + default = "us-central1" +} + +variable "deletion_protection" { + description = "Enable deletion protection for resources (set to true for production)" + type = bool + default = false +} + +# --- Stack Toggles --- +variable "enable_dcp" { + description = "Enable the new Data Commons Platform stack" + type = bool + default = false +} + +variable "enable_cdc" { + description = "Enable the legacy Custom Data Commons stack" + type = bool + default = true +} + +# --- DCP Stack Variables --- +variable "dcp_image_base" { + description = "Docker image base URL for DCP (without tag)" + type = string + default = "us-docker.pkg.dev/datcom-ci/gcr.io/datacommons-platform" +} + +variable "dcp_image_tag" { + description = "Docker image tag for DCP" + type = string + default = "latest" +} + +variable "dcp_service_name" { + description = "Cloud Run service name for DCP" + type = string + default = "dcp-svc" +} + +variable "dcp_service_account_name" { + description = "Service account for DCP" + type = string + default = "dcp-sa" +} + +variable "dcp_create_spanner_instance" { + description = "Create a new Spanner instance for DCP" + type = bool + default = false +} + +variable "dcp_create_spanner_db" { + description = "Create a new Spanner database for DCP" + type = bool + default = true +} + +variable "dcp_spanner_instance_id" { + description = "Spanner instance for DCP" + type = string + default = "" +} + +variable "dcp_spanner_database_id" { + description = "Spanner database for DCP" + type = string + default = "dcp-db" +} + +variable "dcp_spanner_processing_units" { + description = "Spanner units for DCP" + type = number + default = 100 +} + +variable "dcp_service_cpu" { + description = "CPU limit for the DCP service container" + type = string + default = "1000m" +} + +variable "dcp_service_memory" { + description = "Memory limit for the DCP service container" + type = string + default = "512Mi" +} + +variable "dcp_service_min_instances" { + description = "Minimum number of instances for the DCP service" + type = number + default = 0 +} + +variable "dcp_service_max_instances" { + description = "Maximum number of instances for the DCP service" + type = number + default = 10 +} + +variable "dcp_service_concurrency" { + description = "Maximum concurrent requests per instance for the DCP service" + type = number + default = 80 +} + +variable "dcp_service_timeout_seconds" { + description = "Request timeout in seconds for the DCP service" + type = number + default = 300 +} + +variable "namespace" { + description = "Global prefix for all resources" + type = string + default = "" +} + +variable "cdc_dc_api_key" { + description = "DC API Key for CDC" + type = string + default = "" +} + +variable "cdc_maps_api_key" { + description = "Maps API Key for CDC" + type = string + default = null +} + +variable "cdc_disable_google_maps" { + description = "Disable maps in CDC" + type = bool + default = false +} + +variable "cdc_google_analytics_tag_id" { + description = "GA tag for CDC" + type = string + default = null +} + +variable "cdc_gcs_data_bucket_name" { + description = "CDC data bucket" + type = string + default = "" +} + +variable "cdc_gcs_data_bucket_input_folder" { + description = "CDC input folder" + type = string + default = "input" +} + +variable "cdc_gcs_data_bucket_output_folder" { + description = "CDC output folder" + type = string + default = "output" +} + +variable "cdc_gcs_data_bucket_location" { + description = "CDC bucket location" + type = string + default = "US" +} + +variable "cdc_mysql_instance_name" { + description = "CDC MySQL name" + type = string + default = "sql-inst" +} + +variable "cdc_mysql_database_name" { + description = "CDC MySQL DB name" + type = string + default = "datacommons" +} + +variable "cdc_mysql_database_version" { + description = "CDC MySQL version" + type = string + default = "MYSQL_8_0" +} + +variable "cdc_mysql_cpu_count" { + description = "CDC MySQL CPU" + type = number + default = 2 +} + +variable "cdc_mysql_memory_size_mb" { + description = "CDC MySQL RAM" + type = number + default = 7680 +} + +variable "cdc_mysql_storage_size_gb" { + description = "CDC MySQL Disk" + type = number + default = 20 +} + +variable "cdc_mysql_user" { + description = "CDC MySQL user" + type = string + default = "datacommons" +} + +variable "cdc_vpc_connector_cidr" { + description = "CIDR range for the CDC VPC Access Connector" + type = string + default = "10.13.0.0/28" +} + +variable "cdc_web_service_image" { + description = "CDC web image" + type = string + default = "gcr.io/datcom-ci/datacommons-services:stable" +} + +variable "cdc_web_service_min_instance_count" { + description = "CDC min instances" + type = number + default = 1 +} + +variable "cdc_web_service_max_instance_count" { + description = "CDC max instances" + type = number + default = 1 +} + +variable "cdc_web_service_cpu" { + description = "CDC web CPU" + type = string + default = "4" +} + +variable "cdc_web_service_memory" { + description = "CDC web RAM" + type = string + default = "16G" +} + +variable "cdc_make_dc_web_service_public" { + description = "CDC public access" + type = bool + default = true +} + +variable "cdc_data_job_image" { + description = "CDC data job image" + type = string + default = "gcr.io/datcom-ci/datacommons-data:stable" +} + +variable "cdc_data_job_cpu" { + description = "CDC data job CPU" + type = string + default = "2" +} + +variable "cdc_data_job_memory" { + description = "CDC data job RAM" + type = string + default = "8G" +} + +variable "cdc_data_job_timeout" { + description = "CDC data job timeout" + type = string + default = "600s" +} + +variable "cdc_search_scope" { + description = "CDC search scope" + type = string + default = "base_and_custom" +} + +variable "cdc_enable_mcp" { + description = "CDC enable MCP" + type = bool + default = true +} + +variable "cdc_vpc_network_name" { + description = "CDC VPC network" + type = string + default = "default" +} + +variable "cdc_vpc_network_subnet_name" { + description = "CDC VPC subnet" + type = string + default = "default" +} + +variable "cdc_enable_redis" { + description = "CDC enable redis" + type = bool + default = false +} + +variable "cdc_redis_instance_name" { + description = "CDC redis name" + type = string + default = "datacommons-redis-instance" +} + +variable "cdc_redis_memory_size_gb" { + description = "CDC redis size" + type = number + default = 2 +} + +variable "cdc_redis_tier" { + description = "CDC redis tier" + type = string + default = "STANDARD_HA" +} + +variable "cdc_redis_location_id" { + description = "CDC redis zone" + type = string + default = "us-central1-a" +} + +variable "cdc_redis_alternative_location_id" { + description = "CDC redis alt zone" + type = string + default = "us-central1-b" +} + +variable "cdc_redis_replica_count" { + description = "CDC redis replicas" + type = number + default = 1 +}