Merge branch 'feat/tf-multiple-networks' into feat/k3s-node-ip

Author: sjpb

Diff for: README.md

@@ -82,33 +82,35 @@ And generate secrets for it:
 
 Create an OpenTofu variables file to define the required infrastructure, e.g.:
 
-    # environments/$ENV/terraform/terraform.tfvars:
+    # environments/$ENV/tofu/tofu.tfvars:
 
     cluster_name = "mycluster"
     cluster_net = "some_network" # *
     cluster_subnet = "some_subnet" # *
     key_pair = "my_key" # *
     control_node_flavor = "some_flavor_name"
     login = {
+        # Arbitrary group name for these login nodes
         interactive = {
             nodes: ["login-0"]
-            flavor: "login_flavor_name"
+            flavor: "login_flavor_name" # *
         }
     }
     cluster_image_id = "rocky_linux_9_image_uuid"
     compute = {
+        # Group name used for compute node partition definition
         general = {
             nodes: ["compute-0", "compute-1"]
-            flavor: "compute_flavor_name"
+            flavor: "compute_flavor_name" # *
         }
     }
 
-Variables marked `*` refer to OpenStack resources which must already exist. The above is a minimal configuration - for all variables and descriptions see `environments/$ENV/terraform/terraform.tfvars`.
+Variables marked `*` refer to OpenStack resources which must already exist. The above is a minimal configuration - for all variables and descriptions see `environments/$ENV/tofu/tofu.tfvars`.
 
 To deploy this infrastructure, ensure the venv and the environment are [activated](#create-a-new-environment) and run:
 
     export OS_CLOUD=openstack
-    cd environments/$ENV/terraform/
+    cd environments/$ENV/tofu/
     tofu init
     tofu apply
 

Diff for: ansible/ci/retrieve_inventory.yml

@@ -7,7 +7,7 @@
   gather_facts: no
   vars:
     cluster_prefix: "{{ undef(hint='cluster_prefix must be defined') }}" # e.g. ci4005969475
-    ci_vars_file: "{{ appliances_environment_root + '/terraform/' + lookup('env', 'CI_CLOUD') }}.tfvars"
+    ci_vars_file: "{{ appliances_environment_root + '/tofu/' + lookup('env', 'CI_CLOUD') }}.tfvars"
     cluster_network: "{{ lookup('ansible.builtin.ini', 'cluster_net', file=ci_vars_file, type='properties') | trim('\"') }}"
   tasks:
     - name: Get control host IP

Diff for: ansible/roles/block_devices/README.md

@@ -11,7 +11,7 @@ This is a convenience wrapper around the ansible modules:
 
 To avoid issues with device names changing after e.g. reboots, devices are identified by serial number and mounted by filesystem UUID.
 
-**NB:** This role is ignored[^1] during Packer builds as block devices will not be attached to the Packer build VMs. This role is therefore deprecated and it is suggested that `cloud-init` is used instead. See e.g. `environments/skeleton/{{cookiecutter.environment}}/terraform/control.userdata.tpl`.
+**NB:** This role is ignored[^1] during Packer builds as block devices will not be attached to the Packer build VMs. This role is therefore deprecated and it is suggested that `cloud-init` is used instead. See e.g. `environments/skeleton/{{cookiecutter.environment}}/tofu/control.userdata.tpl`.
 
 [^1]: See `environments/common/inventory/group_vars/builder/defaults.yml`
 

Diff for: ansible/roles/compute_init/README.md

@@ -43,7 +43,7 @@ The following roles/groups are currently fully functional:
 - `openhpc`: all functionality
 
 The above may be enabled by setting the compute_init_enable property on the
-terraform compute variable.
+tofu compute variable.
 
 # Development/debugging
 

Diff for: ansible/roles/freeipa/README.md

@@ -7,7 +7,7 @@ Support FreeIPA in the appliance. In production use it is expected the FreeIPA s
 
 ## Usage
 - Add hosts to the `freeipa_client` group and run (at a minimum) the `ansible/iam.yml` playbook.
-- Host names must match the domain name. By default (using the skeleton Terraform) hostnames are of the form `nodename.cluster_name.cluster_domain_suffix` where `cluster_name` and `cluster_domain_suffix` are Terraform variables.
+- Host names must match the domain name. By default (using the skeleton OpenTofu) hostnames are of the form `nodename.cluster_name.cluster_domain_suffix` where `cluster_name` and `cluster_domain_suffix` are OpenTofu variables.
 - Hosts discover the FreeIPA server FQDN (and their own domain) from DNS records. If DNS servers are not set this is not set from DHCP, then use the `resolv_conf` role to configure this. For example when using the in-appliance FreeIPA development server:
 
   ```ini
@@ -28,7 +28,7 @@ Support FreeIPA in the appliance. In production use it is expected the FreeIPA s
 - For production use with an external FreeIPA server, a random one-time password (OTP) must be generated when adding hosts to FreeIPA (e.g. using `ipa host-add --random ...`). This password should be set as a hostvar `freeipa_host_password`. Initial host enrolment will use this OTP to enrol the host. After this it becomes irrelevant so it does not need to be committed to git. This approach means the appliance does not require the FreeIPA administrator password.
 - For development use with the in-appliance FreeIPA server, `freeipa_host_password` will be automatically generated in memory.
 - The `control` host must define `appliances_state_dir` (on persistent storage). This is used to back-up keytabs to allow FreeIPA clients to automatically re-enrol after e.g. reimaging. Note that:
-  - This is implemented when using the skeleton Terraform; on the control node `appliances_state_dir` defaults to `/var/lib/state` which is mounted from a volume.
+  - This is implemented when using the skeleton OpenTofu; on the control node `appliances_state_dir` defaults to `/var/lib/state` which is mounted from a volume.
   - Nodes are not re-enroled by a [Slurm-driven reimage](../../collections/ansible_collections/stackhpc/slurm_openstack_tools/roles/rebuild/README.md) (as that does not run this role).
   - If both a backed-up keytab and `freeipa_host_password` exist, the former is used.
 

Diff for: docs/networks.md

@@ -0,0 +1,102 @@
+# Networking
+
+The default OpenTofu configurations in the appliance do not provision networks,
+subnets or associated infrastructure such as routers. The requirements are that:
+1. At least one network exists.
+2. The first network defined spans all nodes, referred to as the "access network".
+3. Only one subnet per network is attached to nodes.
+4. At least one network on each node provides outbound internet access (either
+directly, or via a proxy).
+
+Futhermore, it is recommended that the deploy host has an interface on the
+access network. While it is possible to e.g. use a floating IP on a login node
+as an SSH proxy to access the other nodes, this can create problems in recovering
+the cluster if the login node is unavailable and can make Ansible problems harder
+to debug.
+
+This page describes supported configurations and how to implement them using
+the OpenTofu variables. These will normally be set in
+`environments/site/tofu/terraform.tfvars` for the site base environment. If they
+need to be overriden for specific environments, this can be done via an OpenTofu
+module as discussed [here](./production.md).
+
+Note that if an OpenStack subnet has a gateway IP defined then nodes with ports
+attached to that subnet will get a default route set via that gateway.
+
+## Single network
+This is the simplest possible configuration. A single network and subnet is
+used for all nodes. The subnet provides outbound internet access via the default
+route defined by the subnet gateway (often an OpenStack router to an external
+network).
+
+```terraform
+cluster_networks = [
+  {
+    network = "netA"
+    subnet = "subnetA"
+  }
+]
+...
+```
+
+## Multiple homogenous networks
+This is similar to the above, except each node has multiple networks. The first
+network, "netA" is the access network. Note that only one subnet must have a
+gateway defined, else default routes via both subnets will be present causing
+routing problems. It also shows the second network (netB) using direct-type
+vNICs for RDMA.
+
+```terraform
+cluster_networks = [
+  {
+    network = "netA"
+    subnet = "subnetA"
+  },
+  {
+    network = "netB"
+    subnet = "subnetB"
+  },
+]
+
+vnic_types = {
+    netB = "direct"
+}
+...
+```
+
+
+## Additional networks on some nodes
+
+This example shows how to modify variables for specific node groups. In this
+case a baremetal node group has a second network attached. As above, only a
+single subnet can have a gateway IP.
+
+```terraform
+cluster_networks = [
+  {
+    network = "netA"
+    subnet = "subnetA"
+  }
+]
+
+compute = {
+  general = {
+    nodes = ["general-0", "general-1"]
+  }
+  baremetal = {
+    nodes = ["baremetal-0", "baremetal-1"]
+    extra_networks = [
+      {
+        network = "netB"
+        subnet = "subnetB"
+      }
+    ]
+    vnic_types = {
+        netA = "baremetal"
+        netB = "baremetal"
+    ...
+    }
+  }
+}
+...
+```

Diff for: docs/openondemand.md

@@ -33,7 +33,7 @@ See the [ansible/roles/openondemand/README.md](../ansible/roles/openondemand/REA
 The following variables have been given default values to allow Open OnDemand to work in a newly created environment without additional configuration, but generally should be overridden in `environment/site/inventory/group_vars/all/` with site-specific values:
 - `openondemand_servername` - this must be defined for both `openondemand` and `grafana` hosts (when Grafana is enabled). Default is `ansible_host` (i.e. the IP address) of the first host in the `openondemand` group.
 - `openondemand_auth` and any corresponding options. Defaults to `basic_pam`.
-- `openondemand_desktop_partition` and `openondemand_jupyter_partition` if the corresponding inventory groups are defined. Defaults to the first compute group defined in the `compute` Terraform variable in `environments/$ENV/terraform`.
+- `openondemand_desktop_partition` and `openondemand_jupyter_partition` if the corresponding inventory groups are defined. Defaults to the first compute group defined in the `compute` OpenTofu variable in `environments/$ENV/tofu`.
 
 It is also recommended to set:
 - `openondemand_dashboard_support_url`

Diff for: docs/operations.md

@@ -57,10 +57,10 @@ This is a usually a two-step process:
 
 - If new nodes are required, define a new node group by adding an entry to the `compute` mapping in `environments/$ENV/tofu/main.tf` assuming the default OpenTofu configuration:
     - The key is the partition name.
-    - The value should be a mapping, with the parameters defined in `environments/$SITE_ENV/terraform/compute/variables.tf`, but in brief will need at least `flavor` (name) and `nodes` (a list of node name suffixes).
+    - The value should be a mapping, with the parameters defined in `environments/$SITE_ENV/tofu/compute/variables.tf`, but in brief will need at least `flavor` (name) and `nodes` (a list of node name suffixes).
 - Add a new partition to the partition configuration as described under [Modifying Slurm Partition-specific Configuration](#Modifying-Slurm-Partition-specific-Configuration).
 
-Deploying the additional nodes and applying these changes requires rerunning both Terraform and the Ansible site.yml playbook - follow [Deploying a Cluster](#Deploying-a-Cluster).
+Deploying the additional nodes and applying these changes requires rerunning both OpenTofu and the Ansible site.yml playbook - follow [Deploying a Cluster](#Deploying-a-Cluster).
 
 # Adding Additional Packages
 By default, the following utility packages are installed during the StackHPC image build:

Diff for: docs/persistent-state.md

@@ -13,14 +13,14 @@ If using the `environments/common/layout/everything` Ansible groups template (wh
 
 Note that if `appliances_state_dir` is defined, the path it gives must exist and should be owned by root. Directories will be created within this with appropriate permissions for each item of state defined above. Additionally, the systemd units for the services listed above will be modified to require `appliances_state_dir` to be mounted before service start (via the `systemd` role).
 
-A new cookiecutter-produced environment supports persistent state in the default Terraform (see `environments/skeleton/{{cookiecutter.environment}}/terraform/`) by:
+A new cookiecutter-produced environment supports persistent state in the default OpenTofu (see `environments/skeleton/{{cookiecutter.environment}}/tofu/`) by:
 
-- Defining a volume with a default size of 150GB - this can be controlled by the Terraform variable `state_volume_size`.
+- Defining a volume with a default size of 150GB - this can be controlled by the OpenTofu variable `state_volume_size`.
 - Attaching it to the control node.
 - Defining cloud-init userdata for the control node which formats and mounts this volume at `/var/lib/state`.
-- Defining `appliances_state_dir: /var/lib/state` for the control node in the (Terraform-templated) `inventory/hosts` file.
+- Defining `appliances_state_dir: /var/lib/state` for the control node in the (OpenTofu-templated) `inventory/hosts` file.
 
-**NB: The default Terraform is provided as a working example and for internal CI use - therefore this volume is deleted when running `terraform destroy` - this may not be appropriate for a production environment.**
+**NB: The default OpenTofu is provided as a working example and for internal CI use - therefore this volume is deleted when running `tofu destroy` - this may not be appropriate for a production environment.**
 
 In general, the Prometheus data is likely to be the only sizeable state stored. The size of this can be influenced through [Prometheus role variables](https://github.com/cloudalchemy/ansible-prometheus#role-variables), e.g.:
 - `prometheus_storage_retention` - [default](../environments/common/inventory/group_vars/all/prometheus.yml) 31d

Diff for: docs/production.md

@@ -41,15 +41,15 @@ and referenced from the `site` and `production` environments, e.g.:
 - OpenTofu configurations should be defined in the `site` environment and used
   as a module from the other environments. This can be done with the
   cookie-cutter generated configurations:
-  - Delete the *contents* of the cookie-cutter generated `terraform/` directories
+  - Delete the *contents* of the cookie-cutter generated `tofu/` directories
     from the `production` and `staging` environments.
-  - Create a `main.tf` in those directories which uses `site/terraform/` as a
+  - Create a `main.tf` in those directories which uses `site/tofu/` as a
     [module](https://opentofu.org/docs/language/modules/), e.g. :
 
     ```
     ...
     module "cluster" {
-        source = "../../site/terraform/"
+        source = "../../site/tofu/"
 
         cluster_name = "foo"
         ...
@@ -61,7 +61,7 @@ and referenced from the `site` and `production` environments, e.g.:
           into the module block.
         - Environment-independent variables (e.g. maybe `cluster_net` if the
           same is used for staging and production) should be set as *defaults*
-          in `environments/site/terraform/variables.tf`, and then don't need to
+          in `environments/site/tofu/variables.tf`, and then don't need to
           be passed in to the module.
 
 - Vault-encrypt secrets. Running the `generate-passwords.yml` playbook creates
@@ -102,7 +102,7 @@ and referenced from the `site` and `production` environments, e.g.:
 
 - Consider whether having (read-only) access to Grafana without login is OK. If not, remove `grafana_auth_anonymous` in `environments/$ENV/inventory/group_vars/all/grafana.yml`
 
-- Modify `environments/site/terraform/nodes.tf` to provide fixed IPs for at least
+- Modify `environments/site/tofu/nodes.tf` to provide fixed IPs for at least
   the control node, and (if not using FIPs) the login node(s):
 
     ```

Diff for: docs/upgrades.md

@@ -62,7 +62,7 @@ All other commands should be run on the Ansible deploy host.
 
 1. If required, build an "extra" image with local modifications, see [docs/image-build.md](./image-build.md).
 
-1. Modify your site-specific environment to use this image, e.g. via `cluster_image_id` in `environments/$SITE_ENV/terraform/variables.tf`.
+1. Modify your site-specific environment to use this image, e.g. via `cluster_image_id` in `environments/$SITE_ENV/tofu/variables.tf`.
 
 1. Test this in your staging cluster.
 

Diff for: environments/README.md

@@ -7,7 +7,7 @@ typically contains all the environment specific config. It must output an ansibl
 that conforms to the structure we expect. Providing that the inventory conforms to this
 structure, the ansible code will still be able to interface with that inventory.
 This allows the ansible code to be decoupled from the code that deployed the infrastructure
-and can therefore be tool and cloud agnostic i.e we don't care if you use terraform or ansible.
+and can therefore be tool and cloud agnostic.
 
 A pattern we use is to chain multiple ansible inventories to provide a crude form of inheritance. e.g
 

Diff for: environments/common/README.md

@@ -2,7 +2,7 @@
 
 This contains an inventory that defines variables which are common between the
 `production` and `development` environments. It is not intended to be used in
-a standalone fashion to deploy infrastructure (i.e no terraform), but is instead
+a standalone fashion to deploy infrastructure, but is instead
 referenced in `ansible.cfg` from the `production` and `development` configurations.
 
 The pattern we use is that all resources referenced in the inventory

Diff for: environments/skeleton/{{cookiecutter.environment}}/tofu/compute.tf

@@ -20,6 +20,9 @@ module "compute" {
   vnic_profiles = lookup(each.value, "vnic_profiles", var.vnic_profiles)
   volume_backed_instances = lookup(each.value, "volume_backed_instances", var.volume_backed_instances)
   root_volume_size = lookup(each.value, "root_volume_size", var.root_volume_size)
+  
+  # optionally set for group
+  networks = concat(var.cluster_networks, lookup(each.value, "extra_networks", []))
   extra_volumes = lookup(each.value, "extra_volumes", {})
   compute_init_enable = lookup(each.value, "compute_init_enable", [])
   ignore_image_changes = lookup(each.value, "ignore_image_changes", false)

Diff for: environments/skeleton/{{cookiecutter.environment}}/tofu/control.tf

@@ -1,7 +1,5 @@
 locals {
   control_volumes = concat([openstack_blockstorage_volume_v3.state], var.home_volume_size > 0 ? [openstack_blockstorage_volume_v3.home][0] : [])
-
-  access_network_name = length(var.cluster_networks) == 1 ? var.cluster_networks[0].network : [for n in var.cluster_networks: n if lookup(n, "access_network", false)][0].network
 }
 
 resource "openstack_networking_port_v2" "control" {
@@ -55,14 +53,14 @@ resource "openstack_compute_instance_v2" "control" {
     for_each = {for net in var.cluster_networks: net.network => net}
     content {
       port = openstack_networking_port_v2.control[network.key].id
-      access_network = network.key == local.access_network_name
+      access_network = network.key == var.cluster_networks[0].network
     }
   }
 
   metadata = {
     environment_root = var.environment_root
     k3s_token = local.k3s_token
-    access_ip = openstack_networking_port_v2.control[local.access_network_name].all_fixed_ips[0]
+    access_ip = openstack_networking_port_v2.control[var.cluster_networks[0].network].all_fixed_ips[0]
   }
 
   user_data = <<-EOF

Diff for: environments/skeleton/{{cookiecutter.environment}}/tofu/login.tf

@@ -11,14 +11,17 @@ module "login" {
   cluster_domain_suffix = var.cluster_domain_suffix
 
   # can be set for group, defaults to top-level value:
-  networks = lookup(each.value, "networks", var.cluster_networks)
   image_id = lookup(each.value, "image_id", var.cluster_image_id)
   vnic_types = lookup(each.value, "vnic_types", var.vnic_types)
   vnic_profiles = lookup(each.value, "vnic_profiles", var.vnic_profiles)
   volume_backed_instances = lookup(each.value, "volume_backed_instances", var.volume_backed_instances)
   root_volume_size = lookup(each.value, "root_volume_size", var.root_volume_size)
+  
+  # optionally set for group
+  networks = concat(var.cluster_networks, lookup(each.value, "extra_networks", []))
   extra_volumes = lookup(each.value, "extra_volumes", {})
 
+  # can't be set for login
   compute_init_enable = []
   ignore_image_changes = false