Feat/templatetf

.github/workflows/stackhpc.yml

@@ -14,7 +14,7 @@ jobs:
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
-      TF_VAR_cluster_name: ci${{ github.run_id }}
+      CI_CLUSTER_NAME: ci${{ github.run_id }}
       CI_CLOUD: ${{ vars.CI_CLOUD }}
     steps:
       - uses: actions/checkout@v2
@@ -43,10 +43,6 @@ jobs:
         with:
           terraform: v1.5.5
 
-      - name: Initialise terraform
-        run: terraform init
-        working-directory: ${{ github.workspace }}/environments/.stackhpc/terraform
-
       - name: Write clouds.yaml
         run: |
           mkdir -p ~/.config/openstack/
@@ -67,8 +63,7 @@ jobs:
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
-          cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
-          terraform apply -auto-approve -var-file="${{ vars.CI_CLOUD }}.tfvars"
+          ansible-playbook -v ansible/infra.yml -e terraform_autoapprove=true
 
       - name: Delete infrastructure if provisioning failed
         run: |

ansible/.gitignore

@@ -55,3 +55,5 @@ roles/*
 !roles/persist_hostkeys/
 !roles/persist_hostkeys/**
 !roles/requirements.yml
+!roles/terraform/
+!roles/terraform/**

ansible/infra.yml

@@ -0,0 +1,6 @@
+- hosts: localhost
+  become: no
+  gather_facts: no
+  tasks:
+    - import_role:
+        name: terraform

ansible/roles/terraform/defaults/main.yml

@@ -0,0 +1,7 @@
+terraform_templates: [main.tf.j2]
+terraform_project_path:
+terraform_autoapprove: false
+terraform_binary_path:
+terraform_backend_config: {}
+terraform_variables: {}
+terraform_state: present

ansible/roles/terraform/filter_plugins/terraform.py

@@ -0,0 +1,17 @@
+import re
+
+def expand_hostlist(hostlist):
+    match =  re.search(r'(\w+)-\[(\d+)-(\d+)\]', hostlist)
+    if match:
+        prefix = match.groups()[0]
+        start, end = [int(v) for v in match.groups()[1:]]
+        hosts = [f'{prefix}-{n}' for n in range(start, end+1)]
+        return hosts
+    else:
+        return [hostlist,]
+
+class FilterModule(object):
+    def filters(self):
+        return {
+            'expand_hostlist': expand_hostlist,
+        }

ansible/roles/terraform/tasks/apply.yml

@@ -0,0 +1,44 @@
+- name: Create Terraform plan
+  community.general.terraform:
+    binary_path: "{{ terraform_binary_path or omit }}"
+    project_path: "{{ terraform_project_path }}"
+    state: planned
+    backend_config: "{{ terraform_backend_config }}"
+    plan_file: terraform.plan
+    force_init: yes
+    init_reconfigure: yes
+    variables: "{{ terraform_variables }}"
+  register: _tf_plan
+
+- name: Show Terraform plan
+  debug:
+    msg: "{{ _tf_plan.stdout }}"
+
+- name: Prompt to approve Terraform plan execution
+  pause:
+    prompt: "Do you want to execute this plan? (Only 'yes' executes)"
+  register: _tf_approve_plan
+  when:
+    - "'No changes. Your infrastructure matches the configuration.' not in _tf_plan.stdout"
+    - 'not terraform_autoapprove | bool'
+
+- name: End host if Terraform plan is not approved
+  ansible.builtin.meta: end_host
+  when: "not (( terraform_autoapprove | bool ) or ( _tf_approve_plan.user_input | default(false) | bool ))"
+
+- name: Provision infrastructure using Terraform
+  community.general.terraform:
+    binary_path: "{{ terraform_binary_path or omit }}"
+    project_path: "{{ terraform_project_path }}"
+    state: "{{ terraform_state }}"
+    backend_config: "{{ terraform_backend_config }}"
+    force_init: yes
+    init_reconfigure: yes
+    variables: "{{ terraform_variables }}"
+    plan_file: terraform.plan
+  register: terraform_provision
+
+- name: Show Terraform provision output
+  debug:
+    msg: "{{ terraform_provision.stdout }}"
+  when: "'stdout' in terraform_provision"

ansible/roles/terraform/tasks/main.yml

@@ -0,0 +1,7 @@
+- name: Template Terraform configurations
+  template:
+    src: "{{ item }}"
+    dest: "{{ terraform_project_path }}/{{ (item | splitext | first) if item.endswith('.j2') else item }}"
+  loop: "{{ terraform_templates }}"
+
+- include_tasks: apply.yml

ansible/roles/terraform/templates/main.tf.j2

@@ -0,0 +1,148 @@
+#jinja2:lstrip_blocks: True
+terraform {
+  required_version = ">= 0.14"
+  required_providers {
+    openstack = {
+      source = "terraform-provider-openstack/openstack"
+    }
+  }
+}
+
+# --- volumes ---
+{% for volume_name, volume in cluster_volumes.items() %}
+resource "openstack_blockstorage_volume_v3" "{{ volume_name }}" {
+    name = "{{ cluster_name }}-{{ volume_name }}"
+    description = "{{ volume.description }}"
+    size = "{{ volume.size }}"
+}
+{% endfor %}
+
+{% for instance_hostlist, _instance in cluster_instances.items() %}
+{%  set hostgroup = instance_hostlist.split('-')[0] %}{# NB: assumes prefix- format #}
+### --- hostgroup {{ hostgroup }} ---
+{%  set instance = cluster_instance_defaults | combine(_instance) %}
+
+{# NB: Currently secgroups apply to all ports on each instance #}
+data "openstack_networking_secgroup_v2" "{{ hostgroup }}" {
+  for_each = toset({{ instance.secgroup_names | to_json }})
+
+  name = each.key
+}
+
+{%    for port in instance.ports %}
+{%      set port_tf_name = (hostgroup, port.network_name) | join('_') | replace('-', '_') %}
+
+data "openstack_networking_network_v2" "{{ port_tf_name }}" {
+  name = "{{ port.network_name }}"
+}
+
+{% if 'subnet_name' in port %}
+data "openstack_networking_subnet_v2" "{{ port_tf_name }}" {
+  name = "{{ port.subnet_name }}"
+}
+{% endif %}
+
+resource "openstack_networking_port_v2" "{{ port_tf_name }}" {
+    for_each = toset({{ instance_hostlist | expand_hostlist | to_json }})
+
+    name = "{{ cluster_name }}-${each.key}-{{ port.network_name }}"
+    network_id = data.openstack_networking_network_v2.{{ port_tf_name }}.id
+
+    {% if 'subnet_name' in port %}
+    fixed_ip {
+      subnet_id = data.openstack_networking_subnet_v2.{{ port_tf_name }}.id
+    }
+    {% endif %}
+
+    security_group_ids = [for sg in data.openstack_networking_secgroup_v2.{{ hostgroup }}: sg.id]
+
+    binding {
+      vnic_type = "{{ port.vnic_type | default('normal') }}"
+      profile = {{ port.binding_profile | to_json if 'binding_profile' in port else 'null' }}
+    }
+}
+{%    endfor %}{# instance.ports #}
+
+data "openstack_images_image_v2"  "{{ hostgroup }}" {
+  name = "{{ instance.image_name }}"
+}
+
+resource "openstack_compute_instance_v2" "{{ hostgroup }}" {
+    for_each = toset({{ instance_hostlist | expand_hostlist | to_json }})
+
+    name = "{{ cluster_name }}-${each.key}"
+    image_name = data.openstack_images_image_v2.{{ hostgroup }}.name
+    {% if instance and 'flavor_name' in instance %}
+    flavor_name = "{{ instance.flavor_name }}"
+    {% else %}
+    flavor_id = "{{ instance.flavor_id }}"
+    {% endif %}
+    key_pair = "{{ instance.key_pair }}"
+    {% if instance.volumes | default([]) | length or instance.root_volume_size | default(None) %}
+    # root disk:
+    block_device {
+        uuid = data.openstack_images_image_v2.{{ hostgroup }}.id
+        source_type = "image"
+        destination_type = "{{ 'volume' if instance.root_volume_size | default(None) else 'local' }}"
+        volume_size = {{ instance.root_volume_size if instance.root_volume_size | default(None) else 'null' }}
+        boot_index = 0
+        delete_on_termination = true
+     }
+    {%  for volume_name in instance.volumes | default([]) %}
+    block_device {
+      destination_type = "volume"
+      source_type = "volume"
+      boot_index = -1
+      uuid = openstack_blockstorage_volume_v3.{{ volume_name }}.id
+    }
+    {%  endfor %}
+    {% endif %}
+    {% for port in instance.ports %}
+    {%   set port_tf_name = (hostgroup, port.network_name) | join('_') | replace('-', '_') %}
+    network {
+      port = openstack_networking_port_v2.{{ port_tf_name }}[each.key].id
+    }
+    {% endfor %}
+
+    metadata = {
+      environment_root = "{{ appliances_environment_root }}"
+    }
+
+    user_data = <<-EOF
+    #cloud-config
+    fs_setup:
+    {% for volume_name, volume in instance.get('volumes', {}).items() %}
+    - label: {{ volume_name }}
+      filesystem: ext4
+      device: {{ volume.device_path }}
+      partition: auto
+    {% endfor %}
+    mounts:
+    {% for volume_name, volume in instance.get('volumes', {}).items() %}
+    - [LABEL={{ volume_name }}, {{ volume.mount_point }}, auto, "{{ volume.mount_options | default('') }}" ]
+    {% endfor %}
+    EOF
+
+}
+
+{% endfor %}{# cluster_instances.items() #}
+
+resource "local_file" "hosts" {
+  content = <<-EOF
+    {% for instance_hostlist, _instance in cluster_instances.items() %}
+    {% set hostgroup = instance_hostlist.split('-')[0] %}
+    [{{ hostgroup }}]
+    {%  for hostkey in instance_hostlist | expand_hostlist %}
+    {% set inventory_hostname = cluster_name + '-' + hostkey %}
+    {{ inventory_hostname }} ansible_host=${openstack_compute_instance_v2.{{ hostgroup }}["{{ hostkey }}"].network[0].fixed_ip_v4}
+    {%  endfor %}
+
+    {% for extra_group in _instance.extra_groups | default([]) %}
+    [{{ extra_group }}:children]
+    {{ hostgroup }}
+    {%  endfor %}
+
+    {% endfor %}
+  EOF
+  filename = "../inventory/hosts"
+}

environments/.stackhpc/inventory/group_vars/all/cluster.yml

@@ -0,0 +1,36 @@
+ci_cluster_name: "{{ lookup('env', 'CI_CLUSTER_NAME') }}"
+dev_cluster_name: "{{ lookup('env', 'APPLIANCES_REPO_ROOT') | basename | split('-') | last }}" # dev directories use slurm-app-$FEATURE for directories
+cluster_name: "{{ ci_cluster_name | default(dev_cluster_name, true) }}" # true means use default if value is empty string
+
+cluster_volumes: # can reduce the size a lot for dev/CI
+  state:
+    description: Persistent state
+    size: 10 # GB
+  home:
+    description: User home directories
+    size: 10 # GB
+
+# TODO: support SMS-labs here too
+cluster_instance_defaults:
+  image_name: openhpc-230926-1343-e3d3e307 # https://github.com/stackhpc/ansible-slurm-appliance/pull/314
+  flavor_name: vm.ska.cpu.general.small
+  key_pair: slurm-app-ci
+  ports:
+    - network_name: portal-internal # required str (tf standard: network_id)
+  secgroup_names:
+    - default
+
+cluster_instances:
+  login:
+    secgroup_names: [default, SSH]
+  compute-[0-3]:
+  control:
+    flavor_name: vm.ska.cpu.general.quarter
+    volumes:
+      state:
+        device_path: /dev/sdb
+        mount_point: /var/lib/state
+      home:
+        device_path: /dev/sdc
+        mount_point: /exports/home
+        mount_options: x-systemd.required-by=nfs-server.service,x-systemd.before=nfs-server.service