Add scale out functionality for SNO Cluster

README.md

@@ -211,7 +211,8 @@ xxxxx-xxxxx-xxxxx-xxxxx
 * [Deploy Single Node OpenShift (SNO) clusters via jetlag from a Performance Lab Bastion Machine](docs/deploy-sno-performancelab.md)
 * [Deploy Single Node OpenShift (SNO) clusters via jetlag on IBMcloud ](docs/deploy-sno-ibmcloud.md)
 * [Deploy a VMNO](docs/deploy-vmno.md)
-* [Scale up MNO](docs/scale-out-mno.md)
+* [Scale out MNO](docs/scale-out-mno.md)
+* [Scale out SNO](docs/scale-out-sno.md)
 
 ## Tips and Troubleshooting
 

ansible/mno-scale-out.yml → ansible/ocp-scale-out.yml

@@ -6,11 +6,11 @@
   - vars/all.yml
   - vars/scale_out.yml
   roles:
-  - mno-scale-out
+  - ocp-scale-out
   - role: boot-iso
     vars:
       inventory_group: 'worker'
       offset: "{{ current_worker_count }}"
       index: "{{ current_worker_count+scale_out_count }}"
-      virtual_media_iso: "mno-scale-out.x86_64.iso"
-  - mno-scale-out-csr
+      virtual_media_iso: "ocp-scale-out.x86_64.iso"
+  - ocp-scale-out-csr

ansible/roles/create-inventory/tasks/main.yml

@@ -233,8 +233,7 @@
     user: "{{ lab_cloud }}"
     password: "{{ bmc_password }}"
     validate_certs: false
-  with_items:
-    - "{{ ocpinventory_worker_nodes }}"
+  loop: "{{ ocpinventory_worker_nodes }}"
   when: cluster_type == "mno"
   register: mno_foreman_data
 
@@ -262,6 +261,20 @@
     set_fact:
       ocpinventory_sno_nodes: "{{ ocpinventory.json.nodes[1:] }}"
 
+  - name: SNO - Set max number of nodes
+    set_fact:
+      max_nodes: "{{ ocpinventory.json.nodes|length }}"
+    when: worker_node_count == None
+
+  - name: SNO - Set max number of nodes (worker_node_count set)
+    set_fact:
+      max_nodes: "{{ worker_node_count|int + 2 }}"
+    when: worker_node_count != None
+
+  - name: SNO - Set ocpinventory worker nodes (for scale-out scenarios)
+    set_fact:
+      ocpinventory_worker_nodes: "{{ ocpinventory.json.nodes[2:max_nodes|int] }}"
+
   - name: SNO - Get lab data for each sno
     uri:
       url: "https://{{ labs[lab]['foreman'] }}/api/hosts/{{ item.pm_addr | replace('mgmt-','') }}"
@@ -272,6 +285,17 @@
     register: sno_foreman_data
     loop: "{{ ocpinventory_sno_nodes }}"
 
+  - name: SNO - Get lab mac address for worker nodes
+    uri:
+      url: "https://{{ labs[lab]['foreman'] }}/api/hosts/{{ item.pm_addr | replace('mgmt-','') }}"
+      force_basic_auth: yes
+      user: "{{ lab_cloud }}"
+      password: "{{ bmc_password }}"
+      validate_certs: false
+    loop: "{{ ocpinventory_worker_nodes }}"
+    when: cluster_type == "sno" and ocpinventory_worker_nodes|length > 0
+    register: sno_worker_foreman_data
+
   - name: SNO - Add bond MAC addresses to SNO nodes (when bonding is enabled)
     when: enable_bond | default(false) and not (public_vlan | default(false))
     set_fact:

ansible/roles/create-inventory/templates/inventory-sno.j2

@@ -32,11 +32,59 @@ bmc_password={{ bmc_password }}
 [controlplane:vars]
 # Unused
 
+{% if ocpinventory_worker_nodes is defined and ocpinventory_worker_nodes|length > 0 %}
 [worker]
-# Unused
+{% for worker in ocpinventory_worker_nodes %}
+{%   set worker_long_hostname=worker.pm_addr | replace('mgmt-','') %}
+{%   set worker_short_hostname=(worker_long_hostname.split('.')[0]) %}
+{%   set lab_mac=(sno_foreman_data.results|selectattr('json.name', 'eq', worker_long_hostname)|first) | json_query(mac_query) | join(', ') %}
+{%   if sno_use_lab_dhcp %}
+{%     set ip=(sno_foreman_data.results|selectattr('json.name', 'eq', worker_long_hostname)|first).json.ip %}
+{%   else %}
+{%     set ip=controlplane_network | ansible.utils.nthhost(loop.index0 + sno_controlplane_ip_offset) %}
+{%   endif %}
+{% if sno_worker_foreman_data is defined and sno_worker_foreman_data.results is defined %}
+{{ worker_short_hostname }} bmc_address={{ worker.pm_addr }} mac_address={{ worker.mac[controlplane_network_interface_idx|int] }} lab_mac={{ lab_mac }} ip={{ ip }} lab_dhcp_interface={{ controlplane_lab_interface}} vendor={{ hw_vendor[(worker.pm_addr.split('.')[0]).split('-')[-1]] }} install_disk={{ worker_install_disk }}
+{% else %}
+# {{ worker_short_hostname }} bmc_address={{ worker.pm_addr }} mac_address={{ worker.mac[controlplane_network_interface_idx|int] }} lab_mac=UNDEFINED ip={{ ip }} lab_dhcp_interface={{ controlplane_lab_interface}} vendor={{ hw_vendor[(worker.pm_addr.split('.')[0]).split('-')[-1]] }} install_disk={{ worker_install_disk }}
+{% endif %}
+{% endfor %}
 
 [worker:vars]
-# Unused
+role=worker
+bmc_user={{ bmc_user }}
+bmc_password={{ bmc_password }}
+{% if sno_use_lab_dhcp %}
+# Commented out because sno_use_lab_dhcp is set to true
+# lab_interface={{ controlplane_lab_interface }}
+# network_interface={{ controlplane_network_interface }}
+# network_prefix={{ controlplane_network_prefix }}
+# gateway={{ controlplane_network_gateway }}
+{% if controlplane_bastion_as_dns %}
+# dns1={{ bastion_controlplane_ip }}
+{% else %}
+# dns1={{ labs[lab]['dns'][0] }}
+# dns2={{ labs[lab]['dns'][1] | default('') }}
+{% endif %}
+{% else %}
+lab_interface={{ controlplane_lab_interface }}
+network_interface={{ controlplane_network_interface }}
+network_prefix={{ controlplane_network_prefix }}
+gateway={{ controlplane_network_gateway }}
+{% if controlplane_bastion_as_dns %}
+dns1={{ bastion_controlplane_ip }}
+{% else %}
+dns1={{ labs[lab]['dns'][0] }}
+dns2={{ labs[lab]['dns'][1] | default('') }}
+{% endif %}
+{% endif %}
+{% else %}
+[worker]
+# No worker nodes configured
+
+[worker:vars]
+# No worker nodes configured
+{% endif %}
 
 [sno]
 # Only one can be uncommented and deployed at a time, but you can deploy additional SNOs

ansible/roles/mno-scale-out-csr/tasks/check_nodes_joined.yml → ansible/roles/ocp-scale-out-csr/tasks/check_nodes_joined.yml

@@ -4,6 +4,10 @@
     r_qry: "{{ qry }}"
     r_worker_counter: "{{ worker_counter }}"
 
+- name: Set KUBECONFIG path based on cluster type
+  set_fact:
+    cluster_kubeconfig: "{{ bastion_cluster_config_dir }}/{{ 'kubeconfig' if cluster_type != 'sno' else groups['sno'][0] + '/kubeconfig' }}"
+
 - name: approve CSRs and check if nodes have joined the cluster
   block:
     - name: Increment the retry count
@@ -17,19 +21,19 @@
 
     - name: Get CSRs
       shell: |
-        KUBECONFIG={{ bastion_cluster_config_dir }}/kubeconfig oc get csr -o json
+        KUBECONFIG={{ cluster_kubeconfig }} oc get csr -o json
       register: oc_get_csr
 
     - name: Approve pending CSRs
       shell: |
-        KUBECONFIG={{ bastion_cluster_config_dir }}/kubeconfig oc adm certificate approve {{ item.metadata.name }}
+        KUBECONFIG={{ cluster_kubeconfig }} oc adm certificate approve {{ item.metadata.name }}
       loop: "{{ oc_get_csr.stdout | from_json | json_query(qry) }}"
       loop_control:
         label: "{{ item.metadata.name }}"
 
     - name: Get worker node count
       shell: |
-        KUBECONFIG={{ bastion_cluster_config_dir }}/kubeconfig oc get nodes | {{ worker_counter }}
+        KUBECONFIG={{ cluster_kubeconfig }} oc get nodes | {{ worker_counter }}
       register: oc_get_nodes_workers
 
     - name: Current Worker Node Count

ansible/roles/ocp-scale-out/tasks/main.yml

@@ -0,0 +1,38 @@
+---
+- name: Delete ocp-scale-out directory
+  ansible.builtin.file:
+    state: absent
+    path: /root/ocp-scale-out
+
+- name: Create ocp-scale-out directory
+  ansible.builtin.file:
+    state: directory
+    path: /root/ocp-scale-out
+
+- name: Template Scaleout Nodes Config
+  template:
+    src: nodes-config.yml.j2
+    dest: /root/ocp-scale-out/nodes-config.yaml
+  vars:
+    workers: "{{ groups['worker'][current_worker_count:current_worker_count+scale_out_count] }}"
+
+- name: Set KUBECONFIG path based on cluster type
+  set_fact:
+    cluster_kubeconfig: "{{ bastion_cluster_config_dir }}/{{ 'kubeconfig' if cluster_type != 'sno' else groups['sno'][0] + '/kubeconfig' }}"
+
+- name: Add Nodes to cluster and generate boot iso (Takes a min or two)
+  command: oc adm node-image create --kubeconfig {{ cluster_kubeconfig }}
+  args:
+    chdir: /root/ocp-scale-out/
+
+- name: Copy scale out discovery iso to http server
+  ansible.builtin.copy:
+    src: /root/ocp-scale-out/node.x86_64.iso
+    dest: /opt/http_store/data/ocp-scale-out.x86_64.iso
+    remote_src: true
+
+- name: Delete ocp-scale-out directory
+  ansible.builtin.file:
+    state: absent
+    path: /root/ocp-scale-out
+

ansible/roles/mno-scale-out/templates/nodes-config.yml.j2 → ansible/roles/ocp-scale-out/templates/nodes-config.yml.j2

@@ -5,33 +5,54 @@ hosts:
   rootDeviceHints:
    deviceName: {{ hostvars[worker].install_disk }}
   interfaces:
-{% if hostvars[worker].lab_mac | default(False) %}
+{% if hostvars[worker].network_interface | default(False) %}
+   - macAddress: {{ hostvars[worker].mac_address }}
+     name: {{ hostvars[worker].network_interface }}
+{% endif %}
+{% if sno_use_lab_dhcp %}
+   - macAddress: {{ hostvars[worker].lab_mac }}
+     name: {{ hostvars[worker].lab_dhcp_interface }}
+{% elif hostvars[worker].lab_interface | default(False) %}
    - macAddress: {{ hostvars[worker].lab_mac }}
      name: {{ hostvars[worker].lab_interface }}
 {% endif %}
-   - macAddress: {{ hostvars[worker].mac_address }}
-     name: {{ hostvars[worker].network_interface }}
   networkConfig:
    interfaces:
-     - name: {{ hostvars[worker].network_interface }}
+{% if sno_use_lab_dhcp %}
+     - name: {{ hostvars[worker].lab_dhcp_interface }}
        type: ethernet
        state: up
-       mac-address: {{ hostvars[worker].mac_address }}
+       mac-address: {{ hostvars[worker].lab_mac }}
        ipv4:
          enabled: true
-         address:
-           - ip: {{ hostvars[worker].ip }}
-             prefix-length: {{ hostvars[worker].network_prefix }}
+         dhcp: true
          auto-dns: false
-{% if hostvars[worker].lab_mac | default(False) %}
+         auto-gateway: true
+         auto-routes: true
+   dns-resolver:
+     config:
+       server:
+       - {{ hostvars[groups['sno'][0]].ip }}
+{% elif hostvars[worker].lab_interface | default(False) %}
      - name: {{ hostvars[worker].lab_interface }}
        type: ethernet
        state: up
-       mac-address: {{ hostvars[worker].lab_interface }}
+       mac-address: {{ hostvars[worker].lab_mac }}
        ipv4:
          enabled: false
          auto-dns: false
 {% endif %}
+{% if hostvars[worker].network_interface | default(False) %}
+     - name: {{ hostvars[worker].network_interface }}
+       type: ethernet
+       state: up
+       mac-address: {{ hostvars[worker].mac_address }}
+       ipv4:
+         enabled: true
+         address:
+           - ip: {{ hostvars[worker].ip }}
+             prefix-length: {{ hostvars[worker].network_prefix }}
+         auto-dns: false
    dns-resolver:
      config:
        server:
@@ -40,5 +61,6 @@ hosts:
      config:
      - destination: 0.0.0.0/0
        next-hop-address: {{ hostvars[worker].gateway }}
-       next-hop-interface: eth0
-{% endfor %}
+       next-hop-interface: {{ hostvars[worker].network_interface }}
+{% endif %}
+{% endfor %}

docs/scale-out-mno.md

@@ -5,7 +5,7 @@ A JetLag deployed Multi-Node Openshift deployment can be scale out via JetLag. W
 _**Steps to Scale Out:**_
 - [Add New Node Entries to Worker Inventory](#add-new-node-entries-to-worker-inventory)
 - [Update scale_out.yml](#update-scale_out.yml)
-- [Run mno-scale-out.yml](#run-mno-scale-out.yml)
+- [Run ocp-scale-out.yml](#run-ocp-scale-out.yml)
 
 ## Add Nodes to Worker Inventory
 To add new node entries to the worker inventory there are three potential options.
@@ -32,11 +32,11 @@ There are two variables in ansible/vars/scale_out.yml that indicate which entrie
 
 Example: If the initial OCP deployment had three baremetal workers and the intended worker count was ten, current_worker_count would be 3 and scale_out_count would be 7. Scale out from three existing workers, adding seven new workers, for a total of ten worker nodes.
 
-## Run mno-scale-out.yml
+## Run ocp-scale-out.yml
 Once the new worker records are added to the inventory and the scale_out.yml file has the proper values. The final step is to run the mno-scale-out.yml playbook.
 
 ```console
-(.ansible) [root@xxx-h01-000-r650 jetlag]# ansible-playbook -i ansible/inventory/cloud99.local ansible/mno-scale-out.yml
+(.ansible) [root@xxx-h01-000-r650 jetlag]# ansible-playbook -i ansible/inventory/cloud99.local ansible/ocp-scale-out.yml
 ...
 ```