salt/volumes: Detach devices before (up|down)grade

Since the management of loop devices changed between 2.6 and 2.7
(introduction of systemd units for this purpose), we want to ensure
there will not be duplicated devices pointing to the same sparse file.
To do this, we introduce a "cleanup" formula, which can operate in two
modes, 'upgrade' and 'downgrade' (controlled via pillar).
For upgrade, we manage this cleanup in the `deploy_node` orchestrate,
during the drain.
For downgrade, we cannot change the `deploy_node` orchestrate, so we
manually drain and cleanup from the `downgrade` orchestrate.

See: scality#2982

Author: gdemonet

buildchain/buildchain/salt_tree.py

@@ -669,6 +669,7 @@ def _get_parts(self) -> Iterator[str]:
     Path('salt/metalk8s/utils/httpd-tools/init.sls'),
     Path('salt/metalk8s/utils/httpd-tools/installed.sls'),
 
+    Path('salt/metalk8s/volumes/cleanup.sls'),
     Path('salt/metalk8s/volumes/init.sls'),
     Path('salt/metalk8s/volumes/installed.sls'),
     Path('salt/metalk8s/volumes/prepared.sls'),

docs/operation/downgrade.rst

@@ -5,6 +5,13 @@ packaged with your current installation.
 This section describes a reliable downgrade procedure for **MetalK8s**
 including all the components that are included in the stack.
 
+.. warning::
+
+   Downgrading a single-node cluster from 2.7.x to 2.6.y with
+   ``SparseLoopDevice`` volumes will force draining of this single node.
+   Expect an interruption of service during this downgrade (production
+   deployments **should not** rely on ``SparseLoopDevice`` volumes).
+
 Supported Versions
 ******************
 

docs/operation/upgrade.rst

@@ -5,6 +5,13 @@ with every new release.
 This section describes a reliable upgrade procedure for **MetalK8s** including
 all the components that are included in the stack.
 
+.. warning::
+
+   Upgrading a single-node cluster from 2.6.x to 2.7.y with
+   ``SparseLoopDevice`` volumes will force draining of this single node.
+   Expect an interruption of service during this upgrade (production
+   deployments **should not** rely on ``SparseLoopDevice`` volumes).
+
 Supported Versions
 ******************
 .. note::

salt/metalk8s/orchestrate/deploy_node.sls

@@ -1,4 +1,6 @@
 {%- set node_name = pillar.orchestrate.node_name %}
+{%- set run_drain = not pillar.orchestrate.get('skip_draining', False) %}
+{%- set cleanup_loop_devices = pillar.orchestrate.get('cleanup_loop_devices', False) %}
 {%- set version = pillar.metalk8s.nodes[node_name].version %}
 
 {%- set skip_roles = pillar.metalk8s.nodes[node_name].get('skip_roles', []) %}
@@ -70,7 +72,27 @@ Cordon the node:
   metalk8s_cordon.node_cordoned:
     - name: {{ node_name }}
 
-{%- if not pillar.orchestrate.get('skip_draining', False) %}
+{%- if run_drain or cleanup_loop_devices %}
+  {%- if not run_drain %}
+    {# Check if we can avoid the drain (if no loop device needs cleaning up) #}
+    {%- set volumes = salt.metalk8s_kubernetes.list_objects(
+      kind="Volume", apiVersion="storage.metalk8s.scality.com/v1alpha1",
+    ) %}
+    {%- set volumes_to_clean = volumes
+                             | selectattr('spec.nodeName', 'equalto', node_name)
+                             | selectattr('spec.sparseLoopDevice', 'defined')
+                             | list
+    %}
+    {%- if volumes_to_clean %}
+      {%- do salt.log.warning(
+        'Forcing drain to clean up loop devices for the following Volumes: '
+        ~ volumes_to_clean | map(attribute='metadata.name') | join(', ')
+      ) %}
+      {%- set run_drain = True %}
+    {%- endif %}
+  {%- endif %}
+
+  {%- if run_drain %}
 
 Drain the node:
   metalk8s_drain.node_drained:
@@ -83,6 +105,7 @@ Drain the node:
     - require_in:
       - salt: Run the highstate
 
+  {%- endif %}
 {%- endif %}
 
 {%- if node_name in salt.saltutil.runner('manage.up') %}
@@ -131,6 +154,31 @@ Wait minion available:
 
 {%- endif %}
 
+{#- Nodes in 2.6 or lower rely on manual provisioning of loop devices, which
+    we need to clean-up to avoid having duplicates generated by the systemd
+    units, introduced in 2.7
+    Note that we run this before `metalk8s.roles.etcd`, since this would
+    provision the aforementioned loop devices (because `etcd` role includes
+    `node` role, which takes care of volume provisioning) #}
+{%- if cleanup_loop_devices %}
+
+Cleanup existing loop devices:
+  salt.state:
+    - tgt: {{ node_name }}
+    - saltenv: metalk8s-{{ version }}
+    - sls:
+      - metalk8s.volumes.cleanup
+    - require:
+      - salt: Wait minion available
+      - metalk8s_drain: Drain the node
+    - require_in:
+  {%- if 'etcd' in roles and 'etcd' not in skip_roles %}
+      - salt: Check pillar before etcd deployment
+  {%- endif %}
+      - http: Wait for API server to be available before highstate
+
+{%- endif %}
+
 {%- if 'etcd' in roles and 'etcd' not in skip_roles %}
 
 Check pillar before etcd deployment:
@@ -171,7 +219,7 @@ Install etcd node:
 Register the node into etcd cluster:
   salt.runner:
     - name: state.orchestrate
-    - pillar: {{ pillar | json  }}
+    - pillar: {{ pillar | json }}
     - mods:
       - metalk8s.orchestrate.register_etcd
     - require:

salt/metalk8s/orchestrate/downgrade/init.sls

@@ -73,8 +73,60 @@ Upgrade salt-minion on {{ node }}:
           node_name: {{ node }}
     - require:
       - metalk8s_kubernetes: Set node {{ node }} version to {{ dest_version }}
+
+  {%- set force_drain = True %}
+
+  {%- if pillar.metalk8s.nodes|length == 1 %}
+    {# Check if we can avoid the drain (if no loop device needs cleaning up) #}
+    {%- set volumes = salt.metalk8s_kubernetes.list_objects(
+      kind="Volume", apiVersion="storage.metalk8s.scality.com/v1alpha1",
+    ) %}
+    {%- set volumes_to_clean = volumes
+                             | selectattr('spec.nodeName', 'equalto', node)
+                             | selectattr('spec.sparseLoopDevice', 'defined')
+                             | list
+    %}
+    {%- if volumes_to_clean %}
+      {%- do salt.log.warning(
+        'Forcing drain to clean up loop devices for the following Volumes: '
+        ~ volumes_to_clean | map(attribute='metadata.name') | join(', ')
+      ) %}
+    {%- else %}
+      {%- set force_drain = False %}
+    {%- endif %}
+  {%- endif %}
+
+  {%- if force_drain %}
+# We force a node drain manually to handle cleanup of loop devices outside
+# of the `deploy_node` orchestrate, since it comes from `dest_version`.
+# The `deploy_node` orchestrate will take care of uncordoning.
+Cordon node {{ node }}:
+  metalk8s_cordon.node_cordoned:
+    - name: {{ node }}
+    - require:
+      - salt: Upgrade salt-minion on {{ node }}
+
+Drain node {{ node }}:
+  metalk8s_drain.node_drained:
+    - name: {{ node }}
+    - ignore_daemonset: True
+    - delete_local_data: True
+    - force: True
+    - require:
+      - metalk8s_cordon: Cordon node {{ node }}
+
+Cleanup loop devices from node {{ node }}:
+  salt.state:
+    - tgt: {{ node }}
+    - saltenv: {{ saltenv }} {#- Use current version of this formula #}
+    - sls:
+      - metalk8s.volumes.cleanup
+    - require:
+      - metalk8s_drain: Drain node {{ node }}
     - require_in:
       - salt: Deploy node {{ node }}
+
+  {%- endif %}
 {%- endif %}
 
 Deploy node {{ node }}:

salt/metalk8s/orchestrate/upgrade/init.sls

@@ -108,6 +108,11 @@ Deploy node {{ node }}:
           {#- Do not drain if we are in single node cluster #}
           skip_draining: True
           {%- endif %}
+          {%- if salt.pkg.version_cmp(node_version, '2.7.0') < 0 %}
+          {#- Nodes in 2.6 or lower rely on manual provisioning of loop
+              devices, which we need to clean-up while the node is drained #}
+          cleanup_loop_devices: True
+          {%- endif %}
     - require:
       - metalk8s_kubernetes: Set node {{ node }} version to {{ dest_version }}
     - require_in:

salt/metalk8s/volumes/cleanup.sls

@@ -0,0 +1,67 @@
+# Clean already attached loop devices before applying highstate, to avoid
+# creating duplicates for a same sparse file.
+# This state can cleanup either "manually", for upgrade, or by disabling the
+# systemd units, for downgrade.
+{%- set target_version = pillar.metalk8s.nodes[grains.id].version %}
+
+{%- if '_errors' in pillar.metalk8s.volumes %}
+
+Cannot proceed with volume cleanup (pillar errors):
+  test.fail_without_changes:
+    - comment: 'Errors: {{ pillar.metalk8s.volumes._errors | join (", ") }}'
+
+{%- else %}
+  {%- set sparse_volumes = pillar.metalk8s.volumes.values()
+                         | selectattr('spec.sparseLoopDevice', 'defined')
+                         | list %}
+
+  {%- if not sparse_volumes %}
+
+Nothing to cleanup:
+  test.succeed_without_changes: []
+
+  {%- else %}
+    {%- set target_is_2_7 = salt.pkg.version_cmp(target_version, '2.7.0') >= 0 %}
+    {%- if target_is_2_7 %}
+
+include:
+  - .installed
+
+    {%- endif %}
+
+    {%- for volume in sparse_volumes %}
+      {%- set volume_name = volume.metadata.name %}
+      {%- set volume_id = volume.metadata.uid %}
+      {%- if target_is_2_7 %}
+
+Cleanup loop device for Volume {{ volume_name }}:
+  cmd.run:
+    - name: /usr/local/libexec/metalk8s-sparse-volume-cleanup "{{ volume_id }}"
+    # Only cleanup if the systemd unit doesn't exist yet (this command exits
+    # with retcode 3 if the service is dead, 4 if the template does not exist)
+    - unless: systemctl status metalk8s-sparse-volume@{{ volume_id }}
+    - require:
+      - test: Ensure Python 3 is available
+      - file: Install clean-up script
+
+      {%- else %} {# target_version < 2.7.0 #}
+
+Disable systemd unit for Volume {{ volume_name }}:
+  service.dead:
+    - name: metalk8s-sparse-volume@{{ volume_id }}
+    - enable: false
+    - require_in:
+      - file: Remove systemd template unit for sparse loop device provisioning
+
+      {%- endif %}
+    {%- endfor %}
+  {%- endif %}
+
+  {%- if not target_is_2_7 %}
+
+Remove systemd template unit for sparse loop device provisioning:
+  file.absent:
+    - name: /etc/systemd/system/[email protected]
+
+  {%- endif %}
+{%- endif %}