Skip to content

Commit 7ca2336

Browse files
NicolasTNicolasT
committed
salt: use roles retrieved from Node objects
This allows to set `node-role.kubernetes.io/$ROLE` labels on a Node object, where `$ROLE` can be `bootstrap`, `etcd`, `master` and `node`, which will then determine the `highstate` of the node. Furthermore, it requires a `metalk8s.scality.com/version` label to select a MetalK8s version (actually, `saltenv`) to be used. It also removes the `node-role.kubernetes.io/master` Taint from the bootstrap Node, and replaces it with a `node-role.kubernetes.io/bootstrap` Taint, which is tolerated by CoreDNS and the UI. This is a special-case of the `node-role.kubernetes.io/master` Taint which would not permit such services to run on the node. See: scality#925 See: scality#925 See: scality#742 See: scality#742 See: scality#903 See: scality#903
1 parent 074180b commit 7ca2336

File tree

19 files changed

+189
-9
lines changed

19 files changed

+189
-9
lines changed

‎buildchain/buildchain/salt_tree.py

+26-1
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,24 @@ def task__deploy_salt_tree() -> Iterator[types.TaskDict]:
7373

7474
# List of salt files to install.
7575
SALT_FILES : Tuple[Union[Path, targets.FileTarget], ...] = (
76+
targets.TemplateFile(
77+
task_name='top.sls',
78+
source=constants.ROOT/'salt'/'top.sls.in',
79+
destination=constants.ISO_ROOT/'salt'/'top.sls',
80+
context={'VERSION': constants.SHORT_VERSION},
81+
file_dep=[constants.VERSION_FILE],
82+
),
83+
84+
Path('salt/metalk8s/roles/minion.sls'),
85+
Path('salt/metalk8s/roles/bootstrap.sls'),
86+
Path('salt/metalk8s/roles/salt-master.sls'),
87+
Path('salt/metalk8s/roles/registry.sls'),
88+
Path('salt/metalk8s/roles/repository.sls'),
89+
Path('salt/metalk8s/roles/ca.sls'),
90+
Path('salt/metalk8s/roles/etcd.sls'),
91+
Path('salt/metalk8s/roles/master.sls'),
92+
Path('salt/metalk8s/roles/node.sls'),
93+
7694
Path('salt/metalk8s/orchestrate/bootstrap_without_master.sls'),
7795
Path('salt/metalk8s/orchestrate/deploy_etcd_on_new_node.sls'),
7896
Path('salt/metalk8s/orchestrate/bootstrap_with_master.sls'),
@@ -156,7 +174,13 @@ def task__deploy_salt_tree() -> Iterator[types.TaskDict]:
156174
Path('salt/metalk8s/kubeadm/init/kubeconfig/scheduler.sls'),
157175

158176
Path('salt/metalk8s/kubeadm/init/mark-control-plane/init.sls'),
159-
Path('salt/metalk8s/kubeadm/init/mark-control-plane/configured.sls'),
177+
targets.TemplateFile(
178+
task_name='configured.sls',
179+
source=constants.ROOT/'salt'/'metalk8s'/'kubeadm'/'init'/'mark-control-plane'/'configured.sls.in',
180+
destination=constants.ISO_ROOT/'salt'/'metalk8s'/'kubeadm'/'init'/'mark-control-plane'/'configured.sls',
181+
context={'VERSION': constants.SHORT_VERSION},
182+
file_dep=[constants.VERSION_FILE],
183+
),
160184

161185
Path('salt/metalk8s/kubeadm/init/addons/init.sls'),
162186
Path('salt/metalk8s/kubeadm/init/addons/kube-proxy.sls'),
@@ -213,6 +237,7 @@ def task__deploy_salt_tree() -> Iterator[types.TaskDict]:
213237
Path('salt/_modules/metalk8s.py'),
214238

215239
Path('salt/_pillar/metalk8s.py'),
240+
Path('salt/_pillar/metalk8s_nodes.py'),
216241

217242
Path('salt/_roster/kubernetes_nodes.py'),
218243

‎examples/new-node.yaml

+5-1
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,12 @@ metadata:
1212
metalk8s.scality.com/ssh-key-path: <PATH/TO/SSH/KEY>
1313
metalk8s.scality.com/ssh-sudo: <true if the user is not root>
1414
labels:
15-
beta.kubernetes.io/os: linux
15+
metalk8s.scality.com/version: '2.0'
16+
node-role.kubernetes.io/master: ''
17+
node-role.kubernetes.io/etcd: ''
1618
spec:
1719
taints:
1820
- effect: NoSchedule
1921
key: node-role.kubernetes.io/master
22+
- effect: NoSchedule
23+
key: node-role.kubernetes.io/etcd

‎examples/new-node_vagrant.yaml

+5-1
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,12 @@ metadata:
1212
metalk8s.scality.com/ssh-key-path: /etc/metalk8s/pki/preshared_key_for_k8s_nodes
1313
metalk8s.scality.com/ssh-sudo: "true"
1414
labels:
15-
beta.kubernetes.io/os: linux
15+
metalk8s.scality.com/version: '2.0'
16+
node-role.kubernetes.io/master: ''
17+
node-role.kubernetes.io/etcd: ''
1618
spec:
1719
taints:
1820
- effect: NoSchedule
1921
key: node-role.kubernetes.io/master
22+
- effect: NoSchedule
23+
key: node-role.kubernetes.io/etcd

‎salt/_pillar/metalk8s_nodes.py

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import os.path
2+
import logging
3+
4+
try:
5+
import kubernetes.client
6+
import kubernetes.config
7+
HAS_DEPS = True
8+
except ImportError:
9+
HAS_DEPS = False
10+
11+
12+
VERSION_LABEL = 'metalk8s.scality.com/version'
13+
ROLE_LABEL_PREFIX = 'node-role.kubernetes.io/'
14+
15+
16+
log = logging.getLogger(__name__)
17+
18+
__virtualname__ = 'metalk8s_nodes'
19+
20+
def __virtual__():
21+
if HAS_DEPS:
22+
return __virtualname__
23+
else:
24+
return False, 'Missing Kubernetes client library dependency'
25+
26+
27+
def node_info(node, ca_minion):
28+
result = {
29+
'roles': [],
30+
'version': None,
31+
}
32+
33+
roles = set()
34+
35+
if VERSION_LABEL in node.metadata.labels:
36+
result['version'] = node.metadata.labels[VERSION_LABEL]
37+
38+
for (label, value) in node.metadata.labels.items():
39+
if label.startswith(ROLE_LABEL_PREFIX):
40+
role = label[len(ROLE_LABEL_PREFIX):]
41+
if role:
42+
roles.add(role)
43+
44+
if node.metadata.name == ca_minion:
45+
roles.add('ca')
46+
47+
result['roles'] = list(roles)
48+
49+
return result
50+
51+
52+
def ext_pillar(minion_id, pillar, kubeconfig):
53+
if not os.path.isfile(kubeconfig):
54+
log.warning(
55+
'%s: kubeconfig not found at %s', __virtualname__, kubeconfig)
56+
return {}
57+
58+
ca_minion = None
59+
if 'metalk8s' in pillar:
60+
if 'ca' in pillar['metalk8s']:
61+
ca_minion = pillar['metalk8s']['ca'].get('minion', None)
62+
63+
client = kubernetes.config.new_client_from_config(
64+
config_file=kubeconfig,
65+
)
66+
67+
coreV1 = kubernetes.client.CoreV1Api(api_client=client)
68+
69+
node_list = coreV1.list_node()
70+
71+
pillar_nodes = dict(
72+
(node.metadata.name, node_info(node, ca_minion))
73+
for node in node_list.items)
74+
75+
return {
76+
'metalk8s': {
77+
'nodes': pillar_nodes,
78+
'node': pillar_nodes.get(minion_id, None),
79+
},
80+
}

‎salt/metalk8s/kubeadm/init/addons/files/coredns_deployment.yaml.j2

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ spec:
2323
tolerations:
2424
- key: "CriticalAddonsOnly"
2525
operator: "Exists"
26-
- key: "node-role.kubernetes.io/master"
26+
- key: "node-role.kubernetes.io/bootstrap"
2727
operator: "Exists"
2828
effect: "NoSchedule"
2929
nodeSelector:

‎salt/metalk8s/kubeadm/init/mark-control-plane/configured.sls ‎salt/metalk8s/kubeadm/init/mark-control-plane/configured.sls.in

+27-3
Original file line numberDiff line numberDiff line change
@@ -3,23 +3,47 @@
33
{% set kubeconfig = "/etc/kubernetes/admin.conf" %}
44
{% set context = "kubernetes-admin@kubernetes" %}
55

6-
Apply control-plane role label:
6+
Apply control-plane master role label:
77
kubernetes.node_label_present:
88
- name: "node-role.kubernetes.io/master"
99
- kubeconfig: {{ kubeconfig }}
1010
- context: {{ context }}
1111
- node: {{ hostname }}
1212
- value: ""
1313

14-
Apply control-plane taints:
14+
Apply control-plane etcd role label:
15+
kubernetes.node_label_present:
16+
- name: "node-role.kubernetes.io/etcd"
17+
- kubeconfig: {{ kubeconfig }}
18+
- context: {{ context }}
19+
- node: {{ hostname }}
20+
- value: ""
21+
22+
Apply control-plane bootstrap role label:
23+
kubernetes.node_label_present:
24+
- name: "node-role.kubernetes.io/bootstrap"
25+
- kubeconfig: {{ kubeconfig }}
26+
- context: {{ context }}
27+
- node: {{ hostname }}
28+
- value: ""
29+
30+
Apply control-plane bootstrap taint:
1531
kubernetes.node_taints_present:
1632
- name: {{ hostname }}
1733
- kubeconfig: {{ kubeconfig }}
1834
- context: {{ context }}
1935
- taints:
20-
- key: "node-role.kubernetes.io/master"
36+
- key: "node-role.kubernetes.io/bootstrap"
2137
effect: "NoSchedule"
2238

39+
Apply node version label:
40+
kubernetes.node_label_present:
41+
- kubeconfig: {{ kubeconfig }}
42+
- context: {{ context }}
43+
- name: "metalk8s.scality.com/version"
44+
- node: {{ hostname }}
45+
- value: "@@VERSION"
46+
2347
Annotate with CRI socket information:
2448
kubernetes.node_annotation_present:
2549
- name: "kubeadm.alpha.kubernetes.io/cri-socket"

‎salt/metalk8s/roles/bootstrap.sls

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Deploy bootstrap:
2+
test.succeed_without_changes

‎salt/metalk8s/roles/ca.sls

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Deploy CA:
2+
test.succeed_without_changes

‎salt/metalk8s/roles/etcd.sls

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Deploy etcd member:
2+
test.succeed_without_changes

‎salt/metalk8s/roles/master.sls

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Deploy control plane node:
2+
test.succeed_without_changes

‎salt/metalk8s/roles/minion.sls

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Deploy Salt minion:
2+
test.succeed_without_changes

‎salt/metalk8s/roles/node.sls

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Deploy workload plane node:
2+
test.succeed_without_changes

‎salt/metalk8s/roles/registry.sls

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Deploy container image registry:
2+
test.succeed_without_changes

‎salt/metalk8s/roles/repository.sls

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Deploy package repository:
2+
test.succeed_without_changes

‎salt/metalk8s/roles/salt-master.sls

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Deploy Salt master:
2+
test.succeed_without_changes

‎salt/metalk8s/salt/master/files/master_99-metalk8s.conf

+3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
11
peer:
22
.*:
33
- x509.sign_remote_certificate
4+
45
ext_pillar:
56
- metalk8s: /etc/metalk8s/bootstrap.yaml
7+
- metalk8s_nodes: /etc/kubernetes/admin.conf
8+
69
roster_defaults:
710
minion_opts:
811
use_superseded:

‎salt/metalk8s/ui/files/metalk8s-ui_deployment.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ spec:
1515
k8s-app: ui
1616
spec:
1717
tolerations:
18-
- key: "node-role.kubernetes.io/master"
18+
- key: "node-role.kubernetes.io/bootstrap"
1919
operator: "Exists"
2020
effect: "NoSchedule"
2121
nodeSelector:

‎salt/top.sls.in

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
metalk8s-@@VERSION:
2+
'metalk8s:node:version:@@VERSION':
3+
- match: pillar
4+
- metalk8s.roles.minion
5+
'I@metalk8s:node:version:@@VERSION and I@metalk8s:node:roles:bootstrap':
6+
- match: compound
7+
- metalk8s.roles.bootstrap
8+
- metalk8s.roles.salt-master
9+
- metalk8s.roles.registry
10+
- metalk8s.roles.repository
11+
'I@metalk8s:node:version:@@VERSION and I@metalk8s:node:roles:ca':
12+
- match: compound
13+
- metalk8s.roles.ca
14+
'I@metalk8s:node:version:@@VERSION and I@metalk8s:node:roles:etcd':
15+
- match: compound
16+
- metalk8s.roles.etcd
17+
'I@metalk8s:node:version:@@VERSION and I@metalk8s:node:roles:master':
18+
- match: compound
19+
- metalk8s.roles.master
20+
'I@metalk8s:node:version:@@VERSION and I@metalk8s:node:roles:node':
21+
- match: compound
22+
- metalk8s.roles.node

‎tests/post/steps/files/busybox.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ metadata:
55
namespace: default
66
spec:
77
tolerations:
8-
- key: "node-role.kubernetes.io/master"
8+
- key: "node-role.kubernetes.io/bootstrap"
99
operator: "Equal"
1010
effect: "NoSchedule"
1111
containers:

0 commit comments

Comments
 (0)