Skip to content

Commit 17ae114

Browse files
derekbitderekbit
committed
fix(v2): compute host CPU mask from online CPU set
get_online_cpu_count returned max_cpu_id + 1 and derived "all online CPUs" as (1 << n) - 1, which is wrong whenever CPU ids do not start at 0 or are non-contiguous (e.g. CPU 0 hot-unplugged so online='1-4', or online='0,2-3,5'). The derived mask then set bits for offline CPUs, which can cause writes to default_smp_affinity to fail with EINVAL and abort spdk_tgt start. Replace it with get_online_cpu_mask, which parses /sys/devices/system/cpu/online into an actual bitmask. Callers now compute the inverse as `online & ~spdk`, so offline CPU bits are never set and the "SPDK covers every online CPU" guard is exact. Use string comparison against "0" so masks wider than 64 bits do not overflow bash arithmetic. Longhorn 13417 Signed-off-by: Derek Su <derek.su@suse.com>
1 parent 918eaa5 commit 17ae114

1 file changed

Lines changed: 61 additions & 48 deletions

File tree

‎package/start-spdk-tgt‎

Lines changed: 61 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -29,25 +29,39 @@ host_workqueue_dir="${host_mount_prefix}/sys/devices/virtual/workqueue"
2929
host_workqueue_cpumask_file="${host_workqueue_dir}/cpumask"
3030
instance_manager_options=()
3131

32-
# Determine the number of online CPUs on the host (falls back to nproc).
33-
function get_online_cpu_count() {
32+
# Compute a bitmask (as a hex "0x..." string) of all currently-online host CPUs
33+
# by parsing /sys/devices/system/cpu/online. Handles arbitrary CPU id sets that
34+
# are non-contiguous and/or do not start at 0 (e.g. "1-4", "0,2-3,5"). If the
35+
# sysfs file cannot be read, falls back to a mask covering CPUs 0..nproc-1.
36+
function get_online_cpu_mask() {
3437
if [ -r "$host_online_cpus_file" ]; then
3538
local range
3639
range=$(cat "$host_online_cpus_file")
3740
python3 -c "
3841
import sys
3942
s = sys.argv[1].strip()
40-
mx = 0
43+
mask = 0
4144
for part in s.split(','):
45+
part = part.strip()
46+
if not part:
47+
continue
4248
if '-' in part:
4349
a, b = part.split('-', 1)
44-
mx = max(mx, int(b))
45-
elif part:
46-
mx = max(mx, int(part))
47-
print(mx + 1)
50+
a, b = int(a), int(b)
51+
if a > b:
52+
sys.exit('ERROR: invalid CPU range: ' + part)
53+
for i in range(a, b + 1):
54+
mask |= (1 << i)
55+
else:
56+
mask |= (1 << int(part))
57+
if mask == 0:
58+
sys.exit('ERROR: parsed empty online CPU set from: ' + repr(sys.argv[1]))
59+
print('0x{:x}'.format(mask))
4860
" "$range"
4961
else
50-
nproc
62+
local n
63+
n=$(nproc)
64+
python3 -c "print('0x{:x}'.format((1 << ${n}) - 1))"
5165
fi
5266
}
5367

@@ -111,44 +125,41 @@ function apply_irq_affinity() {
111125
# Restore IRQ affinity to "all online CPUs", i.e. remove any restriction we may
112126
# have installed for spdk_tgt on a previous run.
113127
function clear_irq_affinity() {
114-
local cpu_count
115-
cpu_count=$(get_online_cpu_count)
116-
if ! [[ "$cpu_count" =~ ^[0-9]+$ ]] || [ "$cpu_count" -le 0 ]; then
117-
log "ERROR: Unable to determine online CPU count (got: $cpu_count)"
128+
local online_mask_hex
129+
online_mask_hex=$(get_online_cpu_mask) || return 1
130+
if ! [[ "$online_mask_hex" =~ ^0x[0-9a-fA-F]+$ ]] || [ "$online_mask_hex" = "0x0" ]; then
131+
log "ERROR: Unable to determine online CPU mask (got: $online_mask_hex)"
118132
return 1
119133
fi
120-
local all_mask_int
121-
all_mask_int=$(python3 -c "print((1 << $cpu_count) - 1)")
122134
local linux_mask
123-
linux_mask=$(format_affinity_mask "$all_mask_int")
124-
log "Clearing IRQ affinity: restoring all ${cpu_count} CPUs (mask=${linux_mask})"
135+
linux_mask=$(format_affinity_mask "$online_mask_hex")
136+
log "Clearing IRQ affinity: restoring all online CPUs (online_mask=${online_mask_hex}, mask=${linux_mask})"
125137
apply_irq_affinity "$linux_mask"
126138
}
127139

128140
# Restrict IRQs to the CPUs that are NOT used by spdk_tgt.
129141
function set_irq_affinity_for_spdk_mask() {
130142
local spdk_mask_hex="$1"
131-
local cpu_count
132-
cpu_count=$(get_online_cpu_count)
133-
if ! [[ "$cpu_count" =~ ^[0-9]+$ ]] || [ "$cpu_count" -le 0 ]; then
134-
log "ERROR: Unable to determine online CPU count (got: $cpu_count)"
143+
local online_mask_hex
144+
online_mask_hex=$(get_online_cpu_mask) || return 1
145+
if ! [[ "$online_mask_hex" =~ ^0x[0-9a-fA-F]+$ ]] || [ "$online_mask_hex" = "0x0" ]; then
146+
log "ERROR: Unable to determine online CPU mask (got: $online_mask_hex)"
135147
return 1
136148
fi
137149
local inv_mask_int
138150
inv_mask_int=$(python3 -c "
151+
online = int('${online_mask_hex}', 0)
139152
spdk = int('${spdk_mask_hex}', 0)
140-
n = ${cpu_count}
141-
full = (1 << n) - 1
142-
inv = full & ~spdk
143-
print(inv)
153+
print(online & ~spdk)
144154
")
145-
if [ "$inv_mask_int" -eq 0 ]; then
146-
log "ERROR: SPDK cpu mask ${spdk_mask_hex} covers every online CPU; refusing to set IRQ affinity to an empty set"
155+
# inv_mask_int may exceed 64 bits, so use string comparison rather than -eq.
156+
if [ "$inv_mask_int" = "0" ]; then
157+
log "ERROR: SPDK cpu mask ${spdk_mask_hex} covers every online CPU (online_mask=${online_mask_hex}); refusing to set IRQ affinity to an empty set"
147158
return 1
148159
fi
149160
local linux_mask
150161
linux_mask=$(format_affinity_mask "$inv_mask_int")
151-
log "Setting IRQ affinity to exclude SPDK CPUs (spdk_mask=${spdk_mask_hex}, irq_mask=${linux_mask})"
162+
log "Setting IRQ affinity to exclude SPDK CPUs (spdk_mask=${spdk_mask_hex}, online_mask=${online_mask_hex}, irq_mask=${linux_mask})"
152163
apply_irq_affinity "$linux_mask"
153164
}
154165

@@ -210,44 +221,41 @@ function clear_workqueue_affinity() {
210221
log "Workqueue cpumask file $host_workqueue_cpumask_file not present; skipping workqueue clear"
211222
return 0
212223
fi
213-
local cpu_count
214-
cpu_count=$(get_online_cpu_count)
215-
if ! [[ "$cpu_count" =~ ^[0-9]+$ ]] || [ "$cpu_count" -le 0 ]; then
216-
log "ERROR: Unable to determine online CPU count (got: $cpu_count)"
224+
local online_mask_hex
225+
online_mask_hex=$(get_online_cpu_mask) || return 1
226+
if ! [[ "$online_mask_hex" =~ ^0x[0-9a-fA-F]+$ ]] || [ "$online_mask_hex" = "0x0" ]; then
227+
log "ERROR: Unable to determine online CPU mask (got: $online_mask_hex)"
217228
return 1
218229
fi
219-
local all_mask_int
220-
all_mask_int=$(python3 -c "print((1 << $cpu_count) - 1)")
221230
local linux_mask
222-
linux_mask=$(format_affinity_mask "$all_mask_int")
223-
log "Clearing workqueue cpumask: restoring all ${cpu_count} CPUs (mask=${linux_mask})"
231+
linux_mask=$(format_affinity_mask "$online_mask_hex")
232+
log "Clearing workqueue cpumask: restoring all online CPUs (online_mask=${online_mask_hex}, mask=${linux_mask})"
224233
apply_workqueue_affinity "$linux_mask"
225234
}
226235

227236
# Restrict unbound kernel workqueues to the CPUs that are NOT used by spdk_tgt.
228237
function set_workqueue_affinity_for_spdk_mask() {
229238
local spdk_mask_hex="$1"
230-
local cpu_count
231-
cpu_count=$(get_online_cpu_count)
232-
if ! [[ "$cpu_count" =~ ^[0-9]+$ ]] || [ "$cpu_count" -le 0 ]; then
233-
log "ERROR: Unable to determine online CPU count (got: $cpu_count)"
239+
local online_mask_hex
240+
online_mask_hex=$(get_online_cpu_mask) || return 1
241+
if ! [[ "$online_mask_hex" =~ ^0x[0-9a-fA-F]+$ ]] || [ "$online_mask_hex" = "0x0" ]; then
242+
log "ERROR: Unable to determine online CPU mask (got: $online_mask_hex)"
234243
return 1
235244
fi
236245
local inv_mask_int
237246
inv_mask_int=$(python3 -c "
247+
online = int('${online_mask_hex}', 0)
238248
spdk = int('${spdk_mask_hex}', 0)
239-
n = ${cpu_count}
240-
full = (1 << n) - 1
241-
inv = full & ~spdk
242-
print(inv)
249+
print(online & ~spdk)
243250
")
244-
if [ "$inv_mask_int" -eq 0 ]; then
245-
log "ERROR: SPDK cpu mask ${spdk_mask_hex} covers every online CPU; refusing to set workqueue cpumask to an empty set"
251+
# inv_mask_int may exceed 64 bits, so use string comparison rather than -eq.
252+
if [ "$inv_mask_int" = "0" ]; then
253+
log "ERROR: SPDK cpu mask ${spdk_mask_hex} covers every online CPU (online_mask=${online_mask_hex}); refusing to set workqueue cpumask to an empty set"
246254
return 1
247255
fi
248256
local linux_mask
249257
linux_mask=$(format_affinity_mask "$inv_mask_int")
250-
log "Setting workqueue cpumask to exclude SPDK CPUs (spdk_mask=${spdk_mask_hex}, wq_mask=${linux_mask})"
258+
log "Setting workqueue cpumask to exclude SPDK CPUs (spdk_mask=${spdk_mask_hex}, online_mask=${online_mask_hex}, wq_mask=${linux_mask})"
251259
apply_workqueue_affinity "$linux_mask"
252260
}
253261

@@ -267,9 +275,14 @@ function apply_cpu_isolation_for_spdk_mask() {
267275
# Orchestrator: undo any IRQ / workqueue affinity we may have installed. Always
268276
# attempts both because the on-disk marker (cpu_mask_file) does not record which
269277
# mechanism was active last time; individual clears tolerate "nothing to do".
278+
# A failure in one clear must NOT prevent the other from running, otherwise a
279+
# transient IRQ clear failure would leave a stale workqueue restriction (or
280+
# vice versa) in place across the whole node.
270281
function clear_cpu_isolation() {
271-
clear_irq_affinity || return 1
272-
clear_workqueue_affinity || return 1
282+
local rc=0
283+
clear_irq_affinity || rc=1
284+
clear_workqueue_affinity || rc=1
285+
return "$rc"
273286
}
274287

275288
# Handle leftover state from a previous (possibly crashed) instance-manager run.

0 commit comments

Comments
 (0)