|
| 1 | +#!/usr/bin/env python |
| 2 | + |
| 3 | +# This program is free software; you can redistribute it and/or modify |
| 4 | +# it under the terms of the GNU General Public License as published by |
| 5 | +# the Free Software Foundation; either version 2 of the License, or |
| 6 | +# (at your option) any later version. |
| 7 | +# |
| 8 | +# This program is distributed in the hope that it will be useful, |
| 9 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 10 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
| 11 | +# |
| 12 | +# See LICENSE for more details. |
| 13 | +# |
| 14 | +# Copyright: 2025 Advanced Micro Devices, Inc. |
| 15 | +# Author: Dheeraj Kumar Srivastava <[email protected]> |
| 16 | + |
| 17 | +""" |
| 18 | +Testing interrupt allocation and IOMMU remapping for PCI devices. |
| 19 | +""" |
| 20 | + |
| 21 | +import os |
| 22 | +import errno |
| 23 | +import struct |
| 24 | +from fcntl import ioctl |
| 25 | +from avocado import Test, skipIf |
| 26 | +from avocado.utils.software_manager.manager import SoftwareManager |
| 27 | +from avocado.utils import cpu, process, dmesg, linux_modules, pci, vfio |
| 28 | + |
| 29 | + |
| 30 | +def get_2k_irq_sup_bits(): |
| 31 | + """ |
| 32 | + Extracts bits 9:8 from the AMD IOMMU extended feature register2, |
| 33 | + which indicates whether IOMMU supports remapping of 2k interrupt per |
| 34 | + device function or not. |
| 35 | +
|
| 36 | + :return: bits 9:8 value from the AMD IOMMU extended feature register2 |
| 37 | + :rtype: int |
| 38 | + :raise: raise exception if sysfs entry for IOMMU extended feature register/2 |
| 39 | + is not present or not readable. |
| 40 | + """ |
| 41 | + |
| 42 | + feature_regs = "/sys/class/iommu/ivhd0/amd-iommu/features" |
| 43 | + |
| 44 | + try: |
| 45 | + with open(feature_regs, "r", encoding="utf-8") as f: |
| 46 | + features = f.read().strip() |
| 47 | + except FileNotFoundError as e: |
| 48 | + raise ValueError( |
| 49 | + f"{feature_regs} not found. Is AMD IOMMU enabled on this system?" |
| 50 | + ) from e |
| 51 | + |
| 52 | + if ":" not in features: |
| 53 | + raise ValueError(f"Unexpected format in {feature_regs}: {features}") |
| 54 | + |
| 55 | + part1, part2 = features.split(":") |
| 56 | + ext2 = part2 if len(part1) > len(part2) else part1 |
| 57 | + |
| 58 | + reg_val = int(ext2, 16) |
| 59 | + # Extract bits 9:8 -> Whether IOMMU supports 2k interrupt remapping per device function) |
| 60 | + return (reg_val >> 8) & 0b11 |
| 61 | + |
| 62 | + |
| 63 | +@skipIf(cpu.get_vendor() != "amd", "Requires AMD platform") |
| 64 | +# pylint: disable=C0103 |
| 65 | +class VFIOInterruptTest(Test): # pylint: disable=too-many-instance-attributes |
| 66 | + """ |
| 67 | + Testing interrupt allocation and IOMMU remapping for PCI devices. |
| 68 | +
|
| 69 | + :param device: full pci address including domain (0000:03:00.0) |
| 70 | + :param count: number of interrupt/s to be allocated. |
| 71 | + """ |
| 72 | + |
| 73 | + def setUp(self): |
| 74 | + """ |
| 75 | + Test initialisation, setup and pre checks |
| 76 | + """ |
| 77 | + |
| 78 | + # VFIO IOCTL constants |
| 79 | + self.pci_device = self.params.get("pci_device", default=None) |
| 80 | + self.initial_driver = pci.get_driver(self.pci_device) |
| 81 | + self.count = self.params.get("count", default=1) |
| 82 | + |
| 83 | + self.container_fd = None |
| 84 | + self.group_fd = None |
| 85 | + self.device_fd = None |
| 86 | + |
| 87 | + self.vfio_ioctls = {} |
| 88 | + |
| 89 | + smm = SoftwareManager() |
| 90 | + if not smm.check_installed("pciutils") and not smm.install("pciutils"): |
| 91 | + self.cancel("pciutils package not found and installing failed") |
| 92 | + |
| 93 | + try: |
| 94 | + self.count = int(self.count) |
| 95 | + except (ValueError, TypeError): |
| 96 | + self.cancel("'Count' input has to be an integer value") |
| 97 | + |
| 98 | + try: |
| 99 | + # Check if interrupt remapping is enabled |
| 100 | + if not dmesg.check_kernel_logs("AMD-Vi: Interrupt remapping enabled"): |
| 101 | + self.cancel("IOMMU Interrupt remapping is not enabled on the system") |
| 102 | + |
| 103 | + # Check kernel and hardware support for requested number of interrupt/s |
| 104 | + if self.count > 512 and get_2k_irq_sup_bits() == 0: |
| 105 | + self.cancel( |
| 106 | + "IOMMU HW doesnot supports remapping of more than 512 interrupts per function" |
| 107 | + ) |
| 108 | + |
| 109 | + if self.count > 2048 and ( |
| 110 | + get_2k_irq_sup_bits() == 0 or get_2k_irq_sup_bits() == 1 |
| 111 | + ): |
| 112 | + self.cancel( |
| 113 | + "IOMMU HW doesnot support remapping of more than 2048 irqs per device function" |
| 114 | + ) |
| 115 | + |
| 116 | + # Validate pci_device input |
| 117 | + if ( |
| 118 | + self.pci_device is None |
| 119 | + or self.pci_device not in pci.get_pci_addresses() |
| 120 | + ): |
| 121 | + self.cancel( |
| 122 | + "Please provide full pci address of a valid pci device on system" |
| 123 | + ) |
| 124 | + |
| 125 | + # Check whether device has MSIX capability |
| 126 | + if not pci.check_msix_capability(self.pci_device): |
| 127 | + self.cancel(f"{self.pci_device} doesnot have msix capability") |
| 128 | + |
| 129 | + # Check device for "count" number of interrupt/s support |
| 130 | + if not pci.device_supports_irqs(self.pci_device, self.count): |
| 131 | + self.cancel( |
| 132 | + f"lspci:{self.pci_device} does not support atleast {self.count} interrupts" |
| 133 | + ) |
| 134 | + |
| 135 | + # Load vfio-pci driver |
| 136 | + self.log.info( |
| 137 | + "%s", linux_modules.configure_module("vfio-pci", "CONFIG_VFIO_PCI") |
| 138 | + ) |
| 139 | + |
| 140 | + # Attach PCI device to vfio-pci driver |
| 141 | + pci.attach_driver(self.pci_device, "vfio-pci") |
| 142 | + self.log.info("Attached vfio-pci driver to %s device", self.pci_device) |
| 143 | + |
| 144 | + except Exception as e: # pylint: disable=broad-except |
| 145 | + self.cancel(f"{e}") |
| 146 | + |
| 147 | + self.get_vfio_ioctls() |
| 148 | + |
| 149 | + def get_vfio_ioctls(self): |
| 150 | + """ |
| 151 | + Get VFIO IOCTLS |
| 152 | + """ |
| 153 | + try: |
| 154 | + path = os.path.dirname(os.path.realpath(__file__)) |
| 155 | + os.chdir(f"{path}/interrupt.py.data/") |
| 156 | + cmd = "gcc get_vfio_ioctls.c -o get_vfio_ioctls" |
| 157 | + process.run(cmd, shell=True, sudo=True) |
| 158 | + output = ( |
| 159 | + process.run("./get_vfio_ioctls", shell=True, sudo=True) |
| 160 | + .stdout_text.strip() |
| 161 | + .split("\n") |
| 162 | + ) |
| 163 | + for i in output: |
| 164 | + self.vfio_ioctls[f"{i.split()[0]}"] = int(f"{i.split()[1]}") |
| 165 | + except Exception as e: # pylint: disable=broad-except |
| 166 | + self.cancel(f"Not able to get required vfio IOCTLS. Reason: {e}") |
| 167 | + finally: |
| 168 | + os.chdir(f"{path}") |
| 169 | + |
| 170 | + def test_allocate_interrupts(self): |
| 171 | + """ |
| 172 | + Request and validate interrupt/s allocation and remapping support for PCI device. |
| 173 | + """ |
| 174 | + try: |
| 175 | + # Open VFIO container |
| 176 | + self.container_fd = vfio.get_vfio_container_fd() |
| 177 | + |
| 178 | + # Validate VFIO container support |
| 179 | + vfio.check_vfio_container( |
| 180 | + self.container_fd, |
| 181 | + self.vfio_ioctls["VFIO_GET_API_VERSION"], |
| 182 | + self.vfio_ioctls["VFIO_API_VERSION"], |
| 183 | + self.vfio_ioctls["VFIO_CHECK_EXTENSION"], |
| 184 | + self.vfio_ioctls["VFIO_TYPE1_IOMMU"], |
| 185 | + ) |
| 186 | + |
| 187 | + # Get IOMMU group file descriptor |
| 188 | + self.group_fd = vfio.get_iommu_group_fd( |
| 189 | + self.pci_device, |
| 190 | + self.vfio_ioctls["VFIO_GROUP_GET_STATUS"], |
| 191 | + self.vfio_ioctls["VFIO_GROUP_FLAGS_VIABLE"], |
| 192 | + ) |
| 193 | + |
| 194 | + # Attach the IOMMU group to the VFIO container |
| 195 | + vfio.attach_group_to_container( |
| 196 | + self.group_fd, |
| 197 | + self.container_fd, |
| 198 | + self.vfio_ioctls["VFIO_GROUP_SET_CONTAINER"], |
| 199 | + ) |
| 200 | + self.log.info("Attached PCI device's IOMMU group to the VFIO container") |
| 201 | + |
| 202 | + # Set VFIO IOMMU of type 1 |
| 203 | + ioctl( |
| 204 | + self.container_fd, |
| 205 | + self.vfio_ioctls["VFIO_SET_IOMMU"], |
| 206 | + self.vfio_ioctls["VFIO_TYPE1_IOMMU"], |
| 207 | + ) |
| 208 | + |
| 209 | + # Get device file descriptor |
| 210 | + self.device_fd = vfio.get_device_fd( |
| 211 | + self.pci_device, |
| 212 | + self.group_fd, |
| 213 | + self.vfio_ioctls["VFIO_GROUP_GET_DEVICE_FD"], |
| 214 | + ) |
| 215 | + |
| 216 | + # Validate if input PCI device is capable of 2k interrupts via VFIO_DEVICE_GET_IRQ_INFO |
| 217 | + if not vfio.vfio_device_supports_irq( |
| 218 | + self.device_fd, |
| 219 | + self.vfio_ioctls["VFIO_PCI_MSIX_IRQ_INDEX"], |
| 220 | + self.vfio_ioctls["VFIO_DEVICE_GET_IRQ_INFO"], |
| 221 | + self.count, |
| 222 | + ): |
| 223 | + self.cancel( |
| 224 | + f"ioctls:{self.pci_device} doesnot support atleast {self.count} interrupts" |
| 225 | + ) |
| 226 | + |
| 227 | + except Exception as e: # pylint: disable=broad-except |
| 228 | + self.cancel(f"{e}") |
| 229 | + |
| 230 | + # Request for "count" no. of interrupt allocation for input PCI device |
| 231 | + for i in range(self.count): |
| 232 | + argsz = struct.calcsize("IIIIIi") |
| 233 | + flags = ( |
| 234 | + self.vfio_ioctls["VFIO_IRQ_SET_DATA_EVENTFD"] |
| 235 | + | self.vfio_ioctls["VFIO_IRQ_SET_ACTION_TRIGGER"] |
| 236 | + ) |
| 237 | + index = self.vfio_ioctls["VFIO_PCI_MSIX_IRQ_INDEX"] |
| 238 | + efd = os.eventfd(0, os.EFD_NONBLOCK) |
| 239 | + nirq = 1 |
| 240 | + irq_set = struct.pack("IIIIIi", argsz, flags, index, i, nirq, efd) |
| 241 | + try: |
| 242 | + ioctl(self.device_fd, self.vfio_ioctls["VFIO_DEVICE_SET_IRQS"], irq_set) |
| 243 | + except IOError as e: |
| 244 | + if e.errno == errno.ENOSPC: |
| 245 | + self.cancel("Kernel doesnot support 2k interrupt remapping feature") |
| 246 | + else: |
| 247 | + self.fail( |
| 248 | + f"Failed to allocate {self.count} irqs. Able to allocate upto {i} irqs" |
| 249 | + ) |
| 250 | + finally: |
| 251 | + os.close(efd) |
| 252 | + |
| 253 | + def tearDown(self): |
| 254 | + """ |
| 255 | + Restore PCI device state on test completion |
| 256 | + """ |
| 257 | + try: |
| 258 | + if self.device_fd is not None: |
| 259 | + # Reset VFIO PCI device |
| 260 | + ioctl(self.device_fd, self.vfio_ioctls["VFIO_DEVICE_RESET"]) |
| 261 | + |
| 262 | + # Close device file descriptor |
| 263 | + os.close(self.device_fd) |
| 264 | + |
| 265 | + # Unset container for pci device |
| 266 | + if self.group_fd is not None and self.container_fd is not None: |
| 267 | + vfio.detach_group_from_container( |
| 268 | + self.group_fd, |
| 269 | + self.container_fd, |
| 270 | + self.vfio_ioctls["VFIO_GROUP_UNSET_CONTAINER"], |
| 271 | + ) |
| 272 | + |
| 273 | + # Close PCI device's IOMMU group and container file discriptor |
| 274 | + if self.group_fd is not None: |
| 275 | + os.close(self.group_fd) |
| 276 | + |
| 277 | + if self.container_fd is not None: |
| 278 | + os.close(self.container_fd) |
| 279 | + |
| 280 | + # Attach device back to original driver |
| 281 | + if self.pci_device and self.pci_device in pci.get_pci_addresses(): |
| 282 | + if self.initial_driver is None: |
| 283 | + cur_driver = pci.get_driver(self.pci_device) |
| 284 | + if cur_driver is not None: |
| 285 | + pci.unbind(cur_driver, self.pci_device) |
| 286 | + else: |
| 287 | + pci.attach_driver(self.pci_device, self.initial_driver) |
| 288 | + except Exception as e: # pylint: disable=broad-except |
| 289 | + self.fail(f"TearDown failed: Reason {e}") |
0 commit comments