From 4bad734d5e18fe596cdd90ed610d67923478dc93 Mon Sep 17 00:00:00 2001 From: Dmitry Lopatin Date: Tue, 25 Nov 2025 18:11:22 +0300 Subject: [PATCH 1/3] refactor(kubevirt): bump version to 1.6.2 Signed-off-by: Dmitry Lopatin --- Taskfile.yaml | 8 +- api/client/examples/cancel-evacuation/go.mod | 4 +- api/client/examples/cancel-evacuation/go.sum | 2 + api/client/examples/list-resources/go.mod | 4 +- api/client/examples/list-resources/go.sum | 2 + api/go.mod | 6 +- api/go.sum | 4 + build/components/versions.yml | 2 +- crds/embedded/virtualmachineinstances.yaml | 472 ++++++++++++++++-- crds/embedded/virtualmachines.yaml | 426 ++++++++++++++-- images/hooks/go.mod | 2 +- images/hooks/go.sum | 4 +- images/virt-api/debug/dlv.Dockerfile | 10 +- images/virt-artifact/werf.inc.yaml | 2 +- images/virt-controller/debug/dlv.Dockerfile | 10 +- images/virt-handler/debug/dlv.Dockerfile | 20 +- images/virt-handler/werf.inc.yaml | 4 - images/virtualization-artifact/go.mod | 4 +- images/virtualization-artifact/go.sum | 4 +- .../livemigration/migration_configuration.go | 3 + images/vm-route-forge/go.mod | 5 +- images/vm-route-forge/go.sum | 10 +- src/cli/go.mod | 4 +- src/cli/go.sum | 4 + templates/kubevirt/kubevirt.yaml | 4 - .../kubevirt/virt-operator/rbac-for-us.yaml | 15 + test/e2e/go.mod | 2 +- test/e2e/go.sum | 2 + 28 files changed, 899 insertions(+), 140 deletions(-) diff --git a/Taskfile.yaml b/Taskfile.yaml index 93eef4dcee..020d59f4f7 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -165,8 +165,7 @@ tasks: "ports": [ { "containerPort": 2345, "name": "dlv" } ], "readinessProbe": null, "livenessProbe": null, - "command": null, - "args": [] + "command": null }, { "name": "proxy", @@ -264,8 +263,7 @@ tasks: "ports": [ { "containerPort": 2345, "name": "dlv" } ], "readinessProbe": null, "livenessProbe": null, - "command": null, - "args": [] + "command": null }, { "name": "proxy", @@ -282,5 +280,5 @@ tasks: } } }' - kubectl -n d8-virtualization port-forward deploy/vit-api 2345:2345 + kubectl -n d8-virtualization port-forward deploy/virt-api 2345:2345 EOF diff --git a/api/client/examples/cancel-evacuation/go.mod b/api/client/examples/cancel-evacuation/go.mod index 63f821ce75..86339fe572 100644 --- a/api/client/examples/cancel-evacuation/go.mod +++ b/api/client/examples/cancel-evacuation/go.mod @@ -53,8 +53,8 @@ require ( k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250701173324-9bd5c66d9911 // indirect k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect - kubevirt.io/api v1.3.1 // indirect - kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 // indirect + kubevirt.io/api v1.6.2 // indirect + kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/randfill v1.0.0 // indirect diff --git a/api/client/examples/cancel-evacuation/go.sum b/api/client/examples/cancel-evacuation/go.sum index d7db165b6a..cf69ab2158 100644 --- a/api/client/examples/cancel-evacuation/go.sum +++ b/api/client/examples/cancel-evacuation/go.sum @@ -372,8 +372,10 @@ k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6J k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/api v1.3.1 h1:MoTNo/zvDlZ44c2ocXLPln8XTaQOeUodiYbEKrTCqv4= kubevirt.io/api v1.3.1/go.mod h1:tCn7VAZktEvymk490iPSMPCmKM9UjbbfH2OsFR/IOLU= +kubevirt.io/api v1.6.2/go.mod h1:p66fEy/g79x7VpgUwrkUgOoG2lYs5LQq37WM6JXMwj4= kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 h1:IWo12+ei3jltSN5jQN1xjgakfvRSF3G3Rr4GXVOOy2I= kubevirt.io/containerized-data-importer-api v1.57.0-alpha1/go.mod h1:Y/8ETgHS1GjO89bl682DPtQOYEU/1ctPFBz6Sjxm4DM= +kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9/go.mod h1:SDJjLGhbPyayDqAqawcGmVNapBp0KodOQvhKPLVGCQU= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= diff --git a/api/client/examples/list-resources/go.mod b/api/client/examples/list-resources/go.mod index 8251bcd219..9888494628 100644 --- a/api/client/examples/list-resources/go.mod +++ b/api/client/examples/list-resources/go.mod @@ -49,8 +49,8 @@ require ( k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250701173324-9bd5c66d9911 // indirect k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect - kubevirt.io/api v1.3.1 // indirect - kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 // indirect + kubevirt.io/api v1.6.2 // indirect + kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/randfill v1.0.0 // indirect diff --git a/api/client/examples/list-resources/go.sum b/api/client/examples/list-resources/go.sum index 3dba2c12dd..2e9bbd8060 100644 --- a/api/client/examples/list-resources/go.sum +++ b/api/client/examples/list-resources/go.sum @@ -365,8 +365,10 @@ k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6J k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/api v1.3.1 h1:MoTNo/zvDlZ44c2ocXLPln8XTaQOeUodiYbEKrTCqv4= kubevirt.io/api v1.3.1/go.mod h1:tCn7VAZktEvymk490iPSMPCmKM9UjbbfH2OsFR/IOLU= +kubevirt.io/api v1.6.2/go.mod h1:p66fEy/g79x7VpgUwrkUgOoG2lYs5LQq37WM6JXMwj4= kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 h1:IWo12+ei3jltSN5jQN1xjgakfvRSF3G3Rr4GXVOOy2I= kubevirt.io/containerized-data-importer-api v1.57.0-alpha1/go.mod h1:Y/8ETgHS1GjO89bl682DPtQOYEU/1ctPFBz6Sjxm4DM= +kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9/go.mod h1:SDJjLGhbPyayDqAqawcGmVNapBp0KodOQvhKPLVGCQU= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= diff --git a/api/go.mod b/api/go.mod index 79b0495d1e..a4b3951825 100644 --- a/api/go.mod +++ b/api/go.mod @@ -17,7 +17,8 @@ require ( k8s.io/apiextensions-apiserver v0.33.3 k8s.io/apimachinery v0.33.3 k8s.io/client-go v0.33.3 - kubevirt.io/api v1.3.1 + k8s.io/kube-openapi v0.0.0-20250701173324-9bd5c66d9911 + kubevirt.io/api v1.6.2 sigs.k8s.io/controller-runtime v0.21.0 ) @@ -70,9 +71,8 @@ require ( k8s.io/code-generator v0.33.3 // indirect k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20250701173324-9bd5c66d9911 // indirect k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect - kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 // indirect + kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect sigs.k8s.io/controller-tools v0.18.0 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect diff --git a/api/go.sum b/api/go.sum index fd5f073e7c..620d6a5a4a 100644 --- a/api/go.sum +++ b/api/go.sum @@ -570,8 +570,12 @@ k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6J k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/api v1.3.1 h1:MoTNo/zvDlZ44c2ocXLPln8XTaQOeUodiYbEKrTCqv4= kubevirt.io/api v1.3.1/go.mod h1:tCn7VAZktEvymk490iPSMPCmKM9UjbbfH2OsFR/IOLU= +kubevirt.io/api v1.6.2 h1:aoqZ4KsbOyDjLnuDw7H9wEgE/YTd/q5BBmYeQjJNizc= +kubevirt.io/api v1.6.2/go.mod h1:p66fEy/g79x7VpgUwrkUgOoG2lYs5LQq37WM6JXMwj4= kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 h1:IWo12+ei3jltSN5jQN1xjgakfvRSF3G3Rr4GXVOOy2I= kubevirt.io/containerized-data-importer-api v1.57.0-alpha1/go.mod h1:Y/8ETgHS1GjO89bl682DPtQOYEU/1ctPFBz6Sjxm4DM= +kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9 h1:KTb8wO1Lxj220DX7d2Rdo9xovvlyWWNo3AVm2ua+1nY= +kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9/go.mod h1:SDJjLGhbPyayDqAqawcGmVNapBp0KodOQvhKPLVGCQU= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= diff --git a/build/components/versions.yml b/build/components/versions.yml index 906d683c8a..9e11a9d2b5 100644 --- a/build/components/versions.yml +++ b/build/components/versions.yml @@ -3,7 +3,7 @@ firmware: libvirt: v10.9.0 edk2: stable202411 core: - 3p-kubevirt: v1.3.1-v12n.25 + 3p-kubevirt: v1.6.2-v12n.0.dlopatin4 3p-containerized-data-importer: v1.60.3-v12n.12 distribution: 2.8.3 package: diff --git a/crds/embedded/virtualmachineinstances.yaml b/crds/embedded/virtualmachineinstances.yaml index b9cc1d5949..b676c9ed3a 100644 --- a/crds/embedded/virtualmachineinstances.yaml +++ b/crds/embedded/virtualmachineinstances.yaml @@ -461,7 +461,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -476,7 +476,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -642,7 +642,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -657,7 +657,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -820,7 +820,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -835,7 +835,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1001,7 +1001,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1016,7 +1016,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1127,9 +1127,12 @@ spec: of a pod. properties: name: - description: Required. + description: |- + Name is this DNS resolver option's name. + Required. type: string value: + description: Value is this DNS resolver option's value. type: string type: object type: array @@ -1447,7 +1450,11 @@ spec: cache: description: |- Cache specifies which kvm disk cache mode should be used. - Supported values are: CacheNone, CacheWriteThrough. + Supported values are: + none: Guest I/O not cached on the host, but may be kept in a disk cache. + writethrough: Guest I/O cached on the host but written through to the physical medium. Slowest but with most guarantees. + writeback: Guest I/O cached on the host. + Defaults to none if the storage supports O_DIRECT, otherwise writethrough. type: string cdrom: description: Attach a volume as a cdrom to the vmi. @@ -1567,12 +1574,24 @@ spec: description: Whether to attach a GPU device to the vmi. items: properties: + claimName: + description: |- + ClaimName needs to be provided from the list vmi.spec.resourceClaims[].name where this + device is allocated + type: string deviceName: + description: DeviceName is the name of the device provisioned + by device-plugins type: string name: description: Name of the GPU device as exposed by a device plugin type: string + requestName: + description: |- + RequestName needs to be provided from resourceClaim.spec.devices.requests[].name where this + device is requested + type: string tag: description: If specified, the virtual network interface address and its tag will be provided to the guest @@ -1601,7 +1620,6 @@ spec: type: object type: object required: - - deviceName - name type: object type: array @@ -1610,19 +1628,28 @@ spec: description: Whether to attach a host device to the vmi. items: properties: + claimName: + description: |- + ClaimName needs to be provided from the list vmi.spec.resourceClaims[].name where this + device is allocated + type: string deviceName: - description: DeviceName is the resource name of the - host device exposed by a device plugin + description: DeviceName is the name of the device provisioned + by device-plugins type: string name: type: string + requestName: + description: |- + RequestName needs to be provided from resourceClaim.spec.devices.requests[].name where this + device is requested + type: string tag: description: If specified, the virtual network interface address and its tag will be provided to the guest via config drive type: string required: - - deviceName - name type: object type: array @@ -1743,9 +1770,8 @@ spec: model: description: |- Interface model. - One of: e1000, e1000e, ne2k_pci, pcnet, rtl8139, virtio. + One of: e1000, e1000e, igb, ne2k_pci, pcnet, rtl8139, virtio. Defaults to virtio. - TODO:(ihar) switch to enums once opengen-api supports them. See: https://github.com/kubernetes/kube-openapi/issues/51 type: string name: description: |- @@ -1805,7 +1831,11 @@ spec: state: description: |- State represents the requested operational state of the interface. - The (only) value supported is 'absent', expressing a request to remove the interface. + The supported values are: + 'absent', expressing a request to remove the interface. + 'down', expressing a request to set the link down. + 'up', expressing a request to set the link up. + Empty value functions as 'up'. type: string tag: description: If specified, the virtual network interface @@ -1831,6 +1861,19 @@ spec: depends on additional factors of the VirtualMachineInstance, like the number of guest CPUs. type: boolean + panicDevices: + description: PanicDevices provides additional crash information + when a guest crashes. + items: + properties: + model: + description: |- + Model specifies what type of panic device is provided. + The panic model used when this attribute is missing depends on the hypervisor and guest arch. + One of: isa, hyperv, pvpanic. + type: string + type: object + type: array rng: description: Whether to have random number generator from host @@ -1853,6 +1896,11 @@ spec: tpm: description: Whether to emulate a TPM device. properties: + enabled: + description: |- + Enabled allows a user to explicitly disable the vTPM even when one is enabled by a preference referenced by the VirtualMachine + Defaults to True + type: boolean persistent: description: |- Persistent indicates the state of the TPM device should be kept accross reboots @@ -1865,10 +1913,30 @@ spec: This is helpful for old machines like CentOS6 or RHEL6 which do not understand virtio_non_transitional (virtio 1.0). type: boolean + video: + description: Video describes the video device configuration + for the vmi. + properties: + type: + description: |- + Type specifies the video device type (e.g., virtio, vga, bochs, ramfb). + If not specified, the default is architecture-dependent (VGA for BIOS-based VMs, Bochs for EFI-based VMs on AMD64; virtio for Arm and s390x). + type: string + type: object watchdog: description: Watchdog describes a watchdog device which can be added to the vmi. properties: + diag288: + description: diag288 watchdog device (specific to s390x + architecture). + properties: + action: + description: |- + The action to take. Valid values are poweroff, reset, shutdown. + Defaults to reset. + type: string + type: object i6300esb: description: i6300esb watchdog device. properties: @@ -2138,6 +2206,11 @@ spec: acpi: description: Information that can be set in the ACPI table properties: + msdmNameRef: + description: |- + Similar to SlicNameRef, another ACPI entry that is used in more recent Windows versions. + The above points to the spec of MSDM too. + type: string slicNameRef: description: |- SlicNameRef should match the volume name of a secret object. The data in the secret should @@ -2221,11 +2294,20 @@ spec: Defaults to a random generated uid. type: string type: object + ioThreads: + description: IOThreads specifies the IOThreads options. + properties: + supplementalPoolThreadCount: + description: SupplementalPoolThreadCount specifies how many + iothreads are allocated for the supplementalPool policy. + format: int32 + type: integer + type: object ioThreadsPolicy: description: |- Controls whether or not disks will share IOThreads. Omitting IOThreadsPolicy disables use of IOThreads. - One of: shared, auto + One of: shared, auto, supplementalPool type: string launchSecurity: description: Launch Security setting of the vmi. @@ -2340,7 +2422,7 @@ spec: - "None": No action will be taken, according to the specified 'RunStrategy' the VirtualMachine will be restarted or shutdown. - "LiveMigrate": the VirtualMachineInstance will be migrated instead of being shutdown. - "LiveMigrateIfPossible": the same as "LiveMigrate" but only if the VirtualMachine is Live-Migratable, otherwise it will behave as "None". - - "External": the VirtualMachineInstance will be protected by a PDB and 'vmi.Status.EvacuationNodeName' will be set on eviction. This is mainly useful for cluster-api-provider-kubevirt (capk) which needs a way for VMI's to be blocked from eviction, yet signal capk that eviction has been called on the VMI so the capk controller can handle tearing the VMI down. Details can be found in the commit description https://github.com/kubevirt/kubevirt/commit/c1d77face705c8b126696bac9a3ee3825f27f1fa. + - "External": the VirtualMachineInstance will be protected and 'vmi.Status.EvacuationNodeName' will be set on eviction. This is mainly useful for cluster-api-provider-kubevirt (capk) which needs a way for VMI's to be blocked from eviction, yet signal capk that eviction has been called on the VMI so the capk controller can handle tearing the VMI down. Details can be found in the commit description https://github.com/kubevirt/kubevirt/commit/c1d77face705c8b126696bac9a3ee3825f27f1fa. type: string hostname: description: |- @@ -2453,7 +2535,6 @@ spec: description: |- TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported - TODO: implement a realistic TCP lifecycle hook properties: host: description: 'Optional: Host name to connect to, defaults @@ -2652,7 +2733,6 @@ spec: description: |- TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported - TODO: implement a realistic TCP lifecycle hook properties: host: description: 'Optional: Host name to connect to, defaults @@ -2682,6 +2762,65 @@ spec: format: int32 type: integer type: object + resourceClaims: + description: |- + ResourceClaims define which ResourceClaims must be allocated + and reserved before the VMI, hence virt-launcher pod is allowed to start. The resources + will be made available to the domain which consumes them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate in kubernetes + https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/ + This field should only be configured if one of the feature-gates GPUsWithDRA or HostDevicesWithDRA is enabled. + This feature is in alpha. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map schedulerName: description: |- If specified, the VMI will be dispatched by specified scheduler. @@ -2809,7 +2948,6 @@ spec: Keys that don't exist in the incoming pod labels will be ignored. A null or empty list means only match against labelSelector. - This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). items: type: string @@ -2849,7 +2987,6 @@ spec: Valid values are integers greater than 0. When value is not nil, WhenUnsatisfiable must be DoNotSchedule. - For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same labelSelector spread as 2/2/2: | zone1 | zone2 | zone3 | @@ -2867,7 +3004,6 @@ spec: - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. - If this value is nil, the behavior is equivalent to the Honor policy. This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. type: string @@ -2879,7 +3015,6 @@ spec: has a toleration, are included. - Ignore: node taints are ignored. All nodes are included. - If this value is nil, the behavior is equivalent to the Ignore policy. This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. type: string @@ -2952,10 +3087,13 @@ spec: that contains config drive networkdata. properties: name: + default: "" description: |- Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -2964,10 +3102,13 @@ spec: contains config drive userdata. properties: name: + default: "" description: |- Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -2999,10 +3140,13 @@ spec: that contains NoCloud networkdata. properties: name: + default: "" description: |- Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -3011,10 +3155,13 @@ spec: contains NoCloud userdata. properties: name: + default: "" description: |- Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -3033,10 +3180,13 @@ spec: More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/ properties: name: + default: "" description: |- Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string optional: description: Specify whether the ConfigMap or it's keys @@ -3076,6 +3226,8 @@ spec: description: Path defines the path to disk file in the container type: string hotpluggable: + description: Hotpluggable indicates whether the volume can + be hotplugged and hotunplugged. type: boolean required: - image @@ -3090,9 +3242,8 @@ spec: be hotplugged and hotunplugged. type: boolean name: - description: |- - Name of both the DataVolume and the PVC in the same namespace. - After PVC population the DataVolume is garbage collected by default. + description: Name of both the DataVolume and the PVC in + the same namespace. type: string required: - name @@ -3340,10 +3491,13 @@ spec: be attached as disk of CDROM type. properties: name: + default: "" description: |- Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -3353,10 +3507,13 @@ spec: be attached as disk of CDROM type. properties: name: + default: "" description: |- Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -3436,6 +3593,97 @@ spec: format: int32 type: integer type: object + deviceStatus: + description: |- + DeviceStatus reflects the state of devices requested in spec.domain.devices. This is an optional field available + only when DRA feature gate is enabled + This field will only be populated if one of the feature-gates GPUsWithDRA or HostDevicesWithDRA is enabled. + This feature is in alpha. + properties: + gpuStatuses: + description: GPUStatuses reflects the state of GPUs requested + in spec.domain.devices.gpus + items: + properties: + deviceResourceClaimStatus: + description: DeviceResourceClaimStatus reflects the DRA + related information for the device + properties: + attributes: + description: |- + Attributes are properties of the device that could be used by kubevirt and other copmonents to learn more + about the device, like pciAddress or mdevUUID + properties: + mDevUUID: + description: MDevUUID is the mediated device uuid + of the allocated device + type: string + pciAddress: + description: PCIAddress is the PCIe bus address + of the allocated device + type: string + type: object + name: + description: Name is the name of actual device on the + host provisioned by the driver as reflected in resourceclaim.status + type: string + resourceClaimName: + description: ResourceClaimName is the name of the resource + claims object used to provision this resource + type: string + type: object + name: + description: Name of the device as specified in spec.domain.devices.gpus.name + or spec.domain.devices.hostDevices.name + type: string + required: + - name + type: object + type: array + x-kubernetes-list-type: atomic + hostDeviceStatuses: + description: |- + HostDeviceStatuses reflects the state of GPUs requested in spec.domain.devices.hostDevices + DRA + items: + properties: + deviceResourceClaimStatus: + description: DeviceResourceClaimStatus reflects the DRA + related information for the device + properties: + attributes: + description: |- + Attributes are properties of the device that could be used by kubevirt and other copmonents to learn more + about the device, like pciAddress or mdevUUID + properties: + mDevUUID: + description: MDevUUID is the mediated device uuid + of the allocated device + type: string + pciAddress: + description: PCIAddress is the PCIe bus address + of the allocated device + type: string + type: object + name: + description: Name is the name of actual device on the + host provisioned by the driver as reflected in resourceclaim.status + type: string + resourceClaimName: + description: ResourceClaimName is the name of the resource + claims object used to provision this resource + type: string + type: object + name: + description: Name of the device as specified in spec.domain.devices.gpus.name + or spec.domain.devices.hostDevices.name + type: string + required: + - name + type: object + type: array + x-kubernetes-list-type: atomic + type: object evacuationNodeName: description: |- EvacuationNodeName is used to track the eviction process of a VMI. It stores the name of the node that we want @@ -3443,8 +3691,9 @@ spec: type: string fsFreezeStatus: description: |- - FSFreezeStatus is the state of the fs of the guest - it can be either frozen or thawed + FSFreezeStatus indicates whether a freeze operation was requested for the guest filesystem. + It will be set to "frozen" if the request was made, or unset otherwise. + This does not reflect the actual state of the guest filesystem. type: string guestOSInfo: description: Guest OS Information @@ -3496,6 +3745,10 @@ spec: items: type: string type: array + linkState: + description: 'LinkState Reports the current operational link + state''. values: up, down.' + type: string mac: description: Hardware address of a Virtual Machine interface type: string @@ -3503,6 +3756,10 @@ spec: description: Name of the interface, corresponds to name of the network assigned to the interface type: string + podInterfaceName: + description: PodInterfaceName represents the name of the pod + network interface + type: string queueCount: description: Specifies how many queues are allocated by MultiQueue format: int32 @@ -3731,6 +3988,13 @@ spec: If set to true, migrations will still start in pre-copy, but switch to post-copy when CompletionTimeoutPerGiB triggers. Defaults to false type: boolean + allowWorkloadDisruption: + description: |- + AllowWorkloadDisruption indicates that the migration shouldn't be + canceled after acceptableCompletionTime is exceeded. Instead, if + permitted, migration will be switched to post-copy or the VMI will be + paused to allow the migration to complete + type: boolean bandwidthPerMigration: anyOf: - type: integer @@ -3743,8 +4007,8 @@ spec: completionTimeoutPerGiB: description: |- CompletionTimeoutPerGiB is the maximum number of seconds per GiB a migration is allowed to take. - If a live-migration takes longer to migrate than this value multiplied by the size of the VMI, - the migration will be cancelled, unless AllowPostCopy is true. Defaults to 800 + If the timeout is reached, the migration will be either paused, switched + to post-copy or cancelled depending on other settings. Defaults to 150 format: int64 type: integer disableTLS: @@ -3794,6 +4058,9 @@ spec: indicates the migration will be unsafe to the guest. Defaults to false type: boolean type: object + migrationNetworkType: + description: The type of migration network, either 'pod' or 'migration' + type: string migrationPolicyName: description: Name of the migration policy. If string is empty, no policy is matched @@ -3809,8 +4076,57 @@ spec: sourceNode: description: The source node that the VMI originated on type: string + sourcePersistentStatePVCName: + description: If the VMI being migrated uses persistent features + (backend-storage), its source PVC name is saved here + type: string sourcePod: type: string + sourceState: + description: SourceState contains migration state managed by the + source virt handler + properties: + domainName: + description: The name of the domain on the source libvirt + domain + type: string + domainNamespace: + description: Namespace used in the name of the source libvirt + domain. Can be used to find and modify paths in the domain + type: string + migrationUID: + description: The Source VirtualMachineInstanceMigration object + associated with this migration + type: string + node: + description: The source node that the VMI originated on + type: string + nodeSelectors: + additionalProperties: + type: string + description: Node selectors needed by the target to start + the receiving pod. + type: object + persistentStatePVCName: + description: If the VMI being migrated uses persistent features + (backend-storage), its source PVC name is saved here + type: string + pod: + description: The source pod that the VMI is originated on + type: string + selinuxContext: + description: SELinuxContext is the actual SELinux context + of the pod + type: string + syncAddress: + description: The ip address/fqdn:port combination to use to + synchronize the VMI with the target. + type: string + virtualMachineInstanceUID: + description: VirtualMachineInstanceUID is the UID of the target + virtual machine instance + type: string + type: object startTimestamp: description: The time the migration action began format: date-time @@ -3853,9 +4169,87 @@ spec: If the VMI requires dedicated CPUs, this field will hold the numa topology on the target node type: string + targetPersistentStatePVCName: + description: If the VMI being migrated uses persistent features + (backend-storage), its target PVC name is saved here + type: string targetPod: description: The target pod that the VMI is moving to type: string + targetState: + description: TargetState contains migration state managed by the + target virt handler + properties: + attachmentPodUID: + description: The UID of the target attachment pod for hotplug + volumes + type: string + cpuSet: + description: |- + If the VMI requires dedicated CPUs, this field will + hold the dedicated CPU set on the target node + items: + type: integer + type: array + x-kubernetes-list-type: atomic + directMigrationNodePorts: + additionalProperties: + type: integer + description: The list of ports opened for live migration on + the destination node + type: object + domainDetected: + description: The Target Node has seen the Domain Start Event + type: boolean + domainName: + description: The name of the domain on the source libvirt + domain + type: string + domainNamespace: + description: Namespace used in the name of the source libvirt + domain. Can be used to find and modify paths in the domain + type: string + domainReadyTimestamp: + description: The timestamp at which the target node detects + the domain is active + format: date-time + type: string + migrationUID: + description: The Source VirtualMachineInstanceMigration object + associated with this migration + type: string + node: + description: The source node that the VMI originated on + type: string + nodeAddress: + description: The address of the target node to use for the + migration + type: string + nodeTopology: + description: |- + If the VMI requires dedicated CPUs, this field will + hold the numa topology on the target node + type: string + persistentStatePVCName: + description: If the VMI being migrated uses persistent features + (backend-storage), its source PVC name is saved here + type: string + pod: + description: The source pod that the VMI is originated on + type: string + selinuxContext: + description: SELinuxContext is the actual SELinux context + of the pod + type: string + syncAddress: + description: The ip address/fqdn:port combination to use to + synchronize the VMI with the target. + type: string + virtualMachineInstanceUID: + description: VirtualMachineInstanceUID is the UID of the target + virtual machine instance + type: string + type: object type: object migrationTransport: description: This represents the migration transport diff --git a/crds/embedded/virtualmachines.yaml b/crds/embedded/virtualmachines.yaml index 72d9f865f6..a0bfe284f4 100644 --- a/crds/embedded/virtualmachines.yaml +++ b/crds/embedded/virtualmachines.yaml @@ -319,7 +319,7 @@ spec: set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ - (Alpha) Using this field requires the VolumeAttributesClass feature gate to be enabled. + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -812,6 +812,7 @@ spec: description: |- Running controls whether the associatied VirtualMachineInstance is created or not Mutually exclusive with RunStrategy + Deprecated: VirtualMachineInstance field "Running" is now deprecated, please use RunStrategy instead. type: boolean template: description: Template is the direct specification of VirtualMachineInstance @@ -1219,7 +1220,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1234,7 +1235,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1402,7 +1403,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1417,7 +1418,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1583,7 +1584,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1598,7 +1599,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1766,7 +1767,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1781,7 +1782,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - This is an alpha field and requires enabling MatchLabelKeysInPodAffinity feature gate. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -1894,9 +1895,13 @@ spec: options of a pod. properties: name: - description: Required. + description: |- + Name is this DNS resolver option's name. + Required. type: string value: + description: Value is this DNS resolver option's + value. type: string type: object type: array @@ -2217,7 +2222,11 @@ spec: cache: description: |- Cache specifies which kvm disk cache mode should be used. - Supported values are: CacheNone, CacheWriteThrough. + Supported values are: + none: Guest I/O not cached on the host, but may be kept in a disk cache. + writethrough: Guest I/O cached on the host but written through to the physical medium. Slowest but with most guarantees. + writeback: Guest I/O cached on the host. + Defaults to none if the storage supports O_DIRECT, otherwise writethrough. type: string cdrom: description: Attach a volume as a cdrom to the @@ -2344,12 +2353,24 @@ spec: vmi. items: properties: + claimName: + description: |- + ClaimName needs to be provided from the list vmi.spec.resourceClaims[].name where this + device is allocated + type: string deviceName: + description: DeviceName is the name of the device + provisioned by device-plugins type: string name: description: Name of the GPU device as exposed by a device plugin type: string + requestName: + description: |- + RequestName needs to be provided from resourceClaim.spec.devices.requests[].name where this + device is requested + type: string tag: description: If specified, the virtual network interface address and its tag will be provided @@ -2388,19 +2409,28 @@ spec: vmi. items: properties: + claimName: + description: |- + ClaimName needs to be provided from the list vmi.spec.resourceClaims[].name where this + device is allocated + type: string deviceName: - description: DeviceName is the resource name - of the host device exposed by a device plugin + description: DeviceName is the name of the device + provisioned by device-plugins type: string name: type: string + requestName: + description: |- + RequestName needs to be provided from resourceClaim.spec.devices.requests[].name where this + device is requested + type: string tag: description: If specified, the virtual network interface address and its tag will be provided to the guest via config drive type: string required: - - deviceName - name type: object type: array @@ -2523,9 +2553,8 @@ spec: model: description: |- Interface model. - One of: e1000, e1000e, ne2k_pci, pcnet, rtl8139, virtio. + One of: e1000, e1000e, igb, ne2k_pci, pcnet, rtl8139, virtio. Defaults to virtio. - TODO:(ihar) switch to enums once opengen-api supports them. See: https://github.com/kubernetes/kube-openapi/issues/51 type: string name: description: |- @@ -2587,7 +2616,11 @@ spec: state: description: |- State represents the requested operational state of the interface. - The (only) value supported is 'absent', expressing a request to remove the interface. + The supported values are: + 'absent', expressing a request to remove the interface. + 'down', expressing a request to set the link down. + 'up', expressing a request to set the link up. + Empty value functions as 'up'. type: string tag: description: If specified, the virtual network @@ -2614,6 +2647,19 @@ spec: of the VirtualMachineInstance, like the number of guest CPUs. type: boolean + panicDevices: + description: PanicDevices provides additional crash + information when a guest crashes. + items: + properties: + model: + description: |- + Model specifies what type of panic device is provided. + The panic model used when this attribute is missing depends on the hypervisor and guest arch. + One of: isa, hyperv, pvpanic. + type: string + type: object + type: array rng: description: Whether to have random number generator from host @@ -2637,6 +2683,11 @@ spec: tpm: description: Whether to emulate a TPM device. properties: + enabled: + description: |- + Enabled allows a user to explicitly disable the vTPM even when one is enabled by a preference referenced by the VirtualMachine + Defaults to True + type: boolean persistent: description: |- Persistent indicates the state of the TPM device should be kept accross reboots @@ -2649,10 +2700,30 @@ spec: This is helpful for old machines like CentOS6 or RHEL6 which do not understand virtio_non_transitional (virtio 1.0). type: boolean + video: + description: Video describes the video device configuration + for the vmi. + properties: + type: + description: |- + Type specifies the video device type (e.g., virtio, vga, bochs, ramfb). + If not specified, the default is architecture-dependent (VGA for BIOS-based VMs, Bochs for EFI-based VMs on AMD64; virtio for Arm and s390x). + type: string + type: object watchdog: description: Watchdog describes a watchdog device which can be added to the vmi. properties: + diag288: + description: diag288 watchdog device (specific + to s390x architecture). + properties: + action: + description: |- + The action to take. Valid values are poweroff, reset, shutdown. + Defaults to reset. + type: string + type: object i6300esb: description: i6300esb watchdog device. properties: @@ -2923,6 +2994,11 @@ spec: description: Information that can be set in the ACPI table properties: + msdmNameRef: + description: |- + Similar to SlicNameRef, another ACPI entry that is used in more recent Windows versions. + The above points to the spec of MSDM too. + type: string slicNameRef: description: |- SlicNameRef should match the volume name of a secret object. The data in the secret should @@ -3009,11 +3085,21 @@ spec: Defaults to a random generated uid. type: string type: object + ioThreads: + description: IOThreads specifies the IOThreads options. + properties: + supplementalPoolThreadCount: + description: SupplementalPoolThreadCount specifies + how many iothreads are allocated for the supplementalPool + policy. + format: int32 + type: integer + type: object ioThreadsPolicy: description: |- Controls whether or not disks will share IOThreads. Omitting IOThreadsPolicy disables use of IOThreads. - One of: shared, auto + One of: shared, auto, supplementalPool type: string launchSecurity: description: Launch Security setting of the vmi. @@ -3129,7 +3215,7 @@ spec: - "None": No action will be taken, according to the specified 'RunStrategy' the VirtualMachine will be restarted or shutdown. - "LiveMigrate": the VirtualMachineInstance will be migrated instead of being shutdown. - "LiveMigrateIfPossible": the same as "LiveMigrate" but only if the VirtualMachine is Live-Migratable, otherwise it will behave as "None". - - "External": the VirtualMachineInstance will be protected by a PDB and 'vmi.Status.EvacuationNodeName' will be set on eviction. This is mainly useful for cluster-api-provider-kubevirt (capk) which needs a way for VMI's to be blocked from eviction, yet signal capk that eviction has been called on the VMI so the capk controller can handle tearing the VMI down. Details can be found in the commit description https://github.com/kubevirt/kubevirt/commit/c1d77face705c8b126696bac9a3ee3825f27f1fa. + - "External": the VirtualMachineInstance will be protected and 'vmi.Status.EvacuationNodeName' will be set on eviction. This is mainly useful for cluster-api-provider-kubevirt (capk) which needs a way for VMI's to be blocked from eviction, yet signal capk that eviction has been called on the VMI so the capk controller can handle tearing the VMI down. Details can be found in the commit description https://github.com/kubevirt/kubevirt/commit/c1d77face705c8b126696bac9a3ee3825f27f1fa. type: string hostname: description: |- @@ -3242,7 +3328,6 @@ spec: description: |- TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported - TODO: implement a realistic TCP lifecycle hook properties: host: description: 'Optional: Host name to connect to, defaults @@ -3441,7 +3526,6 @@ spec: description: |- TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported - TODO: implement a realistic TCP lifecycle hook properties: host: description: 'Optional: Host name to connect to, defaults @@ -3471,6 +3555,65 @@ spec: format: int32 type: integer type: object + resourceClaims: + description: |- + ResourceClaims define which ResourceClaims must be allocated + and reserved before the VMI, hence virt-launcher pod is allowed to start. The resources + will be made available to the domain which consumes them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate in kubernetes + https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/ + This field should only be configured if one of the feature-gates GPUsWithDRA or HostDevicesWithDRA is enabled. + This feature is in alpha. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map schedulerName: description: |- If specified, the VMI will be dispatched by specified scheduler. @@ -3600,7 +3743,6 @@ spec: Keys that don't exist in the incoming pod labels will be ignored. A null or empty list means only match against labelSelector. - This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). items: type: string @@ -3640,7 +3782,6 @@ spec: Valid values are integers greater than 0. When value is not nil, WhenUnsatisfiable must be DoNotSchedule. - For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same labelSelector spread as 2/2/2: | zone1 | zone2 | zone3 | @@ -3658,7 +3799,6 @@ spec: - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. - If this value is nil, the behavior is equivalent to the Honor policy. This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. type: string @@ -3670,7 +3810,6 @@ spec: has a toleration, are included. - Ignore: node taints are ignored. All nodes are included. - If this value is nil, the behavior is equivalent to the Ignore policy. This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. type: string @@ -3743,10 +3882,13 @@ spec: secret that contains config drive networkdata. properties: name: + default: "" description: |- Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -3755,10 +3897,13 @@ spec: secret that contains config drive userdata. properties: name: + default: "" description: |- Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -3790,10 +3935,13 @@ spec: secret that contains NoCloud networkdata. properties: name: + default: "" description: |- Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -3802,10 +3950,13 @@ spec: secret that contains NoCloud userdata. properties: name: + default: "" description: |- Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -3824,10 +3975,13 @@ spec: More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/ properties: name: + default: "" description: |- Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string optional: description: Specify whether the ConfigMap or it's @@ -3882,9 +4036,8 @@ spec: volume can be hotplugged and hotunplugged. type: boolean name: - description: |- - Name of both the DataVolume and the PVC in the same namespace. - After PVC population the DataVolume is garbage collected by default. + description: Name of both the DataVolume and the + PVC in the same namespace. type: string required: - name @@ -4137,10 +4290,13 @@ spec: that should be attached as disk of CDROM type. properties: name: + default: "" description: |- Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -4150,10 +4306,13 @@ spec: that should be attached as disk of CDROM type. properties: name: + default: "" description: |- Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid? type: string type: object x-kubernetes-map-type: atomic @@ -4219,6 +4378,35 @@ spec: updated through an Update() before ObservedGeneration in Status. format: int64 type: integer + instancetypeRef: + description: InstancetypeRef captures the state of any referenced + instance type from the VirtualMachine + nullable: true + properties: + controllerRevisionRef: + description: |- + ControllerRef specifies the ControllerRevision storing a copy of the object captured + when it is first seen by the VirtualMachine controller + properties: + name: + description: Name of the ControllerRevision + type: string + type: object + inferFromVolume: + description: InferFromVolume lists the name of a volume that should + be used to infer or discover the resource + type: string + inferFromVolumeFailurePolicy: + description: InferFromVolumeFailurePolicy controls what should + happen on failure when inferring the resource + type: string + kind: + description: Kind specifies the kind of resource + type: string + name: + description: Name is the name of resource + type: string + type: object memoryDumpRequest: description: |- MemoryDumpRequest tracks memory dump request phase and info of getting a memory @@ -4262,6 +4450,35 @@ spec: vmi when started. format: int64 type: integer + preferenceRef: + description: PreferenceRef captures the state of any referenced preference + from the VirtualMachine + nullable: true + properties: + controllerRevisionRef: + description: |- + ControllerRef specifies the ControllerRevision storing a copy of the object captured + when it is first seen by the VirtualMachine controller + properties: + name: + description: Name of the ControllerRevision + type: string + type: object + inferFromVolume: + description: InferFromVolume lists the name of a volume that should + be used to infer or discover the resource + type: string + inferFromVolumeFailurePolicy: + description: InferFromVolumeFailurePolicy controls what should + happen on failure when inferring the resource + type: string + kind: + description: Kind specifies the kind of resource + type: string + name: + description: Name is the name of resource + type: string + type: object printableStatus: default: Stopped description: PrintableStatus is a human readable, high-level representation @@ -4378,7 +4595,11 @@ spec: cache: description: |- Cache specifies which kvm disk cache mode should be used. - Supported values are: CacheNone, CacheWriteThrough. + Supported values are: + none: Guest I/O not cached on the host, but may be kept in a disk cache. + writethrough: Guest I/O cached on the host but written through to the physical medium. Slowest but with most guarantees. + writeback: Guest I/O cached on the host. + Defaults to none if the storage supports O_DIRECT, otherwise writethrough. type: string cdrom: description: Attach a volume as a cdrom to the vmi. @@ -4531,9 +4752,8 @@ spec: volume can be hotplugged and hotunplugged. type: boolean name: - description: |- - Name of both the DataVolume and the PVC in the same namespace. - After PVC population the DataVolume is garbage collected by default. + description: Name of both the DataVolume and the + PVC in the same namespace. type: string required: - name @@ -4615,6 +4835,134 @@ spec: - name type: object type: array + volumeUpdateState: + description: |- + VolumeUpdateState contains the information about the volumes set + updates related to the volumeUpdateStrategy + properties: + volumeMigrationState: + description: VolumeMigrationState tracks the information related + to the volume migration + properties: + migratedVolumes: + description: MigratedVolumes lists the source and destination + volumes during the volume migration + items: + description: StorageMigratedVolumeInfo tracks the information + about the source and destination volumes during the volume + migration + properties: + destinationPVCInfo: + description: DestinationPVCInfo contains the information + about the destination PVC + properties: + accessModes: + description: |- + AccessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Capacity represents the capacity set + on the corresponding PVC status + type: object + claimName: + description: ClaimName is the name of the PVC + type: string + filesystemOverhead: + description: Percentage of filesystem's size to + be reserved when resizing the PVC + pattern: ^(0(?:\.\d{1,3})?|1)$ + type: string + preallocated: + description: Preallocated indicates if the PVC's + storage is preallocated or not + type: boolean + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests represents the resources requested + by the corresponding PVC spec + type: object + volumeMode: + description: |- + VolumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + type: object + sourcePVCInfo: + description: SourcePVCInfo contains the information + about the source PVC + properties: + accessModes: + description: |- + AccessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Capacity represents the capacity set + on the corresponding PVC status + type: object + claimName: + description: ClaimName is the name of the PVC + type: string + filesystemOverhead: + description: Percentage of filesystem's size to + be reserved when resizing the PVC + pattern: ^(0(?:\.\d{1,3})?|1)$ + type: string + preallocated: + description: Preallocated indicates if the PVC's + storage is preallocated or not + type: boolean + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests represents the resources requested + by the corresponding PVC spec + type: object + volumeMode: + description: |- + VolumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + type: object + volumeName: + description: VolumeName is the name of the volume that + is being migrated + type: string + required: + - volumeName + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object type: object required: - spec diff --git a/images/hooks/go.mod b/images/hooks/go.mod index bb211a4a08..3aa044a027 100644 --- a/images/hooks/go.mod +++ b/images/hooks/go.mod @@ -105,7 +105,7 @@ require ( k8s.io/component-base v0.33.3 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250701173324-9bd5c66d9911 // indirect - kubevirt.io/api v1.3.1 // indirect + kubevirt.io/api v1.6.2 // indirect kubevirt.io/containerized-data-importer-api v1.60.3 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect sigs.k8s.io/controller-runtime v0.21.0 // indirect diff --git a/images/hooks/go.sum b/images/hooks/go.sum index 4433af27eb..663744542f 100644 --- a/images/hooks/go.sum +++ b/images/hooks/go.sum @@ -623,8 +623,8 @@ k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -kubevirt.io/api v1.3.1 h1:MoTNo/zvDlZ44c2ocXLPln8XTaQOeUodiYbEKrTCqv4= -kubevirt.io/api v1.3.1/go.mod h1:tCn7VAZktEvymk490iPSMPCmKM9UjbbfH2OsFR/IOLU= +kubevirt.io/api v1.6.2 h1:aoqZ4KsbOyDjLnuDw7H9wEgE/YTd/q5BBmYeQjJNizc= +kubevirt.io/api v1.6.2/go.mod h1:p66fEy/g79x7VpgUwrkUgOoG2lYs5LQq37WM6JXMwj4= kubevirt.io/containerized-data-importer-api v1.60.3 h1:kQEXi7scpzUa0RPf3/3MKk1Kmem0ZlqqiuK3kDF5L2I= kubevirt.io/containerized-data-importer-api v1.60.3/go.mod h1:8mwrkZIdy8j/LmCyKt2wFXbiMavLUIqDaegaIF67CZs= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= diff --git a/images/virt-api/debug/dlv.Dockerfile b/images/virt-api/debug/dlv.Dockerfile index bca504a4d3..fe10ced68b 100644 --- a/images/virt-api/debug/dlv.Dockerfile +++ b/images/virt-api/debug/dlv.Dockerfile @@ -1,10 +1,10 @@ -FROM golang:1.22.7 AS builder +FROM golang:1.23 AS builder RUN go install github.com/go-delve/delve/cmd/dlv@latest -ARG BRANCH="1.3.1-virtualization" -ENV VERSION="1.3.1" -ENV GOVERSION="1.22.7" +ARG BRANCH="1.6.2-virtualization" +ENV VERSION="1.6.2" +ENV GOVERSION="1.23.0" # Copy the git commits for rebuilding the image if the branch changes ADD "https://api.github.com/repos/deckhouse/3p-kubevirt/commits/$BRANCH" /.git-commit-hash.tmp @@ -14,7 +14,7 @@ WORKDIR /kubevirt RUN go mod edit -go=$GOVERSION && \ go mod download -RUN go mod vendor +RUN go work vendor ENV GO111MODULE=on ENV GOOS=linux diff --git a/images/virt-artifact/werf.inc.yaml b/images/virt-artifact/werf.inc.yaml index 33742cfd40..8a5235efe0 100644 --- a/images/virt-artifact/werf.inc.yaml +++ b/images/virt-artifact/werf.inc.yaml @@ -83,7 +83,7 @@ shell: cd /kubevirt go mod download - go mod vendor + go work vendor for p in patches/*.patch ; do echo -n "Apply ${p} ... " diff --git a/images/virt-controller/debug/dlv.Dockerfile b/images/virt-controller/debug/dlv.Dockerfile index ab4f9606db..fec4948301 100644 --- a/images/virt-controller/debug/dlv.Dockerfile +++ b/images/virt-controller/debug/dlv.Dockerfile @@ -1,10 +1,10 @@ -FROM golang:1.22.7 AS builder +FROM golang:1.23 AS builder RUN go install github.com/go-delve/delve/cmd/dlv@latest -ARG BRANCH="1.3.1-virtualization" -ENV VERSION="1.3.1" -ENV GOVERSION="1.22.7" +ARG BRANCH="v1.6.1-virtualization" +ENV VERSION="1.6.1" +ENV GOVERSION="1.23.0" # Copy the git commits for rebuilding the image if the branch changes ADD "https://api.github.com/repos/deckhouse/3p-kubevirt/commits/$BRANCH" /.git-commit-hash.tmp @@ -14,7 +14,7 @@ WORKDIR /kubevirt RUN go mod edit -go=$GOVERSION && \ go mod download -RUN go mod vendor +RUN go work vendor ENV GO111MODULE=on ENV GOOS=linux diff --git a/images/virt-handler/debug/dlv.Dockerfile b/images/virt-handler/debug/dlv.Dockerfile index 12142d5178..86af33b002 100644 --- a/images/virt-handler/debug/dlv.Dockerfile +++ b/images/virt-handler/debug/dlv.Dockerfile @@ -3,6 +3,7 @@ RUN groupadd --gid 1001 nonroot-user && useradd nonroot-user --uid 1001 --gid 10 FROM basealt AS builder +# TODO add pin repository url RUN apt-get update RUN apt-get install -y \ @@ -17,9 +18,9 @@ RUN apt-get install -y \ RUN go install github.com/go-delve/delve/cmd/dlv@latest -ARG BRANCH="1.3.1-virtualization" -ENV VERSION="1.3.1" -ENV GOVERSION="1.22.7" +ARG BRANCH="1.6.2-virtualization" +ENV VERSION="1.6.2" +ENV GOVERSION="1.23.0" RUN mkdir /kubevirt-config-files && echo "v$VERSION-dirty" > /kubevirt-config-files/.version @@ -31,11 +32,11 @@ WORKDIR /kubevirt RUN go mod edit -go=$GOVERSION && \ go mod download -RUN go mod vendor +RUN go work vendor -RUN for p in patches/*.patch ; do \ - echo -n "Apply ${p} ... " \ - git apply --ignore-space-change --ignore-whitespace ${p} && echo OK || (echo FAIL ; exit 1) \ +RUN for p in patches/*.patch; do \ + echo -n "Apply ${p} ... " && \ + git apply --ignore-space-change --ignore-whitespace "${p}" && echo OK || (echo FAIL && exit 1); \ done ENV GO111MODULE=on @@ -52,9 +53,7 @@ FROM basealt RUN apt-get update && apt-get install --yes \ acl \ procps \ - nftables \ - qemu-img==9.1.2-alt1 \ - xorriso==1.5.6-alt1 && \ + nftables && \ apt-get clean && \ rm --recursive --force /var/lib/apt/lists/ftp.altlinux.org* /var/cache/apt/*.bin @@ -63,7 +62,6 @@ RUN echo "qemu:x:107:107::/home/qemu:/bin/bash" >> /etc/passwd && \ mkdir -p /home/qemu && \ chown -R 107:107 /home/qemu -COPY --from=builder /kubevirt/cmd/virt-handler/virt_launcher.cil /virt_launcher.cil COPY --from=builder /kubevirt-config-files/.version /.version COPY --from=builder /kubevirt/cmd/virt-handler/nsswitch.conf /etc/nsswitch.conf diff --git a/images/virt-handler/werf.inc.yaml b/images/virt-handler/werf.inc.yaml index 4888e05684..3db87a474a 100644 --- a/images/virt-handler/werf.inc.yaml +++ b/images/virt-handler/werf.inc.yaml @@ -25,10 +25,6 @@ import: - virt-chroot - virt-handler - container-disk -- image: {{ .ModuleNamePrefix }}virt-artifact - add: /kubevirt/cmd/{{ $.ImageName }}/virt_launcher.cil - to: /virt_launcher.cil - after: install - image: {{ .ModuleNamePrefix }}virt-artifact add: /kubevirt-config-files/.version to: /.version diff --git a/images/virtualization-artifact/go.mod b/images/virtualization-artifact/go.mod index 3ade7764fa..c8ed344d7a 100644 --- a/images/virtualization-artifact/go.mod +++ b/images/virtualization-artifact/go.mod @@ -32,7 +32,7 @@ require ( k8s.io/klog/v2 v2.130.1 k8s.io/kube-openapi v0.0.0-20250701173324-9bd5c66d9911 k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 - kubevirt.io/api v1.3.1 + kubevirt.io/api v1.6.2 kubevirt.io/containerized-data-importer-api v1.60.3 sigs.k8s.io/controller-runtime v0.21.0 sigs.k8s.io/yaml v1.4.0 @@ -165,4 +165,4 @@ replace ( ) // Kubevirt API replaces -replace kubevirt.io/api => github.com/deckhouse/3p-kubevirt/staging/src/kubevirt.io/api v1.3.1-v12n.18 +replace kubevirt.io/api => github.com/deckhouse/3p-kubevirt/staging/src/kubevirt.io/api v1.6.2-v12n.0 diff --git a/images/virtualization-artifact/go.sum b/images/virtualization-artifact/go.sum index 39d45d303a..db865f72ea 100644 --- a/images/virtualization-artifact/go.sum +++ b/images/virtualization-artifact/go.sum @@ -45,8 +45,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckhouse/3p-kubevirt/staging/src/kubevirt.io/api v1.3.1-v12n.18 h1:9pstD3PiPmby/Chh24ickwUNbAcqbceOPp253/jSr8k= -github.com/deckhouse/3p-kubevirt/staging/src/kubevirt.io/api v1.3.1-v12n.18/go.mod h1:tCn7VAZktEvymk490iPSMPCmKM9UjbbfH2OsFR/IOLU= +github.com/deckhouse/3p-kubevirt/staging/src/kubevirt.io/api v1.6.2-v12n.0 h1:Ro9oG/eakqMxfFSEs91BZYhDXggghuJjFvawV6/EKSo= +github.com/deckhouse/3p-kubevirt/staging/src/kubevirt.io/api v1.6.2-v12n.0/go.mod h1:p66fEy/g79x7VpgUwrkUgOoG2lYs5LQq37WM6JXMwj4= github.com/deckhouse/deckhouse/pkg/log v0.0.0-20250226105106-176cd3afcdd5 h1:PsN1E0oxC/+4zdA977txrqUCuObFL3HAuu5Xnud8m8c= github.com/deckhouse/deckhouse/pkg/log v0.0.0-20250226105106-176cd3afcdd5/go.mod h1:Mk5HRzkc5pIcDIZ2JJ6DPuuqnwhXVkb3you8M8Mg+4w= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= diff --git a/images/virtualization-artifact/pkg/livemigration/migration_configuration.go b/images/virtualization-artifact/pkg/livemigration/migration_configuration.go index 21910bc868..0fe5d9851a 100644 --- a/images/virtualization-artifact/pkg/livemigration/migration_configuration.go +++ b/images/virtualization-artifact/pkg/livemigration/migration_configuration.go @@ -38,6 +38,7 @@ const ( MigrationCompletionTimeoutPerGiB int64 = 800 DefaultUnsafeMigrationOverride bool = false MigrationAllowPostCopy bool = false + MigrationAllowWorkloadDisruption bool = false ) func NewMigrationConfiguration(allowAutoConverge bool, kvconfig virtv1.KubeVirt) *virtv1.MigrationConfiguration { @@ -63,6 +64,7 @@ func NewMigrationConfiguration(allowAutoConverge bool, kvconfig virtv1.KubeVirt) completionTimeoutPerGiB := MigrationCompletionTimeoutPerGiB defaultUnsafeMigrationOverride := DefaultUnsafeMigrationOverride allowPostCopy := MigrationAllowPostCopy + allowWorkloadDisruption := MigrationAllowWorkloadDisruption return &virtv1.MigrationConfiguration{ ParallelMigrationsPerCluster: ¶llelMigrationsPerCluster, @@ -74,6 +76,7 @@ func NewMigrationConfiguration(allowAutoConverge bool, kvconfig virtv1.KubeVirt) UnsafeMigrationOverride: &defaultUnsafeMigrationOverride, AllowAutoConverge: &allowAutoConverge, AllowPostCopy: &allowPostCopy, + AllowWorkloadDisruption: &allowWorkloadDisruption, DisableTLS: nil, Network: nil, MatchSELinuxLevelOnMigration: nil, diff --git a/images/vm-route-forge/go.mod b/images/vm-route-forge/go.mod index 57bbcc5dcc..1748f07e51 100644 --- a/images/vm-route-forge/go.mod +++ b/images/vm-route-forge/go.mod @@ -67,7 +67,6 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oklog/ulid v1.3.1 // indirect - github.com/openshift/api v0.0.0-20230503133300-8bbcb7ca7183 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect @@ -123,8 +122,8 @@ require ( k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250701173324-9bd5c66d9911 // indirect k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect - kubevirt.io/api v1.3.1 // indirect - kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 // indirect + kubevirt.io/api v1.6.2 // indirect + kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/randfill v1.0.0 // indirect diff --git a/images/vm-route-forge/go.sum b/images/vm-route-forge/go.sum index 65cc2e5874..3c63121653 100644 --- a/images/vm-route-forge/go.sum +++ b/images/vm-route-forge/go.sum @@ -300,8 +300,6 @@ github.com/onsi/gomega v1.33.0/go.mod h1:+925n5YtiFsLzzafLUHzVMBpvvRAzrydIBiSIxj github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= -github.com/openshift/api v0.0.0-20230503133300-8bbcb7ca7183 h1:t/CahSnpqY46sQR01SoS+Jt0jtjgmhgE6lFmRnO4q70= -github.com/openshift/api v0.0.0-20230503133300-8bbcb7ca7183/go.mod h1:4VWG+W22wrB4HfBL88P40DxLEpSOaiBVxUnfalfJo9k= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= @@ -677,10 +675,10 @@ k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -kubevirt.io/api v1.3.1 h1:MoTNo/zvDlZ44c2ocXLPln8XTaQOeUodiYbEKrTCqv4= -kubevirt.io/api v1.3.1/go.mod h1:tCn7VAZktEvymk490iPSMPCmKM9UjbbfH2OsFR/IOLU= -kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 h1:IWo12+ei3jltSN5jQN1xjgakfvRSF3G3Rr4GXVOOy2I= -kubevirt.io/containerized-data-importer-api v1.57.0-alpha1/go.mod h1:Y/8ETgHS1GjO89bl682DPtQOYEU/1ctPFBz6Sjxm4DM= +kubevirt.io/api v1.6.2 h1:aoqZ4KsbOyDjLnuDw7H9wEgE/YTd/q5BBmYeQjJNizc= +kubevirt.io/api v1.6.2/go.mod h1:p66fEy/g79x7VpgUwrkUgOoG2lYs5LQq37WM6JXMwj4= +kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9 h1:KTb8wO1Lxj220DX7d2Rdo9xovvlyWWNo3AVm2ua+1nY= +kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9/go.mod h1:SDJjLGhbPyayDqAqawcGmVNapBp0KodOQvhKPLVGCQU= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= diff --git a/src/cli/go.mod b/src/cli/go.mod index bd767e6226..d859fe42de 100644 --- a/src/cli/go.mod +++ b/src/cli/go.mod @@ -66,8 +66,8 @@ require ( k8s.io/api v0.33.3 // indirect k8s.io/apiextensions-apiserver v0.33.3 // indirect k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect - kubevirt.io/api v1.3.1 // indirect - kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 // indirect + kubevirt.io/api v1.6.2 // indirect + kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect sigs.k8s.io/controller-runtime v0.21.0 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect diff --git a/src/cli/go.sum b/src/cli/go.sum index 2c8e3bc9b0..8bb70c9518 100644 --- a/src/cli/go.sum +++ b/src/cli/go.sum @@ -589,8 +589,12 @@ k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8 k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/api v1.3.1 h1:MoTNo/zvDlZ44c2ocXLPln8XTaQOeUodiYbEKrTCqv4= kubevirt.io/api v1.3.1/go.mod h1:tCn7VAZktEvymk490iPSMPCmKM9UjbbfH2OsFR/IOLU= +kubevirt.io/api v1.6.2 h1:aoqZ4KsbOyDjLnuDw7H9wEgE/YTd/q5BBmYeQjJNizc= +kubevirt.io/api v1.6.2/go.mod h1:p66fEy/g79x7VpgUwrkUgOoG2lYs5LQq37WM6JXMwj4= kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 h1:IWo12+ei3jltSN5jQN1xjgakfvRSF3G3Rr4GXVOOy2I= kubevirt.io/containerized-data-importer-api v1.57.0-alpha1/go.mod h1:Y/8ETgHS1GjO89bl682DPtQOYEU/1ctPFBz6Sjxm4DM= +kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9 h1:KTb8wO1Lxj220DX7d2Rdo9xovvlyWWNo3AVm2ua+1nY= +kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9/go.mod h1:SDJjLGhbPyayDqAqawcGmVNapBp0KodOQvhKPLVGCQU= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= diff --git a/templates/kubevirt/kubevirt.yaml b/templates/kubevirt/kubevirt.yaml index 2951618f9d..81b71c9949 100644 --- a/templates/kubevirt/kubevirt.yaml +++ b/templates/kubevirt/kubevirt.yaml @@ -52,16 +52,12 @@ spec: virtOperator: {{ $logVerbosity }} featureGates: - HotplugVolumes - - GPU - Snapshot - ExpandDisks - Root - - VMLiveUpdateFeatures - CPUManager - Sidecar - VolumeSnapshotDataSource - - VolumeMigration - - VolumesUpdateStrategy virtualMachineOptions: disableSerialConsoleLog: {} customizeComponents: diff --git a/templates/kubevirt/virt-operator/rbac-for-us.yaml b/templates/kubevirt/virt-operator/rbac-for-us.yaml index d52cec41b0..91a6311478 100644 --- a/templates/kubevirt/virt-operator/rbac-for-us.yaml +++ b/templates/kubevirt/virt-operator/rbac-for-us.yaml @@ -26,6 +26,8 @@ rules: - kubevirt-virt-api-certs - kubevirt-controller-certs - kubevirt-exportproxy-certs + - kubevirt-synchronization-controller-certs + - kubevirt-synchronization-controller-server-certs resources: - secrets verbs: @@ -834,6 +836,9 @@ rules: - virtualmachineinstances/userlist - virtualmachineinstances/sev/fetchcertchain - virtualmachineinstances/sev/querylaunchmeasurement + - virtualmachineinstances/usbredir + - virtualmachines/objectgraph + - virtualmachineinstances/objectgraph verbs: - get - apiGroups: @@ -846,6 +851,7 @@ rules: - virtualmachineinstances/freeze - virtualmachineinstances/unfreeze - virtualmachineinstances/softreboot + - virtualmachineinstances/reset - virtualmachineinstances/sev/setupsession - virtualmachineinstances/sev/injectlaunchsecret verbs: @@ -1214,6 +1220,15 @@ rules: - subjectaccessreviews verbs: - create +- apiGroups: + - resource.k8s.io + resources: + - resourceclaims + - resourceslices + verbs: + - get + - list + - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/test/e2e/go.mod b/test/e2e/go.mod index b727c9ceab..5ddb0f733b 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -16,7 +16,7 @@ require ( k8s.io/cli-runtime v0.33.3 k8s.io/client-go v0.33.3 k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 - kubevirt.io/api v1.3.1 + kubevirt.io/api v1.6.2 sigs.k8s.io/controller-runtime v0.21.0 sigs.k8s.io/yaml v1.4.0 ) diff --git a/test/e2e/go.sum b/test/e2e/go.sum index f72cde1b6d..66999d0350 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -582,6 +582,8 @@ k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8 k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/api v1.3.1 h1:MoTNo/zvDlZ44c2ocXLPln8XTaQOeUodiYbEKrTCqv4= kubevirt.io/api v1.3.1/go.mod h1:tCn7VAZktEvymk490iPSMPCmKM9UjbbfH2OsFR/IOLU= +kubevirt.io/api v1.6.2 h1:aoqZ4KsbOyDjLnuDw7H9wEgE/YTd/q5BBmYeQjJNizc= +kubevirt.io/api v1.6.2/go.mod h1:p66fEy/g79x7VpgUwrkUgOoG2lYs5LQq37WM6JXMwj4= kubevirt.io/containerized-data-importer-api v1.60.3 h1:kQEXi7scpzUa0RPf3/3MKk1Kmem0ZlqqiuK3kDF5L2I= kubevirt.io/containerized-data-importer-api v1.60.3/go.mod h1:8mwrkZIdy8j/LmCyKt2wFXbiMavLUIqDaegaIF67CZs= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= From 72f94267a3965cf1aa9f77649c0d1c84a5d79dfe Mon Sep 17 00:00:00 2001 From: Vladislav Panfilov Date: Tue, 9 Dec 2025 14:04:43 +0400 Subject: [PATCH 2/3] Review embedded YAML descriptions Signed-off-by: Vladislav Panfilov --- crds/embedded/virtualmachineinstances.yaml | 209 ++++++++---------- crds/embedded/virtualmachines.yaml | 235 +++++++++------------ 2 files changed, 188 insertions(+), 256 deletions(-) diff --git a/crds/embedded/virtualmachineinstances.yaml b/crds/embedded/virtualmachineinstances.yaml index b676c9ed3a..a36f250777 100644 --- a/crds/embedded/virtualmachineinstances.yaml +++ b/crds/embedded/virtualmachineinstances.yaml @@ -1128,11 +1128,11 @@ spec: properties: name: description: |- - Name is this DNS resolver option's name. - Required. + DNS resolver option name. + Required field. type: string value: - description: Value is this DNS resolver option's value. + description: DNS resolver option value. type: string type: object type: array @@ -1576,26 +1576,22 @@ spec: properties: claimName: description: |- - ClaimName needs to be provided from the list vmi.spec.resourceClaims[].name where this - device is allocated + Must be provided from `vmi.spec.resourceClaims[].name` where this + device is allocated. type: string deviceName: - description: DeviceName is the name of the device provisioned - by device-plugins + description: Name of the device provisioned by device plugins. type: string name: - description: Name of the GPU device as exposed by a - device plugin + description: Name of the GPU device as exposed by a device plugin. type: string requestName: description: |- - RequestName needs to be provided from resourceClaim.spec.devices.requests[].name where this - device is requested + Must be provided from `resourceClaim.spec.devices.requests[].name` where this + device is requested. type: string tag: - description: If specified, the virtual network interface - address and its tag will be provided to the guest - via config drive + description: If specified, the virtual network interface address and its tag will be provided to the guest via config drive. type: string virtualGPUOptions: properties: @@ -1630,24 +1626,21 @@ spec: properties: claimName: description: |- - ClaimName needs to be provided from the list vmi.spec.resourceClaims[].name where this - device is allocated + Must be provided from `vmi.spec.resourceClaims[].name` where this + device is allocated. type: string deviceName: - description: DeviceName is the name of the device provisioned - by device-plugins + description: Name of the device provisioned by device plugins. type: string name: type: string requestName: description: |- - RequestName needs to be provided from resourceClaim.spec.devices.requests[].name where this - device is requested + Must be provided from `resourceClaim.spec.devices.requests[].name` where this + device is requested. type: string tag: - description: If specified, the virtual network interface - address and its tag will be provided to the guest - via config drive + description: If specified, the virtual network interface address and its tag will be provided to the guest via config drive. type: string required: - name @@ -1862,15 +1855,14 @@ spec: like the number of guest CPUs. type: boolean panicDevices: - description: PanicDevices provides additional crash information - when a guest crashes. + description: Provides additional crash information when a guest crashes. items: properties: model: description: |- - Model specifies what type of panic device is provided. - The panic model used when this attribute is missing depends on the hypervisor and guest arch. - One of: isa, hyperv, pvpanic. + Type of panic device to provide. + If this attribute is missing, the panic model used depends on the hypervisor and guest architecture. + Valid values: `isa`, `hyperv`, `pvpanic`. type: string type: object type: array @@ -1898,13 +1890,13 @@ spec: properties: enabled: description: |- - Enabled allows a user to explicitly disable the vTPM even when one is enabled by a preference referenced by the VirtualMachine - Defaults to True + Allows explicitly disabling the vTPM even when one is enabled by a preference referenced by the VirtualMachine. + Defaults to true. type: boolean persistent: description: |- - Persistent indicates the state of the TPM device should be kept accross reboots - Defaults to false + Indicates whether the state of the TPM device should be kept across reboots. + Defaults to false. type: boolean type: object useVirtioTransitional: @@ -1914,13 +1906,12 @@ spec: do not understand virtio_non_transitional (virtio 1.0). type: boolean video: - description: Video describes the video device configuration - for the vmi. + description: Video device configuration for the VMI. properties: type: description: |- - Type specifies the video device type (e.g., virtio, vga, bochs, ramfb). - If not specified, the default is architecture-dependent (VGA for BIOS-based VMs, Bochs for EFI-based VMs on AMD64; virtio for Arm and s390x). + Video device type (e.g., virtio, vga, bochs, ramfb). + If not specified, the default is architecture-dependent: VGA for BIOS-based VMs, Bochs for EFI-based VMs on AMD64, virtio for Arm and s390x. type: string type: object watchdog: @@ -1928,12 +1919,11 @@ spec: be added to the vmi. properties: diag288: - description: diag288 watchdog device (specific to s390x - architecture). + description: diag288 watchdog device specific to s390x architecture. properties: action: description: |- - The action to take. Valid values are poweroff, reset, shutdown. + Action to take when the watchdog triggers. Valid values: poweroff, reset, shutdown. Defaults to reset. type: string type: object @@ -2295,19 +2285,18 @@ spec: type: string type: object ioThreads: - description: IOThreads specifies the IOThreads options. + description: IOThreads options. properties: supplementalPoolThreadCount: - description: SupplementalPoolThreadCount specifies how many - iothreads are allocated for the supplementalPool policy. + description: Number of IO threads allocated for the `supplementalPool` policy. format: int32 type: integer type: object ioThreadsPolicy: description: |- - Controls whether or not disks will share IOThreads. - Omitting IOThreadsPolicy disables use of IOThreads. - One of: shared, auto, supplementalPool + Controls whether disks share IOThreads. + Omitting this field disables the use of IOThreads. + Valid values: shared, auto, supplementalPool. type: string launchSecurity: description: Launch Security setting of the vmi. @@ -2764,55 +2753,47 @@ spec: type: object resourceClaims: description: |- - ResourceClaims define which ResourceClaims must be allocated - and reserved before the VMI, hence virt-launcher pod is allowed to start. The resources - will be made available to the domain which consumes them - by name. + Defines which ResourceClaims must be allocated and reserved before the VMI, + hence the `virt-launcher` pod is allowed to start. The resources will be made available to the domain + which consumes them by name. - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate in kubernetes - https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/ - This field should only be configured if one of the feature-gates GPUsWithDRA or HostDevicesWithDRA is enabled. + This is an alpha field and requires enabling the `DynamicResourceAllocation` feature gate in Kubernetes. + See https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/ + This field should only be configured if one of the feature gates `GPUsWithDRA` or `HostDevicesWithDRA` is enabled. This feature is in alpha. items: description: |- - PodResourceClaim references exactly one ResourceClaim, either directly + References exactly one ResourceClaim, either directly or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim for the pod. - It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Adds a name that uniquely identifies the ResourceClaim inside the pod. Containers that need access to the ResourceClaim reference it with this name. properties: name: description: |- - Name uniquely identifies this resource claim inside the pod. - This must be a DNS_LABEL. + Uniquely identifies this resource claim inside the pod. + Must be a `DNS_LABEL`. type: string resourceClaimName: description: |- - ResourceClaimName is the name of a ResourceClaim object in the same - namespace as this pod. + Name of a ResourceClaim object in the same namespace as this pod. - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. + Exactly one of `ResourceClaimName` and `ResourceClaimTemplateName` must be set. type: string resourceClaimTemplateName: description: |- - ResourceClaimTemplateName is the name of a ResourceClaimTemplate - object in the same namespace as this pod. + Name of a ResourceClaimTemplate object in the same namespace as this pod. - The template will be used to create a new ResourceClaim, which will - be bound to this pod. When this pod is deleted, the ResourceClaim - will also be deleted. The pod name and resource name, along with a - generated component, will be used to form a unique name for the - ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + The template will be used to create a new ResourceClaim, which will be bound to this pod. + When this pod is deleted, the ResourceClaim will also be deleted. The pod name and resource name, + along with a generated component, will be used to form a unique name for the ResourceClaim, + which will be recorded in `pod.status.resourceClaimStatuses`. - This field is immutable and no changes will be made to the - corresponding ResourceClaim by the control plane after creating the - ResourceClaim. + This field is immutable and no changes will be made to the corresponding ResourceClaim + by the control plane after creating the ResourceClaim. - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. + Exactly one of `ResourceClaimName` and `ResourceClaimTemplateName` must be set. type: string required: - name @@ -3226,8 +3207,7 @@ spec: description: Path defines the path to disk file in the container type: string hotpluggable: - description: Hotpluggable indicates whether the volume can - be hotplugged and hotunplugged. + description: Indicates whether the volume can be hotplugged and hotunplugged. type: boolean required: - image @@ -3238,12 +3218,10 @@ spec: the process of populating that PVC with a disk image. properties: hotpluggable: - description: Hotpluggable indicates whether the volume can - be hotplugged and hotunplugged. + description: Indicates whether the volume can be hotplugged and hotunplugged. type: boolean name: - description: Name of both the DataVolume and the PVC in - the same namespace. + description: Name of both the DataVolume and the PVC in the same namespace. type: string required: - name @@ -3409,8 +3387,7 @@ spec: More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims type: string hotpluggable: - description: Hotpluggable indicates whether the volume can - be hotplugged and hotunplugged. + description: Indicates whether the volume can be hotplugged and hotunplugged. type: boolean readOnly: description: |- @@ -3438,8 +3415,7 @@ spec: More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims type: string hotpluggable: - description: Hotpluggable indicates whether the volume can - be hotplugged and hotunplugged. + description: Indicates whether the volume can be hotplugged and hotunplugged. type: boolean readOnly: description: |- @@ -3601,40 +3577,34 @@ spec: This feature is in alpha. properties: gpuStatuses: - description: GPUStatuses reflects the state of GPUs requested - in spec.domain.devices.gpus + description: State of GPUs requested in `spec.domain.devices.gpus`. items: properties: deviceResourceClaimStatus: - description: DeviceResourceClaimStatus reflects the DRA - related information for the device + description: DRA-related information for the device. properties: attributes: description: |- - Attributes are properties of the device that could be used by kubevirt and other copmonents to learn more - about the device, like pciAddress or mdevUUID + Properties of the device that could be used by kubevirt and other components to learn more + about the device, like `pciAddress` or `mdevUUID`. properties: mDevUUID: - description: MDevUUID is the mediated device uuid - of the allocated device + description: Mediated device UUID of the allocated device. type: string pciAddress: - description: PCIAddress is the PCIe bus address - of the allocated device + description: PCIe bus address of the allocated device. type: string type: object name: - description: Name is the name of actual device on the - host provisioned by the driver as reflected in resourceclaim.status + description: Name of actual device on the host provisioned by the driver as reflected in `resourceclaim.status`. type: string resourceClaimName: - description: ResourceClaimName is the name of the resource - claims object used to provision this resource + description: Name of the resource claims object used to provision this resource. type: string type: object name: - description: Name of the device as specified in spec.domain.devices.gpus.name - or spec.domain.devices.hostDevices.name + description: Name of the device as specified in `spec.domain.devices.gpus.name` + or `spec.domain.devices.hostDevices.name`. type: string required: - name @@ -3643,40 +3613,35 @@ spec: x-kubernetes-list-type: atomic hostDeviceStatuses: description: |- - HostDeviceStatuses reflects the state of GPUs requested in spec.domain.devices.hostDevices + State of host devices requested in `spec.domain.devices.hostDevices`. DRA items: properties: deviceResourceClaimStatus: - description: DeviceResourceClaimStatus reflects the DRA - related information for the device + description: DRA-related information for the device. properties: attributes: description: |- - Attributes are properties of the device that could be used by kubevirt and other copmonents to learn more - about the device, like pciAddress or mdevUUID + Properties of the device that could be used by kubevirt and other components to learn more + about the device, like `pciAddress` or `mdevUUID`. properties: mDevUUID: - description: MDevUUID is the mediated device uuid - of the allocated device + description: Mediated device UUID of the allocated device. type: string pciAddress: - description: PCIAddress is the PCIe bus address - of the allocated device + description: PCIe bus address of the allocated device. type: string type: object name: - description: Name is the name of actual device on the - host provisioned by the driver as reflected in resourceclaim.status + description: Name of actual device on the host provisioned by the driver as reflected in `resourceclaim.status`. type: string resourceClaimName: - description: ResourceClaimName is the name of the resource - claims object used to provision this resource + description: Name of the resource claims object used to provision this resource. type: string type: object name: - description: Name of the device as specified in spec.domain.devices.gpus.name - or spec.domain.devices.hostDevices.name + description: Name of the device as specified in `spec.domain.devices.gpus.name` + or `spec.domain.devices.hostDevices.name`. type: string required: - name @@ -3746,19 +3711,17 @@ spec: type: string type: array linkState: - description: 'LinkState Reports the current operational link - state''. values: up, down.' + description: 'Current operational link state. Values: up, down.' type: string mac: - description: Hardware address of a Virtual Machine interface + description: Hardware address of a Virtual Machine interface. type: string name: description: Name of the interface, corresponds to name of the - network assigned to the interface + network assigned to the interface. type: string podInterfaceName: - description: PodInterfaceName represents the name of the pod - network interface + description: Name of the pod network interface. type: string queueCount: description: Specifies how many queues are allocated by MultiQueue @@ -4059,7 +4022,7 @@ spec: type: boolean type: object migrationNetworkType: - description: The type of migration network, either 'pod' or 'migration' + description: Type of migration network, either 'pod' or 'migration'. type: string migrationPolicyName: description: Name of the migration policy. If string is empty, @@ -4078,7 +4041,7 @@ spec: type: string sourcePersistentStatePVCName: description: If the VMI being migrated uses persistent features - (backend-storage), its source PVC name is saved here + (backend-storage), its source PVC name is saved here. type: string sourcePod: type: string @@ -4171,7 +4134,7 @@ spec: type: string targetPersistentStatePVCName: description: If the VMI being migrated uses persistent features - (backend-storage), its target PVC name is saved here + (backend-storage), its target PVC name is saved here. type: string targetPod: description: The target pod that the VMI is moving to diff --git a/crds/embedded/virtualmachines.yaml b/crds/embedded/virtualmachines.yaml index a0bfe284f4..f8da8608f7 100644 --- a/crds/embedded/virtualmachines.yaml +++ b/crds/embedded/virtualmachines.yaml @@ -1896,12 +1896,11 @@ spec: properties: name: description: |- - Name is this DNS resolver option's name. - Required. + DNS resolver option name. + Required field. type: string value: - description: Value is this DNS resolver option's - value. + description: DNS resolver option value. type: string type: object type: array @@ -2221,12 +2220,12 @@ spec: type: integer cache: description: |- - Cache specifies which kvm disk cache mode should be used. - Supported values are: - none: Guest I/O not cached on the host, but may be kept in a disk cache. - writethrough: Guest I/O cached on the host but written through to the physical medium. Slowest but with most guarantees. - writeback: Guest I/O cached on the host. - Defaults to none if the storage supports O_DIRECT, otherwise writethrough. + Specifies which KVM disk cache mode should be used. Supported values: + + - `none`: Guest I/O is not cached on the host, but may be kept in a disk cache. + - `writethrough`: Guest I/O is cached on the host but written through to the physical medium. This is the slowest mode but provides the most guarantees. + - `writeback`: Guest I/O is cached on the host. + Defaults to `none` if the storage supports O_DIRECT, otherwise `writethrough`. type: string cdrom: description: Attach a volume as a cdrom to the @@ -2355,26 +2354,22 @@ spec: properties: claimName: description: |- - ClaimName needs to be provided from the list vmi.spec.resourceClaims[].name where this - device is allocated + Must be provided from `vmi.spec.resourceClaims[].name` where this + device is allocated. type: string deviceName: - description: DeviceName is the name of the device - provisioned by device-plugins + description: Name of the device provisioned by device plugins. type: string name: - description: Name of the GPU device as exposed - by a device plugin + description: Name of the GPU device as exposed by a device plugin. type: string requestName: description: |- - RequestName needs to be provided from resourceClaim.spec.devices.requests[].name where this - device is requested + Must be provided from `resourceClaim.spec.devices.requests[].name` where this + device is requested. type: string tag: - description: If specified, the virtual network - interface address and its tag will be provided - to the guest via config drive + description: If specified, the virtual network interface address and its tag will be provided to the guest via config drive. type: string virtualGPUOptions: properties: @@ -2411,24 +2406,21 @@ spec: properties: claimName: description: |- - ClaimName needs to be provided from the list vmi.spec.resourceClaims[].name where this - device is allocated + Must be provided from `vmi.spec.resourceClaims[].name` where this + device is allocated. type: string deviceName: - description: DeviceName is the name of the device - provisioned by device-plugins + description: Name of the device provisioned by device plugins. type: string name: type: string requestName: description: |- - RequestName needs to be provided from resourceClaim.spec.devices.requests[].name where this - device is requested + Must be provided from `resourceClaim.spec.devices.requests[].name` where this + device is requested. type: string tag: - description: If specified, the virtual network - interface address and its tag will be provided - to the guest via config drive + description: If specified, the virtual network interface address and its tag will be provided to the guest via config drive. type: string required: - name @@ -2615,17 +2607,15 @@ spec: type: object state: description: |- - State represents the requested operational state of the interface. - The supported values are: - 'absent', expressing a request to remove the interface. - 'down', expressing a request to set the link down. - 'up', expressing a request to set the link up. - Empty value functions as 'up'. + Requested operational state of the interface. Supported values: + + - 'absent': Expresses a request to remove the interface. + - 'down': Expresses a request to set the link down. + - 'up': Expresses a request to set the link up. + An empty value functions as 'up'. type: string tag: - description: If specified, the virtual network - interface address and its tag will be provided - to the guest via config drive + description: If specified, the virtual network interface address and its tag will be provided to the guest via config drive. type: string required: - name @@ -2648,15 +2638,14 @@ spec: guest CPUs. type: boolean panicDevices: - description: PanicDevices provides additional crash - information when a guest crashes. + description: Provides additional crash information when a guest crashes. items: properties: model: description: |- - Model specifies what type of panic device is provided. - The panic model used when this attribute is missing depends on the hypervisor and guest arch. - One of: isa, hyperv, pvpanic. + Type of panic device to provide. + If this attribute is missing, the panic model used depends on the hypervisor and guest architecture. + Valid values: `isa`, `hyperv`, `pvpanic`. type: string type: object type: array @@ -2685,13 +2674,13 @@ spec: properties: enabled: description: |- - Enabled allows a user to explicitly disable the vTPM even when one is enabled by a preference referenced by the VirtualMachine - Defaults to True + Allows explicitly disabling the vTPM even when one is enabled by a preference referenced by the VirtualMachine. + Defaults to true. type: boolean persistent: description: |- - Persistent indicates the state of the TPM device should be kept accross reboots - Defaults to false + Indicates whether the state of the TPM device should be kept across reboots. + Defaults to false. type: boolean type: object useVirtioTransitional: @@ -2701,13 +2690,12 @@ spec: do not understand virtio_non_transitional (virtio 1.0). type: boolean video: - description: Video describes the video device configuration - for the vmi. + description: Video device configuration for the VMI. properties: type: description: |- - Type specifies the video device type (e.g., virtio, vga, bochs, ramfb). - If not specified, the default is architecture-dependent (VGA for BIOS-based VMs, Bochs for EFI-based VMs on AMD64; virtio for Arm and s390x). + Video device type (e.g., virtio, vga, bochs, ramfb). + If not specified, the default is architecture-dependent: VGA for BIOS-based VMs, Bochs for EFI-based VMs on AMD64, virtio for Arm and s390x. type: string type: object watchdog: @@ -2715,12 +2703,11 @@ spec: which can be added to the vmi. properties: diag288: - description: diag288 watchdog device (specific - to s390x architecture). + description: diag288 watchdog device specific to s390x architecture. properties: action: description: |- - The action to take. Valid values are poweroff, reset, shutdown. + Action to take when the watchdog triggers. Valid values: poweroff, reset, shutdown. Defaults to reset. type: string type: object @@ -2996,12 +2983,12 @@ spec: properties: msdmNameRef: description: |- - Similar to SlicNameRef, another ACPI entry that is used in more recent Windows versions. - The above points to the spec of MSDM too. + Similar to `SlicNameRef`, this is another ACPI entry used in more recent Windows versions. + This field points to the MSDM specification as well. type: string slicNameRef: description: |- - SlicNameRef should match the volume name of a secret object. The data in the secret should + Should match the volume name of a secret object. The data in the secret should be a binary blob that follows the ACPI SLIC standard, see: https://learn.microsoft.com/en-us/previous-versions/windows/hardware/design/dn653305(v=vs.85) type: string @@ -3089,17 +3076,15 @@ spec: description: IOThreads specifies the IOThreads options. properties: supplementalPoolThreadCount: - description: SupplementalPoolThreadCount specifies - how many iothreads are allocated for the supplementalPool - policy. + description: Number of IO threads allocated for the supplementalPool policy. format: int32 type: integer type: object ioThreadsPolicy: description: |- - Controls whether or not disks will share IOThreads. - Omitting IOThreadsPolicy disables use of IOThreads. - One of: shared, auto, supplementalPool + Controls whether disks share IOThreads. + Omitting this field disables the use of IOThreads. + Valid values: shared, auto, supplementalPool. type: string launchSecurity: description: Launch Security setting of the vmi. @@ -3557,55 +3542,47 @@ spec: type: object resourceClaims: description: |- - ResourceClaims define which ResourceClaims must be allocated - and reserved before the VMI, hence virt-launcher pod is allowed to start. The resources - will be made available to the domain which consumes them - by name. + Defines which ResourceClaims must be allocated and reserved before the VMI, + hence the `virt-launcher` pod is allowed to start. The resources will be made available to the domain + which consumes them by name. - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate in kubernetes - https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/ - This field should only be configured if one of the feature-gates GPUsWithDRA or HostDevicesWithDRA is enabled. + This is an alpha field and requires enabling the `DynamicResourceAllocation` feature gate in Kubernetes. + See https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/ + This field should only be configured if one of the feature gates `GPUsWithDRA` or `HostDevicesWithDRA` is enabled. This feature is in alpha. items: description: |- - PodResourceClaim references exactly one ResourceClaim, either directly + References exactly one ResourceClaim, either directly or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim for the pod. - It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Adds a name that uniquely identifies the ResourceClaim inside the Pod. Containers that need access to the ResourceClaim reference it with this name. properties: name: description: |- - Name uniquely identifies this resource claim inside the pod. - This must be a DNS_LABEL. + Uniquely identifies this resource claim inside the pod. + This must be a `DNS_LABEL`. type: string resourceClaimName: description: |- - ResourceClaimName is the name of a ResourceClaim object in the same - namespace as this pod. + Name of a ResourceClaim object in the same namespace as this pod. - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. + Exactly one of `ResourceClaimName` and `ResourceClaimTemplateName` must be set. type: string resourceClaimTemplateName: description: |- - ResourceClaimTemplateName is the name of a ResourceClaimTemplate - object in the same namespace as this pod. + Name of a ResourceClaimTemplate object in the same namespace as this pod. - The template will be used to create a new ResourceClaim, which will - be bound to this pod. When this pod is deleted, the ResourceClaim - will also be deleted. The pod name and resource name, along with a - generated component, will be used to form a unique name for the - ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + The template will be used to create a new ResourceClaim, which will be bound to this pod. + When this pod is deleted, the ResourceClaim will also be deleted. The pod name and resource name, + along with a generated component, will be used to form a unique name for the ResourceClaim, + which will be recorded in `pod.status.resourceClaimStatuses`. - This field is immutable and no changes will be made to the - corresponding ResourceClaim by the control plane after creating the - ResourceClaim. + This field is immutable and no changes will be made to the corresponding ResourceClaim + by the control plane after creating the ResourceClaim. - Exactly one of ResourceClaimName and ResourceClaimTemplateName must - be set. + Exactly one of `ResourceClaimName` and `ResourceClaimTemplateName` must be set. type: string required: - name @@ -4036,8 +4013,7 @@ spec: volume can be hotplugged and hotunplugged. type: boolean name: - description: Name of both the DataVolume and the - PVC in the same namespace. + description: Name of both the DataVolume and the PVC in the same namespace. type: string required: - name @@ -4385,26 +4361,25 @@ spec: properties: controllerRevisionRef: description: |- - ControllerRef specifies the ControllerRevision storing a copy of the object captured - when it is first seen by the VirtualMachine controller + ControllerRevision storing a copy of the object captured + when it is first seen by the VirtualMachine controller. properties: name: - description: Name of the ControllerRevision + description: Name of ControllerRevision. type: string type: object inferFromVolume: - description: InferFromVolume lists the name of a volume that should - be used to infer or discover the resource + description: Lists the name of a volume that should + be used to infer or discover the resource. type: string inferFromVolumeFailurePolicy: - description: InferFromVolumeFailurePolicy controls what should - happen on failure when inferring the resource + description: Controls what happens on failure when inferring the resource. type: string kind: - description: Kind specifies the kind of resource + description: Kind of resource. type: string name: - description: Name is the name of resource + description: Name of resource. type: string type: object memoryDumpRequest: @@ -4414,31 +4389,27 @@ spec: nullable: true properties: claimName: - description: ClaimName is the name of the pvc that will contain - the memory dump + description: Name of the PVC that will contain + the memory dump. type: string endTimestamp: - description: EndTimestamp represents the time the memory dump - was completed + description: Time when the memory dump was completed. format: date-time type: string fileName: - description: FileName represents the name of the output file + description: Name of the output file. type: string message: - description: Message is a detailed message about failure of the - memory dump + description: Detailed message about failure of the memory dump. type: string phase: - description: Phase represents the memory dump phase + description: Memory dump phase. type: string remove: - description: Remove represents request of dissociating the memory - dump pvc + description: Request to dissociate the memory dump PVC. type: boolean startTimestamp: - description: StartTimestamp represents the time the memory dump - started + description: Time when the memory dump started. format: date-time type: string required: @@ -4457,26 +4428,25 @@ spec: properties: controllerRevisionRef: description: |- - ControllerRef specifies the ControllerRevision storing a copy of the object captured - when it is first seen by the VirtualMachine controller + Specifies the `ControllerRevision` storing a copy of the object captured + when it is first seen by the VirtualMachine controller. properties: name: - description: Name of the ControllerRevision + description: Name of ControllerRevision. type: string type: object inferFromVolume: - description: InferFromVolume lists the name of a volume that should - be used to infer or discover the resource + description: Lists the name of a volume that should + be used to infer or discover the resource. type: string inferFromVolumeFailurePolicy: - description: InferFromVolumeFailurePolicy controls what should - happen on failure when inferring the resource + description: Controls what happens on failure when inferring the resource. type: string kind: - description: Kind specifies the kind of resource + description: Kind of resource. type: string name: - description: Name is the name of resource + description: Name of resource. type: string type: object printableStatus: @@ -4594,11 +4564,11 @@ spec: type: integer cache: description: |- - Cache specifies which kvm disk cache mode should be used. - Supported values are: - none: Guest I/O not cached on the host, but may be kept in a disk cache. - writethrough: Guest I/O cached on the host but written through to the physical medium. Slowest but with most guarantees. - writeback: Guest I/O cached on the host. + Cache specifies which KVM disk cache mode should be used. Supported values: + + - `none`: Guest I/O is not cached on the host, but may be kept in a disk cache. + - `writethrough`: Guest I/O is cached on the host but written through to the physical medium. This is the slowest mode but provides the most guarantees. + - `writeback`: Guest I/O is cached on the host. Defaults to none if the storage supports O_DIRECT, otherwise writethrough. type: string cdrom: @@ -4752,8 +4722,7 @@ spec: volume can be hotplugged and hotunplugged. type: boolean name: - description: Name of both the DataVolume and the - PVC in the same namespace. + description: Name of both the DataVolume and the PVC in the same namespace. type: string required: - name @@ -4875,7 +4844,7 @@ spec: on the corresponding PVC status type: object claimName: - description: ClaimName is the name of the PVC + description: Name of the PVC. type: string filesystemOverhead: description: Percentage of filesystem's size to @@ -4925,7 +4894,7 @@ spec: on the corresponding PVC status type: object claimName: - description: ClaimName is the name of the PVC + description: Name of the PVC. type: string filesystemOverhead: description: Percentage of filesystem's size to From 3ad4f7b77f0d682a3848e2d2d470612864cce874 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Thu, 13 Nov 2025 19:50:01 +0300 Subject: [PATCH 3/3] impl usb dra Signed-off-by: Yaroslav Borbat --- Taskfile.yaml | 34 ++ build/components/versions.yml | 2 +- crds/embedded/virtualmachineinstances.yaml | 17 + images/virt-artifact/werf.inc.yaml | 1 + images/virt-controller/debug/dlv.Dockerfile | 6 +- images/virt-launcher/werf.inc.yaml | 2 + .../pkg/logger/options.go | 38 ++ .../debug/dlv.Dockerfile | 31 ++ .../mount-points.yaml | 8 + .../virtualization-dra-plugin/werf.inc.yaml | 28 ++ images/virtualization-dra/.golangci.yaml | 108 +++++ images/virtualization-dra/Taskfile.yaml | 59 +++ .../cmd/virtualization-dra-plugin/app/app.go | 182 +++++++++ .../cmd/virtualization-dra-plugin/main.go | 36 ++ images/virtualization-dra/go.mod | 85 ++++ images/virtualization-dra/go.sum | 258 ++++++++++++ images/virtualization-dra/internal/cdi/cdi.go | 166 ++++++++ .../internal/plugin/driver.go | 196 ++++++++++ .../internal/plugin/healthz.go | 158 ++++++++ .../internal/plugin/interfaces.go | 33 ++ .../internal/plugin/nri_hooks.go | 30 ++ .../internal/plugin/plugin.go | 68 ++++ .../internal/usb/convert.go | 72 ++++ .../virtualization-dra/internal/usb/device.go | 250 ++++++++++++ .../internal/usb/discovery.go | 62 +++ .../internal/usb/monitor.go | 169 ++++++++ .../internal/usb/monitor_test.go | 53 +++ .../virtualization-dra/internal/usb/store.go | 369 ++++++++++++++++++ .../virtualization-dra/pkg/logger/logger.go | 88 +++++ .../virtualization-dra/pkg/logger/options.go | 38 ++ images/virtualization-dra/pkg/set/set.go | 64 +++ .../virtualization-dra/test/deviceclass.yaml | 8 + .../test/pod-with-claim.yaml | 16 + .../test/pod-with-template-1.yaml | 16 + .../test/pod-with-template-2.yaml | 22 ++ .../test/resourceclaim-template.yaml | 18 + .../test/resourceclaim.yaml | 17 + images/virtualization-dra/werf.inc.yaml | 41 ++ templates/kubevirt/kubevirt.yaml | 4 + templates/virtualization-dra/_helper.tpl | 5 + templates/virtualization-dra/daemonset.yaml | 132 +++++++ templates/virtualization-dra/rbac-for-us.yaml | 39 ++ tools/kubeconform/fixtures/module-values.yaml | 1 + 43 files changed, 3026 insertions(+), 4 deletions(-) create mode 100644 images/virtualization-artifact/pkg/logger/options.go create mode 100644 images/virtualization-dra-plugin/debug/dlv.Dockerfile create mode 100644 images/virtualization-dra-plugin/mount-points.yaml create mode 100644 images/virtualization-dra-plugin/werf.inc.yaml create mode 100644 images/virtualization-dra/.golangci.yaml create mode 100644 images/virtualization-dra/Taskfile.yaml create mode 100644 images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go create mode 100644 images/virtualization-dra/cmd/virtualization-dra-plugin/main.go create mode 100644 images/virtualization-dra/go.mod create mode 100644 images/virtualization-dra/go.sum create mode 100644 images/virtualization-dra/internal/cdi/cdi.go create mode 100644 images/virtualization-dra/internal/plugin/driver.go create mode 100644 images/virtualization-dra/internal/plugin/healthz.go create mode 100644 images/virtualization-dra/internal/plugin/interfaces.go create mode 100644 images/virtualization-dra/internal/plugin/nri_hooks.go create mode 100644 images/virtualization-dra/internal/plugin/plugin.go create mode 100644 images/virtualization-dra/internal/usb/convert.go create mode 100644 images/virtualization-dra/internal/usb/device.go create mode 100644 images/virtualization-dra/internal/usb/discovery.go create mode 100644 images/virtualization-dra/internal/usb/monitor.go create mode 100644 images/virtualization-dra/internal/usb/monitor_test.go create mode 100644 images/virtualization-dra/internal/usb/store.go create mode 100644 images/virtualization-dra/pkg/logger/logger.go create mode 100644 images/virtualization-dra/pkg/logger/options.go create mode 100644 images/virtualization-dra/pkg/set/set.go create mode 100644 images/virtualization-dra/test/deviceclass.yaml create mode 100644 images/virtualization-dra/test/pod-with-claim.yaml create mode 100644 images/virtualization-dra/test/pod-with-template-1.yaml create mode 100644 images/virtualization-dra/test/pod-with-template-2.yaml create mode 100644 images/virtualization-dra/test/resourceclaim-template.yaml create mode 100644 images/virtualization-dra/test/resourceclaim.yaml create mode 100644 images/virtualization-dra/werf.inc.yaml create mode 100644 templates/virtualization-dra/_helper.tpl create mode 100644 templates/virtualization-dra/daemonset.yaml create mode 100644 templates/virtualization-dra/rbac-for-us.yaml diff --git a/Taskfile.yaml b/Taskfile.yaml index 020d59f4f7..6703c2ad6e 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -282,3 +282,37 @@ tasks: }' kubectl -n d8-virtualization port-forward deploy/virt-api 2345:2345 EOF + + dlv:virtualization-dra-plugin:build: + desc: "Build image virtualization-dra-plugin with dlv" + cmds: + - docker build --build-arg BRANCH=$BRANCH -f ./images/virtualization-dra-plugin/debug/dlv.Dockerfile -t "{{ .DLV_IMAGE }}" --platform linux/amd64 . + + dlv:virtualization-dra-plugin:build-push: + desc: "Build and Push image virtualization-dra-plugin with dlv" + cmds: + - task: dlv:virtualization-dra-plugin:build + - docker push "{{ .DLV_IMAGE }}" + - task: dlv:virtualization-dra-plugin:print + + dlv:virtualization-dra-plugin:print: + desc: "Print commands for debug" + env: + IMAGE: "{{ .DLV_IMAGE }}" + cmd: | + cat <" + cmds: + - | + {{if .CLI_ARGS}} + go run ../../tools/addlicense/{main,variables,msg,utils}.go -directory {{ .CLI_ARGS }} + {{else}} + go run ../../tools/addlicense/{main,variables,msg,utils}.go -directory ./ + {{end}} + + test:unit: + desc: "Run go unit tests" + cmds: + - | + go tool ginkgo -v -r pkg/ + + lint: + desc: "Run linters locally" + cmds: + - task: lint:go + + lint:go: + desc: "Run golangci-lint" + deps: + - _ensure:golangci-lint + cmds: + - | + golangci-lint run diff --git a/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go b/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go new file mode 100644 index 0000000000..088c76c204 --- /dev/null +++ b/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go @@ -0,0 +1,182 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "log/slog" + "os" + "strconv" + "time" + + "github.com/spf13/cobra" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/component-base/cli/flag" + + "github.com/deckhouse/virtualization-dra/internal/cdi" + "github.com/deckhouse/virtualization-dra/internal/plugin" + "github.com/deckhouse/virtualization-dra/internal/usb" + "github.com/deckhouse/virtualization-dra/pkg/logger" +) + +func NewVirtualizationDraPluginCommand() *cobra.Command { + o := newDraOptions() + + cmd := &cobra.Command{ + Use: "virtualization-dra-plugin", + Short: "virtualization-dra-plugin", + SilenceUsage: true, + SilenceErrors: true, + PreRunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + log := o.Logging.Complete() + logger.SetDefaultLogger(log) + return nil + }, + RunE: o.Run, + } + + fs := cmd.Flags() + for _, f := range o.NamedFlags().FlagSets { + fs.AddFlagSet(f) + } + + return cmd +} + +func newDraOptions() *draOptions { + withDefault := func(env, defaultValue string) string { + if env, ok := os.LookupEnv(env); ok { + return env + } + return defaultValue + } + + o := &draOptions{ + Kubeconfig: os.Getenv("KUBECONFIG"), + NodeName: os.Getenv("NODE_NAME"), + CDIRoot: withDefault("CDI_ROOT", cdi.SpecDir), + KubeletRegisterDirectoryPath: os.Getenv("KUBELET_REGISTER_DIRECTORY_PATH"), + KubeletPluginsDirectoryPath: os.Getenv("KUBELET_PLUGINS_DIRECTORY_PATH"), + USBDevicesPath: withDefault("USB_DEVICES_PATH", usb.PathToUSBDevices), + HealthzPort: 51515, + USBResyncPeriod: usb.DefaultResyncPeriod, + Logging: &logger.Options{}, + } + + if healthzPort := os.Getenv("HEALTHZ_PORT"); healthzPort != "" { + port, err := strconv.Atoi(healthzPort) + if err == nil { + o.HealthzPort = port + } + } + + return o +} + +type draOptions struct { + Kubeconfig string + NodeName string + CDIRoot string + KubeletRegisterDirectoryPath string + KubeletPluginsDirectoryPath string + USBDevicesPath string + HealthzPort int + USBResyncPeriod time.Duration + + Logging *logger.Options +} + +func (o *draOptions) NamedFlags() (fs flag.NamedFlagSets) { + mfs := fs.FlagSet("virtualization-dra plugin") + mfs.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "Path to kubeconfig file") + mfs.StringVar(&o.NodeName, "node-name", o.NodeName, "Node name") + mfs.StringVar(&o.CDIRoot, "cdi-root", o.CDIRoot, "CDI root") + mfs.StringVar(&o.KubeletRegisterDirectoryPath, "kubelet-register-directory-path", o.KubeletRegisterDirectoryPath, "Kubelet register directory path") + mfs.StringVar(&o.KubeletPluginsDirectoryPath, "kubelet-plugins-directory-path", o.KubeletPluginsDirectoryPath, "Kubelet plugins directory path") + mfs.StringVar(&o.USBDevicesPath, "usb-devices-path", o.USBDevicesPath, "USB Devices path") + mfs.IntVar(&o.HealthzPort, "healthz-port", o.HealthzPort, "Healthz port") + mfs.DurationVar(&o.USBResyncPeriod, "usb-resync-period", o.USBResyncPeriod, "USB resync period") + + o.Logging.AddFlags(fs.FlagSet("logging")) + + return fs +} + +func (o *draOptions) Validate() error { + if o.NodeName == "" { + return fmt.Errorf("NodeName is required") + } + if o.CDIRoot == "" { + return fmt.Errorf("CDIRoot is required") + } + if o.HealthzPort <= 0 { + return fmt.Errorf("HealthzPort is required") + } + + return nil +} + +func (o *draOptions) Run(cmd *cobra.Command, _ []string) error { + err := plugin.InitPluginDirs(o.KubeletPluginsDirectoryPath, o.KubeletRegisterDirectoryPath) + if err != nil { + return err + } + + cfg, err := clientcmd.BuildConfigFromFlags("", o.Kubeconfig) + if err != nil { + return fmt.Errorf("failed to get rest config: %w", err) + } + + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + return fmt.Errorf("failed to create kubernetes client: %w", err) + } + + usbCDIManager, err := cdi.NewCDIManager(o.CDIRoot, "usb", plugin.DriverName, o.NodeName, "DRA_USB") + if err != nil { + return fmt.Errorf("failed to create CDI manager: %w", err) + } + + usbStore := usb.NewAllocationStore(o.NodeName, o.USBDevicesPath, o.USBResyncPeriod, usbCDIManager, slog.Default()) + + driver := plugin.NewDriver(o.NodeName, client, usbStore, slog.Default()) + err = driver.Start(cmd.Context()) + if err != nil { + return fmt.Errorf("failed to start driver: %w", err) + } + + healthCheck := plugin.NewHealthCheck(o.HealthzPort, slog.Default()) + err = healthCheck.Start() + if err != nil { + return fmt.Errorf("failed to start health check: %w", err) + } + + err = usbStore.Start(cmd.Context()) + if err != nil { + return fmt.Errorf("failed to start usb store: %w", err) + } + + driver.Wait() + driver.Shutdown() + healthCheck.Stop() + + return nil +} diff --git a/images/virtualization-dra/cmd/virtualization-dra-plugin/main.go b/images/virtualization-dra/cmd/virtualization-dra-plugin/main.go new file mode 100644 index 0000000000..863a7faf03 --- /dev/null +++ b/images/virtualization-dra/cmd/virtualization-dra-plugin/main.go @@ -0,0 +1,36 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "log/slog" + "os" + "os/signal" + "syscall" + + "github.com/deckhouse/virtualization-dra/cmd/virtualization-dra-plugin/app" +) + +func main() { + ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + + if err := app.NewVirtualizationDraPluginCommand().ExecuteContext(ctx); err != nil { + slog.Error("failed to execute command", slog.Any("err", err)) + os.Exit(1) + } +} diff --git a/images/virtualization-dra/go.mod b/images/virtualization-dra/go.mod new file mode 100644 index 0000000000..a6363e2f74 --- /dev/null +++ b/images/virtualization-dra/go.mod @@ -0,0 +1,85 @@ +module github.com/deckhouse/virtualization-dra + +go 1.24.7 + +tool github.com/onsi/ginkgo/v2/ginkgo + +require ( + github.com/containerd/nri v0.10.0 + github.com/deckhouse/deckhouse/pkg/log v0.1.0 + github.com/go-logr/logr v1.4.2 + github.com/godbus/dbus/v5 v5.2.0 + github.com/onsi/ginkgo/v2 v2.21.0 + github.com/onsi/gomega v1.35.1 + github.com/spf13/cobra v1.10.1 + github.com/spf13/pflag v1.0.9 + google.golang.org/grpc v1.72.1 + k8s.io/api v0.34.2 + k8s.io/apimachinery v0.34.2 + k8s.io/client-go v0.34.2 + k8s.io/component-base v0.34.2 + k8s.io/dynamic-resource-allocation v0.34.2 + k8s.io/klog/v2 v2.130.1 + k8s.io/kubelet v0.34.2 + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 + tags.cncf.io/container-device-interface v1.0.1 + tags.cncf.io/container-device-interface/specs-go v1.0.0 +) + +require ( + github.com/DataDog/gostackparse v0.7.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/ttrpc v1.2.7 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/knqyf263/go-plugin v0.9.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/opencontainers/runtime-spec v1.1.0 // indirect + github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect + github.com/tetratelabs/wazero v1.9.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.6.4 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/mod v0.21.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.9.0 // indirect + golang.org/x/tools v0.26.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) diff --git a/images/virtualization-dra/go.sum b/images/virtualization-dra/go.sum new file mode 100644 index 0000000000..dea9d5d69f --- /dev/null +++ b/images/virtualization-dra/go.sum @@ -0,0 +1,258 @@ +github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= +github.com/DataDog/gostackparse v0.7.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/nri v0.10.0 h1:bt2NzfvlY6OJE0i+fB5WVeGQEycxY7iFVQpEbh7J3Go= +github.com/containerd/nri v0.10.0/go.mod h1:5VyvLa/4uL8FjyO8nis1UjbCutXDpngil17KvBSL6BU= +github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRqQ= +github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckhouse/deckhouse/pkg/log v0.1.0 h1:2aPfyiHHSIJlX4x7ysyPOaIb7CLmyY+hUf9uDb8TYd8= +github.com/deckhouse/deckhouse/pkg/log v0.1.0/go.mod h1:pbAxTSDcPmwyl3wwKDcEB3qdxHnRxqTV+J0K+sha8bw= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/godbus/dbus/v5 v5.2.0 h1:3WexO+U+yg9T70v9FdHr9kCxYlazaAXUhx2VMkbfax8= +github.com/godbus/dbus/v5 v5.2.0/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knqyf263/go-plugin v0.9.0 h1:CQs2+lOPIlkZVtcb835ZYDEoyyWJWLbSTWeCs0EwTwI= +github.com/knqyf263/go-plugin v0.9.0/go.mod h1:2z5lCO1/pez6qGo8CvCxSlBFSEat4MEp1DrnA+f7w8Q= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= +github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 h1:DmNGcqH3WDbV5k8OJ+esPWbqUOX5rMLR2PMvziDMJi0= +github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go.mod h1:BRHJJd0E+cx42OybVYSgUvZmU0B8P9gZuRXlZUP7TKI= +github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK99DRLDhyU= +github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I= +github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM= +github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/etcd/client/pkg/v3 v3.6.4 h1:9HBYrjppeOfFjBjaMTRxT3R7xT0GLK8EJMVC4xg6ok0= +go.etcd.io/etcd/client/pkg/v3 v3.6.4/go.mod h1:sbdzr2cl3HzVmxNw//PH7aLGVtY4QySjQFuaCgcRFAI= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1:TLPQVbx1GJ8VKZxz52VAxl1EBgKXXbTiU9Fc5fZeLn4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= +google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= +google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.34.2 h1:fsSUNZhV+bnL6Aqrp6O7lMTy6o5x2C4XLjnh//8SLYY= +k8s.io/api v0.34.2/go.mod h1:MMBPaWlED2a8w4RSeanD76f7opUoypY8TFYkSM+3XHw= +k8s.io/apimachinery v0.34.2 h1:zQ12Uk3eMHPxrsbUJgNF8bTauTVR2WgqJsTmwTE/NW4= +k8s.io/apimachinery v0.34.2/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/client-go v0.34.2 h1:Co6XiknN+uUZqiddlfAjT68184/37PS4QAzYvQvDR8M= +k8s.io/client-go v0.34.2/go.mod h1:2VYDl1XXJsdcAxw7BenFslRQX28Dxz91U9MWKjX97fE= +k8s.io/component-base v0.34.2 h1:HQRqK9x2sSAsd8+R4xxRirlTjowsg6fWCPwWYeSvogQ= +k8s.io/component-base v0.34.2/go.mod h1:9xw2FHJavUHBFpiGkZoKuYZ5pdtLKe97DEByaA+hHbM= +k8s.io/dynamic-resource-allocation v0.34.2 h1:SjlRGSWl6CZXoJwQNL+Y0wRfdH8PkJ4mHRNK6MMj0bY= +k8s.io/dynamic-resource-allocation v0.34.2/go.mod h1:ul6I+gfrCmC+OCuVdN0/iykyB2sPrIqh2WyKQ3RQPCU= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/kubelet v0.34.2 h1:Dl+1uh7xwJr70r+SHKyIpvu6XvzuoPu0uDIC4cqgJUs= +k8s.io/kubelet v0.34.2/go.mod h1:RfwR03iuKeVV7Z1qD9XKH98c3tlPImJpQ3qHIW40htM= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= +tags.cncf.io/container-device-interface v1.0.1 h1:KqQDr4vIlxwfYh0Ed/uJGVgX+CHAkahrgabg6Q8GYxc= +tags.cncf.io/container-device-interface v1.0.1/go.mod h1:JojJIOeW3hNbcnOH2q0NrWNha/JuHoDZcmYxAZwb2i0= +tags.cncf.io/container-device-interface/specs-go v1.0.0 h1:8gLw29hH1ZQP9K1YtAzpvkHCjjyIxHZYzBAvlQ+0vD8= +tags.cncf.io/container-device-interface/specs-go v1.0.0/go.mod h1:u86hoFWqnh3hWz3esofRFKbI261bUlvUfLKGrDhJkgQ= diff --git a/images/virtualization-dra/internal/cdi/cdi.go b/images/virtualization-dra/internal/cdi/cdi.go new file mode 100644 index 0000000000..ca42ed55bc --- /dev/null +++ b/images/virtualization-dra/internal/cdi/cdi.go @@ -0,0 +1,166 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cdi + +import ( + "fmt" + + drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1beta1" + cdiapi "tags.cncf.io/container-device-interface/pkg/cdi" + cdiparser "tags.cncf.io/container-device-interface/pkg/parser" + cdispec "tags.cncf.io/container-device-interface/specs-go" +) + +const SpecDir = cdiapi.DefaultDynamicDir + +const ( + cdiVendor = "dra.virtualization.deckhouse.io" + cdiCommonDeviceName = "common" +) + +type Manager interface { + CreateCommonSpecFile() error + CreateClaimSpecFile(claimUID string, devices PreparedDevices) error + DeleteClaimSpecFile(claimUID string) error + GetClaimDevices(claimUID string, devices ...string) []string +} +type manager struct { + cache *cdiapi.Cache + cdiClass string + cdiKind string + driverName string + nodeName string + cdiEnvPrefix string +} + +func NewCDIManager(cdiSpecDir, cdiClass, driverName, nodeName, cdiEnvPrefix string) (Manager, error) { + if cdiSpecDir == "" { + cdiSpecDir = SpecDir + } + + cache, err := cdiapi.NewCache(cdiapi.WithSpecDirs(cdiSpecDir)) + if err != nil { + return nil, err + } + + return &manager{ + cache: cache, + cdiClass: cdiClass, + cdiKind: fmt.Sprintf("%s/%s", cdiVendor, cdiClass), + driverName: driverName, + nodeName: nodeName, + cdiEnvPrefix: cdiEnvPrefix, + }, nil +} + +func (cdi *manager) CreateCommonSpecFile() error { + spec := &cdispec.Spec{ + Kind: cdi.cdiKind, + Devices: []cdispec.Device{ + { + Name: cdiCommonDeviceName, + ContainerEdits: cdispec.ContainerEdits{ + Env: []string{ + fmt.Sprintf("KUBERNETES_NODE_NAME=%s", cdi.nodeName), + fmt.Sprintf("DRA_RESOURCE_DRIVER_NAME=%s", cdi.driverName), + }, + }, + }, + }, + } + + minVersion, err := cdispec.MinimumRequiredVersion(spec) + if err != nil { + return fmt.Errorf("failed to get minimum required CDI spec version: %v", err) + } + spec.Version = minVersion + + specName, err := cdiapi.GenerateNameForTransientSpec(spec, cdiCommonDeviceName) + if err != nil { + return fmt.Errorf("failed to generate Spec name: %w", err) + } + + return cdi.cache.WriteSpec(spec, specName) +} + +func (cdi *manager) CreateClaimSpecFile(claimUID string, devices PreparedDevices) error { + specName := cdiapi.GenerateTransientSpecName(cdiVendor, cdi.cdiClass, claimUID) + + spec := &cdispec.Spec{ + Kind: cdi.cdiKind, + Devices: []cdispec.Device{}, + } + + for _, device := range devices { + claimEdits := cdiapi.ContainerEdits{ + ContainerEdits: &cdispec.ContainerEdits{ + Env: []string{ + fmt.Sprintf("%s_%s_RESOURCE_CLAIM=%s", cdi.cdiEnvPrefix, device.DeviceName[4:], claimUID), + }, + }, + } + claimEdits.Append(device.ContainerEdits) + + cdiDevice := cdispec.Device{ + Name: fmt.Sprintf("%s-%s", claimUID, device.DeviceName), + ContainerEdits: *claimEdits.ContainerEdits, + } + + spec.Devices = append(spec.Devices, cdiDevice) + } + + minVersion, err := cdispec.MinimumRequiredVersion(spec) + if err != nil { + return fmt.Errorf("failed to get minimum required CDI spec version: %v", err) + } + spec.Version = minVersion + + return cdi.cache.WriteSpec(spec, specName) +} + +func (cdi *manager) DeleteClaimSpecFile(claimUID string) error { + specName := cdiapi.GenerateTransientSpecName(cdiVendor, cdi.cdiClass, claimUID) + return cdi.cache.RemoveSpec(specName) +} + +func (cdi *manager) GetClaimDevices(claimUID string, devices ...string) []string { + cdiDevices := []string{ + cdiparser.QualifiedName(cdiVendor, cdi.cdiClass, cdiCommonDeviceName), + } + + for _, device := range devices { + cdiDevice := cdiparser.QualifiedName(cdiVendor, cdi.cdiClass, fmt.Sprintf("%s-%s", claimUID, device)) + cdiDevices = append(cdiDevices, cdiDevice) + } + + return cdiDevices +} + +type PreparedDevices []*PreparedDevice + +type PreparedDevice struct { + drapbv1.Device + ContainerEdits *cdiapi.ContainerEdits +} + +func (pds PreparedDevices) GetDevices() []*drapbv1.Device { + var devices []*drapbv1.Device + for _, pd := range pds { + devices = append(devices, &pd.Device) + } + return devices +} diff --git a/images/virtualization-dra/internal/plugin/driver.go b/images/virtualization-dra/internal/plugin/driver.go new file mode 100644 index 0000000000..c8251e2020 --- /dev/null +++ b/images/virtualization-dra/internal/plugin/driver.go @@ -0,0 +1,196 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "context" + "errors" + "fmt" + "log/slog" + + "github.com/deckhouse/deckhouse/pkg/log" + "k8s.io/api/resource/v1" + resourceapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/dynamic-resource-allocation/kubeletplugin" + "k8s.io/dynamic-resource-allocation/resourceslice" +) + +const DriverName = "virtualization-dra" + +func NewDriver(nodeName string, kubeClient kubernetes.Interface, allocator Allocator, log *slog.Logger) *Driver { + return &Driver{ + nodeName: nodeName, + kubeClient: kubeClient, + allocator: allocator, + log: log.With(slog.String("driver", DriverName), slog.String("component", "driver")), + } +} + +type Driver struct { + nodeName string + + kubeClient kubernetes.Interface + allocator Allocator + log *slog.Logger + + helper *kubeletplugin.Helper + pluginCtx context.Context + pluginCancel context.CancelCauseFunc +} + +func (d *Driver) Start(ctx context.Context) error { + ctx, cancel := context.WithCancelCause(ctx) + d.pluginCtx = ctx + d.pluginCancel = cancel + + log.Info("Starting dra plugin") + helper, err := kubeletplugin.Start( + ctx, + d, + kubeletplugin.KubeClient(d.kubeClient), + kubeletplugin.NodeName(d.nodeName), + kubeletplugin.DriverName(DriverName), + kubeletplugin.RegistrarDirectoryPath(virtualizationRegistrarDirPath()), + kubeletplugin.RegistrarSocketFilename(virtualizationRegistrarSocketFilename), + kubeletplugin.PluginDataDirectoryPath(virtualizationPluginDirPath()), + ) + if err != nil { + return fmt.Errorf("failed to start kubelet plugin: %w", err) + } + + d.helper = helper + d.startPublisher(ctx) + + return err +} + +func (d *Driver) Wait() { + if d.pluginCtx != nil { + <-d.pluginCtx.Done() + } +} + +func (d *Driver) Shutdown() { + if d.helper != nil { + d.log.Info("Stopping dra plugin") + d.helper.Stop() + } +} + +func (d *Driver) PrepareResourceClaims(ctx context.Context, claims []*v1.ResourceClaim) (map[types.UID]kubeletplugin.PrepareResult, error) { + d.log.Info("Preparing resource claims") + + result := make(map[types.UID]kubeletplugin.PrepareResult, len(claims)) + + for _, claim := range claims { + result[claim.UID] = d.prepareResourceClaim(ctx, claim) + } + + return result, nil +} + +func (d *Driver) prepareResourceClaim(ctx context.Context, claim *resourceapi.ResourceClaim) kubeletplugin.PrepareResult { + if claim.Status.Allocation == nil { + return kubeletplugin.PrepareResult{ + Err: fmt.Errorf("claim %s/%s has no allocation", claim.Namespace, claim.Name), + } + } + + preparedPBs, err := d.allocator.Prepare(ctx, claim) + if err != nil { + return kubeletplugin.PrepareResult{ + Err: fmt.Errorf("error preparing devices for claim %v: %w", claim.UID, err), + } + } + var prepared []kubeletplugin.Device + for _, preparedPB := range preparedPBs { + prepared = append(prepared, kubeletplugin.Device{ + Requests: preparedPB.GetRequestNames(), + PoolName: preparedPB.GetPoolName(), + DeviceName: preparedPB.GetDeviceName(), + CDIDeviceIDs: preparedPB.GetCDIDeviceIDs(), + }) + } + + d.log.Info("Returning newly prepared devices", slog.String("uid", string(claim.UID)), slog.Any("devices", prepared)) + return kubeletplugin.PrepareResult{Devices: prepared} +} + +func (d *Driver) UnprepareResourceClaims(ctx context.Context, claims []kubeletplugin.NamespacedObject) (map[types.UID]error, error) { + d.log.Info("Unpreparing resource claims") + + result := make(map[types.UID]error) + + for _, claim := range claims { + result[claim.UID] = d.unprepareResourceClaim(ctx, claim) + } + + return result, nil +} + +func (d *Driver) unprepareResourceClaim(ctx context.Context, claim kubeletplugin.NamespacedObject) error { + if err := d.allocator.Unprepare(ctx, claim.UID); err != nil { + return fmt.Errorf("error unpreparing devices for claim %v: %w", claim.UID, err) + } + + return nil +} + +func (d *Driver) HandleError(ctx context.Context, err error, msg string) { + utilruntime.HandleErrorWithContext(ctx, err, msg) + if !errors.Is(err, kubeletplugin.ErrRecoverable) && d.pluginCancel != nil { + d.pluginCancel(fmt.Errorf("fatal background error: %w", err)) + } +} + +func (d *Driver) startPublisher(ctx context.Context) { + go func() { + ch := d.allocator.UpdateChannel() + for { + select { + case <-ctx.Done(): + return + case devices := <-ch: + d.log.Info("Publishing devices", slog.Any("devices", devices)) + if len(devices) > 0 { + } + resources := d.makeResources(devices) + err := d.helper.PublishResources(ctx, resources) + if err != nil { + d.log.Error("Failed to publish devices", slog.Any("err", err)) + } + } + } + }() +} + +func (d *Driver) makeResources(devices []resourceapi.Device) resourceslice.DriverResources { + return resourceslice.DriverResources{ + Pools: map[string]resourceslice.Pool{ + d.nodeName: { + Slices: []resourceslice.Slice{ + { + Devices: devices, + }, + }, + }, + }, + } +} diff --git a/images/virtualization-dra/internal/plugin/healthz.go b/images/virtualization-dra/internal/plugin/healthz.go new file mode 100644 index 0000000000..089e2ef8b7 --- /dev/null +++ b/images/virtualization-dra/internal/plugin/healthz.go @@ -0,0 +1,158 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "context" + "fmt" + "log/slog" + "net" + "net/url" + "strconv" + "sync" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" + drapb "k8s.io/kubelet/pkg/apis/dra/v1" + registerapi "k8s.io/kubelet/pkg/apis/pluginregistration/v1" +) + +type HealthCheck struct { + grpc_health_v1.UnimplementedHealthServer + server *grpc.Server + log *slog.Logger + wg sync.WaitGroup + regSockPath string + draSockPath string + port int +} + +func NewHealthCheck(port int, log *slog.Logger) *HealthCheck { + regSockPath := (&url.URL{ + Scheme: "unix", + Path: virtualizationRegistrarSocketPath(), + }).String() + + draSockPath := (&url.URL{ + Scheme: "unix", + Path: virtualizationPluginSocketPath(), + }).String() + + h := &HealthCheck{ + server: grpc.NewServer(), + log: log.With(slog.String("component", "healthcheck")), + regSockPath: regSockPath, + draSockPath: draSockPath, + port: port, + } + grpc_health_v1.RegisterHealthServer(h.server, h) + + return h +} + +func (h *HealthCheck) Start() error { + addr := net.JoinHostPort("", strconv.Itoa(h.port)) + lis, err := net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("failed to listen for healthcheck service at %s: %w", addr, err) + } + + h.wg.Add(1) + go func() { + defer h.wg.Done() + h.log.Info("starting healthcheck service", slog.String("addr", lis.Addr().String())) + if err := h.server.Serve(lis); err != nil { + h.log.Error("failed to serve healthcheck service", slog.String("addr", addr), slog.Any("err", err)) + } + }() + + return nil +} + +func (h *HealthCheck) Stop() { + if h.server != nil { + h.log.Info("stopping healthcheck service") + h.server.GracefulStop() + } + h.wg.Wait() +} + +// Check implements [grpc_health_v1.HealthServer]. +func (h *HealthCheck) Check(ctx context.Context, req *grpc_health_v1.HealthCheckRequest) (*grpc_health_v1.HealthCheckResponse, error) { + knownServices := map[string]struct{}{"": {}, "liveness": {}} + if _, known := knownServices[req.GetService()]; !known { + return nil, status.Error(codes.NotFound, "unknown service") + } + + status := &grpc_health_v1.HealthCheckResponse{ + Status: grpc_health_v1.HealthCheckResponse_NOT_SERVING, + } + + regClient, err := h.newRegClient() + if err != nil { + h.log.Error("failed to create registration client", slog.Any("err", err)) + return status, err + } + + info, err := regClient.GetInfo(ctx, ®isterapi.InfoRequest{}) + if err != nil { + h.log.Error("failed to call GetInfo", slog.Any("err", err)) + return status, nil + } + h.log.Info("Successfully invoked GetInfo", "info", info) + + draClient, err := h.newDraConn() + if err != nil { + h.log.Error("failed to create DRA client", slog.Any("err", err)) + return status, err + } + + _, err = draClient.NodePrepareResources(ctx, &drapb.NodePrepareResourcesRequest{}) + if err != nil { + h.log.Error("failed to call NodePrepareResources", slog.Any("err", err)) + return status, nil + } + h.log.Info("Successfully invoked NodePrepareResources") + + status.Status = grpc_health_v1.HealthCheckResponse_SERVING + return status, nil +} + +func (h *HealthCheck) newRegClient() (registerapi.RegistrationClient, error) { + regConn, err := grpc.NewClient( + h.regSockPath, + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + return nil, fmt.Errorf("connect to registration socket: %w", err) + } + return registerapi.NewRegistrationClient(regConn), nil +} + +func (h *HealthCheck) newDraConn() (drapb.DRAPluginClient, error) { + draConn, err := grpc.NewClient( + h.draSockPath, + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + return nil, fmt.Errorf("connect to DRA socket: %w", err) + } + return drapb.NewDRAPluginClient(draConn), nil +} diff --git a/images/virtualization-dra/internal/plugin/interfaces.go b/images/virtualization-dra/internal/plugin/interfaces.go new file mode 100644 index 0000000000..4ef9d1bf9d --- /dev/null +++ b/images/virtualization-dra/internal/plugin/interfaces.go @@ -0,0 +1,33 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "context" + + "github.com/containerd/nri/pkg/api" + resourceapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/types" + drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1beta1" +) + +type Allocator interface { + UpdateChannel() chan []resourceapi.Device + Prepare(ctx context.Context, claim *resourceapi.ResourceClaim) ([]*drapbv1.Device, error) + Unprepare(ctx context.Context, claimUID types.UID) error + Synchronize(ctx context.Context, pods []*api.PodSandbox, containers []*api.Container) ([]*api.ContainerUpdate, error) +} diff --git a/images/virtualization-dra/internal/plugin/nri_hooks.go b/images/virtualization-dra/internal/plugin/nri_hooks.go new file mode 100644 index 0000000000..4999d3c86f --- /dev/null +++ b/images/virtualization-dra/internal/plugin/nri_hooks.go @@ -0,0 +1,30 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "context" + "log/slog" + + "github.com/containerd/nri/pkg/api" +) + +// Synchronize is called by the NRI to synchronize the state of the driver during bootstrap. +func (d *Driver) Synchronize(ctx context.Context, pods []*api.PodSandbox, containers []*api.Container) ([]*api.ContainerUpdate, error) { + d.log.Info("Synchronizing state with the runtime...", slog.Int("pods", len(pods)), slog.Int("containers", len(containers))) + return d.allocator.Synchronize(ctx, pods, containers) +} diff --git a/images/virtualization-dra/internal/plugin/plugin.go b/images/virtualization-dra/internal/plugin/plugin.go new file mode 100644 index 0000000000..7aad9b0a03 --- /dev/null +++ b/images/virtualization-dra/internal/plugin/plugin.go @@ -0,0 +1,68 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "fmt" + "os" + "path" + + "k8s.io/dynamic-resource-allocation/kubeletplugin" +) + +var ( + kubeletPluginsDir = kubeletplugin.KubeletPluginsDir + kubeletRegistryDir = kubeletplugin.KubeletRegistryDir +) + +const ( + virtualizationPluginSocketFilename = "dra.sock" + virtualizationRegistrarSocketFilename = DriverName + "-reg.sock" +) + +func virtualizationPluginDirPath() string { + return path.Join(kubeletPluginsDir, DriverName) +} + +func virtualizationPluginSocketPath() string { + return path.Join(virtualizationPluginDirPath(), virtualizationPluginSocketFilename) +} + +func virtualizationRegistrarDirPath() string { + return kubeletRegistryDir +} + +func virtualizationRegistrarSocketPath() string { + return path.Join(virtualizationRegistrarDirPath(), virtualizationRegistrarSocketFilename) +} + +func InitPluginDirs(setKubeletPluginsDir, setKubeletRegistryDir string) error { + if setKubeletPluginsDir != "" { + kubeletPluginsDir = setKubeletPluginsDir + } + + pluginDir := virtualizationPluginDirPath() + if err := os.MkdirAll(pluginDir, 0700); err != nil { + return fmt.Errorf("failed to create directory %s: %w", pluginDir, err) + } + + if setKubeletRegistryDir != "" { + kubeletRegistryDir = setKubeletRegistryDir + } + + return nil +} diff --git a/images/virtualization-dra/internal/usb/convert.go b/images/virtualization-dra/internal/usb/convert.go new file mode 100644 index 0000000000..95f4cb65aa --- /dev/null +++ b/images/virtualization-dra/internal/usb/convert.go @@ -0,0 +1,72 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +import ( + resourceapi "k8s.io/api/resource/v1" + "k8s.io/utils/ptr" +) + +func convertToAPIDevice(usbDevice Device) *resourceapi.Device { + return &resourceapi.Device{ + Name: usbDevice.GetName(), + Attributes: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ + "name": { + StringValue: ptr.To(usbDevice.Name), + }, + "manufacturer": { + StringValue: ptr.To(usbDevice.Manufacturer), + }, + "product": { + StringValue: ptr.To(usbDevice.Product), + }, + "vendorID": { + StringValue: ptr.To(usbDevice.VendorID.String()), + }, + "productID": { + StringValue: ptr.To(usbDevice.ProductID.String()), + }, + "bcd": { + StringValue: ptr.To(usbDevice.BCD.String()), + }, + "bus": { + StringValue: ptr.To(usbDevice.Bus.String()), + }, + "resource.kubernetes.io/usbAddressBus": { + IntValue: ptr.To(int64(usbDevice.Bus)), + }, + "deviceNumber": { + StringValue: ptr.To(usbDevice.DeviceNumber.String()), + }, + "resource.kubernetes.io/usbAddressDeviceNumber": { + IntValue: ptr.To(int64(usbDevice.DeviceNumber)), + }, + "major": { + IntValue: ptr.To(int64(usbDevice.Major)), + }, + "minor": { + IntValue: ptr.To(int64(usbDevice.Minor)), + }, + "serial": { + StringValue: ptr.To(usbDevice.Serial), + }, + "devicePath": { + StringValue: ptr.To(usbDevice.DevicePath), + }, + }, + } +} diff --git a/images/virtualization-dra/internal/usb/device.go b/images/virtualization-dra/internal/usb/device.go new file mode 100644 index 0000000000..534a9d3465 --- /dev/null +++ b/images/virtualization-dra/internal/usb/device.go @@ -0,0 +1,250 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +import ( + "bufio" + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/deckhouse/virtualization-dra/pkg/set" +) + +type DeviceSet = set.Set[Device] + +func NewDeviceSet() *DeviceSet { + return set.New[Device]() +} + +type Device struct { + Name string + Manufacturer string + Product string + VendorID int4x + ProductID int4x + BCD int4x + Bus int3d + DeviceNumber int3d + Major int + Minor int + Serial string + DevicePath string +} + +func (d *Device) GetName() string { + // usb---- + // usb-003-005-e39-f100 + return fmt.Sprintf("usb-%s-%s-%s-%s", d.Bus.String(), d.DeviceNumber.String(), d.VendorID.String(), d.ProductID.String()) +} + +func (d *Device) Validate() error { + if d.VendorID == 0 { + return fmt.Errorf("VendorID is required") + } + if d.ProductID == 0 { + return fmt.Errorf("ProductID is required") + } + if d.Bus == 0 { + return fmt.Errorf("Bus is required") + } + if d.DeviceNumber == 0 { + return fmt.Errorf("DeviceNumber is required") + } + if d.DevicePath == "" { + return fmt.Errorf("DevicePath is required") + } + if d.Major == 0 { + return fmt.Errorf("Major is required") + } + if d.Minor == 0 { + return fmt.Errorf("Minor is required") + } + return nil +} + +func LoadDevice(path string) (device Device, err error) { + if err = parseSysUeventFile(path, &device); err != nil { + return + } + if err = parseSerial(path, &device); err != nil { + return + } + if err = parseManufacturer(path, &device); err != nil { + return + } + if err = parseProduct(path, &device); err != nil { + return + } + return +} + +func parseSysUeventFile(path string, device *Device) error { + // Example uevent file: + // MAJOR=189 + // MINOR=257 + // DEVNAME=bus/usb/003/002 + // DEVTYPE=usb_device + // DRIVER=usb + // PRODUCT=e39/f100/35d + // TYPE=0/0/0 + // BUSNUM=003 + // DEVNUM=002 + file, err := os.Open(filepath.Join(path, "uevent")) + if err != nil { + return fmt.Errorf("unable to open the file %s: %w", path, err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + values := strings.Split(line, "=") + if len(values) != 2 { + slog.Info("Skipping %s due not being key=value", slog.String("line", line)) + continue + } + switch values[0] { + case "MAJOR": + val, err := strconv.ParseInt(values[1], 10, 32) + if err != nil { + slog.Error("Failed to parse MAJOR", slog.String("value", values[1]), slog.Any("err", err)) + return nil + } + device.Major = int(val) + case "MINOR": + val, err := strconv.ParseInt(values[1], 10, 32) + if err != nil { + slog.Error("Failed to parse MINOR", slog.String("value", values[1]), slog.Any("err", err)) + return nil + } + device.Minor = int(val) + case "BUSNUM": + val, err := strconv.ParseInt(values[1], 10, 32) + if err != nil { + slog.Error("Failed to parse BUSNUM", slog.String("value", values[1]), slog.Any("err", err)) + return nil + } + device.Bus = int3d(val) + case "DEVNUM": + val, err := strconv.ParseInt(values[1], 10, 32) + if err != nil { + slog.Error("Failed to parse DEVNUM", slog.String("value", values[1]), slog.Any("err", err)) + return nil + } + device.DeviceNumber = int3d(val) + case "PRODUCT": + products := strings.Split(values[1], "/") + if len(products) != 3 { + slog.Error("Failed to parse PRODUCT", slog.String("value", values[1]), slog.Any("err", err)) + return nil + } + + val, err := strconv.ParseInt(products[0], 16, 32) + if err != nil { + slog.Error("Failed to parse PRODUCT", slog.String("value", values[1]), slog.Any("err", err)) + return nil + } + device.VendorID = int4x(val) + + val, err = strconv.ParseInt(products[1], 16, 32) + if err != nil { + slog.Error("Failed to parse PRODUCT", slog.String("value", values[1]), slog.Any("err", err)) + return nil + } + device.ProductID = int4x(val) + + val, err = strconv.ParseInt(products[2], 16, 32) + if err != nil { + slog.Error("Failed to parse PRODUCT", slog.String("value", values[1]), slog.Any("err", err)) + return nil + } + device.BCD = int4x(val) + case "DEVNAME": + device.DevicePath = filepath.Join("/dev", values[1]) + default: + slog.Info("Skipping unhandled line", slog.String("line", line)) + } + } + return nil +} + +func parseSerial(path string, device *Device) error { + b, err := os.ReadFile(filepath.Join(path, "serial")) + if err != nil { + return err + } + lines := strings.Split(string(b), "\n") + if len(lines) >= 1 { + device.Serial = strings.TrimSpace(lines[0]) + } else { + device.Serial = "unknown" + } + + return nil +} + +func parseManufacturer(path string, device *Device) error { + b, err := os.ReadFile(filepath.Join(path, "manufacturer")) + if err != nil { + return err + } + lines := strings.Split(string(b), "\n") + if len(lines) >= 1 { + device.Manufacturer = strings.TrimSpace(lines[0]) + } else { + device.Manufacturer = "unknown" + } + return nil +} + +func parseProduct(path string, device *Device) error { + b, err := os.ReadFile(filepath.Join(path, "product")) + if err != nil { + return err + } + lines := strings.Split(string(b), "\n") + if len(lines) >= 1 { + device.Product = strings.TrimSpace(lines[0]) + } else { + device.Product = "unknown" + } + return nil +} + +type int4x int + +func (i int4x) String() string { + s := strconv.FormatInt(int64(i), 16) + if len(s) < 4 { + return strings.Repeat("0", 4-len(s)) + s + } + return s +} + +type int3d int + +func (i int3d) String() string { + s := strconv.FormatInt(int64(i), 10) + if len(s) < 3 { + return strings.Repeat("0", 3-len(s)) + s + } + return s +} diff --git a/images/virtualization-dra/internal/usb/discovery.go b/images/virtualization-dra/internal/usb/discovery.go new file mode 100644 index 0000000000..304ea83d88 --- /dev/null +++ b/images/virtualization-dra/internal/usb/discovery.go @@ -0,0 +1,62 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" +) + +const PathToUSBDevices = "/sys/bus/usb/devices" + +func discoverPluggedUSBDevices(pathToUSBDevices string) (*DeviceSet, error) { + usbDeviceSet := NewDeviceSet() + err := filepath.Walk(pathToUSBDevices, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + // Ignore named usb controllers + if strings.HasPrefix(info.Name(), "usb") { + return nil + } + // We are interested in actual USB devices information that + // contains idVendor and idProduct. We can skip all others. + if _, err := os.Stat(filepath.Join(path, "idVendor")); err != nil { + return nil + } + + // Get device information + device, err := LoadDevice(path) + if err = device.Validate(); err != nil { + slog.Error("failed to validate device, skip...", slog.Any("device", device), slog.String("error", err.Error())) + return nil + } + if err != nil { + return err + } + usbDeviceSet.Add(device) + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed when walking usb devices tree: %w", err) + } + return usbDeviceSet, nil +} diff --git a/images/virtualization-dra/internal/usb/monitor.go b/images/virtualization-dra/internal/usb/monitor.go new file mode 100644 index 0000000000..356f7967be --- /dev/null +++ b/images/virtualization-dra/internal/usb/monitor.go @@ -0,0 +1,169 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +import ( + "context" + "log/slog" + "time" + + "github.com/godbus/dbus/v5" +) + +type monitor struct { + callback monitorCallback + log *slog.Logger +} + +type monitorCallback struct { + Add func() + Update func() + Delete func() +} + +func newUSBMonitor(callback monitorCallback) *monitor { + return &monitor{ + callback: callback, + log: slog.With(slog.String("component", "usb-monitor")), + } +} + +func (m *monitor) Start(ctx context.Context) { + go func() { + for { + select { + case <-ctx.Done(): + return + case <-time.After(5 * time.Second): + if err := m.run(ctx); err != nil { + m.log.Error("failed to run monitor", slog.Any("err", err)) + } + } + } + }() +} + +func (m *monitor) run(ctx context.Context) error { + conn, err := dbus.ConnectSystemBus(dbus.WithContext(ctx)) + if err != nil { + return err + } + + rules := []string{ + "type='signal',interface='org.freedesktop.DBus.ObjectManager',member='InterfacesAdded'", + "type='signal',interface='org.freedesktop.DBus.ObjectManager',member='InterfacesRemoved'", + } + + for _, rule := range rules { + call := conn.BusObject().Call("org.freedesktop.DBus.AddMatch", 0, rule) + if call.Err != nil { + m.log.Error("Failed to add rule", slog.String("rule", rule), slog.Any("err", call.Err)) + } + } + + signals := make(chan *dbus.Signal, 100) + conn.Signal(signals) + + m.log.Info("Starting USB Monitor...") + defer m.log.Info("Stopping USB Monitor...") + + for { + select { + case <-ctx.Done(): + return nil + case signal := <-signals: + m.handleSignal(signal) + } + } +} + +func (m *monitor) handleSignal(signal *dbus.Signal) { + switch signal.Name { + case "org.freedesktop.DBus.ObjectManager.InterfacesAdded": + if len(signal.Body) >= 2 { + path, ok := signal.Body[0].(dbus.ObjectPath) + if !ok { + return + } + interfaces, ok := signal.Body[1].(map[string]map[string]dbus.Variant) + if !ok { + return + } + + if m.isUSBDevice(interfaces) { + slog.Info("USB Device connected", slog.String("path", string(path))) + if m.callback.Add != nil { + m.callback.Add() + } + } + } + + case "org.freedesktop.DBus.ObjectManager.InterfacesRemoved": + // Sender = {string} ":1.10" + // Path = {dbus.ObjectPath} "/org/freedesktop/UDisks2" + // Name = {string} "org.freedesktop.DBus.ObjectManager.InterfacesRemoved" + // Body = {[]interface{}} + // Body[0] = interface{} | dbus.ObjectPath "/org/freedesktop/UDisks2/block_devices/sda2" + // Body[1] = interface{} | []string{"org.freedesktop.UDisks2.Filesystem","org.freedesktop.UDisks2.Partition","org.freedesktop.UDisks2.Block","org.freedesktop.UDisks2.Drive"} + + if len(signal.Body) >= 2 { + path, ok := signal.Body[0].(dbus.ObjectPath) + if !ok { + return + } + interfaces, ok := signal.Body[1].([]string) + if !ok { + return + } + + if m.containsUSBInterface(interfaces) { + slog.Info("USB Device disconnected", slog.String("path", string(path))) + if m.callback.Delete != nil { + m.callback.Delete() + } + } + } + } +} + +func (m *monitor) isUSBDevice(interfaces map[string]map[string]dbus.Variant) bool { + if drive, ok := interfaces["org.freedesktop.UDisks2.Drive"]; ok { + if connectionBus, ok := drive["ConnectionBus"]; ok { + bus, _ := connectionBus.Value().(string) + return bus == "usb" + } + } + + if block, ok := interfaces["org.freedesktop.UDisks2.Block"]; ok { + if _, ok := block["Drive"]; ok { + // if it has a link to drive, it can be USB + return true + } + } + + return false +} + +func (m *monitor) containsUSBInterface(interfaces []string) bool { + for _, iface := range interfaces { + if iface == "org.freedesktop.UDisks2.Drive" || + iface == "org.freedesktop.UDisks2.Block" { + return true + } + } + return false +} diff --git a/images/virtualization-dra/internal/usb/monitor_test.go b/images/virtualization-dra/internal/usb/monitor_test.go new file mode 100644 index 0000000000..d01ea54377 --- /dev/null +++ b/images/virtualization-dra/internal/usb/monitor_test.go @@ -0,0 +1,53 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +import ( + "context" + "fmt" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMonitor(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "USB Monitor Suite") +} + +var _ = Describe("USB Monitor", func() { + + It("should add a new device", func() { + + monitor := newUSBMonitor(monitorCallback{ + Add: func() { + fmt.Println("USB Device connected") + }, + Update: func() { + fmt.Println("USB Device updated") + }, + Delete: func() { + fmt.Println("USB Device disconnected") + }, + }) + + monitor.Start(context.Background()) + time.Sleep(10 * time.Minute) + }) +}) diff --git a/images/virtualization-dra/internal/usb/store.go b/images/virtualization-dra/internal/usb/store.go new file mode 100644 index 0000000000..5ae5d2b50d --- /dev/null +++ b/images/virtualization-dra/internal/usb/store.go @@ -0,0 +1,369 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +import ( + "context" + "fmt" + "log/slog" + "strings" + "sync" + "time" + + "github.com/containerd/nri/pkg/api" + resourceapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/types" + drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1beta1" + "k8s.io/utils/ptr" + cdiapi "tags.cncf.io/container-device-interface/pkg/cdi" + cdispec "tags.cncf.io/container-device-interface/specs-go" + + "github.com/deckhouse/virtualization-dra/internal/cdi" + "github.com/deckhouse/virtualization-dra/pkg/set" +) + +const DefaultResyncPeriod = 10 * time.Minute + +func NewAllocationStore(nodeName, devicesPath string, resyncPeriod time.Duration, cdiManager cdi.Manager, log *slog.Logger) *AllocationStore { + if resyncPeriod == 0 { + resyncPeriod = DefaultResyncPeriod + } + if devicesPath == "" { + devicesPath = PathToUSBDevices + } + store := &AllocationStore{ + nodeName: nodeName, + devicesPath: devicesPath, + resyncPeriod: resyncPeriod, + cdi: cdiManager, + log: log.With(slog.String("component", "usb-allocation-store")), + updateChannel: make(chan []resourceapi.Device, 2), + discoverPluggedUSBDevices: NewDeviceSet(), + allocatableDevices: make(map[string]resourceapi.Device), + allocatedDevices: set.New[string](), + resourceClaimAllocations: make(map[types.UID][]string), + } + + monitor := newUSBMonitor(monitorCallback{ + Add: store.genericCallback, + Update: store.genericCallback, + Delete: store.genericCallback, + }) + + store.monitor = monitor + + return store +} + +type AllocationStore struct { + nodeName string + devicesPath string + resyncPeriod time.Duration + + cdi cdi.Manager + log *slog.Logger + + monitor *monitor + + updateChannel chan []resourceapi.Device + mu sync.RWMutex + + discoverPluggedUSBDevices *DeviceSet + allocatableDevices map[string]resourceapi.Device + allocatedDevices *set.Set[string] + resourceClaimAllocations map[types.UID][]string +} + +func (s *AllocationStore) sync() error { + s.mu.Lock() + defer s.mu.Unlock() + + discoverPluggedUSBDevices, err := discoverPluggedUSBDevices(s.devicesPath) + if err != nil { + return err + } + if discoverPluggedUSBDevices.Equal(s.discoverPluggedUSBDevices) { + return nil + } + s.discoverPluggedUSBDevices = discoverPluggedUSBDevices + + allocatableDevices := make([]resourceapi.Device, discoverPluggedUSBDevices.Len()) + for i, usbDevice := range discoverPluggedUSBDevices.Slice() { + allocatableDevices[i] = *convertToAPIDevice(usbDevice) + } + + allocatableDevicesByName := make(map[string]resourceapi.Device, len(allocatableDevices)) + for _, device := range allocatableDevices { + allocatableDevicesByName[device.Name] = device + } + + s.allocatableDevices = allocatableDevicesByName + + s.updateChannel <- allocatableDevices + + return nil +} + +func (s *AllocationStore) genericCallback() { + if err := s.sync(); err != nil { + s.log.Error("failed to sync usb state", slog.Any("err", err)) + } +} + +func (s *AllocationStore) Start(ctx context.Context) error { + if err := s.cdi.CreateCommonSpecFile(); err != nil { + return fmt.Errorf("failed to create CDI common spec file: %w", err) + } + + doSync := func() { + err := s.sync() + if err != nil { + s.log.Error("failed to sync usb state", slog.Any("err", err)) + } + } + ticker := time.NewTicker(s.resyncPeriod) + go func() { + doSync() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + doSync() + } + } + }() + + s.monitor.Start(ctx) + + return nil +} + +func (s *AllocationStore) UpdateChannel() chan []resourceapi.Device { + return s.updateChannel +} + +func (s *AllocationStore) Prepare(_ context.Context, claim *resourceapi.ResourceClaim) ([]*drapbv1.Device, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if claim.Status.Allocation == nil { + return nil, fmt.Errorf("claim %s/%s has no allocation", claim.Namespace, claim.Name) + } + + claimUID := string(claim.UID) + + preparedDevices := make(cdi.PreparedDevices, len(claim.Status.Allocation.Devices.Results)) + for i, result := range claim.Status.Allocation.Devices.Results { + usbDevice, exists := s.allocatableDevices[result.Device] + if !exists { + return nil, fmt.Errorf("requested device is not allocatable: %v", result.Device) + } + // TODO: unnecessary? + // kubernetes check allocatable devices + // Warning FailedScheduling 8s default-scheduler 0/3 nodes are available: + // 1 node(s) had tolerated taint {node-role.kubernetes.io/control-plane: }, + // 2 cannot allocate all claims. + // still not schedulable, preemption: 0/3 nodes are available: 3 Preemption is not helpful for scheduling. + if s.allocatedDevices.Contains(result.Device) { + return nil, fmt.Errorf("device %v is already allocated", result.Device) + } + + edits, err := s.makeContainerEdits(claimUID, &usbDevice) + if err != nil { + return nil, err + } + device := cdi.PreparedDevice{ + Device: drapbv1.Device{ + RequestNames: []string{result.Request}, + PoolName: result.Pool, + DeviceName: result.Device, + CDIDeviceIDs: s.cdi.GetClaimDevices(claimUID, result.Device), + }, + ContainerEdits: edits, + } + preparedDevices[i] = &device + } + + err := s.cdi.CreateClaimSpecFile(claimUID, preparedDevices) + if err != nil { + return nil, fmt.Errorf("unable to create CDI spec file for claim: %v", err) + } + + devices := preparedDevices.GetDevices() + for _, device := range devices { + s.allocatedDevices.Add(device.DeviceName) + s.resourceClaimAllocations[claim.UID] = append(s.resourceClaimAllocations[claim.UID], device.DeviceName) + } + + return devices, nil +} + +// TODO: refactor me +func (s *AllocationStore) makeContainerEdits(claimUID string, device *resourceapi.Device) (*cdiapi.ContainerEdits, error) { + var ( + devicePath string + deviceNum string + bus string + major int64 + minor int64 + ) + + if attr, ok := device.Attributes["devicePath"]; ok { + if val := attr.StringValue; val != nil { + devicePath = *val + } else { + return nil, fmt.Errorf("devicePath attribute is not exist") + } + } + + if attr, ok := device.Attributes["deviceNumber"]; ok { + if val := attr.StringValue; val != nil { + deviceNum = *val + } else { + return nil, fmt.Errorf("deviceNum attribute is not exist") + } + } + + if attr, ok := device.Attributes["bus"]; ok { + if val := attr.StringValue; val != nil { + bus = *val + } else { + return nil, fmt.Errorf("bus attribute is not exist") + } + } + + if attr, ok := device.Attributes["major"]; ok { + if val := attr.IntValue; val != nil { + major = *val + } else { + return nil, fmt.Errorf("major attribute is not exist") + } + } + + if attr, ok := device.Attributes["minor"]; ok { + if val := attr.IntValue; val != nil { + minor = *val + } else { + return nil, fmt.Errorf("minor attribute is not exist") + } + } + + claimUIDUpper := strings.ToUpper(claimUID) + deviceNameUpper := strings.ToUpper(device.Name) + + edits := &cdiapi.ContainerEdits{ + ContainerEdits: &cdispec.ContainerEdits{ + Env: []string{ + fmt.Sprintf("DRA_USB_CLAIM_UID_%s=%s", claimUIDUpper, claimUID), + fmt.Sprintf("DRA_USB_DEVICE_NAME_%s=%s", deviceNameUpper, device.Name), + fmt.Sprintf("DRA_USB_CLAIM_UID_%s_DEVICE_NAME=%s", claimUIDUpper, device.Name), + fmt.Sprintf("DRA_USB_%s_DEVICE_PATH=%s", deviceNameUpper, devicePath), + fmt.Sprintf("DRA_USB_%s_BUS_DEVICENUMBER=%s:%s", deviceNameUpper, bus, deviceNum), + }, + DeviceNodes: []*cdispec.DeviceNode{ + { + Path: devicePath, + HostPath: devicePath, + Type: "c", + Major: major, + Minor: minor, + Permissions: "mrw", + UID: ptr.To(uint32(107)), // qemu user. TODO: make this configurable + GID: ptr.To(uint32(107)), // qemu group. TODO: make this configurable + }, + }, + }, + } + + return edits, nil +} + +func (s *AllocationStore) Unprepare(_ context.Context, claimUID types.UID) error { + s.mu.Lock() + defer s.mu.Unlock() + + if err := s.cdi.DeleteClaimSpecFile(string(claimUID)); err != nil { + return fmt.Errorf("unable to delete CDI spec file for claim: %w", err) + } + + allocatedDevices := s.resourceClaimAllocations[claimUID] + for _, device := range allocatedDevices { + s.allocatedDevices.Remove(device) + } + delete(s.resourceClaimAllocations, claimUID) + + return nil +} + +func (s *AllocationStore) Synchronize(_ context.Context, pods []*api.PodSandbox, containers []*api.Container) ([]*api.ContainerUpdate, error) { + s.mu.Lock() + defer s.mu.Unlock() + + containersByPodSandboxId := make(map[string][]*api.Container, len(pods)) + for _, ctr := range containers { + containersByPodSandboxId[ctr.PodSandboxId] = append(containersByPodSandboxId[ctr.PodSandboxId], ctr) + } + + for _, pod := range pods { + s.log.Info("Synchronize pod", slog.String("name", pod.Name), slog.String("namespace", pod.Namespace)) + ctrs := containersByPodSandboxId[pod.Id] + + for _, ctr := range ctrs { + claimUIDDeviceNames, err := parseDraEnvToClaimAllocations(ctr.Env) + if err != nil { + s.log.Error("failed to parse dra env", slog.String("name", pod.Name), slog.String("namespace", pod.Namespace), slog.Any("err", err)) + continue + } + for claimUID, deviceNames := range claimUIDDeviceNames { + s.resourceClaimAllocations[claimUID] = append(s.resourceClaimAllocations[claimUID], deviceNames...) + for _, deviceName := range deviceNames { + s.allocatedDevices.Add(deviceName) + } + } + + } + } + return nil, nil +} + +func parseDraEnvToClaimAllocations(envs []string) (map[types.UID][]string, error) { + result := make(map[types.UID][]string) + + for _, env := range envs { + parts := strings.SplitN(env, "=", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid dra env: %s", env) + } + key := parts[0] + value := parts[1] + + if !strings.HasPrefix(key, "DRA_USB_CLAIM_UID_") || !strings.HasSuffix(key, "_DEVICE_NAME") { + continue + } + uid := strings.TrimPrefix(key, "DRA_USB_CLAIM_UID_") + uid = strings.TrimSuffix(uid, "_DEVICE_NAME") + uid = strings.ToLower(uid) + claimUID := types.UID(uid) + + deviceName := value + + result[claimUID] = append(result[claimUID], deviceName) + } + + return result, nil +} diff --git a/images/virtualization-dra/pkg/logger/logger.go b/images/virtualization-dra/pkg/logger/logger.go new file mode 100644 index 0000000000..12222e88b9 --- /dev/null +++ b/images/virtualization-dra/pkg/logger/logger.go @@ -0,0 +1,88 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logger + +import ( + "io" + "log/slog" + "os" + "strings" + + "github.com/go-logr/logr" + "k8s.io/klog/v2" + + "github.com/deckhouse/deckhouse/pkg/log" +) + +type Output string + +const ( + Stdout Output = "stdout" + Stderr Output = "stderr" + Discard Output = "discard" +) + +const DefaultLogLevel = log.LevelInfo + +var DefaultLogOutput = os.Stdout + +func NewLogger(level, output string, debugVerbosity int) *log.Logger { + return log.NewLogger(log.WithLevel(detectLogLevel(level, debugVerbosity)), log.WithOutput(detectLogOutput(output))) +} + +func detectLogLevel(level string, debugVerbosity int) slog.Level { + switch strings.ToLower(level) { + case "fatal": + return log.LevelFatal.Level() + case "error": + return log.LevelError.Level() + case "warn": + return log.LevelWarn.Level() + case "info": + return log.LevelInfo.Level() + case "debug": + if debugVerbosity != 0 { + return slog.Level(-1 * debugVerbosity) + } + + return log.LevelDebug.Level() + case "trace": + return log.LevelTrace.Level() + default: + return DefaultLogLevel.Level() + } +} + +func detectLogOutput(output string) io.Writer { + switch strings.ToLower(output) { + case string(Stdout): + return os.Stdout + case string(Stderr): + return os.Stderr + case string(Discard): + return io.Discard + default: + return DefaultLogOutput + } +} + +func SetDefaultLogger(l *log.Logger) { + slog.SetDefault(slog.New(l.Handler())) + log.SetDefault(l) + fromSlog := logr.FromSlogHandler(l.Handler()) + klog.SetLogger(fromSlog) +} diff --git a/images/virtualization-dra/pkg/logger/options.go b/images/virtualization-dra/pkg/logger/options.go new file mode 100644 index 0000000000..8199b01609 --- /dev/null +++ b/images/virtualization-dra/pkg/logger/options.go @@ -0,0 +1,38 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logger + +import ( + "github.com/deckhouse/deckhouse/pkg/log" + "github.com/spf13/pflag" +) + +type Options struct { + Level string + Output string + DebugVerbosity int +} + +func (o *Options) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&o.Level, "log-level", o.Level, "Log level") + fs.StringVar(&o.Output, "log-output", o.Output, "Log output") + fs.IntVar(&o.DebugVerbosity, "log-debug-verbosity", o.DebugVerbosity, "Log debug verbosity") +} + +func (o *Options) Complete() *log.Logger { + return NewLogger(o.Level, o.Output, o.DebugVerbosity) +} diff --git a/images/virtualization-dra/pkg/set/set.go b/images/virtualization-dra/pkg/set/set.go new file mode 100644 index 0000000000..4b7f4378a5 --- /dev/null +++ b/images/virtualization-dra/pkg/set/set.go @@ -0,0 +1,64 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package set + +type Set[T comparable] struct { + m map[T]struct{} +} + +func New[T comparable]() *Set[T] { + return &Set[T]{ + m: make(map[T]struct{}), + } +} +func (s *Set[T]) Add(v T) { + s.m[v] = struct{}{} +} + +func (s *Set[T]) Remove(v T) { + delete(s.m, v) +} + +func (s *Set[T]) Contains(v T) bool { + _, ok := s.m[v] + return ok +} + +func (s *Set[T]) Len() int { + return len(s.m) +} + +func (s *Set[T]) Slice() []T { + out := make([]T, 0, len(s.m)) + for k := range s.m { + out = append(out, k) + } + return out +} + +func (s *Set[T]) Equal(other *Set[T]) bool { + if s.Len() != other.Len() { + return false + } + + for k := range s.m { + if !other.Contains(k) { + return false + } + } + return true +} diff --git a/images/virtualization-dra/test/deviceclass.yaml b/images/virtualization-dra/test/deviceclass.yaml new file mode 100644 index 0000000000..195511eca4 --- /dev/null +++ b/images/virtualization-dra/test/deviceclass.yaml @@ -0,0 +1,8 @@ +apiVersion: resource.k8s.io/v1beta1 +kind: DeviceClass +metadata: + name: usb-devices.virtualization.deckhouse.io +spec: + selectors: + - cel: + expression: "device.driver == 'virtualization-dra'" diff --git a/images/virtualization-dra/test/pod-with-claim.yaml b/images/virtualization-dra/test/pod-with-claim.yaml new file mode 100644 index 0000000000..5240037a31 --- /dev/null +++ b/images/virtualization-dra/test/pod-with-claim.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-pod-with-usb-1 + namespace: usb2 +spec: + containers: + - name: test-container + image: nicolaka/netshoot:latest + command: ["sleep", "3600"] + resources: + claims: + - name: usb-device + resourceClaims: + - name: usb-device + resourceClaimName: test-pod-with-usb-1 diff --git a/images/virtualization-dra/test/pod-with-template-1.yaml b/images/virtualization-dra/test/pod-with-template-1.yaml new file mode 100644 index 0000000000..5b44de020b --- /dev/null +++ b/images/virtualization-dra/test/pod-with-template-1.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-pod-with-usb-template-1 + namespace: usb +spec: + containers: + - name: test-container + image: nicolaka/netshoot:latest + command: ["sleep", "3600"] + resources: + claims: + - name: usb-device + resourceClaims: + - name: usb-device + resourceClaimTemplateName: usb-product-f100-vendor-0e39-template diff --git a/images/virtualization-dra/test/pod-with-template-2.yaml b/images/virtualization-dra/test/pod-with-template-2.yaml new file mode 100644 index 0000000000..bcd20b5eff --- /dev/null +++ b/images/virtualization-dra/test/pod-with-template-2.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-pod-with-usb-template-2 + namespace: usb +spec: + containers: + - name: test-container + image: nicolaka/netshoot:latest + command: ["sleep", "3600"] + resources: + claims: + - name: usb-device + volumeMounts: + - name: mnt + mountPath: /mnt + volumes: + - name: mnt + emptyDir: {} + resourceClaims: + - name: usb-device + resourceClaimTemplateName: usb-product-f100-vendor-0e39-template diff --git a/images/virtualization-dra/test/resourceclaim-template.yaml b/images/virtualization-dra/test/resourceclaim-template.yaml new file mode 100644 index 0000000000..3077bd1bf3 --- /dev/null +++ b/images/virtualization-dra/test/resourceclaim-template.yaml @@ -0,0 +1,18 @@ +apiVersion: resource.k8s.io/v1beta1 +kind: ResourceClaimTemplate +metadata: + name: usb-product-f100-vendor-0e39-template + namespace: usb +spec: + spec: + devices: + requests: + - name: req-0 + allocationMode: "ExactCount" + count: 1 + deviceClassName: usb-devices.virtualization.deckhouse.io + selectors: + - cel: + expression: |- + device.attributes["virtualization-dra"].productID == "f100" && + device.attributes["virtualization-dra"].vendorID == "0e39" diff --git a/images/virtualization-dra/test/resourceclaim.yaml b/images/virtualization-dra/test/resourceclaim.yaml new file mode 100644 index 0000000000..66f1a93928 --- /dev/null +++ b/images/virtualization-dra/test/resourceclaim.yaml @@ -0,0 +1,17 @@ +apiVersion: resource.k8s.io/v1beta1 +kind: ResourceClaim +metadata: + name: test-pod-with-usb-1 + namespace: usb2 +spec: + devices: + requests: + - allocationMode: ExactCount + count: 1 + deviceClassName: usb-devices.virtualization.deckhouse.io + name: req-0 + selectors: + - cel: + expression: |- + device.attributes["virtualization-dra"].productID == "f100" && + device.attributes["virtualization-dra"].vendorID == "0e39" diff --git a/images/virtualization-dra/werf.inc.yaml b/images/virtualization-dra/werf.inc.yaml new file mode 100644 index 0000000000..919a0adcbb --- /dev/null +++ b/images/virtualization-dra/werf.inc.yaml @@ -0,0 +1,41 @@ +--- +image: {{ .ModuleNamePrefix }}{{ .ImageName }}-builder +final: false +fromImage: {{ eq $.SVACE_ENABLED "false" | ternary "builder/golang-bookworm-1.24" "builder/golang-alt-svace-1.24.9" }} +git: + - add: {{ .ModuleDir }}/images/{{ .ImageName }} + to: /src/images/virtualization-dra + stageDependencies: + install: + - go.mod + - go.sum + setup: + - "**/*.go" +secrets: + - id: GOPROXY + value: {{ .GOPROXY }} +mount: + - fromPath: ~/go-pkg-cache + to: /go/pkg +shell: + install: + - export GOPROXY=$(cat /run/secrets/GOPROXY) + - cd /src/images/virtualization-dra + - go mod download + setup: + - cd /src/images/virtualization-dra + - mkdir /out + - export GOOS=linux + - export GOARCH=amd64 + - export CGO_ENABLED=0 + + - | + echo "Build virtualization-dra-plugin binary" + {{- $_ := set $ "ProjectName" (list $.ImageName "virtualization-dra-plugin" | join "/") }} + + {{- if eq $.DEBUG_COMPONENT "delve/virtualization-dra-plugin" }} + go build -v -o /out/virtualization-dra-plugin ./cmd/virtualization-dra-plugin + {{- else }} + {{- include "image-build.build" (set $ "BuildCommand" `go build -ldflags="-s -w" -v -o /out/virtualization-dra-plugin ./cmd/virtualization-dra-plugin`) | nindent 4 }} + {{- end }} + diff --git a/templates/kubevirt/kubevirt.yaml b/templates/kubevirt/kubevirt.yaml index 81b71c9949..da46c7be0b 100644 --- a/templates/kubevirt/kubevirt.yaml +++ b/templates/kubevirt/kubevirt.yaml @@ -58,6 +58,10 @@ spec: - CPUManager - Sidecar - VolumeSnapshotDataSource + - VolumeMigration + - VolumesUpdateStrategy + - HostDevicesWithDRA + - HostDevices virtualMachineOptions: disableSerialConsoleLog: {} customizeComponents: diff --git a/templates/virtualization-dra/_helper.tpl b/templates/virtualization-dra/_helper.tpl new file mode 100644 index 0000000000..72f4ff159d --- /dev/null +++ b/templates/virtualization-dra/_helper.tpl @@ -0,0 +1,5 @@ +{{- define "virtualization-dra.isEnabled" -}} +{{- if eq (include "hasValidModuleConfig" .) "true" -}} +true +{{- end -}} +{{- end -}} diff --git a/templates/virtualization-dra/daemonset.yaml b/templates/virtualization-dra/daemonset.yaml new file mode 100644 index 0000000000..47c1e7543f --- /dev/null +++ b/templates/virtualization-dra/daemonset.yaml @@ -0,0 +1,132 @@ +{{- $priorityClassName := include "priorityClassName" . }} +{{- $delve := (include "delve" . | fromYaml) -}} +{{- define "virtualization-dra_resources" }} +cpu: 10m +memory: 25Mi +{{- end }} + + +{{- if eq (include "virtualization-dra.isEnabled" .) "true"}} + +{{- if (.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} +--- +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: virtualization-dra + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra" "workload-resource-policy.deckhouse.io" "every-node")) | nindent 2 }} +spec: + targetRef: + apiVersion: "apps/v1" + kind: DaemonSet + name: virtualization-dra + updatePolicy: + updateMode: "Auto" + resourcePolicy: + containerPolicies: + - containerName: virtualization-dra + minAllowed: + {{- include "virtualization-dra_resources" . | nindent 8 }} + maxAllowed: + cpu: 20m + memory: 25Mi +{{- end }} + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: virtualization-dra + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra")) | nindent 2 }} +spec: + selector: + matchLabels: + app: virtualization-dra + template: + metadata: + labels: + app: virtualization-dra + spec: + {{- include "helm_lib_priority_class" (tuple . $priorityClassName) | nindent 6 }} + {{- include "helm_lib_tolerations" (tuple . "any-node") | nindent 6 }} + {{- include "helm_lib_module_pod_security_context_run_as_user_root" . | nindent 6 }} + imagePullSecrets: + - name: virtualization-module-registry + serviceAccountName: virtualization-dra + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + kubernetes.io/os: linux + containers: + - name: virtualization-dra + {{- include "helm_lib_module_container_security_context_privileged_read_only_root_filesystem" . | nindent 10 }} + image: {{ include "helm_lib_module_image" (list . "virtualizationDraPlugin") }} + imagePullPolicy: "IfNotPresent" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if eq (include "moduleLogLevel" .) "debug" }} + - name: VERBOSITY + value: "10" + {{- end }} + - name: USB_DEVICES_PATH + value: /sys/bus/usb/devices + - name: DBUS_SYSTEM_BUS_ADDRESS + value: "unix:path=/var/run/dbus/system_bus_socket" + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 14 }} + {{- if not ( .Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "virtualization-dra_resources" . | nindent 14 }} + {{- end }} + ports: + - containerPort: 51515 + name: health + protocol: TCP + {{- include "delvePorts" (list $delve "delve/virtualization-dra") | nindent 12 }} + {{- if ne "delve/virtualization-dra" ($delve | dig "debug" "component" "") }} + readinessProbe: + grpc: + port: 51515 + service: liveness + failureThreshold: 3 + periodSeconds: 10 + livenessProbe: + grpc: + port: 51515 + service: liveness + failureThreshold: 3 + periodSeconds: 10 + {{- end }} + volumeMounts: + - name: plugins-registry + mountPath: /var/lib/kubelet/plugins_registry + - name: plugins + mountPath: /var/lib/kubelet/plugins + - name: cdi + mountPath: /var/run/cdi + - name: devices + mountPath: /sys/bus/usb/devices + - name: dbus-socket + mountPath: /var/run/dbus/system_bus_socket + volumes: + - name: plugins-registry + hostPath: + path: /var/lib/kubelet/plugins_registry + - name: plugins + hostPath: + path: /var/lib/kubelet/plugins + - name: cdi + hostPath: + path: /var/run/cdi + - name: devices + hostPath: + path: /sys/bus/usb/devices + - name: dbus-socket + hostPath: + path: /var/run/dbus/system_bus_socket + type: Socket +{{- end }} diff --git a/templates/virtualization-dra/rbac-for-us.yaml b/templates/virtualization-dra/rbac-for-us.yaml new file mode 100644 index 0000000000..8f1405f470 --- /dev/null +++ b/templates/virtualization-dra/rbac-for-us.yaml @@ -0,0 +1,39 @@ +{{- if eq (include "virtualization-dra.isEnabled" .) "true"}} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: virtualization-dra + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra")) | nindent 2 }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ .Chart.Name }}:virtualization-dra + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra")) | nindent 2 }} +rules: + - apiGroups: ["resource.k8s.io"] + resources: ["resourceclaims"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] + - apiGroups: ["resource.k8s.io"] + resources: ["resourceslices"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ .Chart.Name }}:virtualization-dra + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra")) | nindent 2 }} +subjects: + - kind: ServiceAccount + name: virtualization-dra + namespace: d8-{{ .Chart.Name }} +roleRef: + kind: ClusterRole + name: d8:{{ .Chart.Name }}:virtualization-dra + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/tools/kubeconform/fixtures/module-values.yaml b/tools/kubeconform/fixtures/module-values.yaml index 88282bddb2..6dfa410372 100644 --- a/tools/kubeconform/fixtures/module-values.yaml +++ b/tools/kubeconform/fixtures/module-values.yaml @@ -331,6 +331,7 @@ global: virtualizationApi: sha256:0000000000000000000000000000000000000000000000000000000000000000 virtualizationController: sha256:0000000000000000000000000000000000000000000000000000000000000000 vmRouteForge: sha256:0000000000000000000000000000000000000000000000000000000000000000 + virtualizationDraPlugin: sha256:0000000000000000000000000000000000000000000000000000000000000000w registry: CA: "" address: some-registry.io