diff --git a/Taskfile.yaml b/Taskfile.yaml index 93eef4dcee..6703c2ad6e 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -165,8 +165,7 @@ tasks: "ports": [ { "containerPort": 2345, "name": "dlv" } ], "readinessProbe": null, "livenessProbe": null, - "command": null, - "args": [] + "command": null }, { "name": "proxy", @@ -264,8 +263,7 @@ tasks: "ports": [ { "containerPort": 2345, "name": "dlv" } ], "readinessProbe": null, "livenessProbe": null, - "command": null, - "args": [] + "command": null }, { "name": "proxy", @@ -282,5 +280,39 @@ tasks: } } }' - kubectl -n d8-virtualization port-forward deploy/vit-api 2345:2345 + kubectl -n d8-virtualization port-forward deploy/virt-api 2345:2345 + EOF + + dlv:virtualization-dra-plugin:build: + desc: "Build image virtualization-dra-plugin with dlv" + cmds: + - docker build --build-arg BRANCH=$BRANCH -f ./images/virtualization-dra-plugin/debug/dlv.Dockerfile -t "{{ .DLV_IMAGE }}" --platform linux/amd64 . + + dlv:virtualization-dra-plugin:build-push: + desc: "Build and Push image virtualization-dra-plugin with dlv" + cmds: + - task: dlv:virtualization-dra-plugin:build + - docker push "{{ .DLV_IMAGE }}" + - task: dlv:virtualization-dra-plugin:print + + dlv:virtualization-dra-plugin:print: + desc: "Print commands for debug" + env: + IMAGE: "{{ .DLV_IMAGE }}" + cmd: | + cat < /kubevirt-config-files/.version @@ -31,11 +32,11 @@ WORKDIR /kubevirt RUN go mod edit -go=$GOVERSION && \ go mod download -RUN go mod vendor +RUN go work vendor -RUN for p in patches/*.patch ; do \ - echo -n "Apply ${p} ... " \ - git apply --ignore-space-change --ignore-whitespace ${p} && echo OK || (echo FAIL ; exit 1) \ +RUN for p in patches/*.patch; do \ + echo -n "Apply ${p} ... " && \ + git apply --ignore-space-change --ignore-whitespace "${p}" && echo OK || (echo FAIL && exit 1); \ done ENV GO111MODULE=on @@ -52,9 +53,7 @@ FROM basealt RUN apt-get update && apt-get install --yes \ acl \ procps \ - nftables \ - qemu-img==9.1.2-alt1 \ - xorriso==1.5.6-alt1 && \ + nftables && \ apt-get clean && \ rm --recursive --force /var/lib/apt/lists/ftp.altlinux.org* /var/cache/apt/*.bin @@ -63,7 +62,6 @@ RUN echo "qemu:x:107:107::/home/qemu:/bin/bash" >> /etc/passwd && \ mkdir -p /home/qemu && \ chown -R 107:107 /home/qemu -COPY --from=builder /kubevirt/cmd/virt-handler/virt_launcher.cil /virt_launcher.cil COPY --from=builder /kubevirt-config-files/.version /.version COPY --from=builder /kubevirt/cmd/virt-handler/nsswitch.conf /etc/nsswitch.conf diff --git a/images/virt-handler/werf.inc.yaml b/images/virt-handler/werf.inc.yaml index 4888e05684..3db87a474a 100644 --- a/images/virt-handler/werf.inc.yaml +++ b/images/virt-handler/werf.inc.yaml @@ -25,10 +25,6 @@ import: - virt-chroot - virt-handler - container-disk -- image: {{ .ModuleNamePrefix }}virt-artifact - add: /kubevirt/cmd/{{ $.ImageName }}/virt_launcher.cil - to: /virt_launcher.cil - after: install - image: {{ .ModuleNamePrefix }}virt-artifact add: /kubevirt-config-files/.version to: /.version diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index fc98fe7205..0c172ad6d0 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -42,6 +42,7 @@ altLibs: - libvirglrenderer-devel - libdbus - libusb-devel + - libusb - libbpf-devel - libspice-server-devel - ceph-devel @@ -284,6 +285,7 @@ shell: LIBS+=" /usr/lib64/libtpms* /usr/lib64/libjson* /usr/lib64/libfuse*" LIBS+=" /usr/lib64/libxml2.s* /usr/lib64/libgcc_s* /usr/lib64/libaudit*" LIBS+=" /usr/lib64/libisoburn.s* /usr/lib64/libacl.s*" + LIBS+=" /usr/lib64/libusb-*" echo "Relocate additional libs for files in /VBINS" ./relocate_binaries.sh -i "$FILES" -o /VBINS diff --git a/images/virtualization-artifact/go.mod b/images/virtualization-artifact/go.mod index 3ade7764fa..c8ed344d7a 100644 --- a/images/virtualization-artifact/go.mod +++ b/images/virtualization-artifact/go.mod @@ -32,7 +32,7 @@ require ( k8s.io/klog/v2 v2.130.1 k8s.io/kube-openapi v0.0.0-20250701173324-9bd5c66d9911 k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 - kubevirt.io/api v1.3.1 + kubevirt.io/api v1.6.2 kubevirt.io/containerized-data-importer-api v1.60.3 sigs.k8s.io/controller-runtime v0.21.0 sigs.k8s.io/yaml v1.4.0 @@ -165,4 +165,4 @@ replace ( ) // Kubevirt API replaces -replace kubevirt.io/api => github.com/deckhouse/3p-kubevirt/staging/src/kubevirt.io/api v1.3.1-v12n.18 +replace kubevirt.io/api => github.com/deckhouse/3p-kubevirt/staging/src/kubevirt.io/api v1.6.2-v12n.0 diff --git a/images/virtualization-artifact/go.sum b/images/virtualization-artifact/go.sum index 39d45d303a..db865f72ea 100644 --- a/images/virtualization-artifact/go.sum +++ b/images/virtualization-artifact/go.sum @@ -45,8 +45,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckhouse/3p-kubevirt/staging/src/kubevirt.io/api v1.3.1-v12n.18 h1:9pstD3PiPmby/Chh24ickwUNbAcqbceOPp253/jSr8k= -github.com/deckhouse/3p-kubevirt/staging/src/kubevirt.io/api v1.3.1-v12n.18/go.mod h1:tCn7VAZktEvymk490iPSMPCmKM9UjbbfH2OsFR/IOLU= +github.com/deckhouse/3p-kubevirt/staging/src/kubevirt.io/api v1.6.2-v12n.0 h1:Ro9oG/eakqMxfFSEs91BZYhDXggghuJjFvawV6/EKSo= +github.com/deckhouse/3p-kubevirt/staging/src/kubevirt.io/api v1.6.2-v12n.0/go.mod h1:p66fEy/g79x7VpgUwrkUgOoG2lYs5LQq37WM6JXMwj4= github.com/deckhouse/deckhouse/pkg/log v0.0.0-20250226105106-176cd3afcdd5 h1:PsN1E0oxC/+4zdA977txrqUCuObFL3HAuu5Xnud8m8c= github.com/deckhouse/deckhouse/pkg/log v0.0.0-20250226105106-176cd3afcdd5/go.mod h1:Mk5HRzkc5pIcDIZ2JJ6DPuuqnwhXVkb3you8M8Mg+4w= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= diff --git a/images/virtualization-artifact/pkg/livemigration/migration_configuration.go b/images/virtualization-artifact/pkg/livemigration/migration_configuration.go index 21910bc868..0fe5d9851a 100644 --- a/images/virtualization-artifact/pkg/livemigration/migration_configuration.go +++ b/images/virtualization-artifact/pkg/livemigration/migration_configuration.go @@ -38,6 +38,7 @@ const ( MigrationCompletionTimeoutPerGiB int64 = 800 DefaultUnsafeMigrationOverride bool = false MigrationAllowPostCopy bool = false + MigrationAllowWorkloadDisruption bool = false ) func NewMigrationConfiguration(allowAutoConverge bool, kvconfig virtv1.KubeVirt) *virtv1.MigrationConfiguration { @@ -63,6 +64,7 @@ func NewMigrationConfiguration(allowAutoConverge bool, kvconfig virtv1.KubeVirt) completionTimeoutPerGiB := MigrationCompletionTimeoutPerGiB defaultUnsafeMigrationOverride := DefaultUnsafeMigrationOverride allowPostCopy := MigrationAllowPostCopy + allowWorkloadDisruption := MigrationAllowWorkloadDisruption return &virtv1.MigrationConfiguration{ ParallelMigrationsPerCluster: ¶llelMigrationsPerCluster, @@ -74,6 +76,7 @@ func NewMigrationConfiguration(allowAutoConverge bool, kvconfig virtv1.KubeVirt) UnsafeMigrationOverride: &defaultUnsafeMigrationOverride, AllowAutoConverge: &allowAutoConverge, AllowPostCopy: &allowPostCopy, + AllowWorkloadDisruption: &allowWorkloadDisruption, DisableTLS: nil, Network: nil, MatchSELinuxLevelOnMigration: nil, diff --git a/images/virtualization-artifact/pkg/logger/options.go b/images/virtualization-artifact/pkg/logger/options.go new file mode 100644 index 0000000000..8199b01609 --- /dev/null +++ b/images/virtualization-artifact/pkg/logger/options.go @@ -0,0 +1,38 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logger + +import ( + "github.com/deckhouse/deckhouse/pkg/log" + "github.com/spf13/pflag" +) + +type Options struct { + Level string + Output string + DebugVerbosity int +} + +func (o *Options) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&o.Level, "log-level", o.Level, "Log level") + fs.StringVar(&o.Output, "log-output", o.Output, "Log output") + fs.IntVar(&o.DebugVerbosity, "log-debug-verbosity", o.DebugVerbosity, "Log debug verbosity") +} + +func (o *Options) Complete() *log.Logger { + return NewLogger(o.Level, o.Output, o.DebugVerbosity) +} diff --git a/images/virtualization-dra-plugin/debug/dlv.Dockerfile b/images/virtualization-dra-plugin/debug/dlv.Dockerfile new file mode 100644 index 0000000000..9e85ba1d99 --- /dev/null +++ b/images/virtualization-dra-plugin/debug/dlv.Dockerfile @@ -0,0 +1,31 @@ +FROM golang:1.24.7-bookworm@sha256:2c5f7a0c252a17cf6aa30ddee15caa0f485ee29410a6ea64cddb62eea2b07bdf AS builder +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /app/images/virtualization-dra +RUN go install github.com/go-delve/delve/cmd/dlv@latest + +COPY ./images/virtualization-dra/go.mod /app/images/virtualization-dra/ +COPY ./images/virtualization-dra/go.sum /app/images/virtualization-dra/ + +RUN go mod download + +COPY ./images/virtualization-dra/cmd /app/images/virtualization-dra/cmd +COPY ./images/virtualization-dra/internal /app/images/virtualization-dra/internal +COPY ./images/virtualization-dra/pkg /app/images/virtualization-dra/pkg + +ENV GO111MODULE=on +ENV GOOS=${TARGETOS:-linux} +ENV GOARCH=${TARGETARCH:-amd64} +ENV CGO_ENABLED=0 + +RUN go build -tags EE -gcflags "all=-N -l" -a -o virtualization-dra-plugin ./cmd/virtualization-dra-plugin + +FROM busybox:1.36.1-glibc + +WORKDIR /app +COPY --from=builder /go/bin/dlv /app/dlv +COPY --from=builder /app/images/virtualization-dra/virtualization-dra-plugin /app/virtualization-dra-plugin +USER 65532:65532 + +ENTRYPOINT ["./dlv", "--listen=:2345", "--headless=true", "--continue", "--log=true", "--log-output=debugger,debuglineerr,gdbwire,lldbout,rpc", "--accept-multiclient", "--api-version=2", "exec", "./virtualization-dra-plugin", "--"] diff --git a/images/virtualization-dra-plugin/mount-points.yaml b/images/virtualization-dra-plugin/mount-points.yaml new file mode 100644 index 0000000000..9ad5756007 --- /dev/null +++ b/images/virtualization-dra-plugin/mount-points.yaml @@ -0,0 +1,8 @@ +# A list of pre-created mount points for containerd strict mode. + +dirs: + - /var/run/dbus + - /var/run/cdi +# - /sys/bus/usb/devices + - /var/lib/kubelet/plugins_registry + - /var/lib/kubelet/plugins diff --git a/images/virtualization-dra-plugin/werf.inc.yaml b/images/virtualization-dra-plugin/werf.inc.yaml new file mode 100644 index 0000000000..7615500ef5 --- /dev/null +++ b/images/virtualization-dra-plugin/werf.inc.yaml @@ -0,0 +1,28 @@ +--- +image: {{ .ModuleNamePrefix }}{{ .ImageName }} +fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} +import: + - image: {{ .ModuleNamePrefix }}virtualization-dra-builder + add: /out/virtualization-dra-plugin + to: /app/virtualization-dra-plugin + after: install + {{- if eq $.DEBUG_COMPONENT "delve/virtualization-dra-plugin" }} +- image: debugger + add: /app/dlv + to: /app/dlv + after: install + {{- end }} +imageSpec: + config: + user: 64535 + workingDir: "/app" + {{- if eq $.DEBUG_COMPONENT "delve/virtualization-dra-plugin" }} + env: + PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/app/dlv" + XDG_CONFIG_HOME: "/tmp" + entrypoint: ["/app/dlv", "--listen=:2345", "--headless=true", "--continue", "--log=true", "--log-output=debugger,debuglineerr,gdbwire,lldbout,rpc", "--accept-multiclient", "--api-version=2", "exec", "/app/virtualization-dra-plugin", "--", "--leader-election=false"] + {{- else }} + entrypoint: ["/app/virtualization-dra-plugin"] + {{- end }} diff --git a/images/virtualization-dra/.golangci.yaml b/images/virtualization-dra/.golangci.yaml new file mode 100644 index 0000000000..1be21e2a37 --- /dev/null +++ b/images/virtualization-dra/.golangci.yaml @@ -0,0 +1,108 @@ +run: + concurrency: 4 + timeout: 10m +issues: + # Show all errors. + max-issues-per-linter: 0 + max-same-issues: 0 + exclude: + - "don't use an underscore in package name" +output: + sort-results: true + +exclude-files: + - "^zz_generated.*" + +linters-settings: + gofumpt: + extra-rules: true + gci: + sections: + - standard + - default + - prefix(github.com/deckhouse/) + goimports: + local-prefixes: github.com/deckhouse/ + errcheck: + exclude-functions: fmt:.*,[rR]ead|[wW]rite|[cC]lose,io:Copy + revive: + rules: + - name: dot-imports + disabled: true + nolintlint: + # Exclude following linters from requiring an explanation. + # Default: [] + allow-no-explanation: [funlen, gocognit, lll] + # Enable to require an explanation of nonzero length after each nolint directive. + # Default: false + require-explanation: true + # Enable to require nolint directives to mention the specific linter being suppressed. + # Default: false + require-specific: true + importas: + # Do not allow unaliased imports of aliased packages. + # Default: false + no-unaliased: true + # Do not allow non-required aliases. + # Default: false + no-extra-aliases: false + # List of aliases + # Default: [] + alias: + - pkg: github.com/deckhouse/virtualization/api/core/v1alpha2 + alias: "" + - pkg: github.com/deckhouse/virtualization/api/subresources/v1alpha2 + alias: subv1alpha2 + - pkg: kubevirt.io/api/core/v1 + alias: virtv1 + - pkg: k8s.io/api/core/v1 + alias: corev1 + - pkg: k8s.io/api/authentication/v1 + alias: authnv1 + - pkg: k8s.io/api/storage/v1 + alias: storagev1 + - pkg: k8s.io/api/networking/v1 + alias: netv1 + - pkg: k8s.io/api/policy/v1 + alias: policyv1 + - pkg: k8s.io/apimachinery/pkg/apis/meta/v1 + alias: metav1 + +linters: + disable-all: true + enable: + - asciicheck # checks that your code does not contain non-ASCII identifiers + - bidichk # checks for dangerous unicode character sequences + - bodyclose # checks whether HTTP response body is closed successfully + - contextcheck # [maby too many false positives] checks the function whether use a non-inherited context + - dogsled # checks assignments with too many blank identifiers (e.g. x, _, _, _, := f()) + - errcheck # checking for unchecked errors, these unchecked errors can be critical bugs in some cases + - errname # checks that sentinel errors are prefixed with the Err and error types are suffixed with the Error + - errorlint # finds code that will cause problems with the error wrapping scheme introduced in Go 1.13 + - copyloopvar # detects places where loop variables are copied (Go 1.22+) + - gci # controls golang package import order and makes it always deterministic + - gocritic # provides diagnostics that check for bugs, performance and style issues + - gofmt # [replaced by goimports] checks whether code was gofmt-ed + - gofumpt # [replaced by goimports, gofumports is not available yet] checks whether code was gofumpt-ed + - goimports # in addition to fixing imports, goimports also formats your code in the same style as gofmt + - gosimple # specializes in simplifying a code + - govet # reports suspicious constructs, such as Printf calls whose arguments do not align with the format string + - ineffassign # detects when assignments to existing variables are not used + - misspell # finds commonly misspelled English words in comments + - nolintlint # reports ill-formed or insufficient nolint directives + - reassign # Checks that package variables are not reassigned. + - revive # fast, configurable, extensible, flexible, and beautiful linter for Go, drop-in replacement of golint + - stylecheck # is a replacement for golint + - staticcheck # is a go vet on steroids, applying a ton of static analysis checks + - typecheck # like the front-end of a Go compiler, parses and type-checks Go code + - testifylint # checks usage of github.com/stretchr/testify + - unconvert # removes unnecessary type conversions + - unparam # reports unused function parameters + - unused # checks for unused constants, variables, functions and types + - usetesting # reports uses of functions with replacement inside the testing package + - testableexamples # checks if examples are testable (have an expected output) + - thelper # detects golang test helpers without t.Helper() call and checks the consistency of test helpers + - tparallel # detects inappropriate usage of t.Parallel() method in your Go test codes + - whitespace # detects leading and trailing whitespace + - wastedassign # Finds wasted assignment statements. + - importas # checks import aliases against the configured convention diff --git a/images/virtualization-dra/Taskfile.yaml b/images/virtualization-dra/Taskfile.yaml new file mode 100644 index 0000000000..a28e5823b4 --- /dev/null +++ b/images/virtualization-dra/Taskfile.yaml @@ -0,0 +1,59 @@ +version: "3" + +silent: true + +tasks: + fmt: + desc: "Run formatters locally" + cmds: + - task: fmt:gci + - task: fmt:gofumpt + + fmt:gci: + desc: "Format code with gci, important vars: paths." + cmds: + - | + gci write --skip-generated -s standard,default,prefix\(github.com/deckhouse/\) {{.CLI_ARGS}} {{.paths | default "pkg/ cmd/"}} + + fmt:gofumpt: + desc: "Format code with gofumpt, important vars: paths" + cmds: + - | + gofumpt -extra -w {{.CLI_ARGS}} {{.paths | default "cmd/ pkg/"}} + + dev:gogenerate: + desc: |- + Run go generate for all packages. + cmds: + - | + go generate ./... + + dev:addlicense: + desc: |- + Add Flant CE license to files sh,go,py. Default directory is root of project, custom directory path can be passed like: "task dev:addlicense -- " + cmds: + - | + {{if .CLI_ARGS}} + go run ../../tools/addlicense/{main,variables,msg,utils}.go -directory {{ .CLI_ARGS }} + {{else}} + go run ../../tools/addlicense/{main,variables,msg,utils}.go -directory ./ + {{end}} + + test:unit: + desc: "Run go unit tests" + cmds: + - | + go tool ginkgo -v -r pkg/ + + lint: + desc: "Run linters locally" + cmds: + - task: lint:go + + lint:go: + desc: "Run golangci-lint" + deps: + - _ensure:golangci-lint + cmds: + - | + golangci-lint run diff --git a/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go b/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go new file mode 100644 index 0000000000..088c76c204 --- /dev/null +++ b/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go @@ -0,0 +1,182 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "log/slog" + "os" + "strconv" + "time" + + "github.com/spf13/cobra" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/component-base/cli/flag" + + "github.com/deckhouse/virtualization-dra/internal/cdi" + "github.com/deckhouse/virtualization-dra/internal/plugin" + "github.com/deckhouse/virtualization-dra/internal/usb" + "github.com/deckhouse/virtualization-dra/pkg/logger" +) + +func NewVirtualizationDraPluginCommand() *cobra.Command { + o := newDraOptions() + + cmd := &cobra.Command{ + Use: "virtualization-dra-plugin", + Short: "virtualization-dra-plugin", + SilenceUsage: true, + SilenceErrors: true, + PreRunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + log := o.Logging.Complete() + logger.SetDefaultLogger(log) + return nil + }, + RunE: o.Run, + } + + fs := cmd.Flags() + for _, f := range o.NamedFlags().FlagSets { + fs.AddFlagSet(f) + } + + return cmd +} + +func newDraOptions() *draOptions { + withDefault := func(env, defaultValue string) string { + if env, ok := os.LookupEnv(env); ok { + return env + } + return defaultValue + } + + o := &draOptions{ + Kubeconfig: os.Getenv("KUBECONFIG"), + NodeName: os.Getenv("NODE_NAME"), + CDIRoot: withDefault("CDI_ROOT", cdi.SpecDir), + KubeletRegisterDirectoryPath: os.Getenv("KUBELET_REGISTER_DIRECTORY_PATH"), + KubeletPluginsDirectoryPath: os.Getenv("KUBELET_PLUGINS_DIRECTORY_PATH"), + USBDevicesPath: withDefault("USB_DEVICES_PATH", usb.PathToUSBDevices), + HealthzPort: 51515, + USBResyncPeriod: usb.DefaultResyncPeriod, + Logging: &logger.Options{}, + } + + if healthzPort := os.Getenv("HEALTHZ_PORT"); healthzPort != "" { + port, err := strconv.Atoi(healthzPort) + if err == nil { + o.HealthzPort = port + } + } + + return o +} + +type draOptions struct { + Kubeconfig string + NodeName string + CDIRoot string + KubeletRegisterDirectoryPath string + KubeletPluginsDirectoryPath string + USBDevicesPath string + HealthzPort int + USBResyncPeriod time.Duration + + Logging *logger.Options +} + +func (o *draOptions) NamedFlags() (fs flag.NamedFlagSets) { + mfs := fs.FlagSet("virtualization-dra plugin") + mfs.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "Path to kubeconfig file") + mfs.StringVar(&o.NodeName, "node-name", o.NodeName, "Node name") + mfs.StringVar(&o.CDIRoot, "cdi-root", o.CDIRoot, "CDI root") + mfs.StringVar(&o.KubeletRegisterDirectoryPath, "kubelet-register-directory-path", o.KubeletRegisterDirectoryPath, "Kubelet register directory path") + mfs.StringVar(&o.KubeletPluginsDirectoryPath, "kubelet-plugins-directory-path", o.KubeletPluginsDirectoryPath, "Kubelet plugins directory path") + mfs.StringVar(&o.USBDevicesPath, "usb-devices-path", o.USBDevicesPath, "USB Devices path") + mfs.IntVar(&o.HealthzPort, "healthz-port", o.HealthzPort, "Healthz port") + mfs.DurationVar(&o.USBResyncPeriod, "usb-resync-period", o.USBResyncPeriod, "USB resync period") + + o.Logging.AddFlags(fs.FlagSet("logging")) + + return fs +} + +func (o *draOptions) Validate() error { + if o.NodeName == "" { + return fmt.Errorf("NodeName is required") + } + if o.CDIRoot == "" { + return fmt.Errorf("CDIRoot is required") + } + if o.HealthzPort <= 0 { + return fmt.Errorf("HealthzPort is required") + } + + return nil +} + +func (o *draOptions) Run(cmd *cobra.Command, _ []string) error { + err := plugin.InitPluginDirs(o.KubeletPluginsDirectoryPath, o.KubeletRegisterDirectoryPath) + if err != nil { + return err + } + + cfg, err := clientcmd.BuildConfigFromFlags("", o.Kubeconfig) + if err != nil { + return fmt.Errorf("failed to get rest config: %w", err) + } + + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + return fmt.Errorf("failed to create kubernetes client: %w", err) + } + + usbCDIManager, err := cdi.NewCDIManager(o.CDIRoot, "usb", plugin.DriverName, o.NodeName, "DRA_USB") + if err != nil { + return fmt.Errorf("failed to create CDI manager: %w", err) + } + + usbStore := usb.NewAllocationStore(o.NodeName, o.USBDevicesPath, o.USBResyncPeriod, usbCDIManager, slog.Default()) + + driver := plugin.NewDriver(o.NodeName, client, usbStore, slog.Default()) + err = driver.Start(cmd.Context()) + if err != nil { + return fmt.Errorf("failed to start driver: %w", err) + } + + healthCheck := plugin.NewHealthCheck(o.HealthzPort, slog.Default()) + err = healthCheck.Start() + if err != nil { + return fmt.Errorf("failed to start health check: %w", err) + } + + err = usbStore.Start(cmd.Context()) + if err != nil { + return fmt.Errorf("failed to start usb store: %w", err) + } + + driver.Wait() + driver.Shutdown() + healthCheck.Stop() + + return nil +} diff --git a/images/virtualization-dra/cmd/virtualization-dra-plugin/main.go b/images/virtualization-dra/cmd/virtualization-dra-plugin/main.go new file mode 100644 index 0000000000..863a7faf03 --- /dev/null +++ b/images/virtualization-dra/cmd/virtualization-dra-plugin/main.go @@ -0,0 +1,36 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "log/slog" + "os" + "os/signal" + "syscall" + + "github.com/deckhouse/virtualization-dra/cmd/virtualization-dra-plugin/app" +) + +func main() { + ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + + if err := app.NewVirtualizationDraPluginCommand().ExecuteContext(ctx); err != nil { + slog.Error("failed to execute command", slog.Any("err", err)) + os.Exit(1) + } +} diff --git a/images/virtualization-dra/go.mod b/images/virtualization-dra/go.mod new file mode 100644 index 0000000000..a6363e2f74 --- /dev/null +++ b/images/virtualization-dra/go.mod @@ -0,0 +1,85 @@ +module github.com/deckhouse/virtualization-dra + +go 1.24.7 + +tool github.com/onsi/ginkgo/v2/ginkgo + +require ( + github.com/containerd/nri v0.10.0 + github.com/deckhouse/deckhouse/pkg/log v0.1.0 + github.com/go-logr/logr v1.4.2 + github.com/godbus/dbus/v5 v5.2.0 + github.com/onsi/ginkgo/v2 v2.21.0 + github.com/onsi/gomega v1.35.1 + github.com/spf13/cobra v1.10.1 + github.com/spf13/pflag v1.0.9 + google.golang.org/grpc v1.72.1 + k8s.io/api v0.34.2 + k8s.io/apimachinery v0.34.2 + k8s.io/client-go v0.34.2 + k8s.io/component-base v0.34.2 + k8s.io/dynamic-resource-allocation v0.34.2 + k8s.io/klog/v2 v2.130.1 + k8s.io/kubelet v0.34.2 + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 + tags.cncf.io/container-device-interface v1.0.1 + tags.cncf.io/container-device-interface/specs-go v1.0.0 +) + +require ( + github.com/DataDog/gostackparse v0.7.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/ttrpc v1.2.7 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/knqyf263/go-plugin v0.9.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/opencontainers/runtime-spec v1.1.0 // indirect + github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect + github.com/tetratelabs/wazero v1.9.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.6.4 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/mod v0.21.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.9.0 // indirect + golang.org/x/tools v0.26.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) diff --git a/images/virtualization-dra/go.sum b/images/virtualization-dra/go.sum new file mode 100644 index 0000000000..dea9d5d69f --- /dev/null +++ b/images/virtualization-dra/go.sum @@ -0,0 +1,258 @@ +github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= +github.com/DataDog/gostackparse v0.7.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/nri v0.10.0 h1:bt2NzfvlY6OJE0i+fB5WVeGQEycxY7iFVQpEbh7J3Go= +github.com/containerd/nri v0.10.0/go.mod h1:5VyvLa/4uL8FjyO8nis1UjbCutXDpngil17KvBSL6BU= +github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRqQ= +github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckhouse/deckhouse/pkg/log v0.1.0 h1:2aPfyiHHSIJlX4x7ysyPOaIb7CLmyY+hUf9uDb8TYd8= +github.com/deckhouse/deckhouse/pkg/log v0.1.0/go.mod h1:pbAxTSDcPmwyl3wwKDcEB3qdxHnRxqTV+J0K+sha8bw= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/godbus/dbus/v5 v5.2.0 h1:3WexO+U+yg9T70v9FdHr9kCxYlazaAXUhx2VMkbfax8= +github.com/godbus/dbus/v5 v5.2.0/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knqyf263/go-plugin v0.9.0 h1:CQs2+lOPIlkZVtcb835ZYDEoyyWJWLbSTWeCs0EwTwI= +github.com/knqyf263/go-plugin v0.9.0/go.mod h1:2z5lCO1/pez6qGo8CvCxSlBFSEat4MEp1DrnA+f7w8Q= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= +github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= +github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= +github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 h1:DmNGcqH3WDbV5k8OJ+esPWbqUOX5rMLR2PMvziDMJi0= +github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go.mod h1:BRHJJd0E+cx42OybVYSgUvZmU0B8P9gZuRXlZUP7TKI= +github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK99DRLDhyU= +github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I= +github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM= +github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/etcd/client/pkg/v3 v3.6.4 h1:9HBYrjppeOfFjBjaMTRxT3R7xT0GLK8EJMVC4xg6ok0= +go.etcd.io/etcd/client/pkg/v3 v3.6.4/go.mod h1:sbdzr2cl3HzVmxNw//PH7aLGVtY4QySjQFuaCgcRFAI= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1:TLPQVbx1GJ8VKZxz52VAxl1EBgKXXbTiU9Fc5fZeLn4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= +google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= +google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.34.2 h1:fsSUNZhV+bnL6Aqrp6O7lMTy6o5x2C4XLjnh//8SLYY= +k8s.io/api v0.34.2/go.mod h1:MMBPaWlED2a8w4RSeanD76f7opUoypY8TFYkSM+3XHw= +k8s.io/apimachinery v0.34.2 h1:zQ12Uk3eMHPxrsbUJgNF8bTauTVR2WgqJsTmwTE/NW4= +k8s.io/apimachinery v0.34.2/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/client-go v0.34.2 h1:Co6XiknN+uUZqiddlfAjT68184/37PS4QAzYvQvDR8M= +k8s.io/client-go v0.34.2/go.mod h1:2VYDl1XXJsdcAxw7BenFslRQX28Dxz91U9MWKjX97fE= +k8s.io/component-base v0.34.2 h1:HQRqK9x2sSAsd8+R4xxRirlTjowsg6fWCPwWYeSvogQ= +k8s.io/component-base v0.34.2/go.mod h1:9xw2FHJavUHBFpiGkZoKuYZ5pdtLKe97DEByaA+hHbM= +k8s.io/dynamic-resource-allocation v0.34.2 h1:SjlRGSWl6CZXoJwQNL+Y0wRfdH8PkJ4mHRNK6MMj0bY= +k8s.io/dynamic-resource-allocation v0.34.2/go.mod h1:ul6I+gfrCmC+OCuVdN0/iykyB2sPrIqh2WyKQ3RQPCU= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/kubelet v0.34.2 h1:Dl+1uh7xwJr70r+SHKyIpvu6XvzuoPu0uDIC4cqgJUs= +k8s.io/kubelet v0.34.2/go.mod h1:RfwR03iuKeVV7Z1qD9XKH98c3tlPImJpQ3qHIW40htM= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= +tags.cncf.io/container-device-interface v1.0.1 h1:KqQDr4vIlxwfYh0Ed/uJGVgX+CHAkahrgabg6Q8GYxc= +tags.cncf.io/container-device-interface v1.0.1/go.mod h1:JojJIOeW3hNbcnOH2q0NrWNha/JuHoDZcmYxAZwb2i0= +tags.cncf.io/container-device-interface/specs-go v1.0.0 h1:8gLw29hH1ZQP9K1YtAzpvkHCjjyIxHZYzBAvlQ+0vD8= +tags.cncf.io/container-device-interface/specs-go v1.0.0/go.mod h1:u86hoFWqnh3hWz3esofRFKbI261bUlvUfLKGrDhJkgQ= diff --git a/images/virtualization-dra/internal/cdi/cdi.go b/images/virtualization-dra/internal/cdi/cdi.go new file mode 100644 index 0000000000..ca42ed55bc --- /dev/null +++ b/images/virtualization-dra/internal/cdi/cdi.go @@ -0,0 +1,166 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cdi + +import ( + "fmt" + + drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1beta1" + cdiapi "tags.cncf.io/container-device-interface/pkg/cdi" + cdiparser "tags.cncf.io/container-device-interface/pkg/parser" + cdispec "tags.cncf.io/container-device-interface/specs-go" +) + +const SpecDir = cdiapi.DefaultDynamicDir + +const ( + cdiVendor = "dra.virtualization.deckhouse.io" + cdiCommonDeviceName = "common" +) + +type Manager interface { + CreateCommonSpecFile() error + CreateClaimSpecFile(claimUID string, devices PreparedDevices) error + DeleteClaimSpecFile(claimUID string) error + GetClaimDevices(claimUID string, devices ...string) []string +} +type manager struct { + cache *cdiapi.Cache + cdiClass string + cdiKind string + driverName string + nodeName string + cdiEnvPrefix string +} + +func NewCDIManager(cdiSpecDir, cdiClass, driverName, nodeName, cdiEnvPrefix string) (Manager, error) { + if cdiSpecDir == "" { + cdiSpecDir = SpecDir + } + + cache, err := cdiapi.NewCache(cdiapi.WithSpecDirs(cdiSpecDir)) + if err != nil { + return nil, err + } + + return &manager{ + cache: cache, + cdiClass: cdiClass, + cdiKind: fmt.Sprintf("%s/%s", cdiVendor, cdiClass), + driverName: driverName, + nodeName: nodeName, + cdiEnvPrefix: cdiEnvPrefix, + }, nil +} + +func (cdi *manager) CreateCommonSpecFile() error { + spec := &cdispec.Spec{ + Kind: cdi.cdiKind, + Devices: []cdispec.Device{ + { + Name: cdiCommonDeviceName, + ContainerEdits: cdispec.ContainerEdits{ + Env: []string{ + fmt.Sprintf("KUBERNETES_NODE_NAME=%s", cdi.nodeName), + fmt.Sprintf("DRA_RESOURCE_DRIVER_NAME=%s", cdi.driverName), + }, + }, + }, + }, + } + + minVersion, err := cdispec.MinimumRequiredVersion(spec) + if err != nil { + return fmt.Errorf("failed to get minimum required CDI spec version: %v", err) + } + spec.Version = minVersion + + specName, err := cdiapi.GenerateNameForTransientSpec(spec, cdiCommonDeviceName) + if err != nil { + return fmt.Errorf("failed to generate Spec name: %w", err) + } + + return cdi.cache.WriteSpec(spec, specName) +} + +func (cdi *manager) CreateClaimSpecFile(claimUID string, devices PreparedDevices) error { + specName := cdiapi.GenerateTransientSpecName(cdiVendor, cdi.cdiClass, claimUID) + + spec := &cdispec.Spec{ + Kind: cdi.cdiKind, + Devices: []cdispec.Device{}, + } + + for _, device := range devices { + claimEdits := cdiapi.ContainerEdits{ + ContainerEdits: &cdispec.ContainerEdits{ + Env: []string{ + fmt.Sprintf("%s_%s_RESOURCE_CLAIM=%s", cdi.cdiEnvPrefix, device.DeviceName[4:], claimUID), + }, + }, + } + claimEdits.Append(device.ContainerEdits) + + cdiDevice := cdispec.Device{ + Name: fmt.Sprintf("%s-%s", claimUID, device.DeviceName), + ContainerEdits: *claimEdits.ContainerEdits, + } + + spec.Devices = append(spec.Devices, cdiDevice) + } + + minVersion, err := cdispec.MinimumRequiredVersion(spec) + if err != nil { + return fmt.Errorf("failed to get minimum required CDI spec version: %v", err) + } + spec.Version = minVersion + + return cdi.cache.WriteSpec(spec, specName) +} + +func (cdi *manager) DeleteClaimSpecFile(claimUID string) error { + specName := cdiapi.GenerateTransientSpecName(cdiVendor, cdi.cdiClass, claimUID) + return cdi.cache.RemoveSpec(specName) +} + +func (cdi *manager) GetClaimDevices(claimUID string, devices ...string) []string { + cdiDevices := []string{ + cdiparser.QualifiedName(cdiVendor, cdi.cdiClass, cdiCommonDeviceName), + } + + for _, device := range devices { + cdiDevice := cdiparser.QualifiedName(cdiVendor, cdi.cdiClass, fmt.Sprintf("%s-%s", claimUID, device)) + cdiDevices = append(cdiDevices, cdiDevice) + } + + return cdiDevices +} + +type PreparedDevices []*PreparedDevice + +type PreparedDevice struct { + drapbv1.Device + ContainerEdits *cdiapi.ContainerEdits +} + +func (pds PreparedDevices) GetDevices() []*drapbv1.Device { + var devices []*drapbv1.Device + for _, pd := range pds { + devices = append(devices, &pd.Device) + } + return devices +} diff --git a/images/virtualization-dra/internal/plugin/driver.go b/images/virtualization-dra/internal/plugin/driver.go new file mode 100644 index 0000000000..c8251e2020 --- /dev/null +++ b/images/virtualization-dra/internal/plugin/driver.go @@ -0,0 +1,196 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "context" + "errors" + "fmt" + "log/slog" + + "github.com/deckhouse/deckhouse/pkg/log" + "k8s.io/api/resource/v1" + resourceapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/dynamic-resource-allocation/kubeletplugin" + "k8s.io/dynamic-resource-allocation/resourceslice" +) + +const DriverName = "virtualization-dra" + +func NewDriver(nodeName string, kubeClient kubernetes.Interface, allocator Allocator, log *slog.Logger) *Driver { + return &Driver{ + nodeName: nodeName, + kubeClient: kubeClient, + allocator: allocator, + log: log.With(slog.String("driver", DriverName), slog.String("component", "driver")), + } +} + +type Driver struct { + nodeName string + + kubeClient kubernetes.Interface + allocator Allocator + log *slog.Logger + + helper *kubeletplugin.Helper + pluginCtx context.Context + pluginCancel context.CancelCauseFunc +} + +func (d *Driver) Start(ctx context.Context) error { + ctx, cancel := context.WithCancelCause(ctx) + d.pluginCtx = ctx + d.pluginCancel = cancel + + log.Info("Starting dra plugin") + helper, err := kubeletplugin.Start( + ctx, + d, + kubeletplugin.KubeClient(d.kubeClient), + kubeletplugin.NodeName(d.nodeName), + kubeletplugin.DriverName(DriverName), + kubeletplugin.RegistrarDirectoryPath(virtualizationRegistrarDirPath()), + kubeletplugin.RegistrarSocketFilename(virtualizationRegistrarSocketFilename), + kubeletplugin.PluginDataDirectoryPath(virtualizationPluginDirPath()), + ) + if err != nil { + return fmt.Errorf("failed to start kubelet plugin: %w", err) + } + + d.helper = helper + d.startPublisher(ctx) + + return err +} + +func (d *Driver) Wait() { + if d.pluginCtx != nil { + <-d.pluginCtx.Done() + } +} + +func (d *Driver) Shutdown() { + if d.helper != nil { + d.log.Info("Stopping dra plugin") + d.helper.Stop() + } +} + +func (d *Driver) PrepareResourceClaims(ctx context.Context, claims []*v1.ResourceClaim) (map[types.UID]kubeletplugin.PrepareResult, error) { + d.log.Info("Preparing resource claims") + + result := make(map[types.UID]kubeletplugin.PrepareResult, len(claims)) + + for _, claim := range claims { + result[claim.UID] = d.prepareResourceClaim(ctx, claim) + } + + return result, nil +} + +func (d *Driver) prepareResourceClaim(ctx context.Context, claim *resourceapi.ResourceClaim) kubeletplugin.PrepareResult { + if claim.Status.Allocation == nil { + return kubeletplugin.PrepareResult{ + Err: fmt.Errorf("claim %s/%s has no allocation", claim.Namespace, claim.Name), + } + } + + preparedPBs, err := d.allocator.Prepare(ctx, claim) + if err != nil { + return kubeletplugin.PrepareResult{ + Err: fmt.Errorf("error preparing devices for claim %v: %w", claim.UID, err), + } + } + var prepared []kubeletplugin.Device + for _, preparedPB := range preparedPBs { + prepared = append(prepared, kubeletplugin.Device{ + Requests: preparedPB.GetRequestNames(), + PoolName: preparedPB.GetPoolName(), + DeviceName: preparedPB.GetDeviceName(), + CDIDeviceIDs: preparedPB.GetCDIDeviceIDs(), + }) + } + + d.log.Info("Returning newly prepared devices", slog.String("uid", string(claim.UID)), slog.Any("devices", prepared)) + return kubeletplugin.PrepareResult{Devices: prepared} +} + +func (d *Driver) UnprepareResourceClaims(ctx context.Context, claims []kubeletplugin.NamespacedObject) (map[types.UID]error, error) { + d.log.Info("Unpreparing resource claims") + + result := make(map[types.UID]error) + + for _, claim := range claims { + result[claim.UID] = d.unprepareResourceClaim(ctx, claim) + } + + return result, nil +} + +func (d *Driver) unprepareResourceClaim(ctx context.Context, claim kubeletplugin.NamespacedObject) error { + if err := d.allocator.Unprepare(ctx, claim.UID); err != nil { + return fmt.Errorf("error unpreparing devices for claim %v: %w", claim.UID, err) + } + + return nil +} + +func (d *Driver) HandleError(ctx context.Context, err error, msg string) { + utilruntime.HandleErrorWithContext(ctx, err, msg) + if !errors.Is(err, kubeletplugin.ErrRecoverable) && d.pluginCancel != nil { + d.pluginCancel(fmt.Errorf("fatal background error: %w", err)) + } +} + +func (d *Driver) startPublisher(ctx context.Context) { + go func() { + ch := d.allocator.UpdateChannel() + for { + select { + case <-ctx.Done(): + return + case devices := <-ch: + d.log.Info("Publishing devices", slog.Any("devices", devices)) + if len(devices) > 0 { + } + resources := d.makeResources(devices) + err := d.helper.PublishResources(ctx, resources) + if err != nil { + d.log.Error("Failed to publish devices", slog.Any("err", err)) + } + } + } + }() +} + +func (d *Driver) makeResources(devices []resourceapi.Device) resourceslice.DriverResources { + return resourceslice.DriverResources{ + Pools: map[string]resourceslice.Pool{ + d.nodeName: { + Slices: []resourceslice.Slice{ + { + Devices: devices, + }, + }, + }, + }, + } +} diff --git a/images/virtualization-dra/internal/plugin/healthz.go b/images/virtualization-dra/internal/plugin/healthz.go new file mode 100644 index 0000000000..089e2ef8b7 --- /dev/null +++ b/images/virtualization-dra/internal/plugin/healthz.go @@ -0,0 +1,158 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "context" + "fmt" + "log/slog" + "net" + "net/url" + "strconv" + "sync" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" + drapb "k8s.io/kubelet/pkg/apis/dra/v1" + registerapi "k8s.io/kubelet/pkg/apis/pluginregistration/v1" +) + +type HealthCheck struct { + grpc_health_v1.UnimplementedHealthServer + server *grpc.Server + log *slog.Logger + wg sync.WaitGroup + regSockPath string + draSockPath string + port int +} + +func NewHealthCheck(port int, log *slog.Logger) *HealthCheck { + regSockPath := (&url.URL{ + Scheme: "unix", + Path: virtualizationRegistrarSocketPath(), + }).String() + + draSockPath := (&url.URL{ + Scheme: "unix", + Path: virtualizationPluginSocketPath(), + }).String() + + h := &HealthCheck{ + server: grpc.NewServer(), + log: log.With(slog.String("component", "healthcheck")), + regSockPath: regSockPath, + draSockPath: draSockPath, + port: port, + } + grpc_health_v1.RegisterHealthServer(h.server, h) + + return h +} + +func (h *HealthCheck) Start() error { + addr := net.JoinHostPort("", strconv.Itoa(h.port)) + lis, err := net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("failed to listen for healthcheck service at %s: %w", addr, err) + } + + h.wg.Add(1) + go func() { + defer h.wg.Done() + h.log.Info("starting healthcheck service", slog.String("addr", lis.Addr().String())) + if err := h.server.Serve(lis); err != nil { + h.log.Error("failed to serve healthcheck service", slog.String("addr", addr), slog.Any("err", err)) + } + }() + + return nil +} + +func (h *HealthCheck) Stop() { + if h.server != nil { + h.log.Info("stopping healthcheck service") + h.server.GracefulStop() + } + h.wg.Wait() +} + +// Check implements [grpc_health_v1.HealthServer]. +func (h *HealthCheck) Check(ctx context.Context, req *grpc_health_v1.HealthCheckRequest) (*grpc_health_v1.HealthCheckResponse, error) { + knownServices := map[string]struct{}{"": {}, "liveness": {}} + if _, known := knownServices[req.GetService()]; !known { + return nil, status.Error(codes.NotFound, "unknown service") + } + + status := &grpc_health_v1.HealthCheckResponse{ + Status: grpc_health_v1.HealthCheckResponse_NOT_SERVING, + } + + regClient, err := h.newRegClient() + if err != nil { + h.log.Error("failed to create registration client", slog.Any("err", err)) + return status, err + } + + info, err := regClient.GetInfo(ctx, ®isterapi.InfoRequest{}) + if err != nil { + h.log.Error("failed to call GetInfo", slog.Any("err", err)) + return status, nil + } + h.log.Info("Successfully invoked GetInfo", "info", info) + + draClient, err := h.newDraConn() + if err != nil { + h.log.Error("failed to create DRA client", slog.Any("err", err)) + return status, err + } + + _, err = draClient.NodePrepareResources(ctx, &drapb.NodePrepareResourcesRequest{}) + if err != nil { + h.log.Error("failed to call NodePrepareResources", slog.Any("err", err)) + return status, nil + } + h.log.Info("Successfully invoked NodePrepareResources") + + status.Status = grpc_health_v1.HealthCheckResponse_SERVING + return status, nil +} + +func (h *HealthCheck) newRegClient() (registerapi.RegistrationClient, error) { + regConn, err := grpc.NewClient( + h.regSockPath, + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + return nil, fmt.Errorf("connect to registration socket: %w", err) + } + return registerapi.NewRegistrationClient(regConn), nil +} + +func (h *HealthCheck) newDraConn() (drapb.DRAPluginClient, error) { + draConn, err := grpc.NewClient( + h.draSockPath, + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + return nil, fmt.Errorf("connect to DRA socket: %w", err) + } + return drapb.NewDRAPluginClient(draConn), nil +} diff --git a/images/virtualization-dra/internal/plugin/interfaces.go b/images/virtualization-dra/internal/plugin/interfaces.go new file mode 100644 index 0000000000..4ef9d1bf9d --- /dev/null +++ b/images/virtualization-dra/internal/plugin/interfaces.go @@ -0,0 +1,33 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "context" + + "github.com/containerd/nri/pkg/api" + resourceapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/types" + drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1beta1" +) + +type Allocator interface { + UpdateChannel() chan []resourceapi.Device + Prepare(ctx context.Context, claim *resourceapi.ResourceClaim) ([]*drapbv1.Device, error) + Unprepare(ctx context.Context, claimUID types.UID) error + Synchronize(ctx context.Context, pods []*api.PodSandbox, containers []*api.Container) ([]*api.ContainerUpdate, error) +} diff --git a/images/virtualization-dra/internal/plugin/nri_hooks.go b/images/virtualization-dra/internal/plugin/nri_hooks.go new file mode 100644 index 0000000000..4999d3c86f --- /dev/null +++ b/images/virtualization-dra/internal/plugin/nri_hooks.go @@ -0,0 +1,30 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "context" + "log/slog" + + "github.com/containerd/nri/pkg/api" +) + +// Synchronize is called by the NRI to synchronize the state of the driver during bootstrap. +func (d *Driver) Synchronize(ctx context.Context, pods []*api.PodSandbox, containers []*api.Container) ([]*api.ContainerUpdate, error) { + d.log.Info("Synchronizing state with the runtime...", slog.Int("pods", len(pods)), slog.Int("containers", len(containers))) + return d.allocator.Synchronize(ctx, pods, containers) +} diff --git a/images/virtualization-dra/internal/plugin/plugin.go b/images/virtualization-dra/internal/plugin/plugin.go new file mode 100644 index 0000000000..7aad9b0a03 --- /dev/null +++ b/images/virtualization-dra/internal/plugin/plugin.go @@ -0,0 +1,68 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "fmt" + "os" + "path" + + "k8s.io/dynamic-resource-allocation/kubeletplugin" +) + +var ( + kubeletPluginsDir = kubeletplugin.KubeletPluginsDir + kubeletRegistryDir = kubeletplugin.KubeletRegistryDir +) + +const ( + virtualizationPluginSocketFilename = "dra.sock" + virtualizationRegistrarSocketFilename = DriverName + "-reg.sock" +) + +func virtualizationPluginDirPath() string { + return path.Join(kubeletPluginsDir, DriverName) +} + +func virtualizationPluginSocketPath() string { + return path.Join(virtualizationPluginDirPath(), virtualizationPluginSocketFilename) +} + +func virtualizationRegistrarDirPath() string { + return kubeletRegistryDir +} + +func virtualizationRegistrarSocketPath() string { + return path.Join(virtualizationRegistrarDirPath(), virtualizationRegistrarSocketFilename) +} + +func InitPluginDirs(setKubeletPluginsDir, setKubeletRegistryDir string) error { + if setKubeletPluginsDir != "" { + kubeletPluginsDir = setKubeletPluginsDir + } + + pluginDir := virtualizationPluginDirPath() + if err := os.MkdirAll(pluginDir, 0700); err != nil { + return fmt.Errorf("failed to create directory %s: %w", pluginDir, err) + } + + if setKubeletRegistryDir != "" { + kubeletRegistryDir = setKubeletRegistryDir + } + + return nil +} diff --git a/images/virtualization-dra/internal/usb/convert.go b/images/virtualization-dra/internal/usb/convert.go new file mode 100644 index 0000000000..95f4cb65aa --- /dev/null +++ b/images/virtualization-dra/internal/usb/convert.go @@ -0,0 +1,72 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +import ( + resourceapi "k8s.io/api/resource/v1" + "k8s.io/utils/ptr" +) + +func convertToAPIDevice(usbDevice Device) *resourceapi.Device { + return &resourceapi.Device{ + Name: usbDevice.GetName(), + Attributes: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ + "name": { + StringValue: ptr.To(usbDevice.Name), + }, + "manufacturer": { + StringValue: ptr.To(usbDevice.Manufacturer), + }, + "product": { + StringValue: ptr.To(usbDevice.Product), + }, + "vendorID": { + StringValue: ptr.To(usbDevice.VendorID.String()), + }, + "productID": { + StringValue: ptr.To(usbDevice.ProductID.String()), + }, + "bcd": { + StringValue: ptr.To(usbDevice.BCD.String()), + }, + "bus": { + StringValue: ptr.To(usbDevice.Bus.String()), + }, + "resource.kubernetes.io/usbAddressBus": { + IntValue: ptr.To(int64(usbDevice.Bus)), + }, + "deviceNumber": { + StringValue: ptr.To(usbDevice.DeviceNumber.String()), + }, + "resource.kubernetes.io/usbAddressDeviceNumber": { + IntValue: ptr.To(int64(usbDevice.DeviceNumber)), + }, + "major": { + IntValue: ptr.To(int64(usbDevice.Major)), + }, + "minor": { + IntValue: ptr.To(int64(usbDevice.Minor)), + }, + "serial": { + StringValue: ptr.To(usbDevice.Serial), + }, + "devicePath": { + StringValue: ptr.To(usbDevice.DevicePath), + }, + }, + } +} diff --git a/images/virtualization-dra/internal/usb/device.go b/images/virtualization-dra/internal/usb/device.go new file mode 100644 index 0000000000..534a9d3465 --- /dev/null +++ b/images/virtualization-dra/internal/usb/device.go @@ -0,0 +1,250 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +import ( + "bufio" + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/deckhouse/virtualization-dra/pkg/set" +) + +type DeviceSet = set.Set[Device] + +func NewDeviceSet() *DeviceSet { + return set.New[Device]() +} + +type Device struct { + Name string + Manufacturer string + Product string + VendorID int4x + ProductID int4x + BCD int4x + Bus int3d + DeviceNumber int3d + Major int + Minor int + Serial string + DevicePath string +} + +func (d *Device) GetName() string { + // usb---- + // usb-003-005-e39-f100 + return fmt.Sprintf("usb-%s-%s-%s-%s", d.Bus.String(), d.DeviceNumber.String(), d.VendorID.String(), d.ProductID.String()) +} + +func (d *Device) Validate() error { + if d.VendorID == 0 { + return fmt.Errorf("VendorID is required") + } + if d.ProductID == 0 { + return fmt.Errorf("ProductID is required") + } + if d.Bus == 0 { + return fmt.Errorf("Bus is required") + } + if d.DeviceNumber == 0 { + return fmt.Errorf("DeviceNumber is required") + } + if d.DevicePath == "" { + return fmt.Errorf("DevicePath is required") + } + if d.Major == 0 { + return fmt.Errorf("Major is required") + } + if d.Minor == 0 { + return fmt.Errorf("Minor is required") + } + return nil +} + +func LoadDevice(path string) (device Device, err error) { + if err = parseSysUeventFile(path, &device); err != nil { + return + } + if err = parseSerial(path, &device); err != nil { + return + } + if err = parseManufacturer(path, &device); err != nil { + return + } + if err = parseProduct(path, &device); err != nil { + return + } + return +} + +func parseSysUeventFile(path string, device *Device) error { + // Example uevent file: + // MAJOR=189 + // MINOR=257 + // DEVNAME=bus/usb/003/002 + // DEVTYPE=usb_device + // DRIVER=usb + // PRODUCT=e39/f100/35d + // TYPE=0/0/0 + // BUSNUM=003 + // DEVNUM=002 + file, err := os.Open(filepath.Join(path, "uevent")) + if err != nil { + return fmt.Errorf("unable to open the file %s: %w", path, err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + values := strings.Split(line, "=") + if len(values) != 2 { + slog.Info("Skipping %s due not being key=value", slog.String("line", line)) + continue + } + switch values[0] { + case "MAJOR": + val, err := strconv.ParseInt(values[1], 10, 32) + if err != nil { + slog.Error("Failed to parse MAJOR", slog.String("value", values[1]), slog.Any("err", err)) + return nil + } + device.Major = int(val) + case "MINOR": + val, err := strconv.ParseInt(values[1], 10, 32) + if err != nil { + slog.Error("Failed to parse MINOR", slog.String("value", values[1]), slog.Any("err", err)) + return nil + } + device.Minor = int(val) + case "BUSNUM": + val, err := strconv.ParseInt(values[1], 10, 32) + if err != nil { + slog.Error("Failed to parse BUSNUM", slog.String("value", values[1]), slog.Any("err", err)) + return nil + } + device.Bus = int3d(val) + case "DEVNUM": + val, err := strconv.ParseInt(values[1], 10, 32) + if err != nil { + slog.Error("Failed to parse DEVNUM", slog.String("value", values[1]), slog.Any("err", err)) + return nil + } + device.DeviceNumber = int3d(val) + case "PRODUCT": + products := strings.Split(values[1], "/") + if len(products) != 3 { + slog.Error("Failed to parse PRODUCT", slog.String("value", values[1]), slog.Any("err", err)) + return nil + } + + val, err := strconv.ParseInt(products[0], 16, 32) + if err != nil { + slog.Error("Failed to parse PRODUCT", slog.String("value", values[1]), slog.Any("err", err)) + return nil + } + device.VendorID = int4x(val) + + val, err = strconv.ParseInt(products[1], 16, 32) + if err != nil { + slog.Error("Failed to parse PRODUCT", slog.String("value", values[1]), slog.Any("err", err)) + return nil + } + device.ProductID = int4x(val) + + val, err = strconv.ParseInt(products[2], 16, 32) + if err != nil { + slog.Error("Failed to parse PRODUCT", slog.String("value", values[1]), slog.Any("err", err)) + return nil + } + device.BCD = int4x(val) + case "DEVNAME": + device.DevicePath = filepath.Join("/dev", values[1]) + default: + slog.Info("Skipping unhandled line", slog.String("line", line)) + } + } + return nil +} + +func parseSerial(path string, device *Device) error { + b, err := os.ReadFile(filepath.Join(path, "serial")) + if err != nil { + return err + } + lines := strings.Split(string(b), "\n") + if len(lines) >= 1 { + device.Serial = strings.TrimSpace(lines[0]) + } else { + device.Serial = "unknown" + } + + return nil +} + +func parseManufacturer(path string, device *Device) error { + b, err := os.ReadFile(filepath.Join(path, "manufacturer")) + if err != nil { + return err + } + lines := strings.Split(string(b), "\n") + if len(lines) >= 1 { + device.Manufacturer = strings.TrimSpace(lines[0]) + } else { + device.Manufacturer = "unknown" + } + return nil +} + +func parseProduct(path string, device *Device) error { + b, err := os.ReadFile(filepath.Join(path, "product")) + if err != nil { + return err + } + lines := strings.Split(string(b), "\n") + if len(lines) >= 1 { + device.Product = strings.TrimSpace(lines[0]) + } else { + device.Product = "unknown" + } + return nil +} + +type int4x int + +func (i int4x) String() string { + s := strconv.FormatInt(int64(i), 16) + if len(s) < 4 { + return strings.Repeat("0", 4-len(s)) + s + } + return s +} + +type int3d int + +func (i int3d) String() string { + s := strconv.FormatInt(int64(i), 10) + if len(s) < 3 { + return strings.Repeat("0", 3-len(s)) + s + } + return s +} diff --git a/images/virtualization-dra/internal/usb/discovery.go b/images/virtualization-dra/internal/usb/discovery.go new file mode 100644 index 0000000000..304ea83d88 --- /dev/null +++ b/images/virtualization-dra/internal/usb/discovery.go @@ -0,0 +1,62 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" +) + +const PathToUSBDevices = "/sys/bus/usb/devices" + +func discoverPluggedUSBDevices(pathToUSBDevices string) (*DeviceSet, error) { + usbDeviceSet := NewDeviceSet() + err := filepath.Walk(pathToUSBDevices, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + // Ignore named usb controllers + if strings.HasPrefix(info.Name(), "usb") { + return nil + } + // We are interested in actual USB devices information that + // contains idVendor and idProduct. We can skip all others. + if _, err := os.Stat(filepath.Join(path, "idVendor")); err != nil { + return nil + } + + // Get device information + device, err := LoadDevice(path) + if err = device.Validate(); err != nil { + slog.Error("failed to validate device, skip...", slog.Any("device", device), slog.String("error", err.Error())) + return nil + } + if err != nil { + return err + } + usbDeviceSet.Add(device) + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed when walking usb devices tree: %w", err) + } + return usbDeviceSet, nil +} diff --git a/images/virtualization-dra/internal/usb/monitor.go b/images/virtualization-dra/internal/usb/monitor.go new file mode 100644 index 0000000000..356f7967be --- /dev/null +++ b/images/virtualization-dra/internal/usb/monitor.go @@ -0,0 +1,169 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +import ( + "context" + "log/slog" + "time" + + "github.com/godbus/dbus/v5" +) + +type monitor struct { + callback monitorCallback + log *slog.Logger +} + +type monitorCallback struct { + Add func() + Update func() + Delete func() +} + +func newUSBMonitor(callback monitorCallback) *monitor { + return &monitor{ + callback: callback, + log: slog.With(slog.String("component", "usb-monitor")), + } +} + +func (m *monitor) Start(ctx context.Context) { + go func() { + for { + select { + case <-ctx.Done(): + return + case <-time.After(5 * time.Second): + if err := m.run(ctx); err != nil { + m.log.Error("failed to run monitor", slog.Any("err", err)) + } + } + } + }() +} + +func (m *monitor) run(ctx context.Context) error { + conn, err := dbus.ConnectSystemBus(dbus.WithContext(ctx)) + if err != nil { + return err + } + + rules := []string{ + "type='signal',interface='org.freedesktop.DBus.ObjectManager',member='InterfacesAdded'", + "type='signal',interface='org.freedesktop.DBus.ObjectManager',member='InterfacesRemoved'", + } + + for _, rule := range rules { + call := conn.BusObject().Call("org.freedesktop.DBus.AddMatch", 0, rule) + if call.Err != nil { + m.log.Error("Failed to add rule", slog.String("rule", rule), slog.Any("err", call.Err)) + } + } + + signals := make(chan *dbus.Signal, 100) + conn.Signal(signals) + + m.log.Info("Starting USB Monitor...") + defer m.log.Info("Stopping USB Monitor...") + + for { + select { + case <-ctx.Done(): + return nil + case signal := <-signals: + m.handleSignal(signal) + } + } +} + +func (m *monitor) handleSignal(signal *dbus.Signal) { + switch signal.Name { + case "org.freedesktop.DBus.ObjectManager.InterfacesAdded": + if len(signal.Body) >= 2 { + path, ok := signal.Body[0].(dbus.ObjectPath) + if !ok { + return + } + interfaces, ok := signal.Body[1].(map[string]map[string]dbus.Variant) + if !ok { + return + } + + if m.isUSBDevice(interfaces) { + slog.Info("USB Device connected", slog.String("path", string(path))) + if m.callback.Add != nil { + m.callback.Add() + } + } + } + + case "org.freedesktop.DBus.ObjectManager.InterfacesRemoved": + // Sender = {string} ":1.10" + // Path = {dbus.ObjectPath} "/org/freedesktop/UDisks2" + // Name = {string} "org.freedesktop.DBus.ObjectManager.InterfacesRemoved" + // Body = {[]interface{}} + // Body[0] = interface{} | dbus.ObjectPath "/org/freedesktop/UDisks2/block_devices/sda2" + // Body[1] = interface{} | []string{"org.freedesktop.UDisks2.Filesystem","org.freedesktop.UDisks2.Partition","org.freedesktop.UDisks2.Block","org.freedesktop.UDisks2.Drive"} + + if len(signal.Body) >= 2 { + path, ok := signal.Body[0].(dbus.ObjectPath) + if !ok { + return + } + interfaces, ok := signal.Body[1].([]string) + if !ok { + return + } + + if m.containsUSBInterface(interfaces) { + slog.Info("USB Device disconnected", slog.String("path", string(path))) + if m.callback.Delete != nil { + m.callback.Delete() + } + } + } + } +} + +func (m *monitor) isUSBDevice(interfaces map[string]map[string]dbus.Variant) bool { + if drive, ok := interfaces["org.freedesktop.UDisks2.Drive"]; ok { + if connectionBus, ok := drive["ConnectionBus"]; ok { + bus, _ := connectionBus.Value().(string) + return bus == "usb" + } + } + + if block, ok := interfaces["org.freedesktop.UDisks2.Block"]; ok { + if _, ok := block["Drive"]; ok { + // if it has a link to drive, it can be USB + return true + } + } + + return false +} + +func (m *monitor) containsUSBInterface(interfaces []string) bool { + for _, iface := range interfaces { + if iface == "org.freedesktop.UDisks2.Drive" || + iface == "org.freedesktop.UDisks2.Block" { + return true + } + } + return false +} diff --git a/images/virtualization-dra/internal/usb/monitor_test.go b/images/virtualization-dra/internal/usb/monitor_test.go new file mode 100644 index 0000000000..d01ea54377 --- /dev/null +++ b/images/virtualization-dra/internal/usb/monitor_test.go @@ -0,0 +1,53 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +import ( + "context" + "fmt" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMonitor(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "USB Monitor Suite") +} + +var _ = Describe("USB Monitor", func() { + + It("should add a new device", func() { + + monitor := newUSBMonitor(monitorCallback{ + Add: func() { + fmt.Println("USB Device connected") + }, + Update: func() { + fmt.Println("USB Device updated") + }, + Delete: func() { + fmt.Println("USB Device disconnected") + }, + }) + + monitor.Start(context.Background()) + time.Sleep(10 * time.Minute) + }) +}) diff --git a/images/virtualization-dra/internal/usb/store.go b/images/virtualization-dra/internal/usb/store.go new file mode 100644 index 0000000000..5ae5d2b50d --- /dev/null +++ b/images/virtualization-dra/internal/usb/store.go @@ -0,0 +1,369 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +import ( + "context" + "fmt" + "log/slog" + "strings" + "sync" + "time" + + "github.com/containerd/nri/pkg/api" + resourceapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/types" + drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1beta1" + "k8s.io/utils/ptr" + cdiapi "tags.cncf.io/container-device-interface/pkg/cdi" + cdispec "tags.cncf.io/container-device-interface/specs-go" + + "github.com/deckhouse/virtualization-dra/internal/cdi" + "github.com/deckhouse/virtualization-dra/pkg/set" +) + +const DefaultResyncPeriod = 10 * time.Minute + +func NewAllocationStore(nodeName, devicesPath string, resyncPeriod time.Duration, cdiManager cdi.Manager, log *slog.Logger) *AllocationStore { + if resyncPeriod == 0 { + resyncPeriod = DefaultResyncPeriod + } + if devicesPath == "" { + devicesPath = PathToUSBDevices + } + store := &AllocationStore{ + nodeName: nodeName, + devicesPath: devicesPath, + resyncPeriod: resyncPeriod, + cdi: cdiManager, + log: log.With(slog.String("component", "usb-allocation-store")), + updateChannel: make(chan []resourceapi.Device, 2), + discoverPluggedUSBDevices: NewDeviceSet(), + allocatableDevices: make(map[string]resourceapi.Device), + allocatedDevices: set.New[string](), + resourceClaimAllocations: make(map[types.UID][]string), + } + + monitor := newUSBMonitor(monitorCallback{ + Add: store.genericCallback, + Update: store.genericCallback, + Delete: store.genericCallback, + }) + + store.monitor = monitor + + return store +} + +type AllocationStore struct { + nodeName string + devicesPath string + resyncPeriod time.Duration + + cdi cdi.Manager + log *slog.Logger + + monitor *monitor + + updateChannel chan []resourceapi.Device + mu sync.RWMutex + + discoverPluggedUSBDevices *DeviceSet + allocatableDevices map[string]resourceapi.Device + allocatedDevices *set.Set[string] + resourceClaimAllocations map[types.UID][]string +} + +func (s *AllocationStore) sync() error { + s.mu.Lock() + defer s.mu.Unlock() + + discoverPluggedUSBDevices, err := discoverPluggedUSBDevices(s.devicesPath) + if err != nil { + return err + } + if discoverPluggedUSBDevices.Equal(s.discoverPluggedUSBDevices) { + return nil + } + s.discoverPluggedUSBDevices = discoverPluggedUSBDevices + + allocatableDevices := make([]resourceapi.Device, discoverPluggedUSBDevices.Len()) + for i, usbDevice := range discoverPluggedUSBDevices.Slice() { + allocatableDevices[i] = *convertToAPIDevice(usbDevice) + } + + allocatableDevicesByName := make(map[string]resourceapi.Device, len(allocatableDevices)) + for _, device := range allocatableDevices { + allocatableDevicesByName[device.Name] = device + } + + s.allocatableDevices = allocatableDevicesByName + + s.updateChannel <- allocatableDevices + + return nil +} + +func (s *AllocationStore) genericCallback() { + if err := s.sync(); err != nil { + s.log.Error("failed to sync usb state", slog.Any("err", err)) + } +} + +func (s *AllocationStore) Start(ctx context.Context) error { + if err := s.cdi.CreateCommonSpecFile(); err != nil { + return fmt.Errorf("failed to create CDI common spec file: %w", err) + } + + doSync := func() { + err := s.sync() + if err != nil { + s.log.Error("failed to sync usb state", slog.Any("err", err)) + } + } + ticker := time.NewTicker(s.resyncPeriod) + go func() { + doSync() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + doSync() + } + } + }() + + s.monitor.Start(ctx) + + return nil +} + +func (s *AllocationStore) UpdateChannel() chan []resourceapi.Device { + return s.updateChannel +} + +func (s *AllocationStore) Prepare(_ context.Context, claim *resourceapi.ResourceClaim) ([]*drapbv1.Device, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if claim.Status.Allocation == nil { + return nil, fmt.Errorf("claim %s/%s has no allocation", claim.Namespace, claim.Name) + } + + claimUID := string(claim.UID) + + preparedDevices := make(cdi.PreparedDevices, len(claim.Status.Allocation.Devices.Results)) + for i, result := range claim.Status.Allocation.Devices.Results { + usbDevice, exists := s.allocatableDevices[result.Device] + if !exists { + return nil, fmt.Errorf("requested device is not allocatable: %v", result.Device) + } + // TODO: unnecessary? + // kubernetes check allocatable devices + // Warning FailedScheduling 8s default-scheduler 0/3 nodes are available: + // 1 node(s) had tolerated taint {node-role.kubernetes.io/control-plane: }, + // 2 cannot allocate all claims. + // still not schedulable, preemption: 0/3 nodes are available: 3 Preemption is not helpful for scheduling. + if s.allocatedDevices.Contains(result.Device) { + return nil, fmt.Errorf("device %v is already allocated", result.Device) + } + + edits, err := s.makeContainerEdits(claimUID, &usbDevice) + if err != nil { + return nil, err + } + device := cdi.PreparedDevice{ + Device: drapbv1.Device{ + RequestNames: []string{result.Request}, + PoolName: result.Pool, + DeviceName: result.Device, + CDIDeviceIDs: s.cdi.GetClaimDevices(claimUID, result.Device), + }, + ContainerEdits: edits, + } + preparedDevices[i] = &device + } + + err := s.cdi.CreateClaimSpecFile(claimUID, preparedDevices) + if err != nil { + return nil, fmt.Errorf("unable to create CDI spec file for claim: %v", err) + } + + devices := preparedDevices.GetDevices() + for _, device := range devices { + s.allocatedDevices.Add(device.DeviceName) + s.resourceClaimAllocations[claim.UID] = append(s.resourceClaimAllocations[claim.UID], device.DeviceName) + } + + return devices, nil +} + +// TODO: refactor me +func (s *AllocationStore) makeContainerEdits(claimUID string, device *resourceapi.Device) (*cdiapi.ContainerEdits, error) { + var ( + devicePath string + deviceNum string + bus string + major int64 + minor int64 + ) + + if attr, ok := device.Attributes["devicePath"]; ok { + if val := attr.StringValue; val != nil { + devicePath = *val + } else { + return nil, fmt.Errorf("devicePath attribute is not exist") + } + } + + if attr, ok := device.Attributes["deviceNumber"]; ok { + if val := attr.StringValue; val != nil { + deviceNum = *val + } else { + return nil, fmt.Errorf("deviceNum attribute is not exist") + } + } + + if attr, ok := device.Attributes["bus"]; ok { + if val := attr.StringValue; val != nil { + bus = *val + } else { + return nil, fmt.Errorf("bus attribute is not exist") + } + } + + if attr, ok := device.Attributes["major"]; ok { + if val := attr.IntValue; val != nil { + major = *val + } else { + return nil, fmt.Errorf("major attribute is not exist") + } + } + + if attr, ok := device.Attributes["minor"]; ok { + if val := attr.IntValue; val != nil { + minor = *val + } else { + return nil, fmt.Errorf("minor attribute is not exist") + } + } + + claimUIDUpper := strings.ToUpper(claimUID) + deviceNameUpper := strings.ToUpper(device.Name) + + edits := &cdiapi.ContainerEdits{ + ContainerEdits: &cdispec.ContainerEdits{ + Env: []string{ + fmt.Sprintf("DRA_USB_CLAIM_UID_%s=%s", claimUIDUpper, claimUID), + fmt.Sprintf("DRA_USB_DEVICE_NAME_%s=%s", deviceNameUpper, device.Name), + fmt.Sprintf("DRA_USB_CLAIM_UID_%s_DEVICE_NAME=%s", claimUIDUpper, device.Name), + fmt.Sprintf("DRA_USB_%s_DEVICE_PATH=%s", deviceNameUpper, devicePath), + fmt.Sprintf("DRA_USB_%s_BUS_DEVICENUMBER=%s:%s", deviceNameUpper, bus, deviceNum), + }, + DeviceNodes: []*cdispec.DeviceNode{ + { + Path: devicePath, + HostPath: devicePath, + Type: "c", + Major: major, + Minor: minor, + Permissions: "mrw", + UID: ptr.To(uint32(107)), // qemu user. TODO: make this configurable + GID: ptr.To(uint32(107)), // qemu group. TODO: make this configurable + }, + }, + }, + } + + return edits, nil +} + +func (s *AllocationStore) Unprepare(_ context.Context, claimUID types.UID) error { + s.mu.Lock() + defer s.mu.Unlock() + + if err := s.cdi.DeleteClaimSpecFile(string(claimUID)); err != nil { + return fmt.Errorf("unable to delete CDI spec file for claim: %w", err) + } + + allocatedDevices := s.resourceClaimAllocations[claimUID] + for _, device := range allocatedDevices { + s.allocatedDevices.Remove(device) + } + delete(s.resourceClaimAllocations, claimUID) + + return nil +} + +func (s *AllocationStore) Synchronize(_ context.Context, pods []*api.PodSandbox, containers []*api.Container) ([]*api.ContainerUpdate, error) { + s.mu.Lock() + defer s.mu.Unlock() + + containersByPodSandboxId := make(map[string][]*api.Container, len(pods)) + for _, ctr := range containers { + containersByPodSandboxId[ctr.PodSandboxId] = append(containersByPodSandboxId[ctr.PodSandboxId], ctr) + } + + for _, pod := range pods { + s.log.Info("Synchronize pod", slog.String("name", pod.Name), slog.String("namespace", pod.Namespace)) + ctrs := containersByPodSandboxId[pod.Id] + + for _, ctr := range ctrs { + claimUIDDeviceNames, err := parseDraEnvToClaimAllocations(ctr.Env) + if err != nil { + s.log.Error("failed to parse dra env", slog.String("name", pod.Name), slog.String("namespace", pod.Namespace), slog.Any("err", err)) + continue + } + for claimUID, deviceNames := range claimUIDDeviceNames { + s.resourceClaimAllocations[claimUID] = append(s.resourceClaimAllocations[claimUID], deviceNames...) + for _, deviceName := range deviceNames { + s.allocatedDevices.Add(deviceName) + } + } + + } + } + return nil, nil +} + +func parseDraEnvToClaimAllocations(envs []string) (map[types.UID][]string, error) { + result := make(map[types.UID][]string) + + for _, env := range envs { + parts := strings.SplitN(env, "=", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid dra env: %s", env) + } + key := parts[0] + value := parts[1] + + if !strings.HasPrefix(key, "DRA_USB_CLAIM_UID_") || !strings.HasSuffix(key, "_DEVICE_NAME") { + continue + } + uid := strings.TrimPrefix(key, "DRA_USB_CLAIM_UID_") + uid = strings.TrimSuffix(uid, "_DEVICE_NAME") + uid = strings.ToLower(uid) + claimUID := types.UID(uid) + + deviceName := value + + result[claimUID] = append(result[claimUID], deviceName) + } + + return result, nil +} diff --git a/images/virtualization-dra/pkg/logger/logger.go b/images/virtualization-dra/pkg/logger/logger.go new file mode 100644 index 0000000000..12222e88b9 --- /dev/null +++ b/images/virtualization-dra/pkg/logger/logger.go @@ -0,0 +1,88 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logger + +import ( + "io" + "log/slog" + "os" + "strings" + + "github.com/go-logr/logr" + "k8s.io/klog/v2" + + "github.com/deckhouse/deckhouse/pkg/log" +) + +type Output string + +const ( + Stdout Output = "stdout" + Stderr Output = "stderr" + Discard Output = "discard" +) + +const DefaultLogLevel = log.LevelInfo + +var DefaultLogOutput = os.Stdout + +func NewLogger(level, output string, debugVerbosity int) *log.Logger { + return log.NewLogger(log.WithLevel(detectLogLevel(level, debugVerbosity)), log.WithOutput(detectLogOutput(output))) +} + +func detectLogLevel(level string, debugVerbosity int) slog.Level { + switch strings.ToLower(level) { + case "fatal": + return log.LevelFatal.Level() + case "error": + return log.LevelError.Level() + case "warn": + return log.LevelWarn.Level() + case "info": + return log.LevelInfo.Level() + case "debug": + if debugVerbosity != 0 { + return slog.Level(-1 * debugVerbosity) + } + + return log.LevelDebug.Level() + case "trace": + return log.LevelTrace.Level() + default: + return DefaultLogLevel.Level() + } +} + +func detectLogOutput(output string) io.Writer { + switch strings.ToLower(output) { + case string(Stdout): + return os.Stdout + case string(Stderr): + return os.Stderr + case string(Discard): + return io.Discard + default: + return DefaultLogOutput + } +} + +func SetDefaultLogger(l *log.Logger) { + slog.SetDefault(slog.New(l.Handler())) + log.SetDefault(l) + fromSlog := logr.FromSlogHandler(l.Handler()) + klog.SetLogger(fromSlog) +} diff --git a/images/virtualization-dra/pkg/logger/options.go b/images/virtualization-dra/pkg/logger/options.go new file mode 100644 index 0000000000..8199b01609 --- /dev/null +++ b/images/virtualization-dra/pkg/logger/options.go @@ -0,0 +1,38 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package logger + +import ( + "github.com/deckhouse/deckhouse/pkg/log" + "github.com/spf13/pflag" +) + +type Options struct { + Level string + Output string + DebugVerbosity int +} + +func (o *Options) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&o.Level, "log-level", o.Level, "Log level") + fs.StringVar(&o.Output, "log-output", o.Output, "Log output") + fs.IntVar(&o.DebugVerbosity, "log-debug-verbosity", o.DebugVerbosity, "Log debug verbosity") +} + +func (o *Options) Complete() *log.Logger { + return NewLogger(o.Level, o.Output, o.DebugVerbosity) +} diff --git a/images/virtualization-dra/pkg/set/set.go b/images/virtualization-dra/pkg/set/set.go new file mode 100644 index 0000000000..4b7f4378a5 --- /dev/null +++ b/images/virtualization-dra/pkg/set/set.go @@ -0,0 +1,64 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package set + +type Set[T comparable] struct { + m map[T]struct{} +} + +func New[T comparable]() *Set[T] { + return &Set[T]{ + m: make(map[T]struct{}), + } +} +func (s *Set[T]) Add(v T) { + s.m[v] = struct{}{} +} + +func (s *Set[T]) Remove(v T) { + delete(s.m, v) +} + +func (s *Set[T]) Contains(v T) bool { + _, ok := s.m[v] + return ok +} + +func (s *Set[T]) Len() int { + return len(s.m) +} + +func (s *Set[T]) Slice() []T { + out := make([]T, 0, len(s.m)) + for k := range s.m { + out = append(out, k) + } + return out +} + +func (s *Set[T]) Equal(other *Set[T]) bool { + if s.Len() != other.Len() { + return false + } + + for k := range s.m { + if !other.Contains(k) { + return false + } + } + return true +} diff --git a/images/virtualization-dra/test/deviceclass.yaml b/images/virtualization-dra/test/deviceclass.yaml new file mode 100644 index 0000000000..195511eca4 --- /dev/null +++ b/images/virtualization-dra/test/deviceclass.yaml @@ -0,0 +1,8 @@ +apiVersion: resource.k8s.io/v1beta1 +kind: DeviceClass +metadata: + name: usb-devices.virtualization.deckhouse.io +spec: + selectors: + - cel: + expression: "device.driver == 'virtualization-dra'" diff --git a/images/virtualization-dra/test/pod-with-claim.yaml b/images/virtualization-dra/test/pod-with-claim.yaml new file mode 100644 index 0000000000..5240037a31 --- /dev/null +++ b/images/virtualization-dra/test/pod-with-claim.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-pod-with-usb-1 + namespace: usb2 +spec: + containers: + - name: test-container + image: nicolaka/netshoot:latest + command: ["sleep", "3600"] + resources: + claims: + - name: usb-device + resourceClaims: + - name: usb-device + resourceClaimName: test-pod-with-usb-1 diff --git a/images/virtualization-dra/test/pod-with-template-1.yaml b/images/virtualization-dra/test/pod-with-template-1.yaml new file mode 100644 index 0000000000..5b44de020b --- /dev/null +++ b/images/virtualization-dra/test/pod-with-template-1.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-pod-with-usb-template-1 + namespace: usb +spec: + containers: + - name: test-container + image: nicolaka/netshoot:latest + command: ["sleep", "3600"] + resources: + claims: + - name: usb-device + resourceClaims: + - name: usb-device + resourceClaimTemplateName: usb-product-f100-vendor-0e39-template diff --git a/images/virtualization-dra/test/pod-with-template-2.yaml b/images/virtualization-dra/test/pod-with-template-2.yaml new file mode 100644 index 0000000000..bcd20b5eff --- /dev/null +++ b/images/virtualization-dra/test/pod-with-template-2.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-pod-with-usb-template-2 + namespace: usb +spec: + containers: + - name: test-container + image: nicolaka/netshoot:latest + command: ["sleep", "3600"] + resources: + claims: + - name: usb-device + volumeMounts: + - name: mnt + mountPath: /mnt + volumes: + - name: mnt + emptyDir: {} + resourceClaims: + - name: usb-device + resourceClaimTemplateName: usb-product-f100-vendor-0e39-template diff --git a/images/virtualization-dra/test/resourceclaim-template.yaml b/images/virtualization-dra/test/resourceclaim-template.yaml new file mode 100644 index 0000000000..3077bd1bf3 --- /dev/null +++ b/images/virtualization-dra/test/resourceclaim-template.yaml @@ -0,0 +1,18 @@ +apiVersion: resource.k8s.io/v1beta1 +kind: ResourceClaimTemplate +metadata: + name: usb-product-f100-vendor-0e39-template + namespace: usb +spec: + spec: + devices: + requests: + - name: req-0 + allocationMode: "ExactCount" + count: 1 + deviceClassName: usb-devices.virtualization.deckhouse.io + selectors: + - cel: + expression: |- + device.attributes["virtualization-dra"].productID == "f100" && + device.attributes["virtualization-dra"].vendorID == "0e39" diff --git a/images/virtualization-dra/test/resourceclaim.yaml b/images/virtualization-dra/test/resourceclaim.yaml new file mode 100644 index 0000000000..66f1a93928 --- /dev/null +++ b/images/virtualization-dra/test/resourceclaim.yaml @@ -0,0 +1,17 @@ +apiVersion: resource.k8s.io/v1beta1 +kind: ResourceClaim +metadata: + name: test-pod-with-usb-1 + namespace: usb2 +spec: + devices: + requests: + - allocationMode: ExactCount + count: 1 + deviceClassName: usb-devices.virtualization.deckhouse.io + name: req-0 + selectors: + - cel: + expression: |- + device.attributes["virtualization-dra"].productID == "f100" && + device.attributes["virtualization-dra"].vendorID == "0e39" diff --git a/images/virtualization-dra/werf.inc.yaml b/images/virtualization-dra/werf.inc.yaml new file mode 100644 index 0000000000..919a0adcbb --- /dev/null +++ b/images/virtualization-dra/werf.inc.yaml @@ -0,0 +1,41 @@ +--- +image: {{ .ModuleNamePrefix }}{{ .ImageName }}-builder +final: false +fromImage: {{ eq $.SVACE_ENABLED "false" | ternary "builder/golang-bookworm-1.24" "builder/golang-alt-svace-1.24.9" }} +git: + - add: {{ .ModuleDir }}/images/{{ .ImageName }} + to: /src/images/virtualization-dra + stageDependencies: + install: + - go.mod + - go.sum + setup: + - "**/*.go" +secrets: + - id: GOPROXY + value: {{ .GOPROXY }} +mount: + - fromPath: ~/go-pkg-cache + to: /go/pkg +shell: + install: + - export GOPROXY=$(cat /run/secrets/GOPROXY) + - cd /src/images/virtualization-dra + - go mod download + setup: + - cd /src/images/virtualization-dra + - mkdir /out + - export GOOS=linux + - export GOARCH=amd64 + - export CGO_ENABLED=0 + + - | + echo "Build virtualization-dra-plugin binary" + {{- $_ := set $ "ProjectName" (list $.ImageName "virtualization-dra-plugin" | join "/") }} + + {{- if eq $.DEBUG_COMPONENT "delve/virtualization-dra-plugin" }} + go build -v -o /out/virtualization-dra-plugin ./cmd/virtualization-dra-plugin + {{- else }} + {{- include "image-build.build" (set $ "BuildCommand" `go build -ldflags="-s -w" -v -o /out/virtualization-dra-plugin ./cmd/virtualization-dra-plugin`) | nindent 4 }} + {{- end }} + diff --git a/images/vm-route-forge/go.mod b/images/vm-route-forge/go.mod index 57bbcc5dcc..1748f07e51 100644 --- a/images/vm-route-forge/go.mod +++ b/images/vm-route-forge/go.mod @@ -67,7 +67,6 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oklog/ulid v1.3.1 // indirect - github.com/openshift/api v0.0.0-20230503133300-8bbcb7ca7183 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect @@ -123,8 +122,8 @@ require ( k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250701173324-9bd5c66d9911 // indirect k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect - kubevirt.io/api v1.3.1 // indirect - kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 // indirect + kubevirt.io/api v1.6.2 // indirect + kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/randfill v1.0.0 // indirect diff --git a/images/vm-route-forge/go.sum b/images/vm-route-forge/go.sum index 65cc2e5874..3c63121653 100644 --- a/images/vm-route-forge/go.sum +++ b/images/vm-route-forge/go.sum @@ -300,8 +300,6 @@ github.com/onsi/gomega v1.33.0/go.mod h1:+925n5YtiFsLzzafLUHzVMBpvvRAzrydIBiSIxj github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= -github.com/openshift/api v0.0.0-20230503133300-8bbcb7ca7183 h1:t/CahSnpqY46sQR01SoS+Jt0jtjgmhgE6lFmRnO4q70= -github.com/openshift/api v0.0.0-20230503133300-8bbcb7ca7183/go.mod h1:4VWG+W22wrB4HfBL88P40DxLEpSOaiBVxUnfalfJo9k= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= @@ -677,10 +675,10 @@ k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -kubevirt.io/api v1.3.1 h1:MoTNo/zvDlZ44c2ocXLPln8XTaQOeUodiYbEKrTCqv4= -kubevirt.io/api v1.3.1/go.mod h1:tCn7VAZktEvymk490iPSMPCmKM9UjbbfH2OsFR/IOLU= -kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 h1:IWo12+ei3jltSN5jQN1xjgakfvRSF3G3Rr4GXVOOy2I= -kubevirt.io/containerized-data-importer-api v1.57.0-alpha1/go.mod h1:Y/8ETgHS1GjO89bl682DPtQOYEU/1ctPFBz6Sjxm4DM= +kubevirt.io/api v1.6.2 h1:aoqZ4KsbOyDjLnuDw7H9wEgE/YTd/q5BBmYeQjJNizc= +kubevirt.io/api v1.6.2/go.mod h1:p66fEy/g79x7VpgUwrkUgOoG2lYs5LQq37WM6JXMwj4= +kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9 h1:KTb8wO1Lxj220DX7d2Rdo9xovvlyWWNo3AVm2ua+1nY= +kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9/go.mod h1:SDJjLGhbPyayDqAqawcGmVNapBp0KodOQvhKPLVGCQU= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= diff --git a/src/cli/go.mod b/src/cli/go.mod index bd767e6226..d859fe42de 100644 --- a/src/cli/go.mod +++ b/src/cli/go.mod @@ -66,8 +66,8 @@ require ( k8s.io/api v0.33.3 // indirect k8s.io/apiextensions-apiserver v0.33.3 // indirect k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect - kubevirt.io/api v1.3.1 // indirect - kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 // indirect + kubevirt.io/api v1.6.2 // indirect + kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect sigs.k8s.io/controller-runtime v0.21.0 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect diff --git a/src/cli/go.sum b/src/cli/go.sum index 2c8e3bc9b0..8bb70c9518 100644 --- a/src/cli/go.sum +++ b/src/cli/go.sum @@ -589,8 +589,12 @@ k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8 k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/api v1.3.1 h1:MoTNo/zvDlZ44c2ocXLPln8XTaQOeUodiYbEKrTCqv4= kubevirt.io/api v1.3.1/go.mod h1:tCn7VAZktEvymk490iPSMPCmKM9UjbbfH2OsFR/IOLU= +kubevirt.io/api v1.6.2 h1:aoqZ4KsbOyDjLnuDw7H9wEgE/YTd/q5BBmYeQjJNizc= +kubevirt.io/api v1.6.2/go.mod h1:p66fEy/g79x7VpgUwrkUgOoG2lYs5LQq37WM6JXMwj4= kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 h1:IWo12+ei3jltSN5jQN1xjgakfvRSF3G3Rr4GXVOOy2I= kubevirt.io/containerized-data-importer-api v1.57.0-alpha1/go.mod h1:Y/8ETgHS1GjO89bl682DPtQOYEU/1ctPFBz6Sjxm4DM= +kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9 h1:KTb8wO1Lxj220DX7d2Rdo9xovvlyWWNo3AVm2ua+1nY= +kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9/go.mod h1:SDJjLGhbPyayDqAqawcGmVNapBp0KodOQvhKPLVGCQU= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= diff --git a/templates/kubevirt/kubevirt.yaml b/templates/kubevirt/kubevirt.yaml index 2951618f9d..da46c7be0b 100644 --- a/templates/kubevirt/kubevirt.yaml +++ b/templates/kubevirt/kubevirt.yaml @@ -52,16 +52,16 @@ spec: virtOperator: {{ $logVerbosity }} featureGates: - HotplugVolumes - - GPU - Snapshot - ExpandDisks - Root - - VMLiveUpdateFeatures - CPUManager - Sidecar - VolumeSnapshotDataSource - VolumeMigration - VolumesUpdateStrategy + - HostDevicesWithDRA + - HostDevices virtualMachineOptions: disableSerialConsoleLog: {} customizeComponents: diff --git a/templates/kubevirt/virt-operator/rbac-for-us.yaml b/templates/kubevirt/virt-operator/rbac-for-us.yaml index d52cec41b0..91a6311478 100644 --- a/templates/kubevirt/virt-operator/rbac-for-us.yaml +++ b/templates/kubevirt/virt-operator/rbac-for-us.yaml @@ -26,6 +26,8 @@ rules: - kubevirt-virt-api-certs - kubevirt-controller-certs - kubevirt-exportproxy-certs + - kubevirt-synchronization-controller-certs + - kubevirt-synchronization-controller-server-certs resources: - secrets verbs: @@ -834,6 +836,9 @@ rules: - virtualmachineinstances/userlist - virtualmachineinstances/sev/fetchcertchain - virtualmachineinstances/sev/querylaunchmeasurement + - virtualmachineinstances/usbredir + - virtualmachines/objectgraph + - virtualmachineinstances/objectgraph verbs: - get - apiGroups: @@ -846,6 +851,7 @@ rules: - virtualmachineinstances/freeze - virtualmachineinstances/unfreeze - virtualmachineinstances/softreboot + - virtualmachineinstances/reset - virtualmachineinstances/sev/setupsession - virtualmachineinstances/sev/injectlaunchsecret verbs: @@ -1214,6 +1220,15 @@ rules: - subjectaccessreviews verbs: - create +- apiGroups: + - resource.k8s.io + resources: + - resourceclaims + - resourceslices + verbs: + - get + - list + - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/templates/virtualization-dra/_helper.tpl b/templates/virtualization-dra/_helper.tpl new file mode 100644 index 0000000000..72f4ff159d --- /dev/null +++ b/templates/virtualization-dra/_helper.tpl @@ -0,0 +1,5 @@ +{{- define "virtualization-dra.isEnabled" -}} +{{- if eq (include "hasValidModuleConfig" .) "true" -}} +true +{{- end -}} +{{- end -}} diff --git a/templates/virtualization-dra/daemonset.yaml b/templates/virtualization-dra/daemonset.yaml new file mode 100644 index 0000000000..47c1e7543f --- /dev/null +++ b/templates/virtualization-dra/daemonset.yaml @@ -0,0 +1,132 @@ +{{- $priorityClassName := include "priorityClassName" . }} +{{- $delve := (include "delve" . | fromYaml) -}} +{{- define "virtualization-dra_resources" }} +cpu: 10m +memory: 25Mi +{{- end }} + + +{{- if eq (include "virtualization-dra.isEnabled" .) "true"}} + +{{- if (.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} +--- +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: virtualization-dra + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra" "workload-resource-policy.deckhouse.io" "every-node")) | nindent 2 }} +spec: + targetRef: + apiVersion: "apps/v1" + kind: DaemonSet + name: virtualization-dra + updatePolicy: + updateMode: "Auto" + resourcePolicy: + containerPolicies: + - containerName: virtualization-dra + minAllowed: + {{- include "virtualization-dra_resources" . | nindent 8 }} + maxAllowed: + cpu: 20m + memory: 25Mi +{{- end }} + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: virtualization-dra + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra")) | nindent 2 }} +spec: + selector: + matchLabels: + app: virtualization-dra + template: + metadata: + labels: + app: virtualization-dra + spec: + {{- include "helm_lib_priority_class" (tuple . $priorityClassName) | nindent 6 }} + {{- include "helm_lib_tolerations" (tuple . "any-node") | nindent 6 }} + {{- include "helm_lib_module_pod_security_context_run_as_user_root" . | nindent 6 }} + imagePullSecrets: + - name: virtualization-module-registry + serviceAccountName: virtualization-dra + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + kubernetes.io/os: linux + containers: + - name: virtualization-dra + {{- include "helm_lib_module_container_security_context_privileged_read_only_root_filesystem" . | nindent 10 }} + image: {{ include "helm_lib_module_image" (list . "virtualizationDraPlugin") }} + imagePullPolicy: "IfNotPresent" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if eq (include "moduleLogLevel" .) "debug" }} + - name: VERBOSITY + value: "10" + {{- end }} + - name: USB_DEVICES_PATH + value: /sys/bus/usb/devices + - name: DBUS_SYSTEM_BUS_ADDRESS + value: "unix:path=/var/run/dbus/system_bus_socket" + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 14 }} + {{- if not ( .Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "virtualization-dra_resources" . | nindent 14 }} + {{- end }} + ports: + - containerPort: 51515 + name: health + protocol: TCP + {{- include "delvePorts" (list $delve "delve/virtualization-dra") | nindent 12 }} + {{- if ne "delve/virtualization-dra" ($delve | dig "debug" "component" "") }} + readinessProbe: + grpc: + port: 51515 + service: liveness + failureThreshold: 3 + periodSeconds: 10 + livenessProbe: + grpc: + port: 51515 + service: liveness + failureThreshold: 3 + periodSeconds: 10 + {{- end }} + volumeMounts: + - name: plugins-registry + mountPath: /var/lib/kubelet/plugins_registry + - name: plugins + mountPath: /var/lib/kubelet/plugins + - name: cdi + mountPath: /var/run/cdi + - name: devices + mountPath: /sys/bus/usb/devices + - name: dbus-socket + mountPath: /var/run/dbus/system_bus_socket + volumes: + - name: plugins-registry + hostPath: + path: /var/lib/kubelet/plugins_registry + - name: plugins + hostPath: + path: /var/lib/kubelet/plugins + - name: cdi + hostPath: + path: /var/run/cdi + - name: devices + hostPath: + path: /sys/bus/usb/devices + - name: dbus-socket + hostPath: + path: /var/run/dbus/system_bus_socket + type: Socket +{{- end }} diff --git a/templates/virtualization-dra/rbac-for-us.yaml b/templates/virtualization-dra/rbac-for-us.yaml new file mode 100644 index 0000000000..8f1405f470 --- /dev/null +++ b/templates/virtualization-dra/rbac-for-us.yaml @@ -0,0 +1,39 @@ +{{- if eq (include "virtualization-dra.isEnabled" .) "true"}} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: virtualization-dra + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra")) | nindent 2 }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ .Chart.Name }}:virtualization-dra + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra")) | nindent 2 }} +rules: + - apiGroups: ["resource.k8s.io"] + resources: ["resourceclaims"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] + - apiGroups: ["resource.k8s.io"] + resources: ["resourceslices"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ .Chart.Name }}:virtualization-dra + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra")) | nindent 2 }} +subjects: + - kind: ServiceAccount + name: virtualization-dra + namespace: d8-{{ .Chart.Name }} +roleRef: + kind: ClusterRole + name: d8:{{ .Chart.Name }}:virtualization-dra + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/test/e2e/go.mod b/test/e2e/go.mod index b727c9ceab..5ddb0f733b 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -16,7 +16,7 @@ require ( k8s.io/cli-runtime v0.33.3 k8s.io/client-go v0.33.3 k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 - kubevirt.io/api v1.3.1 + kubevirt.io/api v1.6.2 sigs.k8s.io/controller-runtime v0.21.0 sigs.k8s.io/yaml v1.4.0 ) diff --git a/test/e2e/go.sum b/test/e2e/go.sum index f72cde1b6d..66999d0350 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -582,6 +582,8 @@ k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8 k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/api v1.3.1 h1:MoTNo/zvDlZ44c2ocXLPln8XTaQOeUodiYbEKrTCqv4= kubevirt.io/api v1.3.1/go.mod h1:tCn7VAZktEvymk490iPSMPCmKM9UjbbfH2OsFR/IOLU= +kubevirt.io/api v1.6.2 h1:aoqZ4KsbOyDjLnuDw7H9wEgE/YTd/q5BBmYeQjJNizc= +kubevirt.io/api v1.6.2/go.mod h1:p66fEy/g79x7VpgUwrkUgOoG2lYs5LQq37WM6JXMwj4= kubevirt.io/containerized-data-importer-api v1.60.3 h1:kQEXi7scpzUa0RPf3/3MKk1Kmem0ZlqqiuK3kDF5L2I= kubevirt.io/containerized-data-importer-api v1.60.3/go.mod h1:8mwrkZIdy8j/LmCyKt2wFXbiMavLUIqDaegaIF67CZs= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= diff --git a/tools/kubeconform/fixtures/module-values.yaml b/tools/kubeconform/fixtures/module-values.yaml index 88282bddb2..6dfa410372 100644 --- a/tools/kubeconform/fixtures/module-values.yaml +++ b/tools/kubeconform/fixtures/module-values.yaml @@ -331,6 +331,7 @@ global: virtualizationApi: sha256:0000000000000000000000000000000000000000000000000000000000000000 virtualizationController: sha256:0000000000000000000000000000000000000000000000000000000000000000 vmRouteForge: sha256:0000000000000000000000000000000000000000000000000000000000000000 + virtualizationDraPlugin: sha256:0000000000000000000000000000000000000000000000000000000000000000w registry: CA: "" address: some-registry.io