DRA: Add configurable health check timeout per device

Implements device-specific health check timeouts in the DRA health monitoring
system as defined in KEP-4680. This allows DRA drivers to specify custom
timeout values for individual devices through the gRPC health API.

Changes:
- Add HealthCheckTimeout field to state.DeviceHealth struct to store
  device-specific timeout durations
- Add health_check_timeout_seconds field to DeviceHealth proto message
  in the DRA health gRPC API (v1alpha1)
- Update manager.go to extract timeout from gRPC responses and apply
  DefaultHealthTimeout (30s) when not specified
- Handle negative timeout values defensively by logging a warning and
  falling back to the default timeout
- Simplify healthinfo.go by removing redundant fallback logic since
  timeouts are now always set at creation time
- Update tests to include HealthCheckTimeout in test fixtures

The timeout behavior is:
- Positive values: Use the specified timeout in seconds
- Zero or unspecified: Use DefaultHealthTimeout (30 seconds)
- Negative values: Log warning and use DefaultHealthTimeout

This implementation provides flexibility for DRA drivers to define
appropriate health check intervals for different device types while
maintaining backward compatibility through sensible defaults.

Ref: KEP-4680 (Add Resource Health to Pod Status)
Ref: kubernetes/enhancements#5476

Signed-off-by: Carlos Eduardo Arango Gutierrez <[email protected]>

Author: ArangoGutierrez

pkg/kubelet/cm/dra/healthinfo.go

@@ -28,9 +28,9 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/cm/dra/state"
 )
 
-// TODO(#133118): Make health timeout configurable.
 const (
-	healthTimeout = 30 * time.Second
+	// DefaultHealthTimeout is the default timeout for device health checks when not specified by the plugin
+	DefaultHealthTimeout = 30 * time.Second
 )
 
 // healthInfoCache is a cache of known device health.
@@ -129,7 +129,14 @@ func (cache *healthInfoCache) getHealthInfo(driverName, poolName, deviceName str
 		if driver, ok := (*cache.HealthInfo)[driverName]; ok {
 			key := poolName + "/" + deviceName
 			if device, ok := driver.Devices[key]; ok {
-				if now.Sub(device.LastUpdated) > healthTimeout {
+				// Use device-specific timeout if set, otherwise use default
+				timeout := device.HealthCheckTimeout
+				if timeout == 0 {
+					timeout = DefaultHealthTimeout
+				}
+				
+				// Check if device health has timed out
+				if now.Sub(device.LastUpdated) > timeout {
 					res = state.DeviceHealthStatusUnknown
 				} else {
 					res = device.Health
@@ -178,7 +185,14 @@ func (cache *healthInfoCache) updateHealthInfo(driverName string, devices []stat
 		// them. Mark them as "Unknown" if their status has timed out.
 		for key, existingDevice := range currentDriver.Devices {
 			if _, wasReported := reportedKeys[key]; !wasReported {
-				if existingDevice.Health != state.DeviceHealthStatusUnknown && now.Sub(existingDevice.LastUpdated) > healthTimeout {
+				// Use device-specific timeout if set, otherwise use default
+				timeout := existingDevice.HealthCheckTimeout
+				if timeout == 0 {
+					timeout = DefaultHealthTimeout
+				}
+				
+				// Mark as unknown if the device health has timed out
+				if existingDevice.Health != state.DeviceHealthStatusUnknown && now.Sub(existingDevice.LastUpdated) > timeout {
 					existingDevice.Health = state.DeviceHealthStatusUnknown
 					existingDevice.LastUpdated = now
 					currentDriver.Devices[key] = existingDevice

pkg/kubelet/cm/dra/healthinfo_test.go

@@ -38,9 +38,10 @@ const (
 
 var (
 	testDeviceHealth = state.DeviceHealth{
-		PoolName:   testPool,
-		DeviceName: testDevice,
-		Health:     state.DeviceHealthStatusHealthy,
+		PoolName:           testPool,
+		DeviceName:         testDevice,
+		Health:             state.DeviceHealthStatusHealthy,
+		HealthCheckTimeout: DefaultHealthTimeout,
 	}
 )
 
@@ -206,7 +207,7 @@ func TestGetHealthInfo(t *testing.T) {
 		driverState := (*cache.HealthInfo)[testDriver]
 		deviceKey := testPool + "/" + testDevice
 		device := driverState.Devices[deviceKey]
-		device.LastUpdated = time.Now().Add((-healthTimeout) - time.Second)
+		device.LastUpdated = time.Now().Add((-DefaultHealthTimeout) - time.Second)
 		driverState.Devices[deviceKey] = device
 		(*cache.HealthInfo)[testDriver] = driverState
 		return nil
@@ -237,7 +238,7 @@ func TestGetHealthInfoRobust(t *testing.T) {
 			name: "device exists and is healthy",
 			initialState: &state.DevicesHealthMap{
 				testDriver: {Devices: map[string]state.DeviceHealth{
-					testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now()},
+					testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now(), HealthCheckTimeout: DefaultHealthTimeout},
 				}},
 			},
 			driverName:     testDriver,
@@ -249,7 +250,7 @@ func TestGetHealthInfoRobust(t *testing.T) {
 			name: "device exists and is unhealthy",
 			initialState: &state.DevicesHealthMap{
 				testDriver: {Devices: map[string]state.DeviceHealth{
-					testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusUnhealthy, LastUpdated: time.Now()},
+					testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusUnhealthy, LastUpdated: time.Now(), HealthCheckTimeout: DefaultHealthTimeout},
 				}},
 			},
 			driverName:     testDriver,
@@ -261,7 +262,7 @@ func TestGetHealthInfoRobust(t *testing.T) {
 			name: "device exists but timed out",
 			initialState: &state.DevicesHealthMap{
 				testDriver: {Devices: map[string]state.DeviceHealth{
-					testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now().Add((-1 * healthTimeout) - time.Second)},
+					testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now().Add((-1 * DefaultHealthTimeout) - time.Second), HealthCheckTimeout: DefaultHealthTimeout},
 				}},
 			},
 			driverName:     testDriver,
@@ -273,7 +274,7 @@ func TestGetHealthInfoRobust(t *testing.T) {
 			name: "device exists, just within timeout",
 			initialState: &state.DevicesHealthMap{
 				testDriver: {Devices: map[string]state.DeviceHealth{
-					testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now().Add((-1 * healthTimeout) + time.Second)},
+					testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now().Add((-1 * DefaultHealthTimeout) + time.Second), HealthCheckTimeout: DefaultHealthTimeout},
 				}},
 			},
 			driverName:     testDriver,
@@ -285,7 +286,7 @@ func TestGetHealthInfoRobust(t *testing.T) {
 			name: "device does not exist, just outside of timeout",
 			initialState: &state.DevicesHealthMap{
 				testDriver: {Devices: map[string]state.DeviceHealth{
-					testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now().Add((-1 * healthTimeout) - time.Second)},
+					testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now().Add((-1 * DefaultHealthTimeout) - time.Second), HealthCheckTimeout: DefaultHealthTimeout},
 				}},
 			},
 			driverName:     testDriver,
@@ -297,7 +298,7 @@ func TestGetHealthInfoRobust(t *testing.T) {
 			name: "device does not exist",
 			initialState: &state.DevicesHealthMap{
 				testDriver: {Devices: map[string]state.DeviceHealth{
-					testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now()},
+					testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now(), HealthCheckTimeout: DefaultHealthTimeout},
 				}},
 			},
 			driverName:     testDriver,
@@ -309,7 +310,7 @@ func TestGetHealthInfoRobust(t *testing.T) {
 			name: "driver does not exist",
 			initialState: &state.DevicesHealthMap{
 				testDriver: {Devices: map[string]state.DeviceHealth{
-					testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now()},
+					testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now(), HealthCheckTimeout: DefaultHealthTimeout},
 				}},
 			},
 			driverName:     "driver2",
@@ -321,7 +322,7 @@ func TestGetHealthInfoRobust(t *testing.T) {
 			name: "pool does not exist",
 			initialState: &state.DevicesHealthMap{
 				testDriver: {Devices: map[string]state.DeviceHealth{
-					testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now()},
+					testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now(), HealthCheckTimeout: DefaultHealthTimeout},
 				}},
 			},
 			driverName:     testDriver,
@@ -333,8 +334,8 @@ func TestGetHealthInfoRobust(t *testing.T) {
 			name: "multiple devices",
 			initialState: &state.DevicesHealthMap{
 				testDriver: {Devices: map[string]state.DeviceHealth{
-					testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now()},
-					testPool + "/device2":       {PoolName: testPool, DeviceName: "device2", Health: state.DeviceHealthStatusUnhealthy, LastUpdated: time.Now()},
+					testPool + "/" + testDevice: {PoolName: testPool, DeviceName: testDevice, Health: state.DeviceHealthStatusHealthy, LastUpdated: time.Now(), HealthCheckTimeout: DefaultHealthTimeout},
+					testPool + "/device2":       {PoolName: testPool, DeviceName: "device2", Health: state.DeviceHealthStatusUnhealthy, LastUpdated: time.Now(), HealthCheckTimeout: DefaultHealthTimeout},
 				}},
 			},
 			driverName:     testDriver,
@@ -382,7 +383,7 @@ func TestUpdateHealthInfo(t *testing.T) {
 	assert.Equal(t, state.DeviceHealthStatusUnhealthy, cache.getHealthInfo(testDriver, testPool, testDevice))
 
 	// 4 -- Add second device, omit first
-	secondDevice := state.DeviceHealth{PoolName: testPool, DeviceName: "device2", Health: state.DeviceHealthStatusHealthy}
+	secondDevice := state.DeviceHealth{PoolName: testPool, DeviceName: "device2", Health: state.DeviceHealthStatusHealthy, HealthCheckTimeout: DefaultHealthTimeout}
 	// When the first device is omitted, it should be marked as "Unknown" after a timeout.
 	// For this test, we simulate the timeout by not reporting it.
 	firstDeviceAsUnknown := newHealth
@@ -392,7 +393,7 @@ func TestUpdateHealthInfo(t *testing.T) {
 	err = cache.withLock(func() error {
 		deviceKey := testPool + "/" + testDevice
 		device := (*cache.HealthInfo)[testDriver].Devices[deviceKey]
-		device.LastUpdated = time.Now().Add(-healthTimeout * 2)
+		device.LastUpdated = time.Now().Add(-DefaultHealthTimeout * 2)
 		(*cache.HealthInfo)[testDriver].Devices[deviceKey] = device
 		return nil
 	})
@@ -411,7 +412,7 @@ func TestUpdateHealthInfo(t *testing.T) {
 	assert.Equal(t, state.DeviceHealthStatusUnknown, cache2.getHealthInfo(testDriver, testPool, testDevice))
 
 	// 6 -- Test how updateHealthInfo handles device timeouts
-	timeoutDevice := state.DeviceHealth{PoolName: testPool, DeviceName: "timeoutDevice", Health: "Unhealthy"}
+	timeoutDevice := state.DeviceHealth{PoolName: testPool, DeviceName: "timeoutDevice", Health: "Unhealthy", HealthCheckTimeout: DefaultHealthTimeout}
 	_, err = cache.updateHealthInfo(testDriver, []state.DeviceHealth{timeoutDevice})
 	require.NoError(t, err)
 
@@ -420,14 +421,14 @@ func TestUpdateHealthInfo(t *testing.T) {
 		driverState := (*cache.HealthInfo)[testDriver]
 		deviceKey := testPool + "/timeoutDevice"
 		device := driverState.Devices[deviceKey]
-		device.LastUpdated = time.Now().Add((-healthTimeout) - time.Second)
+		device.LastUpdated = time.Now().Add((-DefaultHealthTimeout) - time.Second)
 		driverState.Devices[deviceKey] = device
 		(*cache.HealthInfo)[testDriver] = driverState
 		return nil
 	})
 	require.NoError(t, err)
 
-	expectedTimeoutDeviceUnknown := state.DeviceHealth{PoolName: testPool, DeviceName: "timeoutDevice", Health: state.DeviceHealthStatusUnknown}
+	expectedTimeoutDeviceUnknown := state.DeviceHealth{PoolName: testPool, DeviceName: "timeoutDevice", Health: state.DeviceHealthStatusUnknown, HealthCheckTimeout: DefaultHealthTimeout}
 	expectedChanged6 := []state.DeviceHealth{expectedTimeoutDeviceUnknown}
 	changedDevices, err = cache.updateHealthInfo(testDriver, []state.DeviceHealth{})
 	require.NoError(t, err)

pkg/kubelet/cm/dra/manager.go

@@ -933,11 +933,29 @@ func (m *Manager) HandleWatchResourcesStream(ctx context.Context, stream draheal
 			default:
 				health = state.DeviceHealthStatusUnknown
 			}
+
+			// Extract the health check timeout from the gRPC response
+			// If not specified, zero, or negative, use the default timeout
+			timeout := DefaultHealthTimeout
+			timeoutSeconds := d.GetHealthCheckTimeoutSeconds()
+			if timeoutSeconds > 0 {
+				timeout = time.Duration(timeoutSeconds) * time.Second
+			} else if timeoutSeconds < 0 {
+				// Log warning for negative timeout values and use default
+				logger.V(4).Info("Ignoring negative health check timeout, using default",
+					"pluginName", pluginName,
+					"poolName", d.GetDevice().GetPoolName(),
+					"deviceName", d.GetDevice().GetDeviceName(),
+					"providedTimeout", timeoutSeconds,
+					"defaultTimeout", DefaultHealthTimeout)
+			}
+
 			devices[i] = state.DeviceHealth{
-				PoolName:    d.GetDevice().GetPoolName(),
-				DeviceName:  d.GetDevice().GetDeviceName(),
-				Health:      health,
-				LastUpdated: time.Unix(d.GetLastUpdatedTime(), 0),
+				PoolName:           d.GetDevice().GetPoolName(),
+				DeviceName:         d.GetDevice().GetDeviceName(),
+				Health:             health,
+				LastUpdated:        time.Unix(d.GetLastUpdatedTime(), 0),
+				HealthCheckTimeout: timeout,
 			}
 		}
 

pkg/kubelet/cm/dra/state/state.go

@@ -95,4 +95,9 @@ type DeviceHealth struct {
 
 	// LastUpdated keeps track of the last health status update of this device.
 	LastUpdated time.Time
+
+	// HealthCheckTimeout is the timeout for the health check of the device.
+	// Zero value means use the default timeout (DefaultHealthTimeout).
+	// This ensures backward compatibility with existing data.
+	HealthCheckTimeout time.Duration
 }

staging/src/k8s.io/kubelet/go.mod

@@ -9,6 +9,7 @@ godebug default=go1.24
 require (
 	github.com/emicklei/go-restful/v3 v3.12.2
 	github.com/gogo/protobuf v1.3.2
+	github.com/golang/protobuf v1.5.4
 	github.com/stretchr/testify v1.10.0
 	go.uber.org/goleak v1.3.0
 	google.golang.org/grpc v1.72.2