Add support for new flags in Container node_kubelet_config. (#12985) (#9274)

[upstream:68ab860a2515113f7b839d63212ada3e54d38133]

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

.changelog/12985.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+container: added new fields `container_log_max_size`, `container_log_max_files`, `image_gc_low_threshold_percent`, `image_gc_high_threshold_percent`, `image_minimum_gc_age`, `image_maximum_gc_age`, and `allowed_unsafe_sysctls` to `node_kubelet_config` block. 
+```

google-beta/services/container/node_config.go

@@ -616,10 +616,45 @@ func schemaNodeConfig() *schema.Schema {
 								Optional:    true,
 								Description: `Controls the maximum number of processes allowed to run in a pod.`,
 							},
+							"container_log_max_size": {
+								Type:        schema.TypeString,
+								Optional:    true,
+								Description: `Defines the maximum size of the container log file before it is rotated.`,
+							},
+							"container_log_max_files": {
+								Type:        schema.TypeInt,
+								Optional:    true,
+								Description: `Defines the maximum number of container log files that can be present for a container.`,
+							},
+							"image_gc_low_threshold_percent": {
+								Type:        schema.TypeInt,
+								Optional:    true,
+								Description: `Defines the percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to.`,
+							},
+							"image_gc_high_threshold_percent": {
+								Type:        schema.TypeInt,
+								Optional:    true,
+								Description: `Defines the percent of disk usage after which image garbage collection is always run.`,
+							},
+							"image_minimum_gc_age": {
+								Type:        schema.TypeString,
+								Optional:    true,
+								Description: `Defines the minimum age for an unused image before it is garbage collected.`,
+							},
+							"image_maximum_gc_age": {
+								Type:        schema.TypeString,
+								Optional:    true,
+								Description: `Defines the maximum age an image can be unused before it is garbage collected.`,
+							},
+							"allowed_unsafe_sysctls": {
+								Type:        schema.TypeList,
+								Optional:    true,
+								Description: `Defines a comma-separated allowlist of unsafe sysctls or sysctl patterns which can be set on the Pods.`,
+								Elem:        &schema.Schema{Type: schema.TypeString},
+							},
 						},
 					},
 				},
-
 				"linux_node_config": {
 					Type:        schema.TypeList,
 					Optional:    true,
@@ -1273,6 +1308,31 @@ func expandKubeletConfig(v interface{}) *container.NodeKubeletConfig {
 	if podPidsLimit, ok := cfg["pod_pids_limit"]; ok {
 		kConfig.PodPidsLimit = int64(podPidsLimit.(int))
 	}
+	if containerLogMaxSize, ok := cfg["container_log_max_size"]; ok {
+		kConfig.ContainerLogMaxSize = containerLogMaxSize.(string)
+	}
+	if containerLogMaxFiles, ok := cfg["container_log_max_files"]; ok {
+		kConfig.ContainerLogMaxFiles = int64(containerLogMaxFiles.(int))
+	}
+	if imageGcLowThresholdPercent, ok := cfg["image_gc_low_threshold_percent"]; ok {
+		kConfig.ImageGcLowThresholdPercent = int64(imageGcLowThresholdPercent.(int))
+	}
+	if imageGcHighThresholdPercent, ok := cfg["image_gc_high_threshold_percent"]; ok {
+		kConfig.ImageGcHighThresholdPercent = int64(imageGcHighThresholdPercent.(int))
+	}
+	if imageMinimumGcAge, ok := cfg["image_minimum_gc_age"]; ok {
+		kConfig.ImageMinimumGcAge = imageMinimumGcAge.(string)
+	}
+	if imageMaximumGcAge, ok := cfg["image_maximum_gc_age"]; ok {
+		kConfig.ImageMaximumGcAge = imageMaximumGcAge.(string)
+	}
+	if allowedUnsafeSysctls, ok := cfg["allowed_unsafe_sysctls"]; ok {
+		sysctls := allowedUnsafeSysctls.([]interface{})
+		kConfig.AllowedUnsafeSysctls = make([]string, len(sysctls))
+		for i, s := range sysctls {
+			kConfig.AllowedUnsafeSysctls[i] = s.(string)
+		}
+	}
 	return kConfig
 }
 
@@ -1868,6 +1928,13 @@ func flattenKubeletConfig(c *container.NodeKubeletConfig) []map[string]interface
 			"cpu_manager_policy":                     c.CpuManagerPolicy,
 			"insecure_kubelet_readonly_port_enabled": flattenInsecureKubeletReadonlyPortEnabled(c),
 			"pod_pids_limit":                         c.PodPidsLimit,
+			"container_log_max_size":                 c.ContainerLogMaxSize,
+			"container_log_max_files":                c.ContainerLogMaxFiles,
+			"image_gc_low_threshold_percent":         c.ImageGcLowThresholdPercent,
+			"image_gc_high_threshold_percent":        c.ImageGcHighThresholdPercent,
+			"image_minimum_gc_age":                   c.ImageMinimumGcAge,
+			"image_maximum_gc_age":                   c.ImageMaximumGcAge,
+			"allowed_unsafe_sysctls":                 c.AllowedUnsafeSysctls,
 		})
 	}
 	return result

google-beta/services/container/resource_container_node_pool_test.go

@@ -529,7 +529,7 @@ func TestAccContainerNodePool_withKubeletConfig(t *testing.T) {
 		CheckDestroy:             testAccCheckContainerClusterDestroyProducer(t),
 		Steps: []resource.TestStep{
 			{
-				Config: testAccContainerNodePool_withKubeletConfig(cluster, np, "static", "100ms", networkName, subnetworkName, "TRUE", true, 2048),
+				Config: testAccContainerNodePool_withKubeletConfig(cluster, np, "static", "100ms", networkName, subnetworkName, "TRUE", "100Mi", "1m", "10m", true, 2048, 10, 10, 85),
 				ConfigPlanChecks: resource.ConfigPlanChecks{
 					PreApply: []plancheck.PlanCheck{
 						acctest.ExpectNoDelete(),
@@ -542,6 +542,20 @@ func TestAccContainerNodePool_withKubeletConfig(t *testing.T) {
 						"node_config.0.kubelet_config.0.insecure_kubelet_readonly_port_enabled", "TRUE"),
 					resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config",
 						"node_config.0.kubelet_config.0.pod_pids_limit", "2048"),
+					resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config",
+						"node_config.0.kubelet_config.0.container_log_max_size", "100Mi"),
+					resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config",
+						"node_config.0.kubelet_config.0.container_log_max_files", "10"),
+					resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config",
+						"node_config.0.kubelet_config.0.image_gc_low_threshold_percent", "10"),
+					resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config",
+						"node_config.0.kubelet_config.0.image_gc_high_threshold_percent", "85"),
+					resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config",
+						"node_config.0.kubelet_config.0.image_minimum_gc_age", "1m"),
+					resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config",
+						"node_config.0.kubelet_config.0.image_maximum_gc_age", "10m"),
+					// resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config",
+					//      "node_config.0.kubelet_config.0.allowed_unsafe_sysctls.0", "kernel.shm*"),
 				),
 			},
 			{
@@ -550,7 +564,7 @@ func TestAccContainerNodePool_withKubeletConfig(t *testing.T) {
 				ImportStateVerify: true,
 			},
 			{
-				Config: testAccContainerNodePool_withKubeletConfig(cluster, np, "", "", networkName, subnetworkName, "FALSE", false, 1024),
+				Config: testAccContainerNodePool_withKubeletConfig(cluster, np, "", "", networkName, subnetworkName, "FALSE", "200Mi", "30s", "", false, 1024, 5, 50, 80),
 				ConfigPlanChecks: resource.ConfigPlanChecks{
 					PreApply: []plancheck.PlanCheck{
 						acctest.ExpectNoDelete(),
@@ -588,7 +602,7 @@ func TestAccContainerNodePool_withInvalidKubeletCpuManagerPolicy(t *testing.T) {
 		CheckDestroy:             testAccCheckContainerClusterDestroyProducer(t),
 		Steps: []resource.TestStep{
 			{
-				Config:      testAccContainerNodePool_withKubeletConfig(cluster, np, "dontexist", "100us", networkName, subnetworkName, "TRUE", false, 1024),
+				Config:      testAccContainerNodePool_withKubeletConfig(cluster, np, "dontexist", "100us", networkName, subnetworkName, "TRUE", "", "", "", false, 1024, 2, 70, 75),
 				ExpectError: regexp.MustCompile(`.*to be one of \["?static"? "?none"? "?"?\].*`),
 			},
 		},
@@ -3154,7 +3168,8 @@ resource "google_container_node_pool" "with_sandbox_config" {
 `, cluster, networkName, subnetworkName, np)
 }
 
-func testAccContainerNodePool_withKubeletConfig(cluster, np, policy, period, networkName, subnetworkName, insecureKubeletReadonlyPortEnabled string, quota bool, podPidsLimit int) string {
+// TODO: add allowed_unsafe_sysctls in the test after GKE version 1.32.0-gke.1448000 is default version in regular channel and used in Terraform test.
+func testAccContainerNodePool_withKubeletConfig(cluster, np, policy, period, networkName, subnetworkName, insecureKubeletReadonlyPortEnabled, containerLogMaxSize, imageMinimumGcAge, imageMaximumGcAge string, quota bool, podPidsLimit, containerLogMaxFiles, imageGcLowThresholdPercent, imageGcHighThresholdPercent int) string {
 	return fmt.Sprintf(`
 data "google_container_engine_versions" "central1a" {
   location = "us-central1-a"
@@ -3185,6 +3200,13 @@ resource "google_container_node_pool" "with_kubelet_config" {
       cpu_cfs_quota_period                   = %q
       insecure_kubelet_readonly_port_enabled = "%s"
       pod_pids_limit                         = %d
+      container_log_max_size                 = %q
+      container_log_max_files                = %d
+      image_gc_low_threshold_percent         = %d
+      image_gc_high_threshold_percent        = %d
+      image_minimum_gc_age                   = %q
+      image_maximum_gc_age                   = %q
+      # allowed_unsafe_sysctls               = ["kernel.shm*", "kernel.msg*", "kernel.sem", "fs.mqueue.*", "net.*"]
     }
     oauth_scopes = [
       "https://www.googleapis.com/auth/logging.write",
@@ -3193,7 +3215,7 @@ resource "google_container_node_pool" "with_kubelet_config" {
     logging_variant = "DEFAULT"
   }
 }
-`, cluster, networkName, subnetworkName, np, policy, quota, period, insecureKubeletReadonlyPortEnabled, podPidsLimit)
+`, cluster, networkName, subnetworkName, np, policy, quota, period, insecureKubeletReadonlyPortEnabled, podPidsLimit, containerLogMaxSize, containerLogMaxFiles, imageGcLowThresholdPercent, imageGcHighThresholdPercent, imageMinimumGcAge, imageMaximumGcAge)
 }
 
 func testAccContainerNodePool_withLinuxNodeConfig(cluster, np, tcpMem, networkName, subnetworkName string) string {
@@ -3215,6 +3237,7 @@ func testAccContainerNodePool_withLinuxNodeConfig(cluster, np, tcpMem, networkNa
         "net.ipv4.tcp_rmem"           = "%s"
         "net.ipv4.tcp_wmem"           = "%s"
         "net.ipv4.tcp_tw_reuse"       = 1
+	"kernel.shmmni"               = 8192
       }
     }
 `, tcpMem, tcpMem)

website/docs/r/container_cluster.html.markdown

@@ -1349,6 +1349,24 @@ such as `"300ms"`. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m",
 
 * `pod_pids_limit` - (Optional) Controls the maximum number of processes allowed to run in a pod. The value must be greater than or equal to 1024 and less than 4194304.
 
+* `container_log_max_size` - (Optional) Defines the maximum size of the
+  container log file before it is rotated. Specified as a positive number and a
+  unit suffix, such as `"100Ki"`, `"10Mi"`. Valid units are "Ki", "Mi", "Gi".
+  The value must be between `"10Mi"` and `"500Mi"`, inclusive. And the total container log size
+  (`container_log_max_size` * `container_log_max_files`) cannot exceed 1% of the total storage of the node.
+
+* `container_log_max_files` - (Optional) Defines the maximum number of container log files that can be present for a container. The integer must be between 2 and 10, inclusive.
+
+* `image_gc_low_threshold_percent` - (Optional) Defines the percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to. The integer must be between 10 and 85, inclusive.
+
+* `image_gc_high_threshold_percent` - (Optional) Defines the percent of disk usage after which image garbage collection is always run. The integer must be between 10 and 85, inclusive.
+
+* `image_minimum_gc_age` - (Optional) Defines the minimum age for an unused image before it is garbage collected. Specified as a sequence of decimal numbers, each with optional fraction and a unit suffix, such as `"300s"`, `"1.5m"`. The value cannot be greater than "2m".
+
+* `image_maximum_gc_age` - (Optional) Defines the maximum age an image can be unused before it is garbage collected. Specified as a sequence of decimal numbers, each with optional fraction and a unit suffix, such as `"300s"`, `"1.5m"`, and `"2h45m"`. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". The value must be a positive duration.
+
+* `allowed_unsafe_sysctls` - (Optional) Defines a comma-separated allowlist of unsafe sysctls or sysctl patterns which can be set on the Pods. The allowed sysctl groups are `kernel.shm*`, `kernel.msg*`, `kernel.sem`, `fs.mqueue.*`, and `net.*`.
+
 <a name="nested_linux_node_config"></a>The `linux_node_config` block supports:
 
 * `sysctls` - (Optional) The Linux kernel parameters to be applied to the nodes