diff --git a/examples/network/vars-network.auto.tfvars b/examples/network/vars-network.auto.tfvars index bc9c68d5..c34fae4f 100644 --- a/examples/network/vars-network.auto.tfvars +++ b/examples/network/vars-network.auto.tfvars @@ -99,3 +99,7 @@ nat_gateway_route_rules = [ # description = "Terraformed - User added Routing Rule: To drg provided to this module. drg_id, if available, is automatically retrieved with keyword drg" # }, ] + +# Experimental +use_stateless_rules = true # Use stateless rules for security lists and network security groups instead of the default stateful rules. +# Note that the egress rule to 0.0.0.0/0 from pods and workers will be statefull independent of this setting because of security concerns. \ No newline at end of file diff --git a/module-network.tf b/module-network.tf index dcb243ed..f2d92211 100644 --- a/module-network.tf +++ b/module-network.tf @@ -152,6 +152,7 @@ module "network" { nat_gateway_id = local.nat_gateway_id nat_route_table_id = local.nat_route_table_id subnets = var.subnets + use_stateless_rules = var.use_stateless_rules vcn_cidrs = local.vcn_cidrs vcn_ipv6_cidr = local.vcn_ipv6_cidr vcn_id = local.vcn_id diff --git a/modules/network/nsg-bastion.tf b/modules/network/nsg-bastion.tf index 87e04f25..5ea5e19d 100644 --- a/modules/network/nsg-bastion.tf +++ b/modules/network/nsg-bastion.tf @@ -14,7 +14,9 @@ locals { ]) # Return provided NSG when configured with an existing ID or created resource ID bastion_nsg_id = one(compact([try(var.nsgs.bastion.id, null), one(oci_core_network_security_group.bastion[*].id)])) - bastion_rules = local.bastion_nsg_enabled ? merge( + bastion_rules = local.bastion_nsg_enabled ? ( var.use_stateless_rules ? local.bastion_stateless_rules: local.bastion_stateful_rules ) : {} + + bastion_stateful_rules = merge( { for cidr in var.bastion_allowed_cidrs : "Allow SSH ingress to bastion from ${cidr}" => { protocol = local.tcp_protocol, port = local.ssh_port, source = cidr, source_type = local.rule_type_cidr, @@ -40,7 +42,52 @@ locals { protocol = local.tcp_protocol, port = local.apiserver_port, destination = local.control_plane_nsg_id, destination_type = local.rule_type_nsg, }, } : {}, - ) : {} + ) + + bastion_stateless_rules = merge( + { for cidr in var.bastion_allowed_cidrs : + "Allow SSH ingress to bastion from ${cidr}" => { + protocol = local.tcp_protocol, destination_port_min = local.ssh_port, destination_port_max = local.ssh_port, source = cidr, source_type = local.rule_type_cidr, stateless = true + } + }, + { for cidr in var.bastion_allowed_cidrs : + "Allow SSH egress from bastion to ${cidr}" => { + protocol = local.tcp_protocol, source_port_min = local.ssh_port, source_port_max = local.ssh_port, destination = cidr, destination_type = local.rule_type_cidr, stateless = true + } + }, + { + "Allow TCP egress from bastion to OCI services" : { + protocol = local.all_protocols, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service, stateless = true + }, + "Allow TCP ingress to bastion from OCI services" : { + protocol = local.all_protocols, port = local.all_ports, source = local.osn, source_type = local.rule_type_service, stateless = true + }, + }, + local.operator_nsg_enabled ? { + "Allow SSH egress from bastion to operator" = { + protocol = local.tcp_protocol, destination_port_min = local.ssh_port, destination_port_max = local.ssh_port, destination = local.operator_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + "Allow ingress to bastion from operator SSH" = { + protocol = local.tcp_protocol, source_port_min = local.ssh_port, source_port_max = local.ssh_port, source = local.operator_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + } : {}, + var.allow_worker_ssh_access && local.worker_nsg_enabled ? { + "Allow SSH egress from bastion to workers" = { + protocol = local.tcp_protocol, destination_port_min = local.ssh_port, destination_port_max = local.ssh_port, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + "Allow ingress to bastion from workers SSH" = { + protocol = local.tcp_protocol, source_port_min = local.ssh_port, source_port_max= local.ssh_port , source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + } : {}, + (var.allow_bastion_cluster_access && local.control_plane_nsg_enabled) ? { + "Allow TCP egress from bastion to cluster endpoint" = { + protocol = local.tcp_protocol, destination_port_min = local.apiserver_port, destination_port_max = local.apiserver_port, destination = local.control_plane_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + "Allow TCP ingress to bastion from cluster endpoint" = { + protocol = local.tcp_protocol, source_port_min = local.apiserver_port, source_port_max = local.apiserver_port, source = local.control_plane_nsg_id, source_type = local.rule_type_nsg, stateless = true + } + } : {}, + ) } resource "oci_core_network_security_group" "bastion" { diff --git a/modules/network/nsg-controlplane.tf b/modules/network/nsg-controlplane.tf index bfa2b5f9..bd79926d 100644 --- a/modules/network/nsg-controlplane.tf +++ b/modules/network/nsg-controlplane.tf @@ -14,7 +14,9 @@ locals { ]) # Return provided NSG when configured with an existing ID or created resource ID control_plane_nsg_id = one(compact([try(var.nsgs.cp.id, null), one(oci_core_network_security_group.cp[*].id)])) - control_plane_rules = local.control_plane_nsg_enabled ? merge( + control_plane_rules = local.control_plane_nsg_enabled ? ( var.use_stateless_rules ? local.control_plane_stateless_rules: local.control_plane_stateful_rules ) : {} + + control_plane_stateful_rules= merge( { "Allow TCP egress from OKE control plane to OCI services" : { protocol = local.tcp_protocol, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service, @@ -83,7 +85,83 @@ locals { } }, var.allow_rules_cp - ) : {} + ) + + control_plane_stateless_rules= merge( + { + "Allow TCP egress from OKE control plane to OCI services" : { + protocol = local.all_protocols, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service, stateless = true + }, + "Allow TCP ingress to OKE control plane from OCI services" : { + protocol = local.all_protocols, port = local.all_ports, source = local.osn, source_type = local.rule_type_service, stateless = true + }, + + "Allow TCP ingress to OKE control plane from worker nodes" : { + protocol = local.tcp_protocol, port = local.all_ports, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow TCP egress from OKE control plane to Kubelet on worker nodes" : { + protocol = local.tcp_protocol, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + + "Allow TCP ingress for Kubernetes control plane inter-communication" : { + protocol = local.tcp_protocol, destination_port_min = local.apiserver_port, destination_port_max = local.apiserver_port, source = local.control_plane_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow TCP egress for Kubernetes control plane inter-communication" : { + protocol = local.tcp_protocol, source_port_min = local.apiserver_port, source_port_max = local.apiserver_port, destination = local.control_plane_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + + "Allow ICMP egress for path discovery to worker nodes" : { + protocol = local.icmp_protocol, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, + }, + "Allow ICMP ingress for path discovery from worker nodes" : { + protocol = local.icmp_protocol, source = local.worker_nsg_id, source_type = local.rule_type_nsg, + }, + }, + var.enable_ipv6 ? { + "Allow ICMPv6 egress for path discovery to worker nodes" : { + protocol = local.icmpv6_protocol, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, + }, + "Allow ICMPv6 ingress for path discovery from worker nodes" : { + protocol = local.icmpv6_protocol, source = local.worker_nsg_id, source_type = local.rule_type_nsg, + }, + } : {}, + local.operator_nsg_enabled ? { + "Allow TCP ingress to kube-apiserver from operator instance" : { + protocol = local.tcp_protocol, destination_port_min = local.apiserver_port, destination_port_max = local.apiserver_port, source = local.operator_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow TCP egress from kube-apiserver to operator instance" : { + protocol = local.tcp_protocol, source_port_min = local.apiserver_port, source_port_max = local.apiserver_port, source = local.operator_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + } : {}, + local.pod_nsg_enabled ? { + "Allow TCP ingress to OKE control plane from pods" : { + protocol = local.tcp_protocol, port = local.all_ports, source = local.pod_nsg_id, source_type = local.rule_type_nsg, stateless = true + } + "Allow TCP egress from OKE control plane to pods" : { + protocol = local.tcp_protocol, port = local.all_ports, destination = local.pod_nsg_id, destination_type = local.rule_type_nsg, stateless = true + } + } : {}, + (var.allow_bastion_cluster_access && local.bastion_nsg_enabled) ? { + "Allow TCP ingress to kube-apiserver from bastion host" = { + protocol = local.tcp_protocol, destination_port_min = local.apiserver_port, destination_port_max = local.apiserver_port, source = local.bastion_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow TCP egress from kube-apiserver to bastion host" = { + protocol = local.tcp_protocol, source_port_min = local.apiserver_port, source_port_max = local.apiserver_port, destination = local.bastion_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + } : {}, + + { for allowed_cidr in var.control_plane_allowed_cidrs : + "Allow TCP ingress to kube-apiserver from ${allowed_cidr}" => { + protocol = local.tcp_protocol, destination_port_min = local.apiserver_port, destination_port_max = local.apiserver_port, source = allowed_cidr, source_type = local.rule_type_cidr, stateless = true + } + }, + { for allowed_cidr in var.control_plane_allowed_cidrs : + "Allow TCP egress from kube-apiserver to ${allowed_cidr}" => { + protocol = local.tcp_protocol, source_port_min = local.apiserver_port, source_port_max = local.apiserver_port, destination = allowed_cidr, destination_type = local.rule_type_cidr, stateless = true + } + }, + var.allow_rules_cp + ) } resource "oci_core_network_security_group" "cp" { diff --git a/modules/network/nsg-fss.tf b/modules/network/nsg-fss.tf index 3aba4acf..ae1a5633 100644 --- a/modules/network/nsg-fss.tf +++ b/modules/network/nsg-fss.tf @@ -14,7 +14,9 @@ locals { ]) # Return provided NSG when configured with an existing ID or created resource ID fss_nsg_id = one(compact([try(var.nsgs.fss.id, null), one(oci_core_network_security_group.fss[*].id)])) - fss_rules = local.fss_nsg_enabled ? { + fss_rules = local.fss_nsg_enabled ? ( var.use_stateless_rules ? local.fss_stateless_rules: local.fss_stateful_rules ) : {} + + fss_stateful_rules = { # See https://docs.oracle.com/en-us/iaas/Content/File/Tasks/securitylistsfilestorage.htm # Ingress "Allow UDP ingress for NFS portmapper from workers" : { @@ -40,7 +42,39 @@ locals { "Allow TCP egress for NFS to the workers" : { protocol = local.tcp_protocol, source_port_min = local.fss_nfs_port_min, source_port_max = local.fss_nfs_port_max, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, }, - } : {} + } + + fss_stateless_rules = { + # See https://docs.oracle.com/en-us/iaas/Content/File/Tasks/securitylistsfilestorage.htm + # Ingress + "Allow UDP ingress for NFS portmapper from workers" : { + protocol = local.udp_protocol, destination_port_min = local.fss_nfs_portmapper_port, destination_port_max = local.fss_nfs_portmapper_port, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow UDP egress for NFS portmapper to workers" : { + protocol = local.udp_protocol, source_port_min = local.fss_nfs_portmapper_port, source_port_max = local.fss_nfs_portmapper_port, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + + "Allow TCP ingress for NFS portmapper from workers" : { + protocol = local.tcp_protocol, destination_port_min = local.fss_nfs_portmapper_port, destination_port_max = local.fss_nfs_portmapper_port, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow TCP egress for NFS portmapper to workers" : { + protocol = local.tcp_protocol, source_port_min = local.fss_nfs_portmapper_port, source_port_max = local.fss_nfs_portmapper_port, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + + "Allow UDP ingress for NFS from workers" : { + protocol = local.udp_protocol, destination_port_min = local.fss_nfs_port_min, destination_port_max = local.fss_nfs_port_min, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow UDP egress for NFS to workers" : { + protocol = local.udp_protocol, source_port_min = local.fss_nfs_port_min, source_port_max = local.fss_nfs_port_min, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + + "Allow TCP ingress for NFS from workers" : { + protocol = local.tcp_protocol, destination_port_min = local.fss_nfs_port_min, destination_port_max = local.fss_nfs_port_max, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow TCP egress for NFS to workers" : { + protocol = local.tcp_protocol, source_port_min = local.fss_nfs_port_min, source_port_max = local.fss_nfs_port_max, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + } } resource "oci_core_network_security_group" "fss" { diff --git a/modules/network/nsg-loadbalancers-int.tf b/modules/network/nsg-loadbalancers-int.tf index 957b0d08..9fb0210f 100644 --- a/modules/network/nsg-loadbalancers-int.tf +++ b/modules/network/nsg-loadbalancers-int.tf @@ -14,7 +14,8 @@ locals { ]) # Return provided NSG when configured with an existing ID or created resource ID int_lb_nsg_id = one(compact([try(var.nsgs.int_lb.id, null), one(oci_core_network_security_group.int_lb[*].id)])) - int_lb_rules = local.int_lb_nsg_enabled ? merge( + int_lb_rules = local.int_lb_nsg_enabled ? ( var.use_stateless_rules ? local.int_lb_stateless_rules: local.int_lb_stateful_rules ) : {} + int_lb_stateful_rules = merge( { "Allow TCP egress from internal load balancers to workers for Node Ports" : { protocol = local.tcp_protocol, port_min = local.node_port_min, port_max = local.node_port_max, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, @@ -36,7 +37,43 @@ locals { } : {}, var.enable_waf ? local.waf_rules : {}, var.allow_rules_internal_lb, - ) : {} + ) + + int_lb_stateless_rules = merge( + { + "Allow TCP egress from internal load balancers to workers for Node Ports" : { + protocol = local.tcp_protocol, destination_port_min = local.node_port_min, destination_port_max = local.node_port_max, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + "Allow TCP ingress to internal load balancers from workers for Node Ports" : { + protocol = local.tcp_protocol, source_port_min = local.node_port_min, source_port_max = local.node_port_max, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + + "Allow UDP egress from internal load balancers to workers for Node Ports" : { + protocol = local.udp_protocol, destination_port_min = local.node_port_min, destination_port_max = local.node_port_max, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + "Allow UDP ingress to internal load balancers from workers for Node Ports" : { + protocol = local.udp_protocol, source_port_min = local.node_port_min, source_port_max = local.node_port_max, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + + "Allow TCP egress from internal load balancers to workers for health checks" : { + protocol = local.tcp_protocol, destination_port_min = local.health_check_port, destination_port_max = local.health_check_port, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + "Allow TCP egress to internal load balancers from workers for health checks" : { + protocol = local.tcp_protocol, source_port_min = local.health_check_port, source_port_max = local.health_check_port, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + + "Allow ICMP egress from internal load balancers to worker nodes for path discovery" : { + protocol = local.icmp_protocol, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, + }, + }, + var.enable_ipv6 ? { + "Allow ICMPv6 egress from internal load balancers to worker nodes for path discovery" : { + protocol = local.icmpv6_protocol, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, + }, + } : {}, + var.enable_waf ? local.waf_rules : {}, + var.allow_rules_internal_lb, + ) } resource "oci_core_network_security_group" "int_lb" { diff --git a/modules/network/nsg-loadbalancers-pub.tf b/modules/network/nsg-loadbalancers-pub.tf index 1a5ae8ab..7214f61e 100644 --- a/modules/network/nsg-loadbalancers-pub.tf +++ b/modules/network/nsg-loadbalancers-pub.tf @@ -14,7 +14,9 @@ locals { ]) # Return provided NSG when configured with an existing ID or created resource ID pub_lb_nsg_id = one(compact([try(var.nsgs.pub_lb.id, null), one(oci_core_network_security_group.pub_lb[*].id)])) - pub_lb_rules = local.pub_lb_nsg_enabled ? merge( + pub_lb_rules = local.pub_lb_nsg_enabled ? ( var.use_stateless_rules ? local.pub_lb_stateless_rules: local.pub_lb_stateful_rules ) : {} + + pub_lb_stateful_rules = merge( { "Allow TCP egress from public load balancers to workers nodes for NodePort traffic" : { protocol = local.tcp_protocol, port_min = local.node_port_min, port_max = local.node_port_max, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, @@ -36,7 +38,43 @@ locals { } : {}, var.enable_waf ? local.waf_rules : {}, var.allow_rules_public_lb, - ) : {} + ) + + pub_lb_stateless_rules = merge( + { + "Allow TCP egress from public load balancers to workers nodes for NodePort traffic" : { + protocol = local.tcp_protocol, destination_port_min = local.node_port_min, destination_port_max = local.node_port_max, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + "Allow TCP ingress to public load balancers from workers nodes for NodePort traffic" : { + protocol = local.tcp_protocol, source_port_min = local.node_port_min, source_port_max = local.node_port_max, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + + "Allow UDP egress from public load balancers to workers nodes for NodePort traffic" : { + protocol = local.udp_protocol, destination_port_min = local.node_port_min, destination_port_max = local.node_port_max, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + "Allow UDP ingress to public load balancers from workers nodes for NodePort traffic" : { + protocol = local.udp_protocol, source_port_min = local.node_port_min, source_port_max = local.node_port_max, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + + "Allow TCP egress from public load balancers to worker nodes for health checks" : { + protocol = local.tcp_protocol, destination_port_min = local.health_check_port, destination_port_max = local.health_check_port, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + "Allow TCP ingress to public load balancers from worker nodes for health checks" : { + protocol = local.tcp_protocol, source_port_min = local.health_check_port, source_port_max = local.health_check_port, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + + "Allow ICMP egress from public load balancers to worker nodes for path discovery" : { + protocol = local.icmp_protocol, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, + }, + }, + var.enable_ipv6 ? { + "Allow ICMPv6 egress from public load balancers to worker nodes for path discovery" : { + protocol = local.icmpv6_protocol, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, + }, + } : {}, + var.enable_waf ? local.waf_rules : {}, + var.allow_rules_public_lb, + ) } resource "oci_core_network_security_group" "pub_lb" { diff --git a/modules/network/nsg-operator.tf b/modules/network/nsg-operator.tf index d7045638..a3c8b437 100644 --- a/modules/network/nsg-operator.tf +++ b/modules/network/nsg-operator.tf @@ -14,7 +14,9 @@ locals { ]) # Return provided NSG when configured with an existing ID or created resource ID operator_nsg_id = one(compact([try(var.nsgs.operator.id, null), one(oci_core_network_security_group.operator[*].id)])) - operator_rules = local.operator_nsg_enabled ? merge( + operator_rules = local.operator_nsg_enabled ? ( var.use_stateless_rules ? local.operator_stateless_rules: local.operator_stateful_rules ) : {} + + operator_stateful_rules = merge( { "Allow TCP egress from operator to OCI services" : { protocol = local.tcp_protocol, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service, @@ -41,7 +43,51 @@ locals { protocol = local.tcp_protocol, port = local.ssh_port, source = local.bastion_nsg_id, source_type = local.rule_type_nsg, } }) : {}, - ) : {} + ) + + operator_stateless_rules = merge( + { + "Allow TCP egress from operator to OCI services" : { + protocol = local.all_protocols, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service, stateless = true + }, + "Allow TCP ingress to operator from OCI services" : { + protocol = local.all_protocols, port = local.all_ports, source = local.osn, source_type = local.rule_type_service, stateless = true + }, + + "Allow TCP egress from operator to Kubernetes API server" : { + protocol = local.tcp_protocol, destination_port_min = local.apiserver_port, destination_port_max = local.apiserver_port, destination = local.control_plane_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + "Allow TCP ingress to operator from Kubernetes API server" : { + protocol = local.tcp_protocol, source_port_min = local.apiserver_port, source_port_max = local.apiserver_port, source = local.control_plane_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + + "Allow ALL egress from operator to all" : { + protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr, stateless = true + }, + "Allow ALL ingress to operator from all" : { + protocol = local.all_protocols, port = local.all_ports, source = local.anywhere, source_type = local.rule_type_cidr, stateless = true + }, + }, + + local.bastion_nsg_enabled ? merge( + var.enable_ipv6 ? { + "Allow ICMPv6 ingress to operator from bastion for path discovery" : { + protocol = local.icmpv6_protocol, source = local.bastion_nsg_id, source_type = local.rule_type_nsg, + } + } : {}, + { + "Allow ICMP ingress to operator from bastion for path discovery" : { + protocol = local.icmp_protocol, source = local.bastion_nsg_id, source_type = local.rule_type_nsg, + } + "Allow ingress to operator SSH from bastion" : { + protocol = local.tcp_protocol, destination_port_min = local.ssh_port, destination_port_max = local.ssh_port, source = local.bastion_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow egress from operator SSH to bastion " : { + protocol = local.tcp_protocol, source_port_min = local.ssh_port, source_port_max = local.ssh_port, destination = local.bastion_nsg_id, destination_type = local.rule_type_nsg, stateless = true + } + + }) : {}, + ) } resource "oci_core_network_security_group" "operator" { diff --git a/modules/network/nsg-pods.tf b/modules/network/nsg-pods.tf index 9d5f0ed0..689f4276 100644 --- a/modules/network/nsg-pods.tf +++ b/modules/network/nsg-pods.tf @@ -14,7 +14,9 @@ locals { ]) # Return provided NSG when configured with an existing ID or created resource ID pod_nsg_id = one(compact([try(var.nsgs.pods.id, null), one(oci_core_network_security_group.pods[*].id)])) - pods_rules = local.pod_nsg_enabled ? merge( + pods_rules = local.pod_nsg_enabled ? ( var.use_stateless_rules ? local.pod_stateless_rules: local.pod_stateful_rules ) : {} + + pod_stateful_rules = merge( { "Allow TCP egress from pods to OCI Services" : { protocol = local.tcp_protocol, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service, @@ -71,7 +73,73 @@ locals { } }) : {}, var.allow_rules_pods - ) : {} + ) + + + pod_stateless_rules = merge( + { + "Allow TCP egress from pods to OCI Services" : { + protocol = local.all_protocols, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service, stateless = true + }, + "Allow TCP egress to pods from OCI Services" : { + protocol = local.all_protocols, port = local.all_ports, source = local.osn, source_type = local.rule_type_service, stateless = true + }, + + "Allow ALL egress from pods to other pods" = { + protocol = local.all_protocols, port = local.all_ports, destination = local.pod_nsg_id, destination_type = local.rule_type_nsg, stateless = true + } + "Allow ALL ingress to pods from other pods" = { + protocol = local.all_protocols, port = local.all_ports, source = local.pod_nsg_id, source_type = local.rule_type_nsg, stateless = true + } + + "Allow TCP egress from pods to Kubernetes control_plane" = { + protocol = local.tcp_protocol, port = local.all_ports, destination = local.control_plane_nsg_id, destination_type = local.rule_type_nsg, stateless = true + } + "Allow TCP ingress to pods from Kubernetes control_plane" = { + protocol = local.tcp_protocol, port = local.all_ports, source = local.control_plane_nsg_id, source_type = local.rule_type_nsg, stateless = true + } + + "Allow ALL egress from pods for cross-node pod communication when using NodePorts or hostNetwork: true" = { + protocol = local.all_protocols, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true + } + "Allow ALL ingress to pods for cross-node pod communication when using NodePorts or hostNetwork: true" = { + protocol = local.all_protocols, port = local.all_ports, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true + } + + "Allow ICMP egress from pods for path discovery" = { + protocol = local.icmp_protocol, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr, + } + "Allow ICMP ingress to pods for path discovery" = { + protocol = local.icmp_protocol, port = local.all_ports, source = local.anywhere, source_type = local.rule_type_cidr, + } + }, + + var.enable_ipv6 ? { + "Allow ICMPv6 ingress to pods for path discovery" : { + protocol = local.icmpv6_protocol, port = local.all_ports, source = local.anywhere_ipv6, source_type = local.rule_type_cidr, + }, + "Allow ICMPv6 egress from pods for path discovery" : { + protocol = local.icmpv6_protocol, port = local.all_ports, destination = local.anywhere_ipv6, destination_type = local.rule_type_cidr, + }, + } : {}, + + var.allow_pod_internet_access ? + merge( + var.enable_ipv6 ? { + "Allow ALL IPv6 egress from pods to internet (not using stateless rules because of security concern with IPv6 GUA and routing over IGW)" = { + protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere_ipv6, destination_type = local.rule_type_cidr, + } + } : {}, + { + "Allow ALL egress from pods to anywhere" = { + protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr, stateless = true + } + "Allow ALL ingress to pods from anywhere" = { + protocol = local.all_protocols, port = local.all_ports, source = local.anywhere, source_type = local.rule_type_cidr, stateless = true + } + }) : {}, + var.allow_rules_pods + ) } resource "oci_core_network_security_group" "pods" { diff --git a/modules/network/nsg-workers.tf b/modules/network/nsg-workers.tf index 43dd2fa8..ba7aecc6 100644 --- a/modules/network/nsg-workers.tf +++ b/modules/network/nsg-workers.tf @@ -14,7 +14,9 @@ locals { ]) # Return provided NSG when configured with an existing ID or created resource ID worker_nsg_id = one(compact([try(var.nsgs.workers.id, null), one(oci_core_network_security_group.workers[*].id)])) - workers_rules = local.worker_nsg_enabled ? merge( + workers_rules = local.worker_nsg_enabled ? ( var.use_stateless_rules ? local.workers_stateless_rules: local.workers_stateful_rules ) : {} + + workers_stateful_rules = merge( { "Allow TCP egress from workers to OCI Services" : { protocol = local.tcp_protocol, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service, @@ -136,7 +138,159 @@ locals { }, } : {}, var.allow_rules_workers - ) : {} + ) + + workers_stateless_rules = merge( + { + "Allow TCP egress from workers to OCI Services" : { + protocol = local.all_protocols, port = local.all_ports, destination = local.osn, destination_type = local.rule_type_service, stateless = true + }, + "Allow TCP ingress to workers from OCI Services" : { + protocol = local.all_protocols, port = local.all_ports, source = local.osn, source_type = local.rule_type_service, stateless = true + }, + + "Allow ALL egress from workers to other workers" : { + protocol = local.all_protocols, port = local.all_ports, destination = local.worker_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + "Allow ALL ingress to workers from other workers" : { + protocol = local.all_protocols, port = local.all_ports, source = local.worker_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + + "Allow ALL egress from workers to Kubernetes control plane" : { + protocol = local.all_protocols, port = local.all_ports, destination = local.control_plane_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + "Allow ALL ingress to workers from Kubernetes control plane" : { + protocol = local.all_protocols, port = local.all_ports, source = local.control_plane_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + + "Allow ICMP egress from workers for path discovery" : { + protocol = local.icmp_protocol, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr, + }, + "Allow ICMP ingress to workers for path discovery" : { + protocol = local.icmp_protocol, port = local.all_ports, source = local.anywhere, source_type = local.rule_type_cidr, + }, + }, + + var.enable_ipv6 ? { + "Allow ICMPv6 ingress to workers for path discovery" : { + protocol = local.icmpv6_protocol, port = local.all_ports, source = local.anywhere_ipv6, source_type = local.rule_type_cidr, + }, + "Allow ICMPv6 egress from workers for path discovery" : { + protocol = local.icmpv6_protocol, port = local.all_ports, destination = local.anywhere_ipv6, destination_type = local.rule_type_cidr, + }, + } : {}, + + local.pod_nsg_enabled ? { + "Allow ALL egress from workers to pods" : { + protocol = local.all_protocols, port = local.all_ports, destination = local.pod_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + "Allow ALL ingress to workers from pods" : { + protocol = local.all_protocols, port = local.all_ports, source = local.pod_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + } : {}, + + var.allow_worker_internet_access ? + merge( + var.enable_ipv6 ? { + "Allow ALL IPv6 egress from workers to internet (not using stateless rules because of security concern with IPv6 GUA and routing over IGW)" = { + protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere_ipv6, destination_type = local.rule_type_cidr, + } + } : {}, + { + "Allow ALL egress from workers to internet (not using stateless rules because of security concern with Public IPs and routing over IGW)" : { + protocol = local.all_protocols, port = local.all_ports, destination = local.anywhere, destination_type = local.rule_type_cidr, + }, + }) : {}, + + local.int_lb_nsg_enabled ? { + "Allow TCP ingress to workers from internal load balancers" : { + protocol = local.tcp_protocol, destination_port_min = local.node_port_min, destination_port_max = local.node_port_max, source = local.int_lb_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow TCP egress from workers to internal load balancers" : { + protocol = local.tcp_protocol, source_port_min = local.node_port_min, source_port_max = local.node_port_max, destination = local.int_lb_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + + "Allow UDP ingress to workers from internal load balancers" : { + protocol = local.udp_protocol, destination_port_min = local.node_port_min, destination_port_max = local.node_port_max, source = local.int_lb_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow UDP egress from workers to internal load balancers" : { + protocol = local.udp_protocol, source_port_min = local.node_port_min, source_port_max = local.node_port_max, destination = local.int_lb_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + + "Allow TCP ingress to workers for health check from internal load balancers" : { + protocol = local.tcp_protocol, destination_port_min = local.health_check_port, destination_port_max = local.health_check_port, source = local.int_lb_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow TCP egress from workers for health check to internal load balancers" : { + protocol = local.tcp_protocol, source_port_min = local.health_check_port, source_port_max = local.health_check_port, destination = local.int_lb_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + } : {}, + + local.pub_lb_nsg_enabled ? { + "Allow TCP ingress to workers from public load balancers" : { + protocol = local.tcp_protocol, destination_port_min = local.node_port_min, destination_port_max = local.node_port_max, source = local.pub_lb_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow TCP egress from workers to public load balancers" : { + protocol = local.tcp_protocol, source_port_min = local.node_port_min, source_port_max = local.node_port_max, destination = local.pub_lb_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + + "Allow UDP ingress to workers from public load balancers" : { + protocol = local.udp_protocol, destination_port_min = local.node_port_min, destination_port_max = local.node_port_max, source = local.pub_lb_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow UDP egress from workers to public load balancers" : { + protocol = local.udp_protocol, source_port_min = local.node_port_min, source_port_max = local.node_port_max, destination = local.pub_lb_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + + "Allow TCP ingress to workers for health check from public load balancers" : { + protocol = local.tcp_protocol, destination_port_min = local.health_check_port, destination_port_max = local.health_check_port, source = local.pub_lb_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow TCP egress from workers for health check to public load balancers" : { + protocol = local.tcp_protocol, source_port_min = local.health_check_port, source_port_max = local.health_check_port, destination = local.pub_lb_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + } : {}, + + local.bastion_nsg_enabled && var.allow_worker_ssh_access ? { + "Allow ingress to workers SSH from bastion" : { + protocol = local.tcp_protocol, destination_port_min = local.ssh_port, destination_port_max = local.ssh_port, source = local.bastion_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow egress from workers SSH to bastion" : { + protocol = local.tcp_protocol, source_port_min = local.ssh_port, source_port_max = local.ssh_port, destination = local.bastion_nsg_id, destination_type = local.rule_type_nsg, stateless = true + } + } : {}, + + local.fss_nsg_enabled ? { + # See https://docs.oracle.com/en-us/iaas/Content/File/Tasks/securitylistsfilestorage.htm + # Ingress + "Allow UDP ingress to workers for NFS portmapper from FSS mounts" : { + protocol = local.udp_protocol, source_port_min = local.fss_nfs_portmapper_port, source_port_max = local.fss_nfs_portmapper_port, source = local.fss_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow UDP egress from workers for NFS portmapper to FSS mounts" : { + protocol = local.udp_protocol, destination_port_min = local.fss_nfs_portmapper_port, destination_port_max = local.fss_nfs_portmapper_port, destination = local.fss_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + + "Allow TCP ingress to workers for NFS portmapper from FSS mounts" : { + protocol = local.tcp_protocol, source_port_min = local.fss_nfs_portmapper_port, source_port_max = local.fss_nfs_portmapper_port, source = local.fss_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow TCP egress from workers for NFS portmapper to FSS mounts" : { + protocol = local.tcp_protocol, destination_port_min = local.fss_nfs_portmapper_port, destination_port_max = local.fss_nfs_portmapper_port, destination = local.fss_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + + + "Allow TCP ingress to workers for NFS from FSS mounts" : { + protocol = local.tcp_protocol, source_port_min = local.fss_nfs_port_min, source_port_max = local.fss_nfs_port_max, source = local.fss_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow TCP egress from workers for NFS to FSS mounts" : { + protocol = local.tcp_protocol, destination_port_min = local.fss_nfs_port_min, destination_port_max = local.fss_nfs_port_max, destination = local.fss_nsg_id, destination_type = local.rule_type_nsg, stateless = true + }, + + "Allow UDP ingress to workers for NFS from FSS mounts" : { + protocol = local.udp_protocol, source_port_min = local.fss_nfs_port_min, source_port_max = local.fss_nfs_port_min, source = local.fss_nsg_id, source_type = local.rule_type_nsg, stateless = true + }, + "Allow UDP egress from workers for NFS to FSS mounts" : { + protocol = local.udp_protocol, destination_port_min = local.fss_nfs_port_min, destination_port_max = local.fss_nfs_port_min, destination = local.fss_nsg_id, destination_type = local.rule_type_nsg, stateless = true + } + } : {}, + var.allow_rules_workers + ) } resource "oci_core_network_security_group" "workers" { diff --git a/modules/network/rules.tf b/modules/network/rules.tf index 5dd4a9fd..92736f32 100644 --- a/modules/network/rules.tf +++ b/modules/network/rules.tf @@ -191,7 +191,7 @@ resource "oci_core_network_security_group_security_rule" "oke" { # Extra precaution against unexpected allow-all ingress rules created by the module # Generated rules will produce errors unless any of the follow conditions are true precondition { - condition = anytrue([ + condition = var.use_stateless_rules || anytrue([ tostring(each.value.protocol) == tostring(local.icmp_protocol), # Traffic is ICMP each.value.direction == "EGRESS", # Traffic is outbound each.value.source != local.anywhere, # Rule does not allow all traffic diff --git a/modules/network/variables.tf b/modules/network/variables.tf index bc8f3d08..71e73290 100644 --- a/modules/network/variables.tf +++ b/modules/network/variables.tf @@ -66,3 +66,5 @@ variable "nsgs" { id = optional(string) })) } + +variable "use_stateless_rules" { type = bool } diff --git a/variables-network.tf b/variables-network.tf index 43a671ec..031833de 100644 --- a/variables-network.tf +++ b/variables-network.tf @@ -324,3 +324,8 @@ variable "enable_waf" { default = false } +variable "use_stateless_rules" { + description = "(experimental) Create NSGs with stateless rules instead of the default stateful rules." + type = bool + default = false +} \ No newline at end of file