Skip to content

dns-controller: use the pod IPs when creating records for pods annotated with AnnotationNameDNSInternal #17791

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dezmodue wants to merge 1 commit into
base: master
Choose a base branch
from
Open

dns-controller: use the pod IPs when creating records for pods annotated with AnnotationNameDNSInternal #17791

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 64 additions & 17 deletions dns-controller/pkg/watchers/pod.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"k8s.io/klog/v2"
"k8s.io/kops/dns-controller/pkg/dns"
"k8s.io/kops/dns-controller/pkg/util"
"k8s.io/kops/upup/pkg/fi/utils"
)

// PodController watches for Pods with dns annotations
Expand Down Expand Up @@ -142,6 +143,25 @@ func (c *PodController) runWatcher(stopCh <-chan struct{}) {
}
}

// podIPsMatchingNodeInternalIP checks which pod IPs match a NodeInternalIP address on the specified node
func (c *PodController) podIPsMatchingNodeInternalIP(pod *v1.Pod) []string {
ctx := context.TODO()
var internalIPs []string
node, err := c.client.CoreV1().Nodes().Get(ctx, pod.Spec.NodeName, metav1.GetOptions{})
if err != nil {
klog.Warningf("Failed to get node %q: %v", pod.Spec.NodeName, err)
return []string{}
}
for _, podIP := range pod.Status.PodIPs {
for _, addr := range node.Status.Addresses {
if addr.Type == v1.NodeInternalIP && addr.Address == podIP.IP {
internalIPs = append(internalIPs, addr.Address)
}
}
}
return internalIPs
}

// updatePodRecords will apply the records for the specified pod. It returns the key that was set.
func (c *PodController) updatePodRecords(pod *v1.Pod) string {
var records []dns.Record
Expand Down Expand Up @@ -175,28 +195,55 @@ func (c *PodController) updatePodRecords(pod *v1.Pod) string {
}

specInternal := pod.Annotations[AnnotationNameDNSInternal]
var recordType dns.RecordType

if specInternal != "" {
var aliases []string
if pod.Spec.HostNetwork {
if pod.Spec.NodeName != "" {
aliases = append(aliases, "node/"+pod.Spec.NodeName+"/internal")
}
} else {
klog.V(4).Infof("Pod %q had %s=%s, but was not HostNetwork", pod.Name, AnnotationNameDNSInternal, specInternal)
}

tokens := strings.Split(specInternal, ",")
for _, token := range tokens {
token = strings.TrimSpace(token)

fqdn := dns.EnsureDotSuffix(token)
for _, alias := range aliases {
records = append(records, dns.Record{
RecordType: dns.RecordTypeAlias,
FQDN: fqdn,
Value: alias,
})
if pod.Spec.NodeName != "" && pod.Spec.HostNetwork {
if len(pod.Status.PodIPs) == 1 {
// The pod has a single IP, we will use it without checking against NodeInternalIP
if utils.IsIPv6IP(pod.Status.PodIP) {
recordType = dns.RecordTypeAAAA
} else {
recordType = dns.RecordTypeA
}
for _, token := range tokens {
token = strings.TrimSpace(token)
fqdn := dns.EnsureDotSuffix(token)
records = append(records, dns.Record{
RecordType: recordType,
FQDN: fqdn,
Value: pod.Status.PodIP,
})
}
} else {
// The pod has multiple IPs, we need to check which match NodeInternalIP
podInternalIPs := c.podIPsMatchingNodeInternalIP(pod)
if len(podInternalIPs) == 0 {
klog.V(4).Infof("Pod %q IPs do not match any NodeInternalIP on node %q", pod.Name, pod.Spec.NodeName)
} else {
for _, token := range tokens {
token = strings.TrimSpace(token)
fqdn := dns.EnsureDotSuffix(token)
for _, podIP := range podInternalIPs {
if utils.IsIPv6IP(podIP) {
recordType = dns.RecordTypeAAAA
} else {
recordType = dns.RecordTypeA
}
records = append(records, dns.Record{
RecordType: recordType,
FQDN: fqdn,
Value: podIP,
})
}
}
}
}
} else {
klog.V(4).Infof("Pod %q had %s=%s, but was not HostNetwork or NodeName not set", pod.Name, AnnotationNameDNSInternal, specInternal)
}
} else {
klog.V(4).Infof("Pod %q did not have %s label", pod.Name, AnnotationNameDNSInternal)
Expand Down
107 changes: 103 additions & 4 deletions dns-controller/pkg/watchers/pod_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,34 @@ import (
"k8s.io/kops/dns-controller/pkg/dns"
)

func TestPodController(t *testing.T) {
func TestPodControllerMultipleIPs(t *testing.T) {
ctx := context.Background()

nspec := &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "my-node",
},
Status: corev1.NodeStatus{
Addresses: []corev1.NodeAddress{
// Internal IPs allocated to the Pod
{Type: corev1.NodeInternalIP, Address: "10.0.0.1"},
{Type: corev1.NodeInternalIP, Address: "fd00:1234:5678:abcd::1"},
// The following 2 addresses should be ignored, as they are not associated allocated to the Pod
{Type: corev1.NodeInternalIP, Address: "10.0.0.3"},
{Type: corev1.NodeInternalIP, Address: "10.0.0.4"},
// External IPs
{Type: corev1.NodeExternalIP, Address: "2001:db8:0:0:0:ff00:42:8329"},
{Type: corev1.NodeExternalIP, Address: "54.100.0.1"},
},
},
}

pspec := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "somepod",
Namespace: "kube-system",
Annotations: map[string]string{
"dns.alpha.kubernetes.io/internal": "internal.a.foo.com",
"dns.alpha.kubernetes.io/internal": "internal.a.foo.com,internal.b.foo.com",
"dns.alpha.kubernetes.io/external": "a.foo.com",
},
},
Expand All @@ -47,14 +67,93 @@ func TestPodController(t *testing.T) {
PodIP: "10.0.0.1",
PodIPs: []corev1.PodIP{
{IP: "10.0.0.1"},
{IP: "fd00:1234:5678:abcd::1"},
{IP: "2001:db8:0:0:0:ff00:42:8329"},
{IP: "54.100.0.1"},
},
},
}

client := fake.NewClientset()

nodes := client.CoreV1().Nodes()
_, err := nodes.Create(ctx, nspec, metav1.CreateOptions{})
if err != nil {
t.Fatalf("unexpected error creating node: %v", err)
}

pods := client.CoreV1().Pods("kube-system")
_, err = pods.Create(ctx, pspec, metav1.CreateOptions{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}

ch := make(chan struct{})
scope := &fakeScope{
readyCh: ch,
records: make(map[string][]dns.Record),
}

dnsctx := &fakeDNSContext{
scope: scope,
}

c, err := NewPodController(client, dnsctx, "kube-system")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}

go c.Run()

select {
case <-ch:
case <-time.After(time.Second):
t.Fatalf("update was not marked as complete")
}

c.Stop()

want := map[string][]dns.Record{
"kube-system/somepod": {
{RecordType: "_alias", FQDN: "a.foo.com.", Value: "node/my-node/external"},
{RecordType: "A", FQDN: "internal.a.foo.com.", Value: "10.0.0.1"},
{RecordType: "AAAA", FQDN: "internal.a.foo.com.", Value: "fd00:1234:5678:abcd::1"},
{RecordType: "A", FQDN: "internal.b.foo.com.", Value: "10.0.0.1"},
{RecordType: "AAAA", FQDN: "internal.b.foo.com.", Value: "fd00:1234:5678:abcd::1"},
},
}
if diff := cmp.Diff(scope.records, want); diff != "" {
t.Fatalf("generated records did not match expected; diff=%s", diff)
}
}

func TestPodControllerSingleIP(t *testing.T) {
// Validates that if a pod has a single IP address, no call to the API server will be made to fetch Node information
ctx := context.Background()

pspec := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "somepod",
Namespace: "kube-system",
Annotations: map[string]string{
"dns.alpha.kubernetes.io/internal": "internal.a.foo.com,internal.b.foo.com",
},
},
Spec: corev1.PodSpec{
HostNetwork: true,
NodeName: "my-node",
},
Status: corev1.PodStatus{
PodIP: "fd00:1234:5678:abcd::1",
PodIPs: []corev1.PodIP{
{IP: "fd00:1234:5678:abcd::1"},
},
},
}

client := fake.NewClientset()

pods := client.CoreV1().Pods("kube-system")
_, err := pods.Create(ctx, pspec, metav1.CreateOptions{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
Expand Down Expand Up @@ -87,8 +186,8 @@ func TestPodController(t *testing.T) {

want := map[string][]dns.Record{
"kube-system/somepod": {
{RecordType: "_alias", FQDN: "a.foo.com.", Value: "node/my-node/external"},
{RecordType: "_alias", FQDN: "internal.a.foo.com.", Value: "node/my-node/internal"},
{RecordType: "AAAA", FQDN: "internal.a.foo.com.", Value: "fd00:1234:5678:abcd::1"},
{RecordType: "AAAA", FQDN: "internal.b.foo.com.", Value: "fd00:1234:5678:abcd::1"},
},
}
if diff := cmp.Diff(scope.records, want); diff != "" {
Expand Down
Loading