Skip to content

Commit 3dba296

Browse files
mattrandallbeckermattrandallbecker
authored
adding hostnetwork to daemonset pod spec for anth and ec2mp, to support imdsv2 and suggested eks iptables drops (#85)
1 parent a07c611 commit 3dba296

File tree

9 files changed

+43
-32
lines changed

9 files changed

+43
-32
lines changed

‎config/helm/aws-node-termination-handler/templates/daemonset.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ spec:
4646
values:
4747
- amd64
4848
serviceAccountName: {{ template "aws-node-termination-handler.serviceAccountName" . }}
49+
hostNetwork: true
4950
containers:
5051
- name: {{ include "aws-node-termination-handler.name" . }}
5152
image: {{ .Values.image.repository}}:{{ .Values.image.tag }}

‎config/helm/ec2-metadata-test-proxy/templates/daemonset.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,14 @@ spec:
1414
labels:
1515
app: {{ .Values.ec2MetadataTestProxy.label }}
1616
spec:
17+
hostNetwork: true
1718
containers:
1819
- name: {{ .Values.ec2MetadataTestProxy.label }}
1920
image: {{ .Values.ec2MetadataTestProxy.image.repository }}:{{ .Values.ec2MetadataTestProxy.image.tag }}
2021
imagePullPolicy: IfNotPresent
2122
ports:
2223
- containerPort: {{ .Values.ec2MetadataTestProxy.port }}
24+
hostPort: {{ .Values.ec2MetadataTestProxy.port }}
2325
env:
2426
- name: INTERRUPTION_NOTICE_DELAY
2527
value: {{ .Values.ec2MetadataTestProxy.interruptionNoticeDelay | quote }}

‎test/e2e/maintenance-event-cancellation-test‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ helm upgrade --install $CLUSTER_NAME-anth $SCRIPTPATH/../../config/helm/aws-node
1818
--wait \
1919
--force \
2020
--namespace kube-system \
21-
--set instanceMetadataURL="http://ec2-metadata-test-proxy.default.svc.cluster.local:1338" \
21+
--set instanceMetadataURL="http://localhost:1338" \
2222
--set image.repository="$NODE_TERMINATION_HANDLER_DOCKER_REPO" \
2323
--set image.tag="$NODE_TERMINATION_HANDLER_DOCKER_TAG" \
2424
--set enableSpotInterruptionDraining="true" \
@@ -39,14 +39,14 @@ TAINT_CHECK_SLEEP=15
3939
DEPLOYED=0
4040
CORDONED=0
4141

42-
for i in `seq 1 10`; do
42+
for i in `seq 1 10`; do
4343
if [[ $(kubectl get deployments regular-pod-test -o jsonpath='{.status.unavailableReplicas}') -eq 0 ]]; then
4444
echo "✅ Verified regular-pod-test pod was scheduled and started!"
4545
DEPLOYED=1
4646
break
4747
fi
4848
sleep 5
49-
done
49+
done
5050

5151
if [[ $DEPLOYED -eq 0 ]]; then
5252
echo "❌ Failed test setup for regular-pod"
@@ -65,19 +65,20 @@ for i in `seq 1 $TAINT_CHECK_CYCLES`; do
6565
sleep $TAINT_CHECK_SLEEP
6666
done
6767

68-
if [[ $CORDONED -eq 0 ]]; then
68+
if [[ $CORDONED -eq 0 ]]; then
6969
echo "❌ Failed cordoning node for scheduled maintenance event"
7070
exit 3
7171
fi
7272

7373
helm upgrade --install $CLUSTER_NAME-emtp $SCRIPTPATH/../../config/helm/ec2-metadata-test-proxy/ \
74+
--wait \
7475
--force \
7576
--namespace default \
7677
--set ec2MetadataTestProxy.image.repository="$EC2_METADATA_DOCKER_REPO" \
7778
--set ec2MetadataTestProxy.image.tag="$EC2_METADATA_DOCKER_TAG" \
7879
--set ec2MetadataTestProxy.enableScheduledMaintenanceEvents="true" \
7980
--set ec2MetadataTestProxy.enableSpotITN="false" \
80-
--set ec2MetadataTestProxy.scheduledEventStatus="cancelled"
81+
--set ec2MetadataTestProxy.scheduledEventStatus="cancelled"
8182

8283
for i in `seq 1 $TAINT_CHECK_CYCLES`; do
8384
if kubectl get nodes $CLUSTER_NAME-worker --no-headers | grep -v SchedulingDisabled; then

‎test/e2e/maintenance-event-dry-run-test‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ helm upgrade --install $CLUSTER_NAME-anth $SCRIPTPATH/../../config/helm/aws-node
1818
--wait \
1919
--force \
2020
--namespace kube-system \
21-
--set instanceMetadataURL="http://ec2-metadata-test-proxy.default.svc.cluster.local:1338" \
21+
--set instanceMetadataURL="http://localhost:1339" \
2222
--set image.repository="$NODE_TERMINATION_HANDLER_DOCKER_REPO" \
2323
--set image.tag="$NODE_TERMINATION_HANDLER_DOCKER_TAG" \
2424
--set dryRun="true" \
@@ -30,11 +30,12 @@ helm upgrade --install $CLUSTER_NAME-emtp $SCRIPTPATH/../../config/helm/ec2-meta
3030
--force \
3131
--namespace default \
3232
--set ec2MetadataTestProxy.image.repository="$EC2_METADATA_DOCKER_REPO" \
33-
--set ec2MetadataTestProxy.image.tag="$EC2_METADATA_DOCKER_TAG"
33+
--set ec2MetadataTestProxy.image.tag="$EC2_METADATA_DOCKER_TAG" \
34+
--set ec2MetadataTestProxy.port=1339
3435

3536
POD_ID=$(kubectl get pods --namespace kube-system | grep -i node-termination-handler | grep Running | cut -d' ' -f1)
3637

37-
for i in $(seq 0 10); do
38+
for i in $(seq 0 10); do
3839
if [[ ! -z $(kubectl logs $POD_ID -n kube-system | grep -i -e 'would have been cordoned and drained') ]]; then
3940
echo "✅ Verified the dryrun logs were executed"
4041
if kubectl get nodes $CLUSTER_NAME-worker --no-headers | grep -v SchedulingDisabled; then

‎test/e2e/maintenance-event-reboot-test‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ helm upgrade --install $CLUSTER_NAME-anth $SCRIPTPATH/../../config/helm/aws-node
2121
--wait \
2222
--force \
2323
--namespace kube-system \
24-
--set instanceMetadataURL="http://ec2-metadata-test-proxy.default.svc.cluster.local:1338" \
24+
--set instanceMetadataURL="http://localhost:1340" \
2525
--set image.repository="$NODE_TERMINATION_HANDLER_DOCKER_REPO" \
2626
--set image.tag="$NODE_TERMINATION_HANDLER_DOCKER_TAG" \
2727
--set enableSpotInterruptionDraining="true" \
@@ -34,22 +34,23 @@ helm upgrade --install $CLUSTER_NAME-emtp $SCRIPTPATH/../../config/helm/ec2-meta
3434
--set ec2MetadataTestProxy.image.repository="$EC2_METADATA_DOCKER_REPO" \
3535
--set ec2MetadataTestProxy.image.tag="$EC2_METADATA_DOCKER_TAG" \
3636
--set ec2MetadataTestProxy.enableScheduledMaintenanceEvents="true" \
37-
--set ec2MetadataTestProxy.enableSpotITN="false"
37+
--set ec2MetadataTestProxy.enableSpotITN="false" \
38+
--set ec2MetadataTestProxy.port=1340
3839

3940
TAINT_CHECK_CYCLES=15
4041
TAINT_CHECK_SLEEP=15
4142

4243
DEPLOYED=0
4344
CORDONED=0
4445

45-
for i in `seq 1 10`; do
46+
for i in `seq 1 10`; do
4647
if [[ $(kubectl get deployments regular-pod-test -o jsonpath='{.status.unavailableReplicas}') -eq 0 ]]; then
4748
echo "✅ Verified regular-pod-test pod was scheduled and started!"
4849
DEPLOYED=1
4950
break
5051
fi
5152
sleep 5
52-
done
53+
done
5354

5455
if [[ $DEPLOYED -eq 0 ]]; then
5556
echo "❌ Failed test setup for regular-pod"
@@ -68,7 +69,7 @@ for i in `seq 1 $TAINT_CHECK_CYCLES`; do
6869
sleep $TAINT_CHECK_SLEEP
6970
done
7071

71-
if [[ $CORDONED -eq 0 ]]; then
72+
if [[ $CORDONED -eq 0 ]]; then
7273
echo "❌ Failed cordoning node for scheduled maintenance event"
7374
exit 3
7475
fi
@@ -79,14 +80,14 @@ docker exec $CLUSTER_NAME-worker sh -c "chmod 0444 /uptime && chown root /uptime
7980

8081
## Remove ec2-metadata-test-proxy to prevent another drain event but keep regular-test-pod
8182
daemonset=$(kubectl get daemonsets | grep 'ec2-metadata-test-proxy' | cut -d' ' -f1)
82-
kubectl delete daemonsets $daemonset
83+
kubectl delete daemonsets $daemonset
8384

8485
## Restart NTH which will simulate a system reboot by mounting a new uptime file
8586
helm upgrade --install $CLUSTER_NAME-anth $SCRIPTPATH/../../config/helm/aws-node-termination-handler/ \
8687
--wait \
8788
--force \
8889
--namespace kube-system \
89-
--set instanceMetadataURL="http://ec2-metadata-test-proxy.default.svc.cluster.local:1338" \
90+
--set instanceMetadataURL="http://ec2-metadata-test-proxy.default.svc.cluster.local:1340" \
9091
--set image.repository="$NODE_TERMINATION_HANDLER_DOCKER_REPO" \
9192
--set image.tag="$NODE_TERMINATION_HANDLER_DOCKER_TAG" \
9293
--set procUptimeFile="/uptime" \
@@ -106,4 +107,4 @@ for i in `seq 1 $TAINT_CHECK_CYCLES`; do
106107
sleep $TAINT_CHECK_SLEEP
107108
done
108109

109-
exit 1
110+
exit 1

‎test/e2e/maintenance-event-test‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,34 +18,36 @@ helm upgrade --install $CLUSTER_NAME-anth $SCRIPTPATH/../../config/helm/aws-node
1818
--wait \
1919
--force \
2020
--namespace kube-system \
21-
--set instanceMetadataURL="http://ec2-metadata-test-proxy.default.svc.cluster.local:1338" \
21+
--set instanceMetadataURL="http://localhost:1341" \
2222
--set image.repository="$NODE_TERMINATION_HANDLER_DOCKER_REPO" \
2323
--set image.tag="$NODE_TERMINATION_HANDLER_DOCKER_TAG" \
2424
--set enableSpotInterruptionDraining="true" \
2525
--set enableScheduledEventDraining="true"
2626

27+
2728
helm upgrade --install $CLUSTER_NAME-emtp $SCRIPTPATH/../../config/helm/ec2-metadata-test-proxy/ \
2829
--wait \
2930
--force \
3031
--namespace default \
3132
--set ec2MetadataTestProxy.image.repository="$EC2_METADATA_DOCKER_REPO" \
3233
--set ec2MetadataTestProxy.image.tag="$EC2_METADATA_DOCKER_TAG" \
3334
--set ec2MetadataTestProxy.enableScheduledMaintenanceEvents="true" \
34-
--set ec2MetadataTestProxy.enableSpotITN="false"
35+
--set ec2MetadataTestProxy.enableSpotITN="false" \
36+
--set ec2MetadataTestProxy.port=1341
3537

3638
TAINT_CHECK_CYCLES=15
3739
TAINT_CHECK_SLEEP=15
3840

3941
DEPLOYED=0
4042

41-
for i in `seq 1 10`; do
43+
for i in `seq 1 10`; do
4244
if [[ $(kubectl get deployments regular-pod-test -o jsonpath='{.status.unavailableReplicas}') -eq 0 ]]; then
4345
echo "✅ Verified regular-pod-test pod was scheduled and started!"
4446
DEPLOYED=1
4547
break
4648
fi
4749
sleep 5
48-
done
50+
done
4951

5052
if [[ $DEPLOYED -eq 0 ]]; then
5153
exit 2
@@ -63,4 +65,4 @@ for i in `seq 1 $TAINT_CHECK_CYCLES`; do
6365
sleep $TAINT_CHECK_SLEEP
6466
done
6567

66-
exit 1
68+
exit 1

‎test/e2e/spot-interruption-dry-run-test‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ helm upgrade --install $CLUSTER_NAME-anth $SCRIPTPATH/../../config/helm/aws-node
1818
--wait \
1919
--force \
2020
--namespace kube-system \
21-
--set instanceMetadataURL="http://ec2-metadata-test-proxy.default.svc.cluster.local:1338" \
21+
--set instanceMetadataURL="http://localhost:1342" \
2222
--set image.repository="$NODE_TERMINATION_HANDLER_DOCKER_REPO" \
2323
--set image.tag="$NODE_TERMINATION_HANDLER_DOCKER_TAG" \
2424
--set dryRun="true" \
@@ -30,11 +30,12 @@ helm upgrade --install $CLUSTER_NAME-emtp $SCRIPTPATH/../../config/helm/ec2-meta
3030
--force \
3131
--namespace default \
3232
--set ec2MetadataTestProxy.image.repository="$EC2_METADATA_DOCKER_REPO" \
33-
--set ec2MetadataTestProxy.image.tag="$EC2_METADATA_DOCKER_TAG"
33+
--set ec2MetadataTestProxy.image.tag="$EC2_METADATA_DOCKER_TAG" \
34+
--set ec2MetadataTestProxy.port=1342
3435

3536
POD_ID=$(kubectl get pods --namespace kube-system | grep -i node-termination-handler | grep Running | cut -d' ' -f1)
3637

37-
for i in $(seq 0 10); do
38+
for i in $(seq 0 10); do
3839
if [[ ! -z $(kubectl logs $POD_ID -n kube-system | grep -i -e 'would have been cordoned and drained') ]]; then
3940
echo "✅ Verified the dryrun logs were executed"
4041
if kubectl get nodes $CLUSTER_NAME-worker --no-headers | grep -v SchedulingDisabled; then

‎test/e2e/spot-interruption-test‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ helm upgrade --install $CLUSTER_NAME-anth $SCRIPTPATH/../../config/helm/aws-node
1818
--wait \
1919
--force \
2020
--namespace kube-system \
21-
--set instanceMetadataURL="http://ec2-metadata-test-proxy.default.svc.cluster.local:1338" \
21+
--set instanceMetadataURL="http://localhost:1343" \
2222
--set image.repository="$NODE_TERMINATION_HANDLER_DOCKER_REPO" \
2323
--set image.tag="$NODE_TERMINATION_HANDLER_DOCKER_TAG" \
2424
--set enableSpotInterruptionDraining="true" \
@@ -33,21 +33,22 @@ helm upgrade --install $CLUSTER_NAME-emtp $SCRIPTPATH/../../config/helm/ec2-meta
3333
--set ec2MetadataTestProxy.image.repository="$EC2_METADATA_DOCKER_REPO" \
3434
--set ec2MetadataTestProxy.image.tag="$EC2_METADATA_DOCKER_TAG" \
3535
--set ec2MetadataTestProxy.enableSpotITN="true" \
36-
--set ec2MetadataTestProxy.enableScheduledMaintenanceEvents="false"
36+
--set ec2MetadataTestProxy.enableScheduledMaintenanceEvents="false" \
37+
--set ec2MetadataTestProxy.port=1343
3738

3839
TAINT_CHECK_CYCLES=15
3940
TAINT_CHECK_SLEEP=15
4041

4142
DEPLOYED=0
4243

43-
for i in `seq 1 10`; do
44+
for i in `seq 1 10`; do
4445
if [[ $(kubectl get deployments regular-pod-test -o jsonpath='{.status.unavailableReplicas}') -eq 0 ]]; then
4546
echo "✅ Verified regular-pod-test pod was scheduled and started!"
4647
DEPLOYED=1
4748
break
4849
fi
4950
sleep 5
50-
done
51+
done
5152

5253
if [[ $DEPLOYED -eq 0 ]]; then
5354
exit 2
@@ -65,4 +66,4 @@ for i in `seq 1 $TAINT_CHECK_CYCLES`; do
6566
sleep $TAINT_CHECK_SLEEP
6667
done
6768

68-
exit 1
69+
exit 1

‎test/e2e/webhook-test‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,14 @@ SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
1616

1717
### LOCAL ONLY TESTS FOR 200 RESPONSE FROM LOCAL CLUSTER, MASTER WILL TEST WITH TRAVIS SECRET URL
1818
if [[ -z $(env | grep "WEBHOOK_URL=") ]]; then
19-
WEBHOOK_URL="http://ec2-metadata-test-proxy.default.svc.cluster.local:1338"
19+
WEBHOOK_URL="http://localhost:1337"
2020
fi
2121

2222
helm upgrade --install $CLUSTER_NAME-anth $SCRIPTPATH/../../config/helm/aws-node-termination-handler/ \
2323
--wait \
2424
--force \
2525
--namespace kube-system \
26-
--set instanceMetadataURL="http://ec2-metadata-test-proxy.default.svc.cluster.local:1338" \
26+
--set instanceMetadataURL="http://localhost:1337" \
2727
--set image.repository="$NODE_TERMINATION_HANDLER_DOCKER_REPO" \
2828
--set image.tag="$NODE_TERMINATION_HANDLER_DOCKER_TAG" \
2929
--set webhookURL="$WEBHOOK_URL" \
@@ -35,7 +35,8 @@ helm upgrade --install $CLUSTER_NAME-emtp $SCRIPTPATH/../../config/helm/ec2-meta
3535
--force \
3636
--namespace default \
3737
--set ec2MetadataTestProxy.image.repository="$EC2_METADATA_DOCKER_REPO" \
38-
--set ec2MetadataTestProxy.image.tag="$EC2_METADATA_DOCKER_TAG"
38+
--set ec2MetadataTestProxy.image.tag="$EC2_METADATA_DOCKER_TAG" \
39+
--set ec2MetadataTestProxy.port=1337
3940

4041
TAINT_CHECK_CYCLES=15
4142
TAINT_CHECK_SLEEP=15

0 commit comments

Comments
 (0)