Skip to content

Commit 2783aa7

Browse files
hedibouattourhedibouattour
committed
converge cnat refactoring
1 parent 24b727f commit 2783aa7

22 files changed

+928
-207
lines changed

‎calico-vpp-agent/cni/pod_interface/common.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ func (i *PodInterfaceDriverData) DoPodIfNatConfiguration(podSpec *storage.LocalP
105105
stack.Push(i.vpp.RemovePodInterface, swIfIndex)
106106
}
107107

108-
err = i.vpp.CnatEnableFeatures(swIfIndex)
108+
err = i.vpp.CnatEnableFeatures(swIfIndex, true, podSpec.GetVrfId(vpplink.IpFamilyV4), podSpec.GetVrfId(vpplink.IpFamilyV6))
109109
if err != nil {
110110
return errors.Wrapf(err, "error configuring nat on pod interface")
111111
}
@@ -129,6 +129,10 @@ func (i *PodInterfaceDriverData) DoPodInterfaceConfiguration(podSpec *storage.Lo
129129
}
130130
}
131131

132+
err = i.vpp.EnableCnatSNATOnInterfaceVRF(swIfIndex)
133+
if err != nil {
134+
return errors.Wrapf(err, "error configuring cnat snat on pod VRF")
135+
}
132136
if !*ifSpec.IsL3 {
133137
/* L2 */
134138
err = i.vpp.SetPromiscOn(swIfIndex)

‎calico-vpp-agent/connectivity/connectivity_server.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,21 @@ func (s *ConnectivityServer) updateAllIPConnectivity() {
137137
}
138138
}
139139

140+
func (s *ConnectivityServer) configureRemoteNodeSnat(node *common.LocalNodeSpec, isAdd bool) {
141+
if node.IPv4Address != nil {
142+
err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv4Address.IP), isAdd)
143+
if err != nil {
144+
s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv4Address.IP, err)
145+
}
146+
}
147+
if node.IPv6Address != nil {
148+
err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv6Address.IP), isAdd)
149+
if err != nil {
150+
s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv6Address.IP, err)
151+
}
152+
}
153+
}
154+
140155
func (s *ConnectivityServer) ServeConnectivity(t *tomb.Tomb) error {
141156
/**
142157
* There might be leftover state in VPP in case we restarted
@@ -189,6 +204,7 @@ func (s *ConnectivityServer) ServeConnectivity(t *tomb.Tomb) error {
189204
if old.IPv6Address != nil {
190205
delete(s.nodeByAddr, old.IPv6Address.IP.String())
191206
}
207+
s.configureRemoteNodeSnat(old, false /* isAdd */)
192208
}
193209
if new != nil {
194210
if new.IPv4Address != nil {
@@ -197,6 +213,7 @@ func (s *ConnectivityServer) ServeConnectivity(t *tomb.Tomb) error {
197213
if new.IPv6Address != nil {
198214
s.nodeByAddr[new.IPv6Address.IP.String()] = *new
199215
}
216+
s.configureRemoteNodeSnat(new, true /* isAdd */)
200217
}
201218
case common.FelixConfChanged:
202219
old, _ := evt.Old.(*felixConfig.Config)

‎calico-vpp-agent/connectivity/ipip.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ func (p *IpipProvider) AddConnectivity(cn *common.NodeConnectivity) error {
127127
return errors.Wrapf(err, "Error enabling gso for ipip interface")
128128
}
129129

130-
err = p.vpp.CnatEnableFeatures(swIfIndex)
130+
err = p.vpp.CnatEnableFeatures(swIfIndex, true, 0, 0)
131131
if err != nil {
132132
p.errorCleanup(tunnel)
133133
return errors.Wrapf(err, "Error enabling nat for ipip interface")

‎calico-vpp-agent/connectivity/ipsec.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ func (p *IpsecProvider) createIPSECTunnel(tunnel *IpsecTunnel, psk string, stack
197197
return errors.Wrapf(err, "Error enabling gso for ipip interface")
198198
}
199199

200-
err = p.vpp.CnatEnableFeatures(swIfIndex)
200+
err = p.vpp.CnatEnableFeatures(swIfIndex, true, 0, 0)
201201
if err != nil {
202202
return errors.Wrapf(err, "Error enabling nat for ipip interface")
203203
}

‎calico-vpp-agent/connectivity/vxlan.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ func (p *VXLanProvider) AddConnectivity(cn *common.NodeConnectivity) error {
177177
return errors.Wrapf(err, "Error enabling gso for vxlan interface")
178178
}
179179

180-
err = p.vpp.CnatEnableFeatures(swIfIndex)
180+
err = p.vpp.CnatEnableFeatures(swIfIndex, true, 0, 0)
181181
if err != nil {
182182
// TODO : delete tunnel
183183
return errors.Wrapf(err, "Error enabling nat for vxlan interface")

‎calico-vpp-agent/connectivity/wireguard.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,7 @@ func (p *WireguardProvider) createWireguardTunnels() error {
217217
return errors.Wrapf(err, "Error enabling gso for wireguard interface")
218218
}
219219

220-
err = p.vpp.CnatEnableFeatures(swIfIndex)
220+
err = p.vpp.CnatEnableFeatures(swIfIndex, true, 0, 0)
221221
if err != nil {
222222
p.errorCleanup(tunnel)
223223
return errors.Wrapf(err, "Error enabling nat for wireguard interface")

‎calico-vpp-agent/policy/policy_server.go‎

Lines changed: 20 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1205,21 +1205,17 @@ func (s *Server) handleWireguardEndpointRemove(msg *proto.WireguardEndpointRemov
12051205
}
12061206

12071207
func (s *Server) onNodeUpdated(old *common.LocalNodeSpec, node *common.LocalNodeSpec) (err error) {
1208-
// This is used by the routing server to process Wireguard key updates
1209-
// As a result we only send an event when a node is updated, not when it is added or deleted
1210-
common.SendEvent(common.CalicoVppEvent{
1211-
Type: common.PeerNodeStateChanged,
1212-
Old: old,
1213-
New: node,
1214-
})
12151208
change := common.GetIpNetChangeType(old.IPv4Address, node.IPv4Address) | common.GetIpNetChangeType(old.IPv6Address, node.IPv6Address)
12161209
if change&(common.ChangeDeleted|common.ChangeUpdated) != 0 && node.Name == *config.NodeName {
12171210
// restart if our BGP config changed
12181211
return NodeWatcherRestartError{}
12191212
}
12201213
if change != common.ChangeSame {
1221-
s.configureRemoteNodeSnat(old, false /* isAdd */)
1222-
s.configureRemoteNodeSnat(node, true /* isAdd */)
1214+
common.SendEvent(common.CalicoVppEvent{
1215+
Type: common.PeerNodeStateChanged,
1216+
Old: old,
1217+
New: node,
1218+
})
12231219
}
12241220

12251221
return nil
@@ -1232,12 +1228,21 @@ func (s *Server) onNodeAdded(node *common.LocalNodeSpec) (err error) {
12321228
/* We found a BGP Spec that seems valid enough */
12331229
s.GotOurNodeBGPchan <- node
12341230
}
1231+
ip4 := net.IP{}
1232+
ip6 := net.IP{}
12351233
if node.IPv4Address != nil {
12361234
s.ip4 = &node.IPv4Address.IP
1235+
ip4 = node.IPv4Address.IP
12371236
}
12381237
if node.IPv6Address != nil {
12391238
s.ip6 = &node.IPv6Address.IP
1239+
ip6 = node.IPv6Address.IP
1240+
}
1241+
err = s.vpp.CnatSetSnatAddresses(ip4, ip6)
1242+
if err != nil {
1243+
s.log.Errorf("Failed to configure SNAT addresses %v", err)
12401244
}
1245+
12411246
err = s.createAllowFromHostPolicy()
12421247
if err != nil {
12431248
return errors.Wrap(err, "Error in creating AllowFromHostPolicy")
@@ -1252,26 +1257,10 @@ func (s *Server) onNodeAdded(node *common.LocalNodeSpec) (err error) {
12521257
Type: common.PeerNodeStateChanged,
12531258
New: node,
12541259
})
1255-
s.configureRemoteNodeSnat(node, true /* isAdd */)
12561260

12571261
return nil
12581262
}
12591263

1260-
func (s *Server) configureRemoteNodeSnat(node *common.LocalNodeSpec, isAdd bool) {
1261-
if node.IPv4Address != nil {
1262-
err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv4Address.IP), isAdd)
1263-
if err != nil {
1264-
s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv4Address.IP, err)
1265-
}
1266-
}
1267-
if node.IPv6Address != nil {
1268-
err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv6Address.IP), isAdd)
1269-
if err != nil {
1270-
s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv6Address.IP, err)
1271-
}
1272-
}
1273-
}
1274-
12751264
func (s *Server) onNodeDeleted(old *common.LocalNodeSpec, node *common.LocalNodeSpec) error {
12761265
common.SendEvent(common.CalicoVppEvent{
12771266
Type: common.PeerNodeStateChanged,
@@ -1282,7 +1271,6 @@ func (s *Server) onNodeDeleted(old *common.LocalNodeSpec, node *common.LocalNode
12821271
return NodeWatcherRestartError{}
12831272
}
12841273

1285-
s.configureRemoteNodeSnat(old, false /* isAdd */)
12861274
return nil
12871275
}
12881276

@@ -1305,8 +1293,8 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate, pending bool) (
13051293
if msg.Pool.Cidr != existing.Pool.Cidr ||
13061294
msg.Pool.Masquerade != existing.Pool.Masquerade {
13071295
var err, err2 error
1308-
err = s.addDelSnatPrefix(&existing, false /* isAdd */)
1309-
err2 = s.addDelSnatPrefix(msg, true /* isAdd */)
1296+
err = s.addDelSnatPrefixForIPPool(&existing, false /* isAdd */)
1297+
err2 = s.addDelSnatPrefixForIPPool(msg, true /* isAdd */)
13101298
if err != nil || err2 != nil {
13111299
return errors.Errorf("error updating snat prefix del:%s, add:%s", err, err2)
13121300
}
@@ -1320,7 +1308,7 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate, pending bool) (
13201308
s.log.Infof("Adding pool: %s, nat:%t", key, msg.Pool.Masquerade)
13211309
s.ippoolmap[key] = *msg
13221310
s.log.Debugf("Pool %v Added, handler called", msg)
1323-
err = s.addDelSnatPrefix(msg, true /* isAdd */)
1311+
err = s.addDelSnatPrefixForIPPool(msg, true /* isAdd */)
13241312
if err != nil {
13251313
return errors.Wrap(err, "error handling ipam add")
13261314
}
@@ -1348,7 +1336,7 @@ func (s *Server) handleIpamPoolRemove(msg *proto.IPAMPoolRemove, pending bool) (
13481336
delete(s.ippoolmap, key)
13491337
s.log.Infof("Deleting pool: %s", key)
13501338
s.log.Debugf("Pool %s deleted, handler called", existing.Pool.Cidr)
1351-
err = s.addDelSnatPrefix(&existing, false /* isAdd */)
1339+
err = s.addDelSnatPrefixForIPPool(&existing, false /* isAdd */)
13521340
if err != nil {
13531341
return errors.Wrap(err, "error handling ipam deletion")
13541342
}
@@ -1386,12 +1374,12 @@ func equalPools(a *proto.IPAMPoolUpdate, b *proto.IPAMPoolUpdate) bool {
13861374
return true
13871375
}
13881376

1389-
// addDelSnatPrefix configures IP Pool prefixes so that we don't source-NAT the packets going
1377+
// addDelSnatPrefixForIPPool configures IP Pool prefixes so that we don't source-NAT the packets going
13901378
// to these addresses. All the IP Pools prefixes are configured that way so that pod <-> pod
13911379
// communications are never source-nated in the cluster
13921380
// Note(aloaugus) - I think the iptables dataplane behaves differently and uses the k8s level
13931381
// pod CIDR for this rather than the individual pool prefixes
1394-
func (s *Server) addDelSnatPrefix(pool *proto.IPAMPoolUpdate, isAdd bool) (err error) {
1382+
func (s *Server) addDelSnatPrefixForIPPool(pool *proto.IPAMPoolUpdate, isAdd bool) (err error) {
13951383
_, ipNet, err := net.ParseCIDR(pool.Pool.Cidr)
13961384
if err != nil {
13971385
return errors.Wrapf(err, "Couldn't parse pool CIDR %s", pool.Pool.Cidr)

‎calico-vpp-agent/services/service_server.go‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -257,10 +257,6 @@ func serviceID(meta *metav1.ObjectMeta) string {
257257
}
258258

259259
func (s *Server) configureSnat() (err error) {
260-
err = s.vpp.CnatSetSnatAddresses(s.getNodeIP(false /* isv6 */), s.getNodeIP(true /* isv6 */))
261-
if err != nil {
262-
s.log.Errorf("Failed to configure SNAT addresses %v", err)
263-
}
264260
nodeIP4, nodeIP6 := common.GetBGPSpecAddresses(s.nodeBGPSpec)
265261
if nodeIP6 != nil {
266262
err = s.vpp.CnatAddSnatPrefix(common.FullyQualified(*nodeIP6))
@@ -280,6 +276,23 @@ func (s *Server) configureSnat() (err error) {
280276
s.log.Errorf("Failed to Add Service CIDR %s %v", serviceCIDR, err)
281277
}
282278
}
279+
err = s.vpp.SetK8sSnatPolicy()
280+
if err != nil {
281+
return errors.Wrap(err, "Error configuring cnat source policy")
282+
}
283+
for _, uplink := range common.VppManagerInfo.UplinkStatuses {
284+
// register vpptap0
285+
err = s.vpp.RegisterPodInterface(uplink.TapSwIfIndex)
286+
if err != nil {
287+
return errors.Wrap(err, "error configuring vpptap0 as pod intf")
288+
}
289+
290+
err = s.vpp.RegisterHostInterface(uplink.TapSwIfIndex)
291+
if err != nil {
292+
return errors.Wrap(err, "error configuring vpptap0 as host intf")
293+
}
294+
}
295+
283296
return nil
284297
}
285298

@@ -442,11 +455,6 @@ func (s *Server) ServeService(t *tomb.Tomb) error {
442455
})
443456
}
444457

445-
err = s.vpp.CnatPurge()
446-
if err != nil {
447-
return err
448-
}
449-
450458
if *config.GetCalicoVppDebug().ServicesEnabled {
451459
s.t.Go(func() error { s.serviceInformer.Run(t.Dying()); return nil })
452460
s.t.Go(func() error { s.endpointInformer.Run(t.Dying()); return nil })

‎vpp-manager/vpp_runner.go‎

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -464,7 +464,7 @@ func (v *VppRunner) configureVppUplinkInterface(
464464
return errors.Wrap(err, "Error disabling ipv6 RA on uplink interface")
465465
}
466466

467-
err = v.vpp.CnatEnableFeatures(ifSpec.SwIfIndex)
467+
err = v.vpp.CnatEnableFeatures(ifSpec.SwIfIndex, true, 0, 0)
468468
if err != nil {
469469
return errors.Wrap(err, "Error configuring NAT on uplink interface")
470470
}
@@ -610,21 +610,11 @@ func (v *VppRunner) configureVppUplinkInterface(
610610
log.Errorf("Error SetInterfaceRxMode on vpptap0 %v", err)
611611
}
612612

613-
err = v.vpp.CnatEnableFeatures(tapSwIfIndex)
613+
err = v.vpp.CnatEnableFeatures(tapSwIfIndex, true, 0, 0)
614614
if err != nil {
615615
return errors.Wrap(err, "Error configuring NAT on vpptap0")
616616
}
617617

618-
err = v.vpp.RegisterPodInterface(tapSwIfIndex)
619-
if err != nil {
620-
return errors.Wrap(err, "error configuring vpptap0 as pod intf")
621-
}
622-
623-
err = v.vpp.RegisterHostInterface(tapSwIfIndex)
624-
if err != nil {
625-
return errors.Wrap(err, "error configuring vpptap0 as host intf")
626-
}
627-
628618
// Linux side tap setup
629619
link, err := netlink.LinkByName(ifSpec.InterfaceName)
630620
if err != nil {
@@ -663,11 +653,6 @@ func (v *VppRunner) doVppGlobalConfiguration() (err error) {
663653
return errors.Wrap(err, "Error creating static VRFs in VPP")
664654
}
665655

666-
err = v.vpp.SetK8sSnatPolicy()
667-
if err != nil {
668-
return errors.Wrap(err, "Error configuring cnat source policy")
669-
}
670-
671656
err = v.vpp.ConfigureNeighborsV4(&types.NeighborConfig{
672657
MaxNumber: *config.GetCalicoVppInitialConfig().IP4NeighborsMaxNumber,
673658
MaxAge: *config.GetCalicoVppInitialConfig().IP4NeighborsMaxAge,

‎vpplink/cnat.go‎

Lines changed: 33 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -59,12 +59,12 @@ func (v *VppLink) CnatTranslateAdd(tr *types.CnatTranslateEntry) (uint32, error)
5959

6060
response, err := client.CnatTranslationUpdate(v.GetContext(), &cnat.CnatTranslationUpdate{
6161
Translation: cnat.CnatTranslation{
62-
Vip: types.ToCnatEndpoint(tr.Endpoint),
63-
IPProto: types.ToVppIPProto(tr.Proto),
64-
Paths: paths,
65-
IsRealIP: BoolToU8(tr.IsRealIP),
66-
Flags: uint8(cnat.CNAT_TRANSLATION_ALLOC_PORT),
67-
LbType: cnat.CnatLbType(tr.LbType),
62+
Vip: types.ToCnatEndpoint(tr.Endpoint),
63+
IPProto: types.ToVppIPProto(tr.Proto),
64+
Paths: paths,
65+
IsRealIP: BoolToU8(tr.IsRealIP),
66+
Flags: uint8(cnat.CNAT_TRANSLATION_ALLOC_PORT | cnat.CNAT_TRANSLATION_NO_CLIENT),
67+
LbType: cnat.CnatLbType(tr.LbType),
6868
},
6969
})
7070
if err != nil {
@@ -95,6 +95,7 @@ func (v *VppLink) CnatSetSnatAddresses(v4, v6 net.IP) error {
9595
SnatIP4: types.ToVppIP4Address(v4),
9696
SnatIP6: types.ToVppIP6Address(v6),
9797
SwIfIndex: types.InvalidInterface,
98+
Flags: cnat.CNAT_TRANSLATION_NO_CLIENT,
9899
})
99100
if err != nil {
100101
return fmt.Errorf("setting SNAT addresses failed: %w", err)
@@ -123,14 +124,18 @@ func (v *VppLink) CnatDelSnatPrefix(prefix *net.IPNet) error {
123124
return v.CnatAddDelSnatPrefix(prefix, false)
124125
}
125126

126-
func (v *VppLink) CnatEnableFeatures(swIfIndex uint32) (err error) {
127-
err = v.EnableFeatureArc46(swIfIndex, FeatureArcCnatInput)
128-
if err != nil {
129-
return fmt.Errorf("enabling arc dnat input failed: %w", err)
127+
func (v *VppLink) CnatEnableFeatures(swIfIndex uint32, isEnable bool, vrfIdV4 uint32, vrfIdV6 uint32) error {
128+
client := cnat.NewServiceClient(v.GetConnection())
129+
130+
request := &cnat.FeatureCnatEnableDisable{
131+
SwIfIndex: interface_types.InterfaceIndex(swIfIndex),
132+
EnableDisable: isEnable,
133+
TableIDIP4: vrfIdV4,
134+
TableIDIP6: vrfIdV6,
130135
}
131-
err = v.EnableFeatureArc46(swIfIndex, FeatureArcCnatOutput)
136+
_, err := client.FeatureCnatEnableDisable(v.GetContext(), request)
132137
if err != nil {
133-
return fmt.Errorf("enabling arc dnat output failed: %w", err)
138+
return fmt.Errorf("FeatureEnableDisable %+v failed: %w", request, err)
134139
}
135140
return nil
136141
}
@@ -187,7 +192,7 @@ func (v *VppLink) disableCnatSNAT(swIfIndex uint32, isIp6 bool) (err error) {
187192
return v.cnatSnatPolicyAddDelPodInterface(swIfIndex, false /* isAdd */, cnat.CNAT_POLICY_INCLUDE_V4)
188193
}
189194

190-
func (v *VppLink) cnatSetSnatPolicy(pol cnat.CnatSnatPolicies) error {
195+
func (v *VppLink) cnatSetSnatPolicyForDefaultVRF(pol cnat.CnatSnatPolicies) error {
191196
client := cnat.NewServiceClient(v.GetConnection())
192197

193198
_, err := client.CnatSetSnatPolicy(v.GetContext(), &cnat.CnatSetSnatPolicy{
@@ -200,9 +205,22 @@ func (v *VppLink) cnatSetSnatPolicy(pol cnat.CnatSnatPolicies) error {
200205
}
201206

202207
func (v *VppLink) SetK8sSnatPolicy() (err error) {
203-
return v.cnatSetSnatPolicy(cnat.CNAT_POLICY_K8S)
208+
return v.cnatSetSnatPolicyForDefaultVRF(cnat.CNAT_POLICY_K8S)
204209
}
205210

206211
func (v *VppLink) ClearSnatPolicy() (err error) {
207-
return v.cnatSetSnatPolicy(cnat.CNAT_POLICY_NONE)
212+
return v.cnatSetSnatPolicyForDefaultVRF(cnat.CNAT_POLICY_NONE)
213+
}
214+
215+
func (v *VppLink) EnableCnatSNATOnInterfaceVRF(swifindex uint32) (err error) {
216+
client := cnat.NewServiceClient(v.GetConnection())
217+
218+
_, err = client.DuplicateCnatSnatDefaultPolicyToVrf(v.GetContext(), &cnat.DuplicateCnatSnatDefaultPolicyToVrf{
219+
SwIfIndex: interface_types.InterfaceIndex(swifindex),
220+
})
221+
if err != nil {
222+
return fmt.Errorf("DuplicateCnatSnatDefaultPolicyToVrf for interface %d failed: %w", swifindex, err)
223+
}
224+
return nil
225+
208226
}

0 commit comments

Comments
 (0)