Skip to content

Commit 6a7cdd8

Browse files
hedibouattourhedibouattour
committed
converge cnat refactoring
1 parent a950c11 commit 6a7cdd8

File tree

17 files changed

+951
-172
lines changed

17 files changed

+951
-172
lines changed

‎calico-vpp-agent/cni/podinterface/common.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ func (i *PodInterfaceDriverData) DoPodIfNatConfiguration(podSpec *storage.LocalP
105105
stack.Push(i.vpp.RemovePodInterface, swIfIndex)
106106
}
107107

108-
err = i.vpp.CnatEnableFeatures(swIfIndex)
108+
err = i.vpp.CnatEnableFeatures(swIfIndex, true, podSpec.GetVrfID(vpplink.IPFamilyV4), podSpec.GetVrfID(vpplink.IPFamilyV6))
109109
if err != nil {
110110
return errors.Wrapf(err, "error configuring nat on pod interface")
111111
}
@@ -129,6 +129,10 @@ func (i *PodInterfaceDriverData) DoPodInterfaceConfiguration(podSpec *storage.Lo
129129
}
130130
}
131131

132+
err = i.vpp.EnableCnatSNATOnInterfaceVRF(swIfIndex)
133+
if err != nil {
134+
return errors.Wrapf(err, "error configuring cnat snat on pod VRF")
135+
}
132136
if !*ifSpec.IsL3 {
133137
/* L2 */
134138
err = i.vpp.SetPromiscOn(swIfIndex)

‎calico-vpp-agent/connectivity/connectivity_server.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,21 @@ func (s *ConnectivityServer) updateAllIPConnectivity() {
137137
}
138138
}
139139

140+
func (s *ConnectivityServer) configureRemoteNodeSnat(node *common.LocalNodeSpec, isAdd bool) {
141+
if node.IPv4Address != nil {
142+
err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv4Address.IP), isAdd)
143+
if err != nil {
144+
s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv4Address.IP, err)
145+
}
146+
}
147+
if node.IPv6Address != nil {
148+
err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv6Address.IP), isAdd)
149+
if err != nil {
150+
s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv6Address.IP, err)
151+
}
152+
}
153+
}
154+
140155
func (s *ConnectivityServer) ServeConnectivity(t *tomb.Tomb) error {
141156
/**
142157
* There might be leftover state in VPP in case we restarted
@@ -214,6 +229,7 @@ func (s *ConnectivityServer) ServeConnectivity(t *tomb.Tomb) error {
214229
delete(s.nodeByAddr, old.IPv6Address.IP.String())
215230
}
216231
}
232+
s.configureRemoteNodeSnat(old, false /* isAdd */)
217233
}
218234
if evt.New != nil {
219235
new, ok := evt.New.(*common.LocalNodeSpec)
@@ -227,6 +243,7 @@ func (s *ConnectivityServer) ServeConnectivity(t *tomb.Tomb) error {
227243
s.nodeByAddr[new.IPv6Address.IP.String()] = *new
228244
}
229245
}
246+
s.configureRemoteNodeSnat(new, true /* isAdd */)
230247
}
231248
case common.FelixConfChanged:
232249
old, ok := evt.Old.(*felixConfig.Config)

‎calico-vpp-agent/connectivity/ipip.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ func (p *IpipProvider) AddConnectivity(cn *common.NodeConnectivity) error {
127127
return errors.Wrapf(err, "Error enabling gso for ipip interface")
128128
}
129129

130-
err = p.vpp.CnatEnableFeatures(swIfIndex)
130+
err = p.vpp.CnatEnableFeatures(swIfIndex, true, 0, 0)
131131
if err != nil {
132132
p.errorCleanup(tunnel)
133133
return errors.Wrapf(err, "Error enabling nat for ipip interface")

‎calico-vpp-agent/connectivity/ipsec.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ func (p *IpsecProvider) createIPSECTunnel(tunnel *IpsecTunnel, psk string, stack
197197
return errors.Wrapf(err, "Error enabling gso for ipip interface")
198198
}
199199

200-
err = p.vpp.CnatEnableFeatures(swIfIndex)
200+
err = p.vpp.CnatEnableFeatures(swIfIndex, true, 0, 0)
201201
if err != nil {
202202
return errors.Wrapf(err, "Error enabling nat for ipip interface")
203203
}

‎calico-vpp-agent/connectivity/vxlan.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ func (p *VXLanProvider) AddConnectivity(cn *common.NodeConnectivity) error {
177177
return errors.Wrapf(err, "Error enabling gso for vxlan interface")
178178
}
179179

180-
err = p.vpp.CnatEnableFeatures(swIfIndex)
180+
err = p.vpp.CnatEnableFeatures(swIfIndex, true, 0, 0)
181181
if err != nil {
182182
// TODO : delete tunnel
183183
return errors.Wrapf(err, "Error enabling nat for vxlan interface")

‎calico-vpp-agent/connectivity/wireguard.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,7 @@ func (p *WireguardProvider) createWireguardTunnels() error {
217217
return errors.Wrapf(err, "Error enabling gso for wireguard interface")
218218
}
219219

220-
err = p.vpp.CnatEnableFeatures(swIfIndex)
220+
err = p.vpp.CnatEnableFeatures(swIfIndex, true, 0, 0)
221221
if err != nil {
222222
p.errorCleanup(tunnel)
223223
return errors.Wrapf(err, "Error enabling nat for wireguard interface")

‎calico-vpp-agent/felix/felix_server.go‎

Lines changed: 20 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1223,21 +1223,17 @@ func (s *Server) handleWireguardEndpointRemove(msg *proto.WireguardEndpointRemov
12231223
}
12241224

12251225
func (s *Server) onNodeUpdated(old *common.LocalNodeSpec, node *common.LocalNodeSpec) (err error) {
1226-
// This is used by the routing server to process Wireguard key updates
1227-
// As a result we only send an event when a node is updated, not when it is added or deleted
1228-
common.SendEvent(common.CalicoVppEvent{
1229-
Type: common.PeerNodeStateChanged,
1230-
Old: old,
1231-
New: node,
1232-
})
12331226
change := common.GetIPNetChangeType(old.IPv4Address, node.IPv4Address) | common.GetIPNetChangeType(old.IPv6Address, node.IPv6Address)
12341227
if change&(common.ChangeDeleted|common.ChangeUpdated) != 0 && node.Name == *config.NodeName {
12351228
// restart if our BGP config changed
12361229
return NodeWatcherRestartError{}
12371230
}
12381231
if change != common.ChangeSame {
1239-
s.configureRemoteNodeSnat(old, false /* isAdd */)
1240-
s.configureRemoteNodeSnat(node, true /* isAdd */)
1232+
common.SendEvent(common.CalicoVppEvent{
1233+
Type: common.PeerNodeStateChanged,
1234+
Old: old,
1235+
New: node,
1236+
})
12411237
}
12421238

12431239
return nil
@@ -1250,12 +1246,21 @@ func (s *Server) onNodeAdded(node *common.LocalNodeSpec) (err error) {
12501246
/* We found a BGP Spec that seems valid enough */
12511247
s.GotOurNodeBGPchan <- node
12521248
}
1249+
ip4 := net.IP{}
1250+
ip6 := net.IP{}
12531251
if node.IPv4Address != nil {
12541252
s.ip4 = &node.IPv4Address.IP
1253+
ip4 = node.IPv4Address.IP
12551254
}
12561255
if node.IPv6Address != nil {
12571256
s.ip6 = &node.IPv6Address.IP
1257+
ip6 = node.IPv6Address.IP
1258+
}
1259+
err = s.vpp.CnatSetSnatAddresses(ip4, ip6)
1260+
if err != nil {
1261+
s.log.Errorf("Failed to configure SNAT addresses %v", err)
12581262
}
1263+
12591264
err = s.createAllowFromHostPolicy()
12601265
if err != nil {
12611266
return errors.Wrap(err, "Error in creating AllowFromHostPolicy")
@@ -1270,26 +1275,10 @@ func (s *Server) onNodeAdded(node *common.LocalNodeSpec) (err error) {
12701275
Type: common.PeerNodeStateChanged,
12711276
New: node,
12721277
})
1273-
s.configureRemoteNodeSnat(node, true /* isAdd */)
12741278

12751279
return nil
12761280
}
12771281

1278-
func (s *Server) configureRemoteNodeSnat(node *common.LocalNodeSpec, isAdd bool) {
1279-
if node.IPv4Address != nil {
1280-
err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv4Address.IP), isAdd)
1281-
if err != nil {
1282-
s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv4Address.IP, err)
1283-
}
1284-
}
1285-
if node.IPv6Address != nil {
1286-
err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv6Address.IP), isAdd)
1287-
if err != nil {
1288-
s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv6Address.IP, err)
1289-
}
1290-
}
1291-
}
1292-
12931282
func (s *Server) onNodeDeleted(old *common.LocalNodeSpec, node *common.LocalNodeSpec) error {
12941283
common.SendEvent(common.CalicoVppEvent{
12951284
Type: common.PeerNodeStateChanged,
@@ -1300,7 +1289,6 @@ func (s *Server) onNodeDeleted(old *common.LocalNodeSpec, node *common.LocalNode
13001289
return NodeWatcherRestartError{}
13011290
}
13021291

1303-
s.configureRemoteNodeSnat(old, false /* isAdd */)
13041292
return nil
13051293
}
13061294

@@ -1323,8 +1311,8 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate, pending bool) (
13231311
if msg.Pool.Cidr != existing.Pool.Cidr ||
13241312
msg.Pool.Masquerade != existing.Pool.Masquerade {
13251313
var err, err2 error
1326-
err = s.addDelSnatPrefix(&existing, false /* isAdd */)
1327-
err2 = s.addDelSnatPrefix(msg, true /* isAdd */)
1314+
err = s.addDelSnatPrefixForIPPool(&existing, false /* isAdd */)
1315+
err2 = s.addDelSnatPrefixForIPPool(msg, true /* isAdd */)
13281316
if err != nil || err2 != nil {
13291317
return errors.Errorf("error updating snat prefix del:%s, add:%s", err, err2)
13301318
}
@@ -1338,7 +1326,7 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate, pending bool) (
13381326
s.log.Infof("Adding pool: %s, nat:%t", key, msg.Pool.Masquerade)
13391327
s.ippoolmap[key] = *msg
13401328
s.log.Debugf("Pool %v Added, handler called", msg)
1341-
err = s.addDelSnatPrefix(msg, true /* isAdd */)
1329+
err = s.addDelSnatPrefixForIPPool(msg, true /* isAdd */)
13421330
if err != nil {
13431331
return errors.Wrap(err, "error handling ipam add")
13441332
}
@@ -1366,7 +1354,7 @@ func (s *Server) handleIpamPoolRemove(msg *proto.IPAMPoolRemove, pending bool) (
13661354
delete(s.ippoolmap, key)
13671355
s.log.Infof("Deleting pool: %s", key)
13681356
s.log.Debugf("Pool %s deleted, handler called", existing.Pool.Cidr)
1369-
err = s.addDelSnatPrefix(&existing, false /* isAdd */)
1357+
err = s.addDelSnatPrefixForIPPool(&existing, false /* isAdd */)
13701358
if err != nil {
13711359
return errors.Wrap(err, "error handling ipam deletion")
13721360
}
@@ -1404,12 +1392,12 @@ func equalPools(a *proto.IPAMPoolUpdate, b *proto.IPAMPoolUpdate) bool {
14041392
return true
14051393
}
14061394

1407-
// addDelSnatPrefix configures IP Pool prefixes so that we don't source-NAT the packets going
1395+
// addDelSnatPrefixForIPPool configures IP Pool prefixes so that we don't source-NAT the packets going
14081396
// to these addresses. All the IP Pools prefixes are configured that way so that pod <-> pod
14091397
// communications are never source-nated in the cluster
14101398
// Note(aloaugus) - I think the iptables dataplane behaves differently and uses the k8s level
14111399
// pod CIDR for this rather than the individual pool prefixes
1412-
func (s *Server) addDelSnatPrefix(pool *proto.IPAMPoolUpdate, isAdd bool) (err error) {
1400+
func (s *Server) addDelSnatPrefixForIPPool(pool *proto.IPAMPoolUpdate, isAdd bool) (err error) {
14131401
_, ipNet, err := net.ParseCIDR(pool.Pool.Cidr)
14141402
if err != nil {
14151403
return errors.Wrapf(err, "Couldn't parse pool CIDR %s", pool.Pool.Cidr)

‎calico-vpp-agent/services/service_server.go‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -299,10 +299,6 @@ func serviceID(meta *metav1.ObjectMeta) string {
299299
}
300300

301301
func (s *Server) configureSnat() (err error) {
302-
err = s.vpp.CnatSetSnatAddresses(s.getNodeIP(false /* isv6 */), s.getNodeIP(true /* isv6 */))
303-
if err != nil {
304-
s.log.Errorf("Failed to configure SNAT addresses %v", err)
305-
}
306302
nodeIP4, nodeIP6 := common.GetBGPSpecAddresses(s.nodeBGPSpec)
307303
if nodeIP6 != nil {
308304
err = s.vpp.CnatAddSnatPrefix(common.FullyQualified(*nodeIP6))
@@ -322,6 +318,23 @@ func (s *Server) configureSnat() (err error) {
322318
s.log.Errorf("Failed to Add Service CIDR %s %v", serviceCIDR, err)
323319
}
324320
}
321+
err = s.vpp.SetK8sSnatPolicy()
322+
if err != nil {
323+
return errors.Wrap(err, "Error configuring cnat source policy")
324+
}
325+
for _, uplink := range common.VppManagerInfo.UplinkStatuses {
326+
// register vpptap0
327+
err = s.vpp.RegisterPodInterface(uplink.TapSwIfIndex)
328+
if err != nil {
329+
return errors.Wrap(err, "error configuring vpptap0 as pod intf")
330+
}
331+
332+
err = s.vpp.RegisterHostInterface(uplink.TapSwIfIndex)
333+
if err != nil {
334+
return errors.Wrap(err, "error configuring vpptap0 as host intf")
335+
}
336+
}
337+
325338
return nil
326339
}
327340

@@ -492,11 +505,6 @@ func (s *Server) ServeService(t *tomb.Tomb) error {
492505
})
493506
}
494507

495-
err = s.vpp.CnatPurge()
496-
if err != nil {
497-
return err
498-
}
499-
500508
if *config.GetCalicoVppDebug().ServicesEnabled {
501509
s.t.Go(func() error { s.serviceInformer.Run(t.Dying()); return nil })
502510
s.t.Go(func() error { s.endpointInformer.Run(t.Dying()); return nil })

‎vpp-manager/vpp_runner.go‎

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -460,7 +460,7 @@ func (v *VppRunner) configureVppUplinkInterface(
460460
return errors.Wrap(err, "Error disabling ipv6 RA on uplink interface")
461461
}
462462

463-
err = v.vpp.CnatEnableFeatures(ifSpec.SwIfIndex)
463+
err = v.vpp.CnatEnableFeatures(ifSpec.SwIfIndex, true, 0, 0)
464464
if err != nil {
465465
return errors.Wrap(err, "Error configuring NAT on uplink interface")
466466
}
@@ -606,21 +606,11 @@ func (v *VppRunner) configureVppUplinkInterface(
606606
log.Errorf("Error SetInterfaceRxMode on vpptap0 %v", err)
607607
}
608608

609-
err = v.vpp.CnatEnableFeatures(tapSwIfIndex)
609+
err = v.vpp.CnatEnableFeatures(tapSwIfIndex, true, 0, 0)
610610
if err != nil {
611611
return errors.Wrap(err, "Error configuring NAT on vpptap0")
612612
}
613613

614-
err = v.vpp.RegisterPodInterface(tapSwIfIndex)
615-
if err != nil {
616-
return errors.Wrap(err, "error configuring vpptap0 as pod intf")
617-
}
618-
619-
err = v.vpp.RegisterHostInterface(tapSwIfIndex)
620-
if err != nil {
621-
return errors.Wrap(err, "error configuring vpptap0 as host intf")
622-
}
623-
624614
// Linux side tap setup
625615
link, err := netlink.LinkByName(ifSpec.InterfaceName)
626616
if err != nil {
@@ -659,11 +649,6 @@ func (v *VppRunner) doVppGlobalConfiguration() (err error) {
659649
return errors.Wrap(err, "Error creating static VRFs in VPP")
660650
}
661651

662-
err = v.vpp.SetK8sSnatPolicy()
663-
if err != nil {
664-
return errors.Wrap(err, "Error configuring cnat source policy")
665-
}
666-
667652
err = v.vpp.ConfigureNeighborsV4(&types.NeighborConfig{
668653
MaxNumber: *config.GetCalicoVppInitialConfig().IP4NeighborsMaxNumber,
669654
MaxAge: *config.GetCalicoVppInitialConfig().IP4NeighborsMaxAge,

‎vpplink/cnat.go‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ func (v *VppLink) CnatTranslateAdd(tr *types.CnatTranslateEntry) (uint32, error)
6464
IPProto: types.ToVppIPProto(tr.Proto),
6565
Paths: paths,
6666
IsRealIP: BoolToU8(tr.IsRealIP),
67-
Flags: uint8(cnat.CNAT_TRANSLATION_ALLOC_PORT),
67+
Flags: uint8(cnat.CNAT_TRANSLATION_ALLOC_PORT | cnat.CNAT_TRANSLATION_NO_CLIENT),
6868
LbType: cnat.CnatLbType(tr.LbType),
6969
FlowHashConfig: ip.IPFlowHashConfigV2(tr.HashConfig),
7070
},
@@ -97,6 +97,7 @@ func (v *VppLink) CnatSetSnatAddresses(v4, v6 net.IP) error {
9797
SnatIP4: types.ToVppIP4Address(v4),
9898
SnatIP6: types.ToVppIP6Address(v6),
9999
SwIfIndex: types.InvalidInterface,
100+
Flags: cnat.CNAT_TRANSLATION_NO_CLIENT,
100101
})
101102
if err != nil {
102103
return fmt.Errorf("setting SNAT addresses failed: %w", err)
@@ -125,14 +126,18 @@ func (v *VppLink) CnatDelSnatPrefix(prefix *net.IPNet) error {
125126
return v.CnatAddDelSnatPrefix(prefix, false)
126127
}
127128

128-
func (v *VppLink) CnatEnableFeatures(swIfIndex uint32) (err error) {
129-
err = v.EnableFeatureArc46(swIfIndex, FeatureArcCnatInput)
130-
if err != nil {
131-
return fmt.Errorf("enabling arc dnat input failed: %w", err)
129+
func (v *VppLink) CnatEnableFeatures(swIfIndex uint32, isEnable bool, vrfIDV4 uint32, vrfIDV6 uint32) error {
130+
client := cnat.NewServiceClient(v.GetConnection())
131+
132+
request := &cnat.FeatureCnatEnableDisable{
133+
SwIfIndex: interface_types.InterfaceIndex(swIfIndex),
134+
EnableDisable: isEnable,
135+
TableIDIP4: vrfIDV4,
136+
TableIDIP6: vrfIDV6,
132137
}
133-
err = v.EnableFeatureArc46(swIfIndex, FeatureArcCnatOutput)
138+
_, err := client.FeatureCnatEnableDisable(v.GetContext(), request)
134139
if err != nil {
135-
return fmt.Errorf("enabling arc dnat output failed: %w", err)
140+
return fmt.Errorf("FeatureEnableDisable %+v failed: %w", request, err)
136141
}
137142
return nil
138143
}
@@ -189,7 +194,7 @@ func (v *VppLink) disableCnatSNAT(swIfIndex uint32, isIP6 bool) (err error) {
189194
return v.cnatSnatPolicyAddDelPodInterface(swIfIndex, false /* isAdd */, cnat.CNAT_POLICY_INCLUDE_V4)
190195
}
191196

192-
func (v *VppLink) cnatSetSnatPolicy(pol cnat.CnatSnatPolicies) error {
197+
func (v *VppLink) cnatSetSnatPolicyForDefaultVRF(pol cnat.CnatSnatPolicies) error {
193198
client := cnat.NewServiceClient(v.GetConnection())
194199

195200
_, err := client.CnatSetSnatPolicy(v.GetContext(), &cnat.CnatSetSnatPolicy{
@@ -202,9 +207,22 @@ func (v *VppLink) cnatSetSnatPolicy(pol cnat.CnatSnatPolicies) error {
202207
}
203208

204209
func (v *VppLink) SetK8sSnatPolicy() (err error) {
205-
return v.cnatSetSnatPolicy(cnat.CNAT_POLICY_K8S)
210+
return v.cnatSetSnatPolicyForDefaultVRF(cnat.CNAT_POLICY_K8S)
206211
}
207212

208213
func (v *VppLink) ClearSnatPolicy() (err error) {
209-
return v.cnatSetSnatPolicy(cnat.CNAT_POLICY_NONE)
214+
return v.cnatSetSnatPolicyForDefaultVRF(cnat.CNAT_POLICY_NONE)
215+
}
216+
217+
func (v *VppLink) EnableCnatSNATOnInterfaceVRF(swifindex uint32) (err error) {
218+
client := cnat.NewServiceClient(v.GetConnection())
219+
220+
_, err = client.DuplicateCnatSnatDefaultPolicyToVrf(v.GetContext(), &cnat.DuplicateCnatSnatDefaultPolicyToVrf{
221+
SwIfIndex: interface_types.InterfaceIndex(swifindex),
222+
})
223+
if err != nil {
224+
return fmt.Errorf("DuplicateCnatSnatDefaultPolicyToVrf for interface %d failed: %w", swifindex, err)
225+
}
226+
return nil
227+
210228
}

0 commit comments

Comments
 (0)