[release/3.26] Converge cnat refactoring

.ci/Dockerfile.depend

@@ -3,15 +3,15 @@ ARG BASE_IMAGE
 #
 # Dependencies image
 #
-FROM ${BASE_IMAGE} as dependencies
+FROM ${BASE_IMAGE:-ubuntu:22.04} AS dependencies
 
 ENV UNATTENDED=y
 
 RUN apt-get update \
  && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
     apt-utils wget cmake curl git
 
-ENV GOVERSION=1.18.2
+ENV GOVERSION=1.24.0
 ENV GOROOT="/root/.go"
 ENV GOPATH="/root/go"
 ENV PATH=$GOROOT/bin:$PATH
@@ -22,9 +22,14 @@ RUN mkdir -p "${GOROOT}" &&\
 RUN wget -nv "https://dl.google.com/go/go${GOVERSION}.linux-amd64.tar.gz" -O "/tmp/go.tar.gz" && \
     tar -C "${GOROOT}" --strip-components=1 -xzf "/tmp/go.tar.gz" && \
     rm -f "/tmp/go.tar.gz" && \
-    curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.50.1
+    curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.64.8
+
+RUN git config --global --add safe.directory /vpp-dataplane
 
 # Get modules used by the source code
-COPY . /vpp-dataplane
+RUN mkdir -p /vpp-dataplane
+RUN git config --global --add safe.directory /vpp-dataplane
+COPY go.mod /vpp-dataplane
+COPY go.sum /vpp-dataplane
 WORKDIR /vpp-dataplane
 RUN go get ./... && rm -fr /vpp-dataplane

.ci/Makefile

@@ -13,7 +13,7 @@ ABS_PATH = $(abspath $(shell pwd)/..)
 # remain stable.
 #
 
-BASE_IMAGE_BUILDER = ubuntu:20.04
+BASE_IMAGE_BUILDER = ubuntu:22.04
 
 # Compute hash to detect any changes and rebuild /push the image
 DEPEND_HASH = $(shell echo "${BASE_IMAGE_BUILDER}-DOCKERFILE:$(shell md5sum Dockerfile.depend)" | md5sum | cut -f1 -d' ')

.ci/common.mk

@@ -15,6 +15,8 @@ SQUASH := --squash
 # push dependency
 PUSH_DEP := image
 
+CGO_ENABLED := 0
+
 # CI specific variables
 ifdef CODEBUILD_BUILD_NUMBER
 	# Define variable when building for CI

calico-vpp-agent/Makefile

@@ -25,9 +25,8 @@ build: felix-api-proxy bin
 	${DOCKER_RUN} go build -o ./bin/calico-vpp-agent ./cmd
 	${DOCKER_RUN} go build -o ./bin/debug ./cmd/debug-state
 
-gobgp: GOBGP_DIR:=$(shell ${DOCKER_RUN} go list -f '{{.Dir}}' -m github.com/osrg/gobgp/v3)
 gobgp: bin
-	${DOCKER_RUN} go build -o ./bin/gobgp $(GOBGP_DIR)/cmd/gobgp/
+	${DOCKER_RUN} go build -o ./bin/gobgp github.com/osrg/gobgp/v3/cmd/gobgp/
 
 image: build gobgp
 	@echo "Image tag                   : $(TAG)"                         > $(VERSION_FILE)

calico-vpp-agent/cni/cni_server.go

@@ -38,6 +38,7 @@ import (
 	"github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common"
 	"github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers"
 	"github.com/projectcalico/vpp-dataplane/v3/config"
+	"github.com/projectcalico/vpp-dataplane/v3/vpp-manager/utils"
 	"github.com/projectcalico/vpp-dataplane/v3/vpplink"
 	"github.com/projectcalico/vpp-dataplane/v3/vpplink/types"
 )
@@ -61,6 +62,8 @@ type Server struct {
 
 	availableBuffers uint64
 
+	RedirectToHostClassifyTableIndex uint32
+
 	networkDefinitions   sync.Map
 	cniMultinetEventChan chan common.CalicoVppEvent
 	nodeBGPSpec          *common.LocalNodeSpec
@@ -245,6 +248,12 @@ func (s *Server) Add(ctx context.Context, request *cniproto.AddRequest) (*cnipro
 			ErrorMessage: err.Error(),
 		}, nil
 	}
+	if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpec.NetworkName == "" {
+		err := s.AddRedirectToHostToInterface(podSpec.TunTapSwIfIndex)
+		if err != nil {
+			return nil, err
+		}
+	}
 
 	s.podInterfaceMap[podSpec.Key()] = *podSpec
 	cniServerStateFile := fmt.Sprintf("%s%d", config.CniServerStateFile, storage.CniServerStateFileVersion)
@@ -315,6 +324,34 @@ func (s *Server) rescanState() {
 		default:
 			s.log.Errorf("Interface add failed %s : %v", podSpecCopy.String(), err)
 		}
+		if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpecCopy.NetworkName == "" {
+			err := s.AddRedirectToHostToInterface(podSpecCopy.TunTapSwIfIndex)
+			if err != nil {
+				s.log.Error(err)
+			}
+		}
+	}
+}
+
+func (s *Server) DelRedirectToHostOnInterface(swIfIndex uint32) error {
+	err := s.vpp.SetClassifyInputInterfaceTables(swIfIndex, s.RedirectToHostClassifyTableIndex, types.InvalidTableId, types.InvalidTableId, false /*isAdd*/)
+	if err != nil {
+		return errors.Wrapf(err, "Error deleting classify input table from interface")
+	} else {
+		s.log.Infof("pod(del) delete input acl table %d from interface %d successfully", s.RedirectToHostClassifyTableIndex, swIfIndex)
+		return nil
+	}
+}
+
+func (s *Server) AddRedirectToHostToInterface(swIfIndex uint32) error {
+	s.log.Infof("Setting classify input acl table %d on interface %d", s.RedirectToHostClassifyTableIndex, swIfIndex)
+	err := s.vpp.SetClassifyInputInterfaceTables(swIfIndex, s.RedirectToHostClassifyTableIndex, types.InvalidTableId, types.InvalidTableId, true)
+	if err != nil {
+		s.log.Warnf("Error setting classify input table: %s, retrying...", err)
+		return errors.Errorf("could not set input acl table %d for interface %d", s.RedirectToHostClassifyTableIndex, swIfIndex)
+	} else {
+		s.log.Infof("set input acl table %d for interface %d successfully", s.RedirectToHostClassifyTableIndex, swIfIndex)
+		return nil
 	}
 }
 
@@ -442,6 +479,47 @@ forloop:
 	return nil
 }
 
+func (s *Server) getMainTap0Info() (tapSwIfIndex uint32, address net.IP) {
+	for _, i := range common.VppManagerInfo.UplinkStatuses {
+		if i.IsMain {
+			tapSwIfIndex = i.TapSwIfIndex
+			break
+		}
+	}
+	address = utils.FakeVppNextHopIP4
+	return
+}
+
+func (s *Server) createRedirectToHostRules() (uint32, error) {
+	var maxNumEntries uint32
+	if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 {
+		maxNumEntries = uint32(2 * len(config.GetCalicoVppInitialConfig().RedirectToHostRules))
+	} else {
+		maxNumEntries = 1
+	}
+	index, err := s.vpp.AddClassifyTable(&types.ClassifyTable{
+		Mask:           types.DstThreeTupleMask,
+		NextTableIndex: types.InvalidID,
+		MaxNumEntries:  maxNumEntries,
+		MissNextIndex:  ^uint32(0),
+	})
+	if err != nil {
+		return types.InvalidID, err
+	}
+	tap0swifindex, tap0nexthop := s.getMainTap0Info()
+	for _, rule := range config.GetCalicoVppInitialConfig().RedirectToHostRules {
+		err = s.vpp.AddSessionRedirect(&types.SessionRedirect{
+			FiveTuple:  types.NewDst3Tuple(rule.Proto, net.ParseIP(rule.Ip), rule.Port),
+			TableIndex: index,
+		}, &types.RoutePath{Gw: tap0nexthop, SwIfIndex: tap0swifindex})
+		if err != nil {
+			return types.InvalidID, err
+		}
+	}
+
+	return index, nil
+}
+
 func (s *Server) ServeCNI(t *tomb.Tomb) error {
 	err := syscall.Unlink(config.CNIServerSocket)
 	if err != nil && !gerrors.Is(err, os.ErrNotExist) {
@@ -453,6 +531,10 @@ func (s *Server) ServeCNI(t *tomb.Tomb) error {
 		return errors.Wrapf(err, "failed to listen on %s", config.CNIServerSocket)
 	}
 
+	s.RedirectToHostClassifyTableIndex, err = s.createRedirectToHostRules()
+	if err != nil {
+		return err
+	}
 	cniproto.RegisterCniDataplaneServer(s.grpcServer, s)
 
 	if *config.GetCalicoVppFeatureGates().MultinetEnabled {

calico-vpp-agent/cni/network_vpp.go

@@ -285,6 +285,12 @@ err:
 
 // CleanUpVPPNamespace deletes the devices in the network namespace.
 func (s *Server) DelVppInterface(podSpec *storage.LocalPodSpec) {
+	if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpec.NetworkName == "" {
+		err := s.DelRedirectToHostOnInterface(podSpec.TunTapSwIfIndex)
+		if err != nil {
+			s.log.Error(err)
+		}
+	}
 	err := ns.IsNSorErr(podSpec.NetnsName)
 	if err != nil {
 		s.log.Infof("pod(del) netns '%s' doesn't exist, skipping", podSpec.NetnsName)

calico-vpp-agent/cni/network_vpp_routes.go

@@ -62,6 +62,7 @@ func (s *Server) RoutePodInterface(podSpec *storage.LocalPodSpec, stack *vpplink
 				SwIfIndex:    swIfIndex,
 				IP:           containerIP.IP,
 				HardwareAddr: common.ContainerSideMacAddress,
+				Flags:        types.IPNeighborStatic,
 			})
 			if err != nil {
 				return errors.Wrapf(err, "Error adding neighbor if[%d] %s", swIfIndex, containerIP.IP.String())
@@ -141,14 +142,15 @@ func (s *Server) RoutePblPortsPodInterface(podSpec *storage.LocalPodSpec, stack
 		} else {
 			stack.Push(s.vpp.DelPblClient, pblIndex)
 		}
-		podSpec.PblIndexes = append(podSpec.PblIndexes, pblIndex)
+		podSpec.PblIndex = pblIndex
 
 		if !isL3 {
 			s.log.Infof("pod(add) neighbor if[%d] %s", swIfIndex, containerIP.IP.String())
 			err = s.vpp.AddNeighbor(&types.Neighbor{
 				SwIfIndex:    swIfIndex,
 				IP:           containerIP.IP,
 				HardwareAddr: common.ContainerSideMacAddress,
+				Flags:        types.IPNeighborStatic,
 			})
 			if err != nil {
 				return errors.Wrapf(err, "Cannot add neighbor if[%d] %s", swIfIndex, containerIP.IP.String())
@@ -159,12 +161,10 @@ func (s *Server) RoutePblPortsPodInterface(podSpec *storage.LocalPodSpec, stack
 }
 
 func (s *Server) UnroutePblPortsPodInterface(podSpec *storage.LocalPodSpec, swIfIndex uint32) {
-	for _, pblIndex := range podSpec.PblIndexes {
-		s.log.Infof("pod(del) PBL client[%d]", pblIndex)
-		err := s.vpp.DelPblClient(pblIndex)
-		if err != nil {
-			s.log.Warnf("Error deleting pbl conf %s", err)
-		}
+	s.log.Infof("pod(del) PBL client[%d]", podSpec.PblIndex)
+	err := s.vpp.DelPblClient(podSpec.PblIndex)
+	if err != nil {
+		s.log.Warnf("Error deleting pbl conf %s", err)
 	}
 }
 

calico-vpp-agent/cni/pod_interface/common.go

@@ -105,7 +105,7 @@ func (i *PodInterfaceDriverData) DoPodIfNatConfiguration(podSpec *storage.LocalP
 		stack.Push(i.vpp.RemovePodInterface, swIfIndex)
 	}
 
-	err = i.vpp.CnatEnableFeatures(swIfIndex)
+	err = i.vpp.CnatEnableFeatures(swIfIndex, true, podSpec.GetVrfId(vpplink.IpFamilyV4), podSpec.GetVrfId(vpplink.IpFamilyV6))
 	if err != nil {
 		return errors.Wrapf(err, "error configuring nat on pod interface")
 	}
@@ -129,6 +129,10 @@ func (i *PodInterfaceDriverData) DoPodInterfaceConfiguration(podSpec *storage.Lo
 		}
 	}
 
+	err = i.vpp.EnableCnatSNATOnInterfaceVRF(swIfIndex)
+	if err != nil {
+		return errors.Wrapf(err, "error configuring cnat snat on pod VRF")
+	}
 	if !*ifSpec.IsL3 {
 		/* L2 */
 		err = i.vpp.SetPromiscOn(swIfIndex)

calico-vpp-agent/cni/pod_interface/memif.go

@@ -93,39 +93,41 @@ func (i *MemifPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSpec,
 	}
 	podSpec.MemifSwIfIndex = memif.SwIfIndex
 
-	watcher, err := i.vpp.WatchInterfaceEvents(memif.SwIfIndex)
-	if err != nil {
-		return err
-	} else {
-		stack.Push(watcher.Stop)
-	}
-	go func() {
-		i.log.WithFields(map[string]interface{}{
-			"swIfIndex": memif.SwIfIndex,
-		}).Infof("begin watching interface events for: %v", i.Name)
-
-		for event := range watcher.Events() {
+	if *(*config.CalicoVppDebug).SpreadTxQueuesOnWorkers {
+		watcher, err := i.vpp.WatchInterfaceEvents(memif.SwIfIndex)
+		if err != nil {
+			return err
+		} else {
+			stack.Push(watcher.Stop)
+		}
+		go func() {
 			i.log.WithFields(map[string]interface{}{
 				"swIfIndex": memif.SwIfIndex,
-			}).Infof("processing interface event for %v: %+v", i.Name, event)
-
-			switch event.Type {
-			case types.InterfaceEventLinkUp:
-				err = i.SpreadTxQueuesOnWorkers(memif.SwIfIndex, memif.NumTxQueues)
-				if err != nil {
-					i.log.Errorf("error spreading tx queues on workers: %v", err)
+			}).Infof("begin watching interface events for: %v", i.Name)
+
+			for event := range watcher.Events() {
+				i.log.WithFields(map[string]interface{}{
+					"swIfIndex": memif.SwIfIndex,
+				}).Infof("processing interface event for %v: %+v", i.Name, event)
+
+				switch event.Type {
+				case types.InterfaceEventLinkUp:
+					err = i.SpreadTxQueuesOnWorkers(memif.SwIfIndex, memif.NumTxQueues)
+					if err != nil {
+						i.log.Errorf("error spreading tx queues on workers: %v", err)
+					}
+					i.SpreadRxQueuesOnWorkers(memif.SwIfIndex, podSpec.IfSpec.NumRxQueues)
+				case types.InterfaceEventDeleted: // this might not be needed here, it could be handled internally in the watcher
+					watcher.Stop()
 				}
-				i.SpreadRxQueuesOnWorkers(memif.SwIfIndex, podSpec.IfSpec.NumRxQueues)
-			case types.InterfaceEventDeleted: // this might not be needed here, it could be handled internally in the watcher
-				watcher.Stop()
 			}
-		}
 
-		i.log.WithFields(map[string]interface{}{
-			"swIfIndex": memif.SwIfIndex,
-		}).Infof("done watching interface events for: %v", i.Name)
+			i.log.WithFields(map[string]interface{}{
+				"swIfIndex": memif.SwIfIndex,
+			}).Infof("done watching interface events for: %v", i.Name)
 
-	}()
+		}()
+	}
 
 	err = i.vpp.SetInterfaceTag(memif.SwIfIndex, podSpec.GetInterfaceTag(i.Name))
 	if err != nil {

calico-vpp-agent/cni/storage/storage.go

@@ -35,7 +35,7 @@ import (
 )
 
 const (
-	CniServerStateFileVersion = 8  // Used to ensure compatibility wen we reload data
+	CniServerStateFileVersion = 9  // Used to ensure compatibility wen we reload data
 	MaxApiTagLen              = 63 /* No more than 64 characters in API tags */
 	VrfTagHashLen             = 8  /* how many hash charatecters (b64) of the name in tag prefix (useful when trucated) */
 )
@@ -126,11 +126,6 @@ func (ps *LocalPodSpec) FullString() string {
 	for _, e := range routes {
 		routesLst = append(routesLst, e.String())
 	}
-	pblIndexes := ps.PblIndexes
-	pblIndexesLst := make([]string, 0, len(pblIndexes))
-	for _, e := range pblIndexes {
-		pblIndexesLst = append(pblIndexesLst, fmt.Sprint(e))
-	}
 	s := fmt.Sprintf("InterfaceName:      %s\n", ps.InterfaceName)
 	s += fmt.Sprintf("NetnsName:          %s\n", ps.NetnsName)
 	s += fmt.Sprintf("AllowIpForwarding:  %t\n", ps.AllowIpForwarding)
@@ -151,7 +146,7 @@ func (ps *LocalPodSpec) FullString() string {
 	s += fmt.Sprintf("TunTapSwIfIndex:    %d\n", ps.TunTapSwIfIndex)
 	s += fmt.Sprintf("MemifSwIfIndex:     %d\n", ps.MemifSwIfIndex)
 	s += fmt.Sprintf("LoopbackSwIfIndex:  %d\n", ps.LoopbackSwIfIndex)
-	s += fmt.Sprintf("PblIndexes:         %s\n", strings.Join(pblIndexesLst, ", "))
+	s += fmt.Sprintf("PblIndex:  %d\n", ps.PblIndex)
 	s += fmt.Sprintf("V4VrfId:            %d\n", ps.V4VrfId)
 	s += fmt.Sprintf("V6VrfId:            %d\n", ps.V6VrfId)
 	return s
@@ -238,8 +233,7 @@ type LocalPodSpec struct {
 	TunTapSwIfIndex   uint32
 	MemifSwIfIndex    uint32
 	LoopbackSwIfIndex uint32
-	PblIndexesLen     int `struc:"int16,sizeof=PblIndexes"`
-	PblIndexes        []uint32
+	PblIndex          uint32
 
 	/**
 	 * These fields are only a runtime cache, but we also store them
@@ -268,7 +262,6 @@ func (ps *LocalPodSpec) Copy() LocalPodSpec {
 	newPs.ContainerIps = append(make([]LocalIP, 0), ps.ContainerIps...)
 	newPs.HostPorts = append(make([]HostPortBinding, 0), ps.HostPorts...)
 	newPs.IfPortConfigs = append(make([]LocalIfPortConfigs, 0), ps.IfPortConfigs...)
-	newPs.PblIndexes = append(make([]uint32, 0), ps.PblIndexes...)
 
 	return newPs
 
@@ -305,7 +298,7 @@ func TruncateStr(text string, size int) string {
 
 func (ps *LocalPodSpec) GetVrfTag(ipFamily vpplink.IpFamily, custom string) string {
 	h := hash(fmt.Sprintf("%s%s%s%s", ipFamily.ShortStr, ps.NetnsName, ps.InterfaceName, custom))
-	s := fmt.Sprintf("%s-%s-%s-%s", h, ipFamily.ShortStr, ps.InterfaceName, filepath.Base(ps.NetnsName))
+	s := fmt.Sprintf("%s-%s-%s%s-%s", h, ipFamily.ShortStr, ps.InterfaceName, custom, filepath.Base(ps.NetnsName))
 	return TruncateStr(s, MaxApiTagLen)
 }
 

calico-vpp-agent/common_tests/common_tests.go

@@ -128,7 +128,7 @@ func AssertRPFVRFExistence(vpp *vpplink.VppLink, interfaceName string, netnsName
 		"Failed to retrieve list of VRFs in VPP")
 	hbytes := sha512.Sum512([]byte(fmt.Sprintf("%s%s%s%s", "4", netnsName, interfaceName, "RPF")))
 	h := base64.StdEncoding.EncodeToString(hbytes[:])[:storage.VrfTagHashLen]
-	s := fmt.Sprintf("%s-%s-%s-%s", h, "4", interfaceName, filepath.Base(netnsName))
+	s := fmt.Sprintf("%s-%s-%sRPF-%s", h, "4", interfaceName, filepath.Base(netnsName))
 	vrfTag := storage.TruncateStr(s, storage.MaxApiTagLen)
 	foundRPFVRF := false
 	var vrfID uint32