diff --git a/calico-vpp-agent/Dockerfile b/calico-vpp-agent/Dockerfile index 700f34f3..57d8730c 100644 --- a/calico-vpp-agent/Dockerfile +++ b/calico-vpp-agent/Dockerfile @@ -3,7 +3,6 @@ FROM ubuntu:22.04 LABEL maintainer="aloaugus@cisco.com" ADD bin/gobgp /bin/gobgp -ADD bin/debug /bin/debug ADD version /etc/calicovppversion ADD bin/felix-api-proxy /bin/felix-api-proxy ADD bin/calico-vpp-agent /bin/calico-vpp-agent diff --git a/calico-vpp-agent/Makefile b/calico-vpp-agent/Makefile index edb68e7d..a1167751 100644 --- a/calico-vpp-agent/Makefile +++ b/calico-vpp-agent/Makefile @@ -23,7 +23,6 @@ felix-api-proxy: bin build: felix-api-proxy bin ${DOCKER_RUN} go build -o ./bin/calico-vpp-agent ./cmd - ${DOCKER_RUN} go build -o ./bin/debug ./cmd/debug-state gobgp: bin ${DOCKER_RUN} go build -o ./bin/gobgp github.com/osrg/gobgp/v3/cmd/gobgp/ diff --git a/calico-vpp-agent/cmd/debug-state/debug-state.go b/calico-vpp-agent/cmd/debug-state/debug-state.go deleted file mode 100644 index bad48956..00000000 --- a/calico-vpp-agent/cmd/debug-state/debug-state.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (C) 2019 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "flag" - "fmt" - - log "github.com/sirupsen/logrus" - - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/storage" - "github.com/projectcalico/vpp-dataplane/v3/config" -) - -func main() { - var fname string - cniServerStateFile := fmt.Sprintf("%s%d", config.CniServerStateFile, storage.CniServerStateFileVersion) - flag.StringVar(&fname, "f", cniServerStateFile, "Pod state path") - flag.Parse() - - st, err := storage.LoadCniServerState(fname) - if err != nil { - log.Errorf("LoadCniServerState errored: %v", err) - return - } - for i, s := range st { - log.Infof("-------- Elem %d--------\n%s", i, s.FullString()) - } - log.Infof("%d Elts", len(st)) -} diff --git a/calico-vpp-agent/cni/cni_pod_test.go b/calico-vpp-agent/cni/cni_pod_test.go index 9bc02675..5fd54bf0 100644 --- a/calico-vpp-agent/cni/cni_pod_test.go +++ b/calico-vpp-agent/cni/cni_pod_test.go @@ -182,7 +182,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { Workload: &cniproto.WorkloadIDs{ Annotations: map[string]string{ // needed just for setting up steering of traffic to default Tun/Tap and to secondary Memif - cni.VppAnnotationPrefix + cni.MemifPortAnnotation: fmt.Sprintf("tcp:%d-%d,udp:%d-%d", + config.VppAnnotationPrefix + config.MemifPortAnnotation: fmt.Sprintf("tcp:%d-%d,udp:%d-%d", memifTCPPortStart, memifTCPPortEnd, memifUDPPortStart, memifUDPPortEnd), }, }, @@ -418,7 +418,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { Workload: &cniproto.WorkloadIDs{ Annotations: map[string]string{ // needed just for setting up steering of traffic to default Tun/Tap and to secondary Memif - cni.VppAnnotationPrefix + cni.MemifPortAnnotation: fmt.Sprintf("tcp:%d-%d,udp:%d-%d", + config.VppAnnotationPrefix + config.MemifPortAnnotation: fmt.Sprintf("tcp:%d-%d,udp:%d-%d", memifTCPPortStart, memifTCPPortEnd, memifUDPPortStart, memifUDPPortEnd), }, }, diff --git a/calico-vpp-agent/cni/cni_server.go b/calico-vpp-agent/cni/cni_server.go index 115aba00..826f5528 100644 --- a/calico-vpp-agent/cni/cni_server.go +++ b/calico-vpp-agent/cni/cni_server.go @@ -72,19 +72,6 @@ func swIfIdxToIfName(idx uint32) string { return fmt.Sprintf("vpp-tun-%d", idx) } -func getHostEndpointProto(proto string) types.IPProto { - switch proto { - case "udp": - return types.UDP - case "sctp": - return types.SCTP - case "tcp": - return types.TCP - default: - return types.TCP - } -} - func (s *Server) SetFelixConfig(felixConfig *felixConfig.Config) { s.tuntapDriver.SetFelixConfig(felixConfig) } @@ -93,83 +80,13 @@ func (s *Server) SetOurBGPSpec(nodeBGPSpec *common.LocalNodeSpec) { s.nodeBGPSpec = nodeBGPSpec } -func (s *Server) newLocalPodSpecFromAdd(request *cniproto.AddRequest) (*storage.LocalPodSpec, error) { - podSpec := storage.LocalPodSpec{ - InterfaceName: request.GetInterfaceName(), - NetnsName: request.GetNetns(), - AllowIpForwarding: request.GetSettings().GetAllowIpForwarding(), - Routes: make([]storage.LocalIPNet, 0), - ContainerIps: make([]storage.LocalIP, 0), - Mtu: int(request.GetSettings().GetMtu()), - - IfPortConfigs: make([]storage.LocalIfPortConfigs, 0), - - OrchestratorID: request.Workload.Orchestrator, - WorkloadID: request.Workload.Namespace + "/" + request.Workload.Pod, - EndpointID: request.Workload.Endpoint, - HostPorts: make([]storage.HostPortBinding, 0), - - /* defaults */ - IfSpec: GetDefaultIfSpec(true /* isL3 */), - PBLMemifSpec: GetDefaultIfSpec(false /* isL3 */), - - V4VrfId: vpplink.InvalidID, - V6VrfId: vpplink.InvalidID, - - MemifSwIfIndex: vpplink.InvalidID, - TunTapSwIfIndex: vpplink.InvalidID, - - NetworkName: request.DataplaneOptions["network_name"], - } - - if podSpec.NetworkName != "" { - if !*config.GetCalicoVppFeatureGates().MultinetEnabled { - return nil, fmt.Errorf("enable multinet in config for multiple networks") - } - if isMemif(podSpec.InterfaceName) { - if !*config.GetCalicoVppFeatureGates().MemifEnabled { - return nil, fmt.Errorf("enable memif in config for memif interfaces") - } - podSpec.EnableMemif = true - podSpec.DefaultIfType = storage.VppIfTypeMemif - podSpec.IfSpec = GetDefaultIfSpec(false) - } - } - - for _, port := range request.Workload.Ports { - hostIP := net.ParseIP(port.HostIp) - hostPort := uint16(port.HostPort) - if hostPort != 0 && hostIP != nil && !hostIP.IsUnspecified() { - podSpec.HostPorts = append(podSpec.HostPorts, storage.HostPortBinding{ - HostPort: hostPort, - HostIP: hostIP, - ContainerPort: uint16(port.Port), - Protocol: getHostEndpointProto(port.Protocol), - }) - } else if hostPort != 0 { - // default to node IP - podSpec.HostPorts = append(podSpec.HostPorts, storage.HostPortBinding{ - HostPort: hostPort, - HostIP: net.ParseIP(s.nodeBGPSpec.IPv4Address.IP.String()), - ContainerPort: uint16(port.Port), - Protocol: getHostEndpointProto(port.Protocol), - }) - } - } - for _, routeStr := range request.GetContainerRoutes() { - _, route, err := net.ParseCIDR(routeStr) - if err != nil { - return nil, errors.Wrapf(err, "Cannot parse container route %s", routeStr) - } - podSpec.Routes = append(podSpec.Routes, storage.LocalIPNet{ - IP: route.IP, - Mask: route.Mask, - }) - } +func (s *Server) Add(ctx context.Context, request *cniproto.AddRequest) (*cniproto.AddReply, error) { + /* We don't support request.GetDesiredHostInterfaceName() */ + podSpec, err := storage.NewLocalPodSpecFromAdd(request, s.nodeBGPSpec) if podSpec.NetworkName != "" { value, ok := s.networkDefinitions.Load(podSpec.NetworkName) if !ok { - s.log.Errorf("trying to create a pod in an unexisting network %s", podSpec.NetworkName) + return nil, fmt.Errorf("trying to create a pod in an unexisting network %s", podSpec.NetworkName) } else { networkDefinition, ok := value.(*watchers.NetworkDefinition) if !ok || networkDefinition == nil { @@ -177,47 +94,10 @@ func (s *Server) newLocalPodSpecFromAdd(request *cniproto.AddRequest) (*storage. } _, route, err := net.ParseCIDR(networkDefinition.Range) if err == nil { - podSpec.Routes = append(podSpec.Routes, storage.LocalIPNet{ - IP: route.IP, - Mask: route.Mask, - }) + podSpec.Routes = append(podSpec.Routes, *route) } } } - for _, requestContainerIP := range request.GetContainerIps() { - containerIp, _, err := net.ParseCIDR(requestContainerIP.GetAddress()) - if err != nil { - return nil, fmt.Errorf("Cannot parse address: %s", requestContainerIP.GetAddress()) - } - // We ignore the prefix len set on the address, - // for a tun it doesn't make sense - podSpec.ContainerIps = append(podSpec.ContainerIps, storage.LocalIP{IP: containerIp}) - } - workload := request.GetWorkload() - if workload != nil { - err := s.ParsePodAnnotations(&podSpec, workload.Annotations) - if err != nil { - return nil, errors.Wrapf(err, "Cannot parse pod Annotations") - } - } - - if podSpec.DefaultIfType == storage.VppIfTypeUnknown { - podSpec.DefaultIfType = storage.VppIfTypeTunTap - } - - return &podSpec, nil -} - -func NewLocalPodSpecFromDel(request *cniproto.DelRequest) *storage.LocalPodSpec { - return &storage.LocalPodSpec{ - InterfaceName: request.GetInterfaceName(), - NetnsName: request.GetNetns(), - } -} - -func (s *Server) Add(ctx context.Context, request *cniproto.AddRequest) (*cniproto.AddReply, error) { - /* We don't support request.GetDesiredHostInterfaceName() */ - podSpec, err := s.newLocalPodSpecFromAdd(request) if err != nil { s.log.Errorf("Error parsing interface add request %v %v", request, err) return &cniproto.AddReply{ @@ -252,7 +132,7 @@ func (s *Server) Add(ctx context.Context, request *cniproto.AddRequest) (*cnipro }, nil } if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpec.NetworkName == "" { - err := s.AddRedirectToHostToInterface(podSpec.TunTapSwIfIndex) + err := s.AddRedirectToHostToInterface(podSpec.Status.TunTapSwIfIndex) if err != nil { return nil, err } @@ -315,7 +195,7 @@ func (s *Server) rescanState() { s.lock.Lock() defer s.lock.Unlock() for _, podSpec := range podSpecs { - /* copy podSpec as a pointer to it will be sent over the event chan */ + // we copy podSpec as a pointer to it will be sent over the event chan podSpecCopy := podSpec.Copy() _, err := s.AddVppInterface(&podSpecCopy, false /* doHostSideConf */) switch err.(type) { @@ -328,7 +208,7 @@ func (s *Server) rescanState() { s.log.Errorf("Interface add failed %s : %v", podSpecCopy.String(), err) } if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpecCopy.NetworkName == "" { - err := s.AddRedirectToHostToInterface(podSpecCopy.TunTapSwIfIndex) + err := s.AddRedirectToHostToInterface(podSpecCopy.Status.TunTapSwIfIndex) if err != nil { s.log.Error(err) } @@ -359,9 +239,9 @@ func (s *Server) AddRedirectToHostToInterface(swIfIndex uint32) error { } func (s *Server) Del(ctx context.Context, request *cniproto.DelRequest) (*cniproto.DelReply, error) { - partialPodSpec := NewLocalPodSpecFromDel(request) + podSpecKey := storage.LocalPodSpecKey(request.GetNetns(), request.GetInterfaceName()) // Only try to delete the device if a namespace was passed in. - if partialPodSpec.NetnsName == "" { + if request.GetNetns() == "" { s.log.Debugf("no netns passed, skipping") return &cniproto.DelReply{ Successful: true, @@ -370,17 +250,17 @@ func (s *Server) Del(ctx context.Context, request *cniproto.DelRequest) (*cnipro s.lock.Lock() defer s.lock.Unlock() - s.log.Infof("pod(del) key=%s", partialPodSpec.Key()) - initialSpec, ok := s.podInterfaceMap[partialPodSpec.Key()] + s.log.Infof("pod(del) key=%s", podSpecKey) + initialSpec, ok := s.podInterfaceMap[podSpecKey] if !ok { - s.log.Warnf("Unknown pod to delete key=%s", partialPodSpec.Key()) + s.log.Warnf("Unknown pod to delete key=%s", podSpecKey) } else { s.log.Infof("pod(del) spec=%s", initialSpec.String()) s.DelVppInterface(&initialSpec) s.log.Infof("pod(del) Done! spec=%s", initialSpec.String()) } - delete(s.podInterfaceMap, initialSpec.Key()) + delete(s.podInterfaceMap, podSpecKey) err := storage.PersistCniServerState(s.podInterfaceMap, config.CniServerStateFile+fmt.Sprint(storage.CniServerStateFileVersion)) if err != nil { s.log.Errorf("CNI state persist errored %v", err) @@ -455,16 +335,16 @@ forloop: } for _, podSpec := range s.podInterfaceMap { - NeededSnat := podSpec.NeedsSnat + NeededSnat := podSpec.Status.NeedsSnat for _, containerIP := range podSpec.GetContainerIps() { - podSpec.NeedsSnat = podSpec.NeedsSnat || s.policyServerIpam.IPNetNeedsSNAT(containerIP) + podSpec.Status.NeedsSnat = podSpec.Status.NeedsSnat || s.policyServerIpam.IPNetNeedsSNAT(containerIP) } - if NeededSnat != podSpec.NeedsSnat { - for _, swIfIndex := range []uint32{podSpec.LoopbackSwIfIndex, podSpec.TunTapSwIfIndex, podSpec.MemifSwIfIndex} { + if NeededSnat != podSpec.Status.NeedsSnat { + for _, swIfIndex := range []uint32{podSpec.Status.LoopbackSwIfIndex, podSpec.Status.TunTapSwIfIndex, podSpec.Status.MemifSwIfIndex} { if swIfIndex != vpplink.InvalidID { s.log.Infof("Enable/Disable interface[%d] SNAT", swIfIndex) for _, ipFamily := range vpplink.IpFamilies { - err := s.vpp.EnableDisableCnatSNAT(swIfIndex, ipFamily.IsIp6, podSpec.NeedsSnat) + err := s.vpp.EnableDisableCnatSNAT(swIfIndex, ipFamily.IsIp6, podSpec.Status.NeedsSnat) if err != nil { return errors.Wrapf(err, "Error enabling/disabling %s snat", ipFamily.Str) } diff --git a/calico-vpp-agent/cni/network_vpp.go b/calico-vpp-agent/cni/network_vpp.go index 0aa0d579..9c93052b 100644 --- a/calico-vpp-agent/cni/network_vpp.go +++ b/calico-vpp-agent/cni/network_vpp.go @@ -59,9 +59,9 @@ func (s *Server) checkAvailableBuffers(podSpec *storage.LocalPodSpec) error { return nil } -func (s *Server) findPodVRFs(podSpec *storage.LocalPodSpec) bool { - podSpec.V4VrfId = types.InvalidID - podSpec.V6VrfId = types.InvalidID +func (s *Server) v4v6VrfsExistInVPP(podSpec *storage.LocalPodSpec) bool { + podSpec.Status.V4VrfId = types.InvalidID + podSpec.Status.V6VrfId = types.InvalidID vrfs, err := s.vpp.ListVRFs() if err != nil { @@ -75,30 +75,39 @@ func (s *Server) findPodVRFs(podSpec *storage.LocalPodSpec) bool { podSpec.SetVrfId(vrf.VrfID, ipFamily) } } - if podSpec.V4VrfId != types.InvalidID && podSpec.V6VrfId != types.InvalidID { + if podSpec.Status.V4VrfId != types.InvalidID && + podSpec.Status.V6VrfId != types.InvalidID { return true } } - if (podSpec.V4VrfId != types.InvalidID) != (podSpec.V6VrfId != types.InvalidID) { - s.log.Errorf("Partial VRF state v4=%d v6=%d key=%s", podSpec.V4VrfId, podSpec.V6VrfId, podSpec.Key()) + if (podSpec.Status.V4VrfId != types.InvalidID) != + (podSpec.Status.V6VrfId != types.InvalidID) { + s.log.Errorf("Partial VRF state v4=%d v6=%d key=%s", + podSpec.Status.V4VrfId, + podSpec.Status.V6VrfId, + podSpec.Key(), + ) } + // We do not have a VRF in VPP for this pod, VPP has probably + // restarted, so we clear the state we have. + podSpec.Status = storage.NewLocalPodSpecStatus() return false } -func (s *Server) removeConflictingContainers(newAddresses []storage.LocalIP, networkName string) { +func (s *Server) removeConflictingContainers(newAddresses []net.IP, networkName string) { addrMap := make(map[string]storage.LocalPodSpec) for _, podSpec := range s.podInterfaceMap { for _, addr := range podSpec.ContainerIps { if podSpec.NetworkName == networkName { - addrMap[addr.IP.String()] = podSpec + addrMap[addr.String()] = podSpec } } } podSpecsToDelete := make(map[string]storage.LocalPodSpec) for _, newAddr := range newAddresses { - podSpec, found := addrMap[newAddr.IP.String()] + podSpec, found := addrMap[newAddr.String()] if found { s.log.Warnf("podSpec conflict newAddr=%s, podSpec=%s", newAddr, podSpec.String()) podSpecsToDelete[podSpec.Key()] = podSpec @@ -117,9 +126,9 @@ func (s *Server) removeConflictingContainers(newAddresses []storage.LocalIP, net // AddVppInterface performs the networking for the given config and IPAM result func (s *Server) AddVppInterface(podSpec *storage.LocalPodSpec, doHostSideConf bool) (tunTapSwIfIndex uint32, err error) { - podSpec.NeedsSnat = false + podSpec.Status.NeedsSnat = false for _, containerIP := range podSpec.GetContainerIps() { - podSpec.NeedsSnat = podSpec.NeedsSnat || s.policyServerIpam.IPNetNeedsSNAT(containerIP) + podSpec.Status.NeedsSnat = podSpec.Status.NeedsSnat || s.policyServerIpam.IPNetNeedsSNAT(containerIP) } err = ns.IsNSorErr(podSpec.NetnsName) @@ -135,23 +144,20 @@ func (s *Server) AddVppInterface(podSpec *storage.LocalPodSpec, doHostSideConf b return vpplink.InvalidID, errors.Errorf("network %s does not exist", podSpec.NetworkName) } } - /** - * Check if the VRFs already exist in VPP, - * if yes we postulate the pod is already well setup - */ - if s.findPodVRFs(podSpec) { + + // Check if the VRFs already exist in VPP, + // if yes we postulate the pod is already well setup + if s.v4v6VrfsExistInVPP(podSpec) { s.log.Infof("VRF already exists in VPP podSpec=%s", podSpec.Key()) - return podSpec.TunTapSwIfIndex, nil + return podSpec.Status.TunTapSwIfIndex, nil } - /** - * Do we already have a pod with this address in VPP ? - * in this case, clean it up otherwise on the other pod's - * deletion our route in the main VRF will be removed - * - * As we did not find the VRF in VPP, we shouldn't find - * ourselves in s.podInterfaceMap - */ + // Do we already have a pod with this address in VPP ? + // in this case, clean it up otherwise on the other pod's + // deletion our route in the main VRF will be removed + // + // As we did not find the VRF in VPP, we shouldn't find + // ourselves in s.podInterfaceMap s.removeConflictingContainers(podSpec.ContainerIps, podSpec.NetworkName) var swIfIndex uint32 var isL3 bool @@ -201,7 +207,7 @@ func (s *Server) AddVppInterface(podSpec *storage.LocalPodSpec, doHostSideConf b /* Routes */ if podSpec.EnableVCL { s.log.Infof("pod(add) Punt routes") - err = s.SetupPuntRoutes(podSpec, stack, podSpec.TunTapSwIfIndex) + err = s.SetupPuntRoutes(podSpec, stack, podSpec.Status.TunTapSwIfIndex) if err != nil { goto err } @@ -269,7 +275,7 @@ func (s *Server) AddVppInterface(podSpec *storage.LocalPodSpec, doHostSideConf b New: podSpec, }) if podSpec.NetworkName != "" && podSpec.EnableMemif { - return podSpec.MemifSwIfIndex, err + return podSpec.Status.MemifSwIfIndex, err } s.log.Infof("pod(add) activate strict RPF on interface") @@ -278,7 +284,7 @@ func (s *Server) AddVppInterface(podSpec *storage.LocalPodSpec, doHostSideConf b s.log.Errorf("failed to activate rpf strict on interface : %s", err) goto err } - return podSpec.TunTapSwIfIndex, err + return podSpec.Status.TunTapSwIfIndex, err err: s.log.Errorf("Error, try a cleanup %+v", err) @@ -290,7 +296,7 @@ err: // CleanUpVPPNamespace deletes the devices in the network namespace. func (s *Server) DelVppInterface(podSpec *storage.LocalPodSpec) { if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpec.NetworkName == "" { - err := s.DelRedirectToHostOnInterface(podSpec.TunTapSwIfIndex) + err := s.DelRedirectToHostOnInterface(podSpec.Status.TunTapSwIfIndex) if err != nil { s.log.Error(err) } @@ -301,8 +307,7 @@ func (s *Server) DelVppInterface(podSpec *storage.LocalPodSpec) { return } - /* At least one VRF does not exist in VPP, still try removing */ - if !s.findPodVRFs(podSpec) { + if !s.v4v6VrfsExistInVPP(podSpec) { s.log.Warnf("pod(del) VRF for netns '%s' doesn't exist, skipping", podSpec.NetnsName) return } @@ -335,11 +340,11 @@ func (s *Server) DelVppInterface(podSpec *storage.LocalPodSpec) { /* Routes */ if podSpec.EnableVCL { - if podSpec.TunTapSwIfIndex != vpplink.InvalidID { + if podSpec.Status.TunTapSwIfIndex != vpplink.InvalidID { s.log.Infof("pod(del) routes to podVRF") s.DeleteVRFRoutesToPod(podSpec) s.log.Infof("pod(del) punt routes") - s.RemovePuntRoutes(podSpec, podSpec.TunTapSwIfIndex) + s.RemovePuntRoutes(podSpec, podSpec.Status.TunTapSwIfIndex) } } else { pblswIfIndex, _ := podSpec.GetParamsForIfType(podSpec.PortFilteredIfType) diff --git a/calico-vpp-agent/cni/network_vpp_hostports.go b/calico-vpp-agent/cni/network_vpp_hostports.go index c4eb2051..d0ebb385 100644 --- a/calico-vpp-agent/cni/network_vpp_hostports.go +++ b/calico-vpp-agent/cni/network_vpp_hostports.go @@ -24,7 +24,7 @@ import ( func (s *Server) AddHostPort(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack) error { for idx, hostPort := range podSpec.HostPorts { for _, containerAddr := range podSpec.ContainerIps { - if !vpplink.AddrFamilyDiffers(containerAddr.IP, hostPort.HostIP) { + if !vpplink.AddrFamilyDiffers(containerAddr, hostPort.HostIP) { continue } entry := &types.CnatTranslateEntry{ @@ -35,7 +35,7 @@ func (s *Server) AddHostPort(podSpec *storage.LocalPodSpec, stack *vpplink.Clean Backends: []types.CnatEndpointTuple{{ DstEndpoint: types.CnatEndpoint{ Port: hostPort.ContainerPort, - IP: containerAddr.IP, + IP: containerAddr, }, }}, IsRealIP: true, diff --git a/calico-vpp-agent/cni/network_vpp_multinet.go b/calico-vpp-agent/cni/network_vpp_multinet.go deleted file mode 100644 index 219b3cde..00000000 --- a/calico-vpp-agent/cni/network_vpp_multinet.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (C) 2019 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cni - -import "strings" - -func isMemif(ifName string) bool { - return strings.HasPrefix(ifName, "memif") -} diff --git a/calico-vpp-agent/cni/network_vpp_routes.go b/calico-vpp-agent/cni/network_vpp_routes.go index 69c31c6b..ea908bd7 100644 --- a/calico-vpp-agent/cni/network_vpp_routes.go +++ b/calico-vpp-agent/cni/network_vpp_routes.go @@ -112,12 +112,12 @@ func (s *Server) UnroutePodInterface(podSpec *storage.LocalPodSpec, swIfIndex ui } func (s *Server) RoutePblPortsPodInterface(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32, isL3 bool) (err error) { - for _, containerIP := range podSpec.GetContainerIps() { + for _, containerIP := range podSpec.ContainerIps { path := types.RoutePath{ SwIfIndex: swIfIndex, } if !isL3 { - path.Gw = containerIP.IP + path.Gw = containerIP } portRanges := make([]types.PblPortRange, 0) @@ -132,8 +132,8 @@ func (s *Server) RoutePblPortsPodInterface(podSpec *storage.LocalPodSpec, stack // See docs/_static/calico_vpp_vrf_layout.drawio client := types.PblClient{ ID: vpplink.InvalidID, - TableId: podSpec.GetVrfId(vpplink.IpFamilyFromIPNet(containerIP)), - Addr: containerIP.IP, + TableId: podSpec.GetVrfId(vpplink.IpFamilyFromIP(containerIP)), + Addr: containerIP, Path: path, PortRanges: portRanges, } @@ -141,25 +141,25 @@ func (s *Server) RoutePblPortsPodInterface(podSpec *storage.LocalPodSpec, stack client.TableId = common.PuntTableId } - vrfId := podSpec.GetVrfId(vpplink.IpFamilyFromIPNet(containerIP)) // pbl only supports v4 ? - s.log.Infof("pod(add) PBL client for %s VRF %d", containerIP.IP, vrfId) + vrfId := podSpec.GetVrfId(vpplink.IpFamilyFromIP(containerIP)) // pbl only supports v4 ? + s.log.Infof("pod(add) PBL client for %s VRF %d", containerIP, vrfId) pblIndex, err := s.vpp.AddPblClient(&client) if err != nil { - return errors.Wrapf(err, "error adding PBL client for %s VRF %d", containerIP.IP, vrfId) + return errors.Wrapf(err, "error adding PBL client for %s VRF %d", containerIP, vrfId) } else { stack.Push(s.vpp.DelPblClient, pblIndex) } - podSpec.PblIndexes = append(podSpec.PblIndexes, pblIndex) + podSpec.Status.PblIndexes[containerIP.String()] = pblIndex if !isL3 { - s.log.Infof("pod(add) neighbor if[%d] %s", swIfIndex, containerIP.IP.String()) + s.log.Infof("pod(add) neighbor if[%d] %s", swIfIndex, containerIP.String()) err = s.vpp.AddNeighbor(&types.Neighbor{ SwIfIndex: swIfIndex, - IP: containerIP.IP, + IP: containerIP, HardwareAddr: common.ContainerSideMacAddress, }) if err != nil { - return errors.Wrapf(err, "Cannot add neighbor if[%d] %s", swIfIndex, containerIP.IP.String()) + return errors.Wrapf(err, "Cannot add neighbor if[%d] %s", swIfIndex, containerIP.String()) } } } @@ -167,7 +167,7 @@ func (s *Server) RoutePblPortsPodInterface(podSpec *storage.LocalPodSpec, stack } func (s *Server) UnroutePblPortsPodInterface(podSpec *storage.LocalPodSpec, swIfIndex uint32) { - for _, pblIndex := range podSpec.PblIndexes { + for _, pblIndex := range podSpec.Status.PblIndexes { s.log.Infof("pod(del) PBL client[%d]", pblIndex) err := s.vpp.DelPblClient(pblIndex) if err != nil { @@ -249,11 +249,13 @@ func (s *Server) ActivateStrictRPF(podSpec *storage.LocalPodSpec, stack *vpplink return errors.Wrapf(err, "failed to add routes for RPF VRF") } s.log.Infof("pod(add) set custom-vrf urpf") - err = s.vpp.SetCustomURPF(podSpec.TunTapSwIfIndex, podSpec.V4RPFVrfId) - if err != nil { - return errors.Wrapf(err, "failed to set urpf strict on interface") - } else { - stack.Push(s.vpp.UnsetURPF, podSpec.TunTapSwIfIndex) + for _, ipFamily := range vpplink.IpFamilies { + err = s.vpp.SetCustomURPF(podSpec.Status.TunTapSwIfIndex, podSpec.GetVrfId(ipFamily), ipFamily) + if err != nil { + return errors.Wrapf(err, "failed to set urpf strict on interface") + } else { + stack.Push(s.vpp.UnsetURPF, podSpec.Status.TunTapSwIfIndex, ipFamily) + } } return nil } @@ -263,13 +265,13 @@ func (s *Server) AddRPFRoutes(podSpec *storage.LocalPodSpec, stack *vpplink.Clea RPFvrfId := podSpec.GetRPFVrfId(vpplink.IpFamilyFromIPNet(containerIP)) // Always there (except multinet memif) pathsToPod := []types.RoutePath{{ - SwIfIndex: podSpec.TunTapSwIfIndex, + SwIfIndex: podSpec.Status.TunTapSwIfIndex, Gw: containerIP.IP, }} // Add pbl memif case - if podSpec.MemifSwIfIndex != vpplink.INVALID_SW_IF_INDEX { + if podSpec.Status.MemifSwIfIndex != vpplink.INVALID_SW_IF_INDEX { pathsToPod = append(pathsToPod, types.RoutePath{ - SwIfIndex: podSpec.MemifSwIfIndex, + SwIfIndex: podSpec.Status.MemifSwIfIndex, Gw: containerIP.IP, }) s.log.Infof("pod(add) add route to %+v in rpfvrf %+v via memif and tun", podSpec.GetContainerIps(), RPFvrfId) @@ -289,25 +291,18 @@ func (s *Server) AddRPFRoutes(podSpec *storage.LocalPodSpec, stack *vpplink.Clea } // Add addresses allowed to be spooofed - if podSpec.AllowedSpoofingPrefixes != "" { - // Parse Annotation data - allowedSources, err := s.ParseSpoofAddressAnnotation(podSpec.AllowedSpoofingPrefixes) - if err != nil { - return errors.Wrapf(err, "error parsing allowSpoofing addresses") + for _, allowedSource := range podSpec.AllowedSpoofingSources { + s.log.Infof("pod(add) add route to %+v in rpfvrf %+v to allow spoofing", allowedSource, RPFvrfId) + route := &types.Route{ + Dst: &allowedSource, + Paths: pathsToPod, + Table: RPFvrfId, } - for _, allowedSource := range allowedSources { - s.log.Infof("pod(add) add route to %+v in rpfvrf %+v to allow spoofing", allowedSource.IPNet, RPFvrfId) - route := &types.Route{ - Dst: &allowedSource.IPNet, - Paths: pathsToPod, - Table: RPFvrfId, - } - err = s.vpp.RouteAdd(route) - if err != nil { - return errors.Wrapf(err, "error adding RPFVRF %d proper route", RPFvrfId) - } else { - stack.Push(s.vpp.RouteDel, route) - } + err = s.vpp.RouteAdd(route) + if err != nil { + return errors.Wrapf(err, "error adding RPFVRF %d proper route", RPFvrfId) + } else { + stack.Push(s.vpp.RouteDel, route) } } } @@ -320,13 +315,13 @@ func (s *Server) DeactivateStrictRPF(podSpec *storage.LocalPodSpec) { RPFvrfId := podSpec.GetRPFVrfId(vpplink.IpFamilyFromIPNet(containerIP)) // Always there (except multinet memif) pathsToPod := []types.RoutePath{{ - SwIfIndex: podSpec.TunTapSwIfIndex, + SwIfIndex: podSpec.Status.TunTapSwIfIndex, Gw: containerIP.IP, }} // pbl memif case - if podSpec.MemifSwIfIndex != vpplink.INVALID_SW_IF_INDEX { + if podSpec.Status.MemifSwIfIndex != vpplink.INVALID_SW_IF_INDEX { pathsToPod = append(pathsToPod, types.RoutePath{ - SwIfIndex: podSpec.MemifSwIfIndex, + SwIfIndex: podSpec.Status.MemifSwIfIndex, Gw: containerIP.IP, }) s.log.Infof("pod(del) del route to %+v in rpfvrf %+v via memif and tun", podSpec.GetContainerIps(), RPFvrfId) @@ -343,22 +338,15 @@ func (s *Server) DeactivateStrictRPF(podSpec *storage.LocalPodSpec) { } // Delete addresses allowed to be spooofed - if podSpec.AllowedSpoofingPrefixes != "" { - // Parse Annotation data - allowedSources, err := s.ParseSpoofAddressAnnotation(podSpec.AllowedSpoofingPrefixes) + for _, allowedSource := range podSpec.AllowedSpoofingSources { + s.log.Infof("pod(del) del route to %+v in rpfvrf %+v used to allow spoofing", allowedSource, RPFvrfId) + err = s.vpp.RouteDel(&types.Route{ + Dst: &allowedSource, + Paths: pathsToPod, + Table: RPFvrfId, + }) if err != nil { - s.log.WithError(err).Error("error parsing allowSpoofing addresses") - } - for _, allowedSource := range allowedSources { - s.log.Infof("pod(del) del route to %+v in rpfvrf %+v used to allow spoofing", allowedSource.IPNet, RPFvrfId) - err = s.vpp.RouteDel(&types.Route{ - Dst: &allowedSource.IPNet, - Paths: pathsToPod, - Table: RPFvrfId, - }) - if err != nil { - s.log.Errorf("error deleting VRF %d route: %s", RPFvrfId, err) - } + s.log.Errorf("error deleting VRF %d route: %s", RPFvrfId, err) } } } diff --git a/calico-vpp-agent/cni/pod_interface/common.go b/calico-vpp-agent/cni/pod_interface/common.go index 04cc794e..caa9492a 100644 --- a/calico-vpp-agent/cni/pod_interface/common.go +++ b/calico-vpp-agent/cni/pod_interface/common.go @@ -86,7 +86,7 @@ func (i *PodInterfaceDriverData) UndoPodIfNatConfiguration(swIfIndex uint32) { } func (i *PodInterfaceDriverData) DoPodIfNatConfiguration(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32) (err error) { - if podSpec.NeedsSnat { + if podSpec.Status.NeedsSnat { i.log.Infof("pod(add) Enable interface[%d] SNAT", swIfIndex) for _, ipFamily := range vpplink.IpFamilies { err = i.vpp.EnableDisableCnatSNAT(swIfIndex, ipFamily.IsIp6, true /*isEnable*/) @@ -152,7 +152,7 @@ func (i *PodInterfaceDriverData) DoPodInterfaceConfiguration(podSpec *storage.Lo return errors.Wrapf(err, "error SetInterfaceRxMode on pod if interface") } - err = i.vpp.InterfaceSetUnnumbered(swIfIndex, podSpec.LoopbackSwIfIndex) + err = i.vpp.InterfaceSetUnnumbered(swIfIndex, podSpec.Status.LoopbackSwIfIndex) if err != nil { return errors.Wrapf(err, "error setting interface unnumbered") } diff --git a/calico-vpp-agent/cni/pod_interface/loopback.go b/calico-vpp-agent/cni/pod_interface/loopback.go index 32445d08..48980a46 100644 --- a/calico-vpp-agent/cni/pod_interface/loopback.go +++ b/calico-vpp-agent/cni/pod_interface/loopback.go @@ -43,7 +43,7 @@ func (i *LoopbackPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSp } else { stack.Push(i.vpp.DeleteLoopback, swIfIndex) } - podSpec.LoopbackSwIfIndex = swIfIndex + podSpec.Status.LoopbackSwIfIndex = swIfIndex for _, ipFamily := range vpplink.IpFamilies { vrfId := podSpec.GetVrfId(ipFamily) @@ -53,7 +53,7 @@ func (i *LoopbackPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSp } } - err = i.DoPodIfNatConfiguration(podSpec, stack, podSpec.LoopbackSwIfIndex) + err = i.DoPodIfNatConfiguration(podSpec, stack, podSpec.Status.LoopbackSwIfIndex) if err != nil { return err } @@ -69,9 +69,9 @@ func (i *LoopbackPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSp } func (i *LoopbackPodInterfaceDriver) DeleteInterface(podSpec *storage.LocalPodSpec) { - i.UndoPodIfNatConfiguration(podSpec.LoopbackSwIfIndex) + i.UndoPodIfNatConfiguration(podSpec.Status.LoopbackSwIfIndex) - err := i.vpp.DeleteLoopback(podSpec.LoopbackSwIfIndex) + err := i.vpp.DeleteLoopback(podSpec.Status.LoopbackSwIfIndex) if err != nil { i.log.Errorf("Error deleting Loopback %s", err) } diff --git a/calico-vpp-agent/cni/pod_interface/memif.go b/calico-vpp-agent/cni/pod_interface/memif.go index 791066e5..b900431f 100644 --- a/calico-vpp-agent/cni/pod_interface/memif.go +++ b/calico-vpp-agent/cni/pod_interface/memif.go @@ -64,7 +64,7 @@ func (i *MemifPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSpec, } else { stack.Push(i.vpp.DelMemifSocketFileName, socketId) } - podSpec.MemifSocketId = socketId + podSpec.Status.MemifSocketId = socketId var usedIfSpec config.InterfaceSpec if podSpec.NetworkName == "" { //PBL case @@ -91,7 +91,7 @@ func (i *MemifPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSpec, } else { stack.Push(i.vpp.DeleteMemif, memif.SwIfIndex) } - podSpec.MemifSwIfIndex = memif.SwIfIndex + podSpec.Status.MemifSwIfIndex = memif.SwIfIndex watcher, err := i.vpp.WatchInterfaceEvents(memif.SwIfIndex) if err != nil { @@ -158,7 +158,7 @@ func (i *MemifPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSpec, } else { stack.Push(i.deleteDummy, podSpec.NetnsName, podSpec.InterfaceName) } - err = ns.WithNetNSPath(podSpec.NetnsName, i.configureDummy(podSpec.MemifSwIfIndex, podSpec)) + err = ns.WithNetNSPath(podSpec.NetnsName, i.configureDummy(podSpec.Status.MemifSwIfIndex, podSpec)) if err != nil { return errors.Wrapf(err, "Error in linux NS config") } @@ -169,26 +169,26 @@ func (i *MemifPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSpec, } func (i *MemifPodInterfaceDriver) DeleteInterface(podSpec *storage.LocalPodSpec) { - if podSpec.MemifSwIfIndex == vpplink.InvalidID { + if podSpec.Status.MemifSwIfIndex == vpplink.InvalidID { return } - i.UndoPodInterfaceConfiguration(podSpec.MemifSwIfIndex) - i.UndoPodIfNatConfiguration(podSpec.MemifSwIfIndex) + i.UndoPodInterfaceConfiguration(podSpec.Status.MemifSwIfIndex) + i.UndoPodIfNatConfiguration(podSpec.Status.MemifSwIfIndex) - err := i.vpp.DeleteMemif(podSpec.MemifSwIfIndex) + err := i.vpp.DeleteMemif(podSpec.Status.MemifSwIfIndex) if err != nil { - i.log.Warnf("Error deleting memif[%d] %s", podSpec.MemifSwIfIndex, err) + i.log.Warnf("Error deleting memif[%d] %s", podSpec.Status.MemifSwIfIndex, err) } - if podSpec.MemifSocketId != 0 { - err = i.vpp.DelMemifSocketFileName(podSpec.MemifSocketId) + if podSpec.Status.MemifSocketId != 0 { + err = i.vpp.DelMemifSocketFileName(podSpec.Status.MemifSocketId) if err != nil { - i.log.Warnf("Error deleting memif[%d] socket[%d] %s", podSpec.MemifSwIfIndex, podSpec.MemifSocketId, err) + i.log.Warnf("Error deleting memif[%d] socket[%d] %s", podSpec.Status.MemifSwIfIndex, podSpec.Status.MemifSocketId, err) } } - i.log.Infof("pod(del) memif swIfIndex=%d", podSpec.MemifSwIfIndex) + i.log.Infof("pod(del) memif swIfIndex=%d", podSpec.Status.MemifSwIfIndex) } @@ -215,7 +215,7 @@ func (i *MemifPodInterfaceDriver) configureDummy(swIfIndex uint32, podSpec *stor } } - for _, route := range podSpec.GetRoutes() { + for _, route := range podSpec.Routes { isV6 := route.IP.To4() == nil if (isV6 && !hasv6) || (!isV6 && !hasv4) { i.log.Infof("Skipping dummy[%d] route for %s", swIfIndex, route.String()) @@ -225,7 +225,7 @@ func (i *MemifPodInterfaceDriver) configureDummy(swIfIndex uint32, podSpec *stor err = netlink.RouteAdd(&netlink.Route{ LinkIndex: contDummy.Attrs().Index, Scope: netlink.SCOPE_UNIVERSE, - Dst: route, + Dst: &route, }) if err != nil { // TODO : in ipv6 '::' already exists diff --git a/calico-vpp-agent/cni/pod_interface/tuntap.go b/calico-vpp-agent/cni/pod_interface/tuntap.go index 875b5fd4..91f5f30a 100644 --- a/calico-vpp-agent/cni/pod_interface/tuntap.go +++ b/calico-vpp-agent/cni/pod_interface/tuntap.go @@ -170,7 +170,7 @@ func (i *TunTapPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSpec return err } - podSpec.TunTapSwIfIndex = swIfIndex + podSpec.Status.TunTapSwIfIndex = swIfIndex i.log.Infof("pod(add) tun swIfIndex=%d", swIfIndex) err = i.DoPodIfNatConfiguration(podSpec, stack, swIfIndex) @@ -197,19 +197,19 @@ func (i *TunTapPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSpec } func (i *TunTapPodInterfaceDriver) DeleteInterface(podSpec *storage.LocalPodSpec) { - if podSpec.TunTapSwIfIndex == vpplink.InvalidID { + if podSpec.Status.TunTapSwIfIndex == vpplink.InvalidID { return } i.unconfigureLinux(podSpec) - i.UndoPodInterfaceConfiguration(podSpec.TunTapSwIfIndex) - i.UndoPodIfNatConfiguration(podSpec.TunTapSwIfIndex) + i.UndoPodInterfaceConfiguration(podSpec.Status.TunTapSwIfIndex) + i.UndoPodIfNatConfiguration(podSpec.Status.TunTapSwIfIndex) - err := i.vpp.DelTap(podSpec.TunTapSwIfIndex) + err := i.vpp.DelTap(podSpec.Status.TunTapSwIfIndex) if err != nil { - i.log.Warnf("Error deleting tun[%d] %s", podSpec.TunTapSwIfIndex, err) + i.log.Warnf("Error deleting tun[%d] %s", podSpec.Status.TunTapSwIfIndex, err) } - i.log.Infof("pod(del) tun swIfIndex=%d", podSpec.TunTapSwIfIndex) + i.log.Infof("pod(del) tun swIfIndex=%d", podSpec.Status.TunTapSwIfIndex) } func (i *TunTapPodInterfaceDriver) configureLinux(podSpec *storage.LocalPodSpec, swIfIndex uint32) error { @@ -317,7 +317,7 @@ func (i *TunTapPodInterfaceDriver) configureNamespaceSideTun(swIfIndex uint32, p } } - for _, route := range podSpec.GetRoutes() { + for _, route := range podSpec.Routes { isV6 := route.IP.To4() == nil if (isV6 && !hasv6) || (!isV6 && !hasv4) { i.log.Infof("pod(add) Skipping tun swIfIndex=%d route=%s", swIfIndex, route.String()) @@ -327,7 +327,7 @@ func (i *TunTapPodInterfaceDriver) configureNamespaceSideTun(swIfIndex uint32, p err = netlink.RouteAdd(&netlink.Route{ LinkIndex: contTun.Attrs().Index, Scope: netlink.SCOPE_UNIVERSE, - Dst: route, + Dst: &route, }) if err != nil { // TODO : in ipv6 '::' already exists diff --git a/calico-vpp-agent/cni/pod_interface/vcl.go b/calico-vpp-agent/cni/pod_interface/vcl.go index 1d5c7065..f10cd15c 100644 --- a/calico-vpp-agent/cni/pod_interface/vcl.go +++ b/calico-vpp-agent/cni/pod_interface/vcl.go @@ -30,7 +30,8 @@ type VclPodInterfaceDriver struct { } func getPodAppNamespaceName(podSpec *storage.LocalPodSpec) string { - return fmt.Sprintf("app-ns-%s", podSpec.Key()) + podSpecKey := storage.LocalPodSpecKey(podSpec.NetnsName, podSpec.InterfaceName) + return fmt.Sprintf("app-ns-%s", podSpecKey) } func NewVclPodInterfaceDriver(vpp *vpplink.VppLink, log *logrus.Entry) *VclPodInterfaceDriver { @@ -60,7 +61,7 @@ func (i *VclPodInterfaceDriver) Init() (err error) { func (i *VclPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { appNamespace := &types.SessionAppNamespace{ NamespaceId: getPodAppNamespaceName(podSpec), - SwIfIndex: podSpec.LoopbackSwIfIndex, + SwIfIndex: podSpec.Status.LoopbackSwIfIndex, SocketName: fmt.Sprintf("abstract:%s,netns_name=%s", "vpp/session", podSpec.NetnsName), Secret: 0, } @@ -71,7 +72,7 @@ func (i *VclPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSpec, s stack.Push(i.vpp.DelSessionAppNamespace, appNamespace) } - err = i.vpp.InterfaceAdminUp(podSpec.LoopbackSwIfIndex) + err = i.vpp.InterfaceAdminUp(podSpec.Status.LoopbackSwIfIndex) if err != nil { return err } diff --git a/calico-vpp-agent/cni/pod_annotations.go b/calico-vpp-agent/cni/storage/pod_annotations.go similarity index 55% rename from calico-vpp-agent/cni/pod_annotations.go rename to calico-vpp-agent/cni/storage/pod_annotations.go index ba719c4f..2736e87a 100644 --- a/calico-vpp-agent/cni/pod_annotations.go +++ b/calico-vpp-agent/cni/storage/pod_annotations.go @@ -13,35 +13,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cni +package storage import ( "encoding/json" "fmt" + "net" "strconv" "strings" "github.com/pkg/errors" cnet "github.com/projectcalico/calico/libcalico-go/lib/net" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/storage" "github.com/projectcalico/vpp-dataplane/v3/config" - "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) -const ( - CalicoAnnotationPrefix string = "cni.projectcalico.org/" - VppAnnotationPrefix string = "cni.projectcalico.org/vpp" - MemifPortAnnotation string = "ExtraMemifPorts" - VclAnnotation string = "Vcl" - SpoofAnnotation string = "AllowedSourcePrefixes" - IfSpecAnnotation string = "InterfacesSpec" - IfSpecPBLAnnotation string = "ExtraMemifSpec" -) - -func (s *Server) ParsePortSpec(value string) (ifPortConfigs *storage.LocalIfPortConfigs, err error) { - ifPortConfigs = &storage.LocalIfPortConfigs{} +func parsePortSpec(value string) (ifPortConfigs *LocalIfPortConfigs, err error) { + ifPortConfigs = &LocalIfPortConfigs{} parts := strings.Split(value, ":") /* tcp:1234[-4567] */ if len(parts) != 2 { return nil, fmt.Errorf("Value should start with protocol e.g. 'tcp:'") @@ -73,15 +62,15 @@ func (s *Server) ParsePortSpec(value string) (ifPortConfigs *storage.LocalIfPort return ifPortConfigs, nil } -func (s *Server) ParsePortMappingAnnotation(podSpec *storage.LocalPodSpec, ifType storage.VppInterfaceType, value string) (err error) { - if podSpec.PortFilteredIfType != storage.VppIfTypeUnknown && podSpec.PortFilteredIfType != ifType { +func parsePortMappingAnnotation(podSpec *LocalPodSpec, ifType VppInterfaceType, value string) (err error) { + if podSpec.PortFilteredIfType != VppIfTypeUnknown && podSpec.PortFilteredIfType != ifType { return fmt.Errorf("Cannot use port filters on different interface type") } podSpec.PortFilteredIfType = ifType // value is expected to be like "tcp:1234-1236,udp:4456" portSpecs := strings.Split(value, ",") for idx, portSpec := range portSpecs { - ifPortConfig, err := s.ParsePortSpec(portSpec) + ifPortConfig, err := parsePortSpec(portSpec) if err != nil { return errors.Wrapf(err, "Error parsing portSpec[%d] %s", idx, portSpec) } @@ -90,15 +79,15 @@ func (s *Server) ParsePortMappingAnnotation(podSpec *storage.LocalPodSpec, ifTyp return nil } -func (s *Server) ParseDefaultIfType(podSpec *storage.LocalPodSpec, ifType storage.VppInterfaceType) (err error) { - if podSpec.DefaultIfType != storage.VppIfTypeUnknown && podSpec.DefaultIfType != ifType { +func parseDefaultIfType(podSpec *LocalPodSpec, ifType VppInterfaceType) (err error) { + if podSpec.DefaultIfType != VppIfTypeUnknown && podSpec.DefaultIfType != ifType { return fmt.Errorf("Cannot set two different default interface type") } podSpec.DefaultIfType = ifType return nil } -func (s *Server) ParseEnableDisableAnnotation(value string) (bool, error) { +func parseEnableDisableAnnotation(value string) (bool, error) { switch value { case "enable": return true, nil @@ -109,53 +98,40 @@ func (s *Server) ParseEnableDisableAnnotation(value string) (bool, error) { } } -func (s *Server) ParseSpoofAddressAnnotation(value string) ([]cnet.IPNet, error) { +func parseSpoofAddressAnnotation(value string) ([]net.IPNet, error) { var requestedSourcePrefixes []string - var allowedSources []cnet.IPNet + allowedSources := make([]net.IPNet, 0) err := json.Unmarshal([]byte(value), &requestedSourcePrefixes) if err != nil { return nil, errors.Errorf("failed to parse '%s' as JSON: %s", value, err) } for _, prefix := range requestedSourcePrefixes { - var ipn *cnet.IPNet - _, ipn, err = cnet.ParseCIDROrIP(prefix) + _, ipn, err := cnet.ParseCIDROrIP(prefix) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "Could not parse %s", prefix) } - allowedSources = append(allowedSources, *(ipn.Network())) + allowedSources = append(allowedSources, ipn.Network().IPNet) } return allowedSources, nil } -func GetDefaultIfSpec(isL3 bool) config.InterfaceSpec { - return config.InterfaceSpec{ - NumRxQueues: config.GetCalicoVppInterfaces().DefaultPodIfSpec.NumRxQueues, - NumTxQueues: config.GetCalicoVppInterfaces().DefaultPodIfSpec.NumTxQueues, - RxQueueSize: vpplink.DefaultIntTo(config.GetCalicoVppInterfaces().DefaultPodIfSpec.RxQueueSize, vpplink.DEFAULT_QUEUE_SIZE), - TxQueueSize: vpplink.DefaultIntTo(config.GetCalicoVppInterfaces().DefaultPodIfSpec.TxQueueSize, vpplink.DEFAULT_QUEUE_SIZE), - IsL3: &isL3, - } -} - -func (s *Server) ParsePodAnnotations(podSpec *storage.LocalPodSpec, annotations map[string]string) (err error) { +func parsePodAnnotations(podSpec *LocalPodSpec, annotations map[string]string) (err error) { for key, value := range annotations { - if key == CalicoAnnotationPrefix+SpoofAnnotation { - podSpec.AllowedSpoofingPrefixes = annotations[CalicoAnnotationPrefix+SpoofAnnotation] - } - if !strings.HasPrefix(key, VppAnnotationPrefix) { - continue - } switch key { - case VppAnnotationPrefix + IfSpecAnnotation: + case config.CalicoAnnotationPrefix + config.SpoofAnnotation: + podSpec.AllowedSpoofingSources, err = parseSpoofAddressAnnotation(value) + if err != nil { + return errors.Wrapf(err, "error parsing allowSpoofing addresses") + } + case config.VppAnnotationPrefix + config.IfSpecAnnotation: var ifSpecs map[string]config.InterfaceSpec err = json.Unmarshal([]byte(value), &ifSpecs) if err != nil { - s.log.Warnf("Error parsing key %s %s", key, err) + return fmt.Errorf("Error parsing key %s %s", key, err) } for _, ifSpec := range ifSpecs { if err := ifSpec.Validate(config.GetCalicoVppInterfaces().MaxPodIfSpec); err != nil { - s.log.Error("Pod interface config exceeds max config") - return err + return errors.Wrap(err, "Pod interface config exceeds max config") } } if ethSpec, found := ifSpecs[podSpec.InterfaceName]; found { @@ -164,35 +140,37 @@ func (s *Server) ParsePodAnnotations(podSpec *storage.LocalPodSpec, annotations podSpec.IfSpec.IsL3 = &isL3 } - case VppAnnotationPrefix + MemifPortAnnotation: + case config.VppAnnotationPrefix + config.MemifPortAnnotation: podSpec.EnableMemif = true - err = s.ParsePortMappingAnnotation(podSpec, storage.VppIfTypeMemif, value) + err = parsePortMappingAnnotation(podSpec, VppIfTypeMemif, value) if err != nil { return err } - err = s.ParseDefaultIfType(podSpec, storage.VppIfTypeTunTap) - case VppAnnotationPrefix + IfSpecPBLAnnotation: + err = parseDefaultIfType(podSpec, VppIfTypeTunTap) + if err != nil { + return errors.Wrapf(err, "Error parsing key %s", key) + } + case config.VppAnnotationPrefix + config.IfSpecPBLAnnotation: var ifSpec *config.InterfaceSpec err := json.Unmarshal([]byte(value), &ifSpec) if err != nil { - s.log.Warnf("Error parsing key %s %s", key, err) + return errors.Wrapf(err, "Error parsing key %s", key) } err = ifSpec.Validate(config.GetCalicoVppInterfaces().MaxPodIfSpec) if err != nil { - s.log.Error("PBL Memif interface config exceeds max config") - return err + return errors.Wrap(err, "PBL Memif interface config exceeds max config") } podSpec.PBLMemifSpec = *ifSpec isL3 := podSpec.PBLMemifSpec.GetIsL3(true) podSpec.PBLMemifSpec.IsL3 = &isL3 - case VppAnnotationPrefix + VclAnnotation: - podSpec.EnableVCL, err = s.ParseEnableDisableAnnotation(value) + case config.VppAnnotationPrefix + config.VclAnnotation: + podSpec.EnableVCL, err = parseEnableDisableAnnotation(value) + if err != nil { + return errors.Wrapf(err, "Error parsing key %s", key) + } default: continue } - if err != nil { - s.log.Warnf("Error parsing key %s %s", key, err) - } } - return nil + return err } diff --git a/calico-vpp-agent/cni/storage/storage.go b/calico-vpp-agent/cni/storage/storage.go index 28b3847f..0e65a742 100644 --- a/calico-vpp-agent/cni/storage/storage.go +++ b/calico-vpp-agent/cni/storage/storage.go @@ -16,17 +16,17 @@ package storage import ( - "bytes" "crypto/sha512" "encoding/base64" + "encoding/json" "fmt" "net" "os" "path/filepath" "strings" - "github.com/lunixbochs/struc" "github.com/pkg/errors" + cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/config" @@ -35,21 +35,13 @@ import ( ) const ( - CniServerStateFileVersion = 8 // Used to ensure compatibility wen we reload data + CniServerStateFileVersion = 9 // Used to ensure compatibility wen we reload data MaxApiTagLen = 63 /* No more than 64 characters in API tags */ VrfTagHashLen = 8 /* how many hash charatecters (b64) of the name in tag prefix (useful when trucated) */ ) -// XXX: Increment CniServerStateFileVersion when changing this struct -type LocalIPNet struct { - MaskSize int `struc:"int8,sizeof=Mask"` - IP net.IP `struc:"[16]byte"` - Mask net.IPMask -} - -// XXX: Increment CniServerStateFileVersion when changing this struct -type LocalIP struct { - IP net.IP `struc:"[16]byte"` +func isMemif(ifName string) bool { + return strings.HasPrefix(ifName, "memif") } type VppInterfaceType uint8 @@ -76,106 +68,55 @@ func (ift VppInterfaceType) String() string { } } -func (n *LocalIPNet) String() string { - ipnet := net.IPNet{ - IP: n.IP, - Mask: n.Mask, +func getHostEndpointProto(proto string) types.IPProto { + switch proto { + case "udp": + return types.UDP + case "sctp": + return types.SCTP + case "tcp": + return types.TCP + default: + return types.TCP } - return ipnet.String() } -func (n *LocalIP) String() string { - return n.IP.String() +func LocalPodSpecKey(netnsName, interfaceName string) string { + return fmt.Sprintf("netns:%s,if:%s", netnsName, interfaceName) } -func (n *LocalIPNet) UpdateSizes() { - n.MaskSize = len(n.Mask) +func (self *LocalPodSpec) Key() string { + return LocalPodSpecKey(self.NetnsName, self.InterfaceName) } -func (ps *LocalPodSpec) UpdateSizes() { - ps.RoutesSize = len(ps.Routes) - ps.ContainerIpsSize = len(ps.ContainerIps) - ps.InterfaceNameSize = len(ps.InterfaceName) - ps.NetnsNameSize = len(ps.NetnsName) - for _, n := range ps.Routes { - n.UpdateSizes() - } -} - -func (ps *LocalPodSpec) Key() string { - return fmt.Sprintf("netns:%s,if:%s", ps.NetnsName, ps.InterfaceName) -} - -func (ps *LocalPodSpec) String() string { - lst := ps.ContainerIps +func (self *LocalPodSpec) String() string { + lst := self.ContainerIps strLst := make([]string, 0, len(lst)) for _, e := range lst { strLst = append(strLst, e.String()) } - return fmt.Sprintf("%s [%s]", ps.Key(), strings.Join(strLst, ", ")) -} - -func (ps *LocalPodSpec) FullString() string { - containerIps := ps.ContainerIps - containerIpsLst := make([]string, 0, len(containerIps)) - for _, e := range containerIps { - containerIpsLst = append(containerIpsLst, e.String()) - } - routes := ps.Routes - routesLst := make([]string, 0, len(routes)) - for _, e := range routes { - routesLst = append(routesLst, e.String()) - } - pblIndexes := ps.PblIndexes - pblIndexesLst := make([]string, 0, len(pblIndexes)) - for _, e := range pblIndexes { - pblIndexesLst = append(pblIndexesLst, fmt.Sprint(e)) - } - s := fmt.Sprintf("InterfaceName: %s\n", ps.InterfaceName) - s += fmt.Sprintf("NetnsName: %s\n", ps.NetnsName) - s += fmt.Sprintf("AllowIpForwarding: %t\n", ps.AllowIpForwarding) - s += fmt.Sprintf("Routes: %s\n", strings.Join(routesLst, ", ")) - s += fmt.Sprintf("ContainerIps: %s\n", strings.Join(containerIpsLst, ", ")) - s += fmt.Sprintf("Mtu: %d\n", ps.Mtu) - s += fmt.Sprintf("OrchestratorID: %s\n", ps.OrchestratorID) - s += fmt.Sprintf("WorkloadID: %s\n", ps.WorkloadID) - s += fmt.Sprintf("EndpointID: %s\n", ps.EndpointID) - s += fmt.Sprintf("HostPorts: %s\n", types.StrableListToString("", ps.HostPorts)) - s += fmt.Sprintf("IfPortConfigs: %s\n", types.StrableListToString("", ps.IfPortConfigs)) - s += fmt.Sprintf("PortFilteredIfType: %s\n", ps.PortFilteredIfType.String()) - s += fmt.Sprintf("DefaultIfType: %s\n", ps.DefaultIfType.String()) - s += fmt.Sprintf("EnableVCL: %t\n", ps.EnableVCL) - s += fmt.Sprintf("EnableMemif: %t\n", ps.EnableMemif) - s += fmt.Sprintf("IsL3: %t\n", *ps.IfSpec.IsL3) - s += fmt.Sprintf("MemifSocketId: %d\n", ps.MemifSocketId) - s += fmt.Sprintf("TunTapSwIfIndex: %d\n", ps.TunTapSwIfIndex) - s += fmt.Sprintf("MemifSwIfIndex: %d\n", ps.MemifSwIfIndex) - s += fmt.Sprintf("LoopbackSwIfIndex: %d\n", ps.LoopbackSwIfIndex) - s += fmt.Sprintf("PblIndexes: %s\n", strings.Join(pblIndexesLst, ", ")) - s += fmt.Sprintf("V4VrfId: %d\n", ps.V4VrfId) - s += fmt.Sprintf("V6VrfId: %d\n", ps.V6VrfId) - return s + return fmt.Sprintf("%s [%s]", self.Key(), strings.Join(strLst, ", ")) } -func (ps *LocalPodSpec) GetParamsForIfType(ifType VppInterfaceType) (swIfIndex uint32, isL3 bool) { +func (self *LocalPodSpec) GetParamsForIfType(ifType VppInterfaceType) (swIfIndex uint32, isL3 bool) { switch ifType { case VppIfTypeTunTap: - return ps.TunTapSwIfIndex, *ps.IfSpec.IsL3 + return self.Status.TunTapSwIfIndex, *self.IfSpec.IsL3 case VppIfTypeMemif: if !*config.GetCalicoVppFeatureGates().MemifEnabled { return types.InvalidID, true } - return ps.MemifSwIfIndex, *ps.PBLMemifSpec.IsL3 + return self.Status.MemifSwIfIndex, *self.PBLMemifSpec.IsL3 default: return types.InvalidID, true } } -func (ps *LocalPodSpec) GetBuffersNeeded() uint64 { +func (self *LocalPodSpec) GetBuffersNeeded() uint64 { var buffersNeededForThisPod uint64 - buffersNeededForThisPod += ps.IfSpec.GetBuffersNeeded() - if ps.NetworkName == "" && ps.EnableMemif { - buffersNeededForThisPod += ps.PBLMemifSpec.GetBuffersNeeded() + buffersNeededForThisPod += self.IfSpec.GetBuffersNeeded() + if self.NetworkName == "" && self.EnableMemif { + buffersNeededForThisPod += self.PBLMemifSpec.GetBuffersNeeded() } return buffersNeededForThisPod } @@ -192,95 +133,231 @@ func (pc *LocalIfPortConfigs) String() string { } // XXX: Increment CniServerStateFileVersion when changing this struct -type LocalPodSpec struct { - InterfaceNameSize int `struc:"int16,sizeof=InterfaceName"` - InterfaceName string - NetnsNameSize int `struc:"int16,sizeof=NetnsName"` - NetnsName string - AllowIpForwarding bool - RoutesSize int `struc:"int16,sizeof=Routes"` - Routes []LocalIPNet - ContainerIpsSize int `struc:"int16,sizeof=ContainerIps"` - ContainerIps []LocalIP - Mtu int - - // Pod identifiers - OrchestratorIDSize int `struc:"int16,sizeof=OrchestratorID"` - OrchestratorID string - WorkloadIDSize int `struc:"int16,sizeof=WorkloadID"` - WorkloadID string - EndpointIDSize int `struc:"int16,sizeof=EndpointID"` - EndpointID string - // HostPort - HostPortsSize int `struc:"int16,sizeof=HostPorts"` - HostPorts []HostPortBinding - - IfPortConfigsLen int `struc:"int16,sizeof=IfPortConfigs"` - IfPortConfigs []LocalIfPortConfigs - /* This interface type will traffic MATCHING the portConfigs */ - PortFilteredIfType VppInterfaceType - /* This interface type will traffic not matching portConfigs */ - DefaultIfType VppInterfaceType - EnableVCL bool - EnableMemif bool - - IfSpec config.InterfaceSpec - PBLMemifSpec config.InterfaceSpec - - /** - * Below are VPP internal ids, mutable fields in AddVppInterface - * We persist them on the disk to avoid rescanning when the agent is restarting. - * - * We should be careful during state-reconciliation as they might not be - * valid anymore. VRF tags should provide this guarantee - */ - MemifSocketId uint32 - TunTapSwIfIndex uint32 - MemifSwIfIndex uint32 - LoopbackSwIfIndex uint32 - PblIndexesLen int `struc:"int16,sizeof=PblIndexes"` - PblIndexes []uint32 - - /** - * These fields are only a runtime cache, but we also store them - * on the disk for debugging purposes. - */ - V4VrfId uint32 - V6VrfId uint32 - NeedsSnat bool - - /* Multi net */ - NetworkNameSize int `struc:"int16,sizeof=NetworkName"` - NetworkName string - - /* rpf check */ - AllowedSpoofingPrefixesSize int `struc:"int16,sizeof=AllowedSpoofingPrefixes"` - AllowedSpoofingPrefixes string - - V4RPFVrfId uint32 - V6RPFVrfId uint32 +type HostPortBinding struct { + HostPort uint16 + HostIP net.IP + ContainerPort uint16 + EntryID uint32 + Protocol types.IPProto } -func (ps *LocalPodSpec) Copy() LocalPodSpec { - newPs := *ps +// LocalPodSpecStatus contains VPP internal ids, mutable fields in AddVppInterface +// We persist them on the disk to avoid rescanning when the agent is restarting. +// +// We should be careful during state-reconciliation as they might not be +// valid anymore. VRF tags should provide this guarantee +// +// +// These fields are only a runtime cache, but we also store them +// on the disk for debugging & gracefull restart. +type LocalPodSpecStatus struct { + MemifSocketId uint32 `json:"memifSocketId"` + MemifSwIfIndex uint32 `json:"memifSwIfIndex"` + TunTapSwIfIndex uint32 `json:"tunTapSwIfIndex"` + LoopbackSwIfIndex uint32 `json:"loopbackSwIfIndex"` + // PblIndexes is a map from containerIP to PBL index in VPP + PblIndexes map[string]uint32 `json:"pblIndexes"` + V4VrfId uint32 `json:"v4VrfId"` + V4RPFVrfId uint32 `json:"v4RPFVrfId"` + V6VrfId uint32 `json:"v6VrfId"` + V6RPFVrfId uint32 `json:"v6RPFVrfId"` + NeedsSnat bool `json:"needsSnat"` +} + +func NewLocalPodSpecStatus() LocalPodSpecStatus { + return LocalPodSpecStatus{ + MemifSocketId: vpplink.InvalidID, + MemifSwIfIndex: vpplink.InvalidID, + TunTapSwIfIndex: vpplink.InvalidID, + LoopbackSwIfIndex: vpplink.InvalidID, + PblIndexes: make(map[string]uint32), + V4VrfId: vpplink.InvalidID, + V4RPFVrfId: vpplink.InvalidID, + V6VrfId: vpplink.InvalidID, + V6RPFVrfId: vpplink.InvalidID, + } +} - newPs.Routes = append(make([]LocalIPNet, 0), ps.Routes...) - newPs.ContainerIps = append(make([]LocalIP, 0), ps.ContainerIps...) - newPs.HostPorts = append(make([]HostPortBinding, 0), ps.HostPorts...) - newPs.IfPortConfigs = append(make([]LocalIfPortConfigs, 0), ps.IfPortConfigs...) - newPs.PblIndexes = append(make([]uint32, 0), ps.PblIndexes...) +// LocalPodSpec represents the configuration and runtime status of +// a given pod & interface couple. It is persisted on disk to allow +// seemless restarts +// +// XXX: Increment CniServerStateFileVersion when changing this struct +type LocalPodSpec struct { + // Status is the runtime Status for this pod & interface couple + Status LocalPodSpecStatus + + // InterfaceName is the name of the interface this podSpec represents + InterfaceName string `json:"interfaceName"` + // NetnsName is the name of the netns mounted on the host + NetnsName string `json:"netnsName"` + // AllowIpForwarding controls whether we allow IP forwarding in the pod + AllowIpForwarding bool `json:"allowIpForwarding"` + // Routes are routes to be configured in the pod + Routes []net.IPNet `json:"routes"` + // ContainerIps are the IPs of the container (typically v4 and v6) + ContainerIps []net.IP `json:"containerIps"` + // Mtu is the MTU to configure in the pod on its interface + Mtu int `json:"mtu"` + // OrchestratorID is a calico/k8s identifier for this pod + OrchestratorID string `json:"orchestratorID"` + // WorkloadID is a calico/k8s identifier for this pod + WorkloadID string `json:"workloadID"` + // EndpointID is a calico/k8s identifier for this pod + EndpointID string `json:"endpointID"` + // HostPorts are the HostPorts configured for this Pod + HostPorts []HostPortBinding `json:"hostPorts"` + // IfPortConfigs specifies a 2-tuple based (port and protocol) set + // of rules allowing to split traffic between two interfaces, + // typically a memif and a tuntap + IfPortConfigs []LocalIfPortConfigs `json:"ifPortConfigs"` + // PortFilteredIfType is the interface type to which we will forward + // traffic MATCHING the portConfigs + PortFilteredIfType VppInterfaceType `json:"portFilteredIfType"` + // DefaultIfType is the interface type to which we will traffic + // not matching portConfigs + DefaultIfType VppInterfaceType `json:"defaultIfType"` + // EnableVCL tells whether the pod asked for VCL + EnableVCL bool `json:"enableVCL"` + // EnableMemif tells whether the pod asked for memif + EnableMemif bool `json:"enableMemif"` + + // IfSpec is the interface specification (rx queues, queue sizes,...) + IfSpec config.InterfaceSpec `json:"ifSpec"` + // PBLMemifSpec is the additional interface specification + // (rx queues, queue sizes,...) + PBLMemifSpec config.InterfaceSpec `json:"pblMemifSpec"` + + // AllowedSpoofingSources is the list of prefixes from which the pod is allowed + // to send traffic + AllowedSpoofingSources []net.IPNet `json:"allowedSpoofingPrefixes"` + + // NetworkName contains the name of the network this podSpec belongs + // to. Keeping in mind that for multi net, PodSpec are duplicated for + // every interface the pod has. + // It is set to the empty string for multinet disabled and to represent + // the default network. + NetworkName string `json:"networkName"` +} + +func (self *LocalPodSpec) Copy() LocalPodSpec { + newPs := *self + newPs.Routes = append(make([]net.IPNet, 0), self.Routes...) + newPs.ContainerIps = append(make([]net.IP, 0), self.ContainerIps...) + newPs.HostPorts = append(make([]HostPortBinding, 0), self.HostPorts...) + newPs.IfPortConfigs = append(make([]LocalIfPortConfigs, 0), self.IfPortConfigs...) + newPs.AllowedSpoofingSources = append(make([]net.IPNet, 0), self.AllowedSpoofingSources...) + newPs.Status.PblIndexes = make(map[string]uint32) + for k, v := range self.Status.PblIndexes { + newPs.Status.PblIndexes[k] = v + } return newPs } -// XXX: Increment CniServerStateFileVersion when changing this struct -type HostPortBinding struct { - HostPort uint16 - HostIP net.IP `struc:"[16]byte"` - ContainerPort uint16 - EntryID uint32 - Protocol types.IPProto +func getDefaultIfSpec(isL3 bool) config.InterfaceSpec { + return config.InterfaceSpec{ + NumRxQueues: config.GetCalicoVppInterfaces().DefaultPodIfSpec.NumRxQueues, + NumTxQueues: config.GetCalicoVppInterfaces().DefaultPodIfSpec.NumTxQueues, + RxQueueSize: vpplink.DefaultIntTo( + config.GetCalicoVppInterfaces().DefaultPodIfSpec.RxQueueSize, + vpplink.DEFAULT_QUEUE_SIZE, + ), + TxQueueSize: vpplink.DefaultIntTo( + config.GetCalicoVppInterfaces().DefaultPodIfSpec.TxQueueSize, + vpplink.DEFAULT_QUEUE_SIZE, + ), + IsL3: &isL3, + } +} + +func NewLocalPodSpecFromAdd(request *cniproto.AddRequest, nodeBGPSpec *common.LocalNodeSpec) (*LocalPodSpec, error) { + podSpec := LocalPodSpec{ + InterfaceName: request.GetInterfaceName(), + NetnsName: request.GetNetns(), + AllowIpForwarding: request.GetSettings().GetAllowIpForwarding(), + Routes: make([]net.IPNet, 0), + ContainerIps: make([]net.IP, 0), + Mtu: int(request.GetSettings().GetMtu()), + + IfPortConfigs: make([]LocalIfPortConfigs, 0), + + OrchestratorID: request.Workload.Orchestrator, + WorkloadID: request.Workload.Namespace + "/" + request.Workload.Pod, + EndpointID: request.Workload.Endpoint, + HostPorts: make([]HostPortBinding, 0), + + /* defaults */ + IfSpec: getDefaultIfSpec(true /* isL3 */), + PBLMemifSpec: getDefaultIfSpec(false /* isL3 */), + + NetworkName: request.DataplaneOptions["network_name"], + Status: NewLocalPodSpecStatus(), + } + + if podSpec.NetworkName != "" { + if !*config.GetCalicoVppFeatureGates().MultinetEnabled { + return nil, fmt.Errorf("enable multinet in config for multiple networks") + } + if isMemif(podSpec.InterfaceName) { + if !*config.GetCalicoVppFeatureGates().MemifEnabled { + return nil, fmt.Errorf("enable memif in config for memif interfaces") + } + podSpec.EnableMemif = true + podSpec.DefaultIfType = VppIfTypeMemif + podSpec.IfSpec = getDefaultIfSpec(false) + } + } + + for _, port := range request.Workload.Ports { + if port.HostPort != 0 { + hostPortBinding := HostPortBinding{ + HostPort: uint16(port.HostPort), + HostIP: net.ParseIP(port.HostIp), + ContainerPort: uint16(port.Port), + } + _ = hostPortBinding.Protocol.UnmarshalText([]byte(port.Protocol)) + if hostPortBinding.HostIP == nil || hostPortBinding.HostIP.IsUnspecified() { + if nodeBGPSpec != nil && nodeBGPSpec.IPv4Address != nil { + // default to node IP + hostPortBinding.HostIP = net.ParseIP( + nodeBGPSpec.IPv4Address.IP.String(), + ) + } + } + podSpec.HostPorts = append(podSpec.HostPorts, hostPortBinding) + } + } + for _, routeStr := range request.GetContainerRoutes() { + _, route, err := net.ParseCIDR(routeStr) + if err != nil { + return nil, errors.Wrapf(err, "Cannot parse container route %s", routeStr) + } + podSpec.Routes = append(podSpec.Routes, *route) + } + for _, requestContainerIP := range request.GetContainerIps() { + containerIp, _, err := net.ParseCIDR(requestContainerIP.GetAddress()) + if err != nil { + return nil, fmt.Errorf("Cannot parse address: %s", requestContainerIP.GetAddress()) + } + // We ignore the prefix len set on the address, + // for a tun it doesn't make sense + podSpec.ContainerIps = append(podSpec.ContainerIps, containerIp) + } + workload := request.GetWorkload() + if workload != nil { + err := parsePodAnnotations(&podSpec, workload.Annotations) + if err != nil { + return nil, errors.Wrapf(err, "Cannot parse pod Annotations") + } + } + + if podSpec.DefaultIfType == VppIfTypeUnknown { + podSpec.DefaultIfType = VppIfTypeTunTap + } + + return &podSpec, nil } func (hp *HostPortBinding) String() string { @@ -303,45 +380,34 @@ func TruncateStr(text string, size int) string { return text } -func (ps *LocalPodSpec) GetVrfTag(ipFamily vpplink.IpFamily, custom string) string { - h := hash(fmt.Sprintf("%s%s%s%s", ipFamily.ShortStr, ps.NetnsName, ps.InterfaceName, custom)) - s := fmt.Sprintf("%s-%s-%s%s-%s", h, ipFamily.ShortStr, ps.InterfaceName, custom, filepath.Base(ps.NetnsName)) +func (self *LocalPodSpec) GetVrfTag(ipFamily vpplink.IpFamily, custom string) string { + h := hash(fmt.Sprintf("%s%s%s%s", ipFamily.ShortStr, self.NetnsName, self.InterfaceName, custom)) + s := fmt.Sprintf("%s-%s-%s%s-%s", h, ipFamily.ShortStr, self.InterfaceName, custom, filepath.Base(self.NetnsName)) return TruncateStr(s, MaxApiTagLen) } -func (ps *LocalPodSpec) GetInterfaceTag(prefix string) string { - h := hash(fmt.Sprintf("%s%s%s", prefix, ps.NetnsName, ps.InterfaceName)) - s := fmt.Sprintf("%s-%s-%s", h, ps.InterfaceName, filepath.Base(ps.NetnsName)) +func (self *LocalPodSpec) GetInterfaceTag(prefix string) string { + h := hash(fmt.Sprintf("%s%s%s", prefix, self.NetnsName, self.InterfaceName)) + s := fmt.Sprintf("%s-%s-%s", h, self.InterfaceName, filepath.Base(self.NetnsName)) return TruncateStr(s, MaxApiTagLen) } -func (ps *LocalPodSpec) GetRoutes() (routes []*net.IPNet) { - routes = make([]*net.IPNet, 0, len(ps.Routes)) - for _, r := range ps.Routes { - routes = append(routes, &net.IPNet{ - IP: r.IP, - Mask: r.Mask, - }) - } - return routes -} - -func (ps *LocalPodSpec) GetContainerIps() (containerIps []*net.IPNet) { - containerIps = make([]*net.IPNet, 0, len(ps.ContainerIps)) - for _, containerIp := range ps.ContainerIps { +func (self *LocalPodSpec) GetContainerIps() (containerIps []*net.IPNet) { + containerIps = make([]*net.IPNet, 0, len(self.ContainerIps)) + for _, containerIp := range self.ContainerIps { containerIps = append(containerIps, &net.IPNet{ - IP: containerIp.IP, - Mask: common.GetMaxCIDRMask(containerIp.IP), + IP: containerIp, + Mask: common.GetMaxCIDRMask(containerIp), }) } return containerIps } -func (ps *LocalPodSpec) Hasv46() (hasv4 bool, hasv6 bool) { +func (self *LocalPodSpec) Hasv46() (hasv4 bool, hasv6 bool) { hasv4 = false hasv6 = false - for _, containerIP := range ps.ContainerIps { - if containerIP.IP.To4() == nil { + for _, containerIP := range self.ContainerIps { + if containerIP.To4() == nil { hasv6 = true } else { hasv4 = true @@ -350,46 +416,45 @@ func (ps *LocalPodSpec) Hasv46() (hasv4 bool, hasv6 bool) { return hasv4, hasv6 } -func (ps *LocalPodSpec) GetVrfId(ipFamily vpplink.IpFamily) uint32 { +func (self *LocalPodSpec) GetVrfId(ipFamily vpplink.IpFamily) uint32 { if ipFamily.IsIp6 { - return ps.V6VrfId + return self.Status.V6VrfId } else { - return ps.V4VrfId + return self.Status.V4VrfId } } -func (ps *LocalPodSpec) GetRPFVrfId(ipFamily vpplink.IpFamily) uint32 { +func (self *LocalPodSpec) GetRPFVrfId(ipFamily vpplink.IpFamily) uint32 { if ipFamily.IsIp6 { - return ps.V6RPFVrfId + return self.Status.V6RPFVrfId } else { - return ps.V4RPFVrfId + return self.Status.V4RPFVrfId } } -func (ps *LocalPodSpec) SetVrfId(id uint32, ipFamily vpplink.IpFamily) { +func (self *LocalPodSpec) SetVrfId(id uint32, ipFamily vpplink.IpFamily) { if ipFamily.IsIp6 { - ps.V6VrfId = id + self.Status.V6VrfId = id } else { - ps.V4VrfId = id + self.Status.V4VrfId = id } } -func (ps *LocalPodSpec) SetRPFVrfId(id uint32, ipFamily vpplink.IpFamily) { +func (self *LocalPodSpec) SetRPFVrfId(id uint32, ipFamily vpplink.IpFamily) { if ipFamily.IsIp6 { - ps.V6RPFVrfId = id + self.Status.V6RPFVrfId = id } else { - ps.V4RPFVrfId = id + self.Status.V4RPFVrfId = id } } type SavedState struct { - Version int `struc:"int32"` - SpecsCount int `struc:"int32,sizeof=Specs"` - Specs []LocalPodSpec + Version int `json:"version"` + SpecsCount int `json:"specsCount"` + Specs []LocalPodSpec `json:"specs"` } func PersistCniServerState(podInterfaceMap map[string]LocalPodSpec, fname string) (err error) { - var buf bytes.Buffer tmpFile := fmt.Sprintf("%s~", fname) state := &SavedState{ Version: CniServerStateFileVersion, @@ -399,12 +464,12 @@ func PersistCniServerState(podInterfaceMap map[string]LocalPodSpec, fname string for _, podSpec := range podInterfaceMap { state.Specs = append(state.Specs, podSpec) } - err = struc.Pack(&buf, state) + data, err := json.Marshal(state) if err != nil { return errors.Wrap(err, "Error encoding pod data") } - err = os.WriteFile(tmpFile, buf.Bytes(), 0200) + err = os.WriteFile(tmpFile, data, 0200) if err != nil { return errors.Wrapf(err, "Error writing file %s", tmpFile) } @@ -416,19 +481,18 @@ func PersistCniServerState(podInterfaceMap map[string]LocalPodSpec, fname string } func LoadCniServerState(fname string) ([]LocalPodSpec, error) { - var state SavedState + state := &SavedState{} data, err := os.ReadFile(fname) if err != nil { if errors.Is(err, os.ErrNotExist) { - return nil, nil // No state to load + return nil, nil } else { return nil, errors.Wrapf(err, "Error reading file %s", fname) } } - buf := bytes.NewBuffer(data) - err = struc.Unpack(buf, &state) + err = json.Unmarshal(data, state) if err != nil { - return nil, errors.Wrapf(err, "Error unpacking") + return nil, errors.Wrapf(err, "Error unmarshaling json state") } if state.Version != CniServerStateFileVersion { // When adding new versions, we need to keep loading old versions or some pods diff --git a/calico-vpp-agent/common_tests/common_tests.go b/calico-vpp-agent/common_tests/common_tests.go index 8814569b..dc334561 100644 --- a/calico-vpp-agent/common_tests/common_tests.go +++ b/calico-vpp-agent/common_tests/common_tests.go @@ -265,17 +265,17 @@ func PodVRFs(podInterface, podNetNSName string, vpp *vpplink.VppLink) (vrf4ID, v podSpec.SetVrfId(vrf.VrfID, ipFamily) } } - if podSpec.V4VrfId != types.InvalidID && podSpec.V6VrfId != types.InvalidID { - return podSpec.V4VrfId, podSpec.V6VrfId, nil + if podSpec.Status.V4VrfId != types.InvalidID && podSpec.Status.V6VrfId != types.InvalidID { + return podSpec.Status.V4VrfId, podSpec.Status.V6VrfId, nil } } - if (podSpec.V4VrfId != types.InvalidID) != (podSpec.V6VrfId != types.InvalidID) { - return podSpec.V4VrfId, podSpec.V6VrfId, - fmt.Errorf("partial VRF state v4=%d v6=%d key=%s", podSpec.V4VrfId, podSpec.V6VrfId, podSpec.Key()) + if (podSpec.Status.V4VrfId != types.InvalidID) != (podSpec.Status.V6VrfId != types.InvalidID) { + return podSpec.Status.V4VrfId, podSpec.Status.V6VrfId, + fmt.Errorf("partial VRF state v4=%d v6=%d key=%s", podSpec.Status.V4VrfId, podSpec.Status.V6VrfId, podSpec.Key()) } - return podSpec.V4VrfId, podSpec.V6VrfId, fmt.Errorf("not VRFs state (key=%s)", podSpec.Key()) + return podSpec.Status.V4VrfId, podSpec.Status.V6VrfId, fmt.Errorf("not VRFs state (key=%s)", podSpec.Key()) } func IpFamilyIndex(ipFamily vpplink.IpFamily) int { diff --git a/calico-vpp-agent/policy/policy_server.go b/calico-vpp-agent/policy/policy_server.go index 64188dd6..9206b681 100644 --- a/calico-vpp-agent/policy/policy_server.go +++ b/calico-vpp-agent/policy/policy_server.go @@ -352,9 +352,9 @@ func (s *Server) handlePolicyServerEvents(evt common.CalicoVppEvent) error { if !ok { return fmt.Errorf("evt.New is not a (*storage.LocalPodSpec) %v", evt.New) } - swIfIndex := podSpec.TunTapSwIfIndex + swIfIndex := podSpec.Status.TunTapSwIfIndex if swIfIndex == vpplink.InvalidID { - swIfIndex = podSpec.MemifSwIfIndex + swIfIndex = podSpec.Status.MemifSwIfIndex } s.workloadAdded(&WorkloadEndpointID{ OrchestratorID: podSpec.OrchestratorID, diff --git a/calico-vpp-agent/prometheus/prometheus.go b/calico-vpp-agent/prometheus/prometheus.go index 3d81b382..ff850130 100644 --- a/calico-vpp-agent/prometheus/prometheus.go +++ b/calico-vpp-agent/prometheus/prometheus.go @@ -233,10 +233,10 @@ func (s *Server) ServePrometheus(t *tomb.Tomb) error { continue } s.lock.Lock() - if podSpec.TunTapSwIfIndex == vpplink.INVALID_SW_IF_INDEX { - s.podInterfacesBySwifIndex[podSpec.MemifSwIfIndex] = *podSpec + if podSpec.Status.TunTapSwIfIndex == vpplink.INVALID_SW_IF_INDEX { + s.podInterfacesBySwifIndex[podSpec.Status.MemifSwIfIndex] = *podSpec } else { - s.podInterfacesBySwifIndex[podSpec.TunTapSwIfIndex] = *podSpec + s.podInterfacesBySwifIndex[podSpec.Status.TunTapSwIfIndex] = *podSpec } s.podInterfacesByKey[podSpec.Key()] = *podSpec s.lock.Unlock() @@ -249,10 +249,10 @@ func (s *Server) ServePrometheus(t *tomb.Tomb) error { } initialPod := s.podInterfacesByKey[podSpec.Key()] delete(s.podInterfacesByKey, initialPod.Key()) - if podSpec.TunTapSwIfIndex == vpplink.INVALID_SW_IF_INDEX { - delete(s.podInterfacesBySwifIndex, initialPod.MemifSwIfIndex) + if podSpec.Status.TunTapSwIfIndex == vpplink.INVALID_SW_IF_INDEX { + delete(s.podInterfacesBySwifIndex, initialPod.Status.MemifSwIfIndex) } else { - delete(s.podInterfacesBySwifIndex, initialPod.TunTapSwIfIndex) + delete(s.podInterfacesBySwifIndex, initialPod.Status.TunTapSwIfIndex) } s.lock.Unlock() } diff --git a/calico-vpp-agent/services/service_server.go b/calico-vpp-agent/services/service_server.go index dafa8cc5..1b7dc2b4 100644 --- a/calico-vpp-agent/services/service_server.go +++ b/calico-vpp-agent/services/service_server.go @@ -40,12 +40,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) -const ( - KeepOriginalPacketAnnotation string = "KeepOriginalPacket" - HashConfigAnnotation string = "HashConfig" - LBTypeAnnotation string = "LBType" -) - /** * Service descriptions from the API are resolved into * slices of LocalService, this allows to diffs between @@ -96,7 +90,7 @@ func (s *Server) ParseServiceAnnotations(annotations map[string]string, name str svc := &serviceInfo{} for key, value := range annotations { switch key { - case cni.VppAnnotationPrefix + LBTypeAnnotation: + case config.VppAnnotationPrefix + config.LBTypeAnnotation: switch strings.ToLower(value) { case "ecmp": svc.lbType = lbTypeECMP @@ -108,7 +102,7 @@ func (s *Server) ParseServiceAnnotations(annotations map[string]string, name str svc.lbType = lbTypeECMP // default value err = append(err, errors.Errorf("Unknown value %s for key %s", value, key)) } - case cni.VppAnnotationPrefix + HashConfigAnnotation: + case config.VppAnnotationPrefix + config.HashConfigAnnotation: hashConfigList := strings.Split(strings.TrimSpace(value), ",") for _, hc := range hashConfigList { switch strings.TrimSpace(strings.ToLower(hc)) { @@ -130,7 +124,7 @@ func (s *Server) ParseServiceAnnotations(annotations map[string]string, name str err = append(err, errors.Errorf("Unknown value %s for key %s", value, key)) } } - case cni.VppAnnotationPrefix + KeepOriginalPacketAnnotation: + case config.VppAnnotationPrefix + config.KeepOriginalPacketAnnotation: var err1 error svc.keepOriginalPacket, err1 = strconv.ParseBool(value) if err1 != nil { diff --git a/config/config.go b/config/config.go index 3fcda0ca..7d5e36b9 100644 --- a/config/config.go +++ b/config/config.go @@ -65,6 +65,20 @@ const ( BaseVppSideHardwareAddress = "02:ca:11:c0:fd:00" ) +const ( + CalicoAnnotationPrefix string = "cni.projectcalico.org/" + VppAnnotationPrefix string = "cni.projectcalico.org/vpp" + MemifPortAnnotation string = "ExtraMemifPorts" + VclAnnotation string = "Vcl" + SpoofAnnotation string = "AllowedSourcePrefixes" + IfSpecAnnotation string = "InterfacesSpec" + IfSpecPBLAnnotation string = "ExtraMemifSpec" + + KeepOriginalPacketAnnotation string = "KeepOriginalPacket" + HashConfigAnnotation string = "HashConfig" + LBTypeAnnotation string = "LBType" +) + var ( // fake constants for place where we need a pointer to true or false True = true diff --git a/go.mod b/go.mod index 63d028d0..65f98fe2 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/google/gopacket v1.1.19 github.com/inconshreveable/mousetrap v1.1.0 github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 - github.com/lunixbochs/struc v0.0.0-20241101090106-8d528fa2c543 + github.com/lunixbochs/struc v0.0.0-20241101090106-8d528fa2c543 // indirect github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.33.1 github.com/orijtech/prometheus-go-metrics-exporter v0.0.6 diff --git a/vpplink/helpers.go b/vpplink/helpers.go index 55b6767d..83468d29 100644 --- a/vpplink/helpers.go +++ b/vpplink/helpers.go @@ -21,6 +21,7 @@ import ( "time" "github.com/pkg/errors" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/ip_types" log "github.com/sirupsen/logrus" ) @@ -31,11 +32,12 @@ type IpFamily struct { IsIp6 bool IsIp4 bool FamilyIdx int + Af ip_types.AddressFamily } var ( - IpFamilyV4 = IpFamily{"ip4", "4", false, true, 0} - IpFamilyV6 = IpFamily{"ip6", "6", true, false, 1} + IpFamilyV4 = IpFamily{"ip4", "4", false, true, 0, ip_types.ADDRESS_IP4} + IpFamilyV6 = IpFamily{"ip6", "6", true, false, 1, ip_types.ADDRESS_IP6} IpFamilies = []IpFamily{IpFamilyV4, IpFamilyV6} ) @@ -49,6 +51,13 @@ func IpFamilyFromIPNet(ipNet *net.IPNet) IpFamily { return IpFamilyV4 } +func IpFamilyFromIP(ipNet net.IP) IpFamily { + if ipNet.To4() == nil { + return IpFamilyV6 + } + return IpFamilyV4 +} + type CleanupCall struct { args []interface{} f interface{} diff --git a/vpplink/types/ip_types.go b/vpplink/types/ip_types.go index 20610dd8..cb758edb 100644 --- a/vpplink/types/ip_types.go +++ b/vpplink/types/ip_types.go @@ -43,6 +43,8 @@ func (mode *IPProto) UnmarshalText(text []byte) error { *mode = TCP case "udp": *mode = UDP + case "sctp": + *mode = SCTP default: *mode = TCP } diff --git a/vpplink/urpf.go b/vpplink/urpf.go index 92c0b5ae..7328ec22 100644 --- a/vpplink/urpf.go +++ b/vpplink/urpf.go @@ -23,13 +23,13 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/urpf" ) -func (v *VppLink) SetCustomURPF(swifindex uint32, tableId uint32) error { +func (v *VppLink) SetCustomURPF(swifindex uint32, tableId uint32, ipFamily IpFamily) error { client := urpf.NewServiceClient(v.GetConnection()) _, err := client.UrpfUpdateV2(v.GetContext(), &urpf.UrpfUpdateV2{ Mode: urpf.URPF_API_MODE_LOOSE, SwIfIndex: interface_types.InterfaceIndex(swifindex), - Af: ip_types.ADDRESS_IP4, + Af: ipFamily.Af, IsInput: true, TableID: tableId, }) @@ -39,7 +39,7 @@ func (v *VppLink) SetCustomURPF(swifindex uint32, tableId uint32) error { return nil } -func (v *VppLink) UnsetURPF(swifindex uint32) error { +func (v *VppLink) UnsetURPF(swifindex uint32, ipFamily IpFamily) error { client := urpf.NewServiceClient(v.GetConnection()) _, err := client.UrpfUpdateV2(v.GetContext(), &urpf.UrpfUpdateV2{