Move to json storage, address pblindex issue

.ci/buildspec.yml

@@ -4,14 +4,18 @@ env:
   shell: bash
 
 phases:
+  install:
+    runtime-versions:
+      golang: latest
+
   pre_build:
     commands:
       - CI_BUILD=1
-      - cd .ci
       - echo Build started on `date`
 
       - docker --version
-      - make ci-install
+      - git config --global user.email "[email protected]"
+      - git config --global user.name "Calico CI"
 
       - echo "Logging to dockerhub"
       - docker login --username ${DOCKERHUB_USERNAME} --password=${DOCKERHUB_TOKEN} || true
@@ -21,13 +25,12 @@ phases:
         docker login --username AWS --password-stdin $ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
 
       - echo "Building dependencies image"
-      - make depend-image
+      - make builder-image
 
   build:
     commands:
       - make go-check
 
-      - cd ..
       - make images
       - docker images
 

.gitignore

@@ -27,3 +27,5 @@ calico-vpp-agent/version
 vpp-manager/images/ubuntu/version
 test/scripts/.buildlogs
 multinet-monitor/version
+.coverage/
+bin/

.golangci.yml

@@ -1,93 +1,56 @@
-linters-settings:
-  errcheck:
-    check-type-assertions: true
-  goconst:
-    min-len: 2
-    min-occurrences: 3
-  gocritic:
-    enabled-tags:
-      - diagnostic
-      - experimental
-      - opinionated
-      - performance
-      - style
-  govet:
-    shadow:
-      # Whether to be strict about shadowing; can be noisy.
-      # Default: false
-      strict: true
-  nolintlint:
-    require-explanation: true
-    require-specific: true
-linters:
-  # Disable all linters.
-  # Default: false
-  # disable-all: true
-  # Enable specific linter
-  # https://golangci-lint.run/usage/linters/#enabled-by-default
-  enable:
-    - errcheck
-    - gofmt
-    - goimports
-    - gosimple
-    - govet
-    - ineffassign
-    - misspell
-    - staticcheck
-    - typecheck
-    - unused
-  # Run only fast linters from enabled linters set (first run won't be fast)
-  # Default: false
-  fast: true
+version: "2"
 run:
-  # The default concurrency value is the number of available CPU.
   concurrency: 4
-  # Timeout for analysis, e.g. 30s, 5m.
-  # Default: 1m
-  timeout: 5m
-  # Include test files or not.
-  # Default: true
   tests: false
-  # Which dirs to skip: issues from them won't be reported.
-  # Can use regexp here: `generated.*`, regexp is applied on full path,
-  # including the path prefix if one is set.
-  # Default value is empty list,
-  # but default dirs are skipped independently of this option's value (see skip-dirs-use-default).
-  # "/" will be replaced by current OS file path separator to properly work on Windows.
-  # skip-dirs:
-  #   - src/external_libs
-  #   - autogenerated_by_my_lib
-  # Enables skipping of directories:
-  # - vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
-  # Default: true
-  # skip-dirs-use-default: false
-  # Which files to skip: they will be analyzed, but issues from them won't be reported.
-  # Default value is empty list,
-  # but there is no need to include all autogenerated files,
-  # we confidently recognize autogenerated files.
-  # If it's not please let us know.
-  # "/" will be replaced by current OS file path separator to properly work on Windows.
-  # skip-files:
-  #   - ".*\\.my\\.go$"
-  #   - lib/bad.go
-  # If set we pass it to "go list -mod={option}". From "go help modules":
-  # If invoked with -mod=readonly, the go command is disallowed from the implicit
-  # automatic updating of go.mod described above. Instead, it fails when any changes
-  # to go.mod are needed. This setting is most useful to check that go.mod does
-  # not need updates, such as in a continuous integration and testing system.
-  # If invoked with -mod=vendor, the go command assumes that the vendor
-  # directory holds the correct copies of dependencies and ignores
-  # the dependency descriptions in go.mod.
-  #
-  # Allowed values: readonly|vendor|mod
-  # By default, it isn't set.
-  # modules-download-mode: readonly
-  # Allow multiple parallel golangci-lint instances running.
-  # If false (default) - golangci-lint acquires file lock on start.
   allow-parallel-runners: false
 output:
   formats:
-    - format: tab
+    tab:
       path: stderr
-  sort-results: true
-
+      colors: false
+linters:
+  enable:
+    - misspell
+  settings:
+    staticcheck:
+      dot-import-whitelist:
+        - "github.com/onsi/ginkgo"
+        - "github.com/onsi/gomega"
+      checks:
+        - all
+    errcheck:
+      check-type-assertions: true
+    goconst:
+      min-len: 2
+      min-occurrences: 3
+    gocritic:
+      enabled-tags:
+        - diagnostic
+        - experimental
+        - opinionated
+        - performance
+        - style
+    nolintlint:
+      require-explanation: true
+      require-specific: true
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  enable:
+    - gofmt
+    - goimports
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

.ci/Dockerfile.depend → Dockerfile.depend

@@ -3,7 +3,7 @@ ARG BASE_IMAGE
 #
 # Dependencies image
 #
-FROM ${BASE_IMAGE} as dependencies
+FROM ${BASE_IMAGE:-ubuntu:22.04} AS dependencies
 
 ENV UNATTENDED=y
 
@@ -22,9 +22,12 @@ RUN mkdir -p "${GOROOT}" &&\
 RUN wget -nv "https://dl.google.com/go/go${GOVERSION}.linux-amd64.tar.gz" -O "/tmp/go.tar.gz" && \
     tar -C "${GOROOT}" --strip-components=1 -xzf "/tmp/go.tar.gz" && \
     rm -f "/tmp/go.tar.gz" && \
-    curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.64.6
+    curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.1.6
 
 # Get modules used by the source code
-COPY . /vpp-dataplane
+RUN mkdir -p /vpp-dataplane
+RUN git config --global --add safe.directory /vpp-dataplane
+COPY go.mod /vpp-dataplane
+COPY go.sum /vpp-dataplane
 WORKDIR /vpp-dataplane
 RUN go get ./... && rm -fr /vpp-dataplane

Makefile

@@ -1,4 +1,4 @@
-include .ci/common.mk
+include common.mk
 
 check-%:
 	@: $(if $(value $*),,$(error $* is undefined))
@@ -160,7 +160,7 @@ cherry-vpp:
 	@echo "Cherry pick VPP ?"
 	@echo "This will reset current branch"
 	@echo "directory : ${VPP_DIR}"
-	@echo "branch    : $(shell cd ${VPP_DIR} && git branch --show-current)"
+	@echo "branch	: $(shell cd ${VPP_DIR} && git branch --show-current)"
 	@echo "Are you sure? [y/N] " && read ans && [ $${ans:-N} = y ]
 	@bash ./vpplink/generated/vpp_clone_current.sh ${VPP_DIR}
 	@make goapi
@@ -204,11 +204,6 @@ release: check-TAG check-CALICO_TAG
 run-integration-tests:
 	$(MAKE) -C test/integration-tests $@
 
-.PHONY: test
-test: go-lint
-	gofmt -s -l . | grep -v generated | grep -v vpp_build | diff -u /dev/null -
-	go test ./...
-
 .PHONY: test-memif-multinet
 test-memif-multinet:
 	kubectl apply -f test/yaml/multinet/network.yaml
@@ -248,21 +243,86 @@ delete-multinet:
 	  do \
 		kubectl exec -it -n calico-vpp-dataplane $$cid -c vpp -- \
 		  rm -rvf /host/etc/cni/net.d/multus.d \
-		          /host/etc/cni/net.d/whereabouts.d \
-		          /host/etc/cni/net.d/00-multus.conf ;\
+				  /host/etc/cni/net.d/whereabouts.d \
+				  /host/etc/cni/net.d/00-multus.conf ;\
 	  done ;\
 	)
 
+.PHONY: lint
+lint:
+	gofmt -s -l . | grep -v binapi | grep -v vpp_build | diff -u /dev/null -
+	test -d vpp-manager/vpp_build && touch vpp-manager/vpp_build/go.mod || true
+	golangci-lint run --color=never
+
+.PHONY: test
+test: go-lint
+	rm -rf .coverage/unit
+	mkdir -p .coverage/unit
+	go test -cover -covermode=atomic ./...  \
+		-args \
+		-test.gocoverdir=$(shell pwd)/.coverage/unit
+
+.PHONY: cov-html
+cov-html:
+	go tool covdata percent -i=.coverage/unit
+	go tool covdata textfmt -i=.coverage/unit -o .coverage/profile
+	go tool cover -html=.coverage/profile
+
+.PHONY: cov
+cov: test
+	go tool covdata percent -i=.coverage/unit
+	@go tool covdata textfmt -i=.coverage/unit -o .coverage/profile
+	@echo "TOTAL:"
+	@go tool cover -func=.coverage/profile | tail -1 | awk '{print $$3}'
+
+#
+# Create a container image image used to build the go code.
+#
+# We're building a hash of that list of dependencies (Go,...). That should give us a deterministic
+# version identifier for any particular list of dependencies -- if, with a future agent version,
+# the dependencies change, then that hash will change. We can use that to minimize downloads
+# from external sources, and maximise re-use of the "dependencies" image for as long as dependencies
+# remain stable.
+#
+
+BASE_IMAGE_BUILDER = ubuntu:22.04
+
+# Compute hash to detect any changes and rebuild/push the image
+DEPEND_HASH = $(shell echo \
+    "${BASE_IMAGE_BUILDER}-DOCKERFILE:$(shell md5sum \
+    	$(CURDIR)/Dockerfile.depend \
+    	$(CURDIR)/go.mod \
+    	$(CURDIR)/go.sum \
+	)" | md5sum | cut -f1 -d' ')
+DEPEND_IMAGE = ${DEPEND_BASE}:${DEPEND_HASH}
+
+ifdef CI_BUILD
+	PUSH_IMAGE = docker image push ${DEPEND_IMAGE}
+else
+	PUSH_IMAGE = echo not pushing image
+endif
+
+.PHONY: builder-image
+builder-image: ## Make dependencies image. (Not required normally; is implied in making any other container image).
+	# Try to pull an existing dependencies image; it's OK if none exists yet.
+	@echo Building depend image
+	docker image pull ${DEPEND_IMAGE} || /bin/true
+	docker image inspect ${DEPEND_IMAGE} >/dev/null 2>/dev/null \
+		  || ( docker image build \
+				-f ./Dockerfile.depend \
+				--build-arg BASE_IMAGE=${BASE_IMAGE_BUILDER} \
+				--tag ${DEPEND_IMAGE} \
+				$(CURDIR) \
+		   && ${PUSH_IMAGE} )
+	docker tag ${DEPEND_IMAGE} ${DEPEND_BASE}:latest
 
 .PHONY: go-check
-go-check: go-lint
-	gofmt -s -l . | grep -v binapi | grep -v vpp_build | diff -u /dev/null -
-	go test ./...
+go-check: builder-image
+	@echo Checking go code
+	docker run -t --rm \
+		-v $(CURDIR):/vpp-dataplane \
+		${DEPEND_IMAGE} \
+		make -C /vpp-dataplane cov
 
 .PHONY: go-lint
-go-lint:
-	golangci-lint run --color=never
-
-.PHONY: go-lint-fix
-go-lint-fix:
-	golangci-lint run --fix
+go-lint: lint

calico-vpp-agent/Dockerfile

@@ -3,7 +3,6 @@ FROM ubuntu:22.04
 LABEL maintainer="[email protected]"
 
 ADD bin/gobgp /bin/gobgp
-ADD bin/debug /bin/debug
 ADD version /etc/calicovppversion
 ADD bin/felix-api-proxy /bin/felix-api-proxy
 ADD bin/calico-vpp-agent /bin/calico-vpp-agent