From 113c0b195dfb9faa88207c46d2b001b0e3b39410 Mon Sep 17 00:00:00 2001 From: Artem Gavrilov Date: Mon, 26 Aug 2024 15:35:31 +0200 Subject: [PATCH 1/2] Migrate to buf schema v2 and move api related tasks to dedicated Makefile --- .github/workflows/main.yml | 2 +- Makefile.include | 135 ++------------------------- api/Makefile | 127 +++++++++++++++++++++++++ api/buf.gen.yaml | 34 +++++++ api/buf.lock | 20 ++-- api/buf.yaml | 37 +++++--- descriptor.bin => api/descriptor.bin | Bin buf.gen.yaml | 44 --------- buf.work.yaml | 3 - 9 files changed, 198 insertions(+), 204 deletions(-) create mode 100644 api/Makefile create mode 100644 api/buf.gen.yaml rename descriptor.bin => api/descriptor.bin (100%) delete mode 100644 buf.gen.yaml delete mode 100644 buf.work.yaml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7c1c51fab8..09de98cf79 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -68,7 +68,7 @@ jobs: - name: Update API compatibility descriptors run: | # log if descriptors changed, useful for "update descriptors" PRs - make descriptors + make -C api descriptors git diff --text - name: Run check-license diff --git a/Makefile.include b/Makefile.include index f329c5aa5d..615f32251b 100644 --- a/Makefile.include +++ b/Makefile.include @@ -1,5 +1,4 @@ -# vim: ts=8:sw=8:ft=make:noai:noet -SWAGGER_UI_VERSION?=latest + .PHONY: default init release gen test clean all default: help @@ -25,8 +24,8 @@ release: ## Build release versions of all components make -C qan-api2 release gen: clean ## Generate files - make gen-api - make clean_swagger + make -C api gen + make -C api clean-swagger make -C agent gen make -C admin gen @@ -38,129 +37,13 @@ gen: clean ## Generate files make format ## TODO: One formatting run is not enough, figure out why. go install -v ./... +clean: ## Remove generated files + make -C api clean + gen-mocks: find . -name mock_*.go -delete ./bin/mockery --config .mockery.yaml -gen-api: ## Generate PMM API - # generated by descriptors target - bin/buf breaking --against descriptor.bin api - - bin/buf generate -v api - - SPECS="\ - api/agentlocal/v1 \ - api/server/v1 \ - api/user/v1 \ - api/inventory/v1 \ - api/management/v1 \ - api/management/v1/service \ - api/actions/v1 \ - api/advisors/v1 \ - api/alerting/v1 \ - api/backup/v1 \ - api/dump/v1beta1 \ - api/accesscontrol/v1beta1 \ - api/qan/v1 \ - api/platform/v1"; \ - for API in $$SPECS; do \ - set -x ; \ - bin/swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \ - bin/swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \ - bin/swagger validate $$API/json/$$(basename $$API).json ; \ - bin/swagger generate client --with-flatten=expand --with-flatten=remove-unused --spec=$$API/json/$$(basename $$API).json --target=$$API/json \ - --additional-initialism=aws \ - --additional-initialism=db \ - --additional-initialism=ok \ - --additional-initialism=pmm \ - --additional-initialism=psmdb \ - --additional-initialism=pxc \ - --additional-initialism=pt \ - --additional-initialism=qan \ - --additional-initialism=rds \ - --additional-initialism=sql \ - --additional-initialism=ha ; \ - done - - # generate public API spec, omit agentlocal and inventory (always private), - # as well as a number of protos that are in beta (not v1 yet, they all go to a similar call below) - bin/swagger mixin --output=api/swagger/swagger.json \ - api/swagger/header.json \ - api/server/v1/json/v1.json \ - api/user/v1/json/v1.json \ - api/inventory/v1/json/v1.json \ - api/management/v1/json/v1.json \ - api/actions/v1/json/v1.json \ - api/alerting/v1/json/v1.json \ - api/advisors/v1/json/v1.json \ - api/backup/v1/json/v1.json \ - api/qan/v1/json/v1.json \ - api/platform/v1/json/v1.json - bin/swagger validate api/swagger/swagger.json - - bin/swagger-order --output=api/swagger/swagger.json api/swagger/swagger.json - - # generate API spec with all PMM Server APIs (omit agentlocal) - bin/swagger mixin --output=api/swagger/swagger-dev.json \ - api/swagger/header-dev.json \ - api/server/v1/json/v1.json \ - api/user/v1/json/v1.json \ - api/inventory/v1/json/v1.json \ - api/management/v1/json/v1.json \ - api/actions/v1/json/v1.json \ - api/alerting/v1/json/v1.json \ - api/advisors/v1/json/v1.json \ - api/backup/v1/json/v1.json \ - api/dump/v1beta1/json/v1beta1.json \ - api/accesscontrol/v1beta1/json/v1beta1.json \ - api/qan/v1/json/v1.json \ - api/platform/v1/json/v1.json - - bin/swagger validate api/swagger/swagger-dev.json - - bin/swagger-order --output=api/swagger/swagger-dev.json api/swagger/swagger-dev.json - -clean_swagger: - find api -name '*.swagger.json' -print -delete - - -ifeq ($(shell test "${SWAGGER_UI_VERSION}" = "latest" && echo 1 || echo 0), 1) -get_swagger_version: -override SWAGGER_UI_VERSION = $(shell curl --silent --head https://github.com/swagger-api/swagger-ui/releases/latest | grep -E '^[lL]ocation' | sed 's;^.*/;;') -else -get_swagger_version: -endif - -update-swagger: get_swagger_version ## Update Swagger UI in api/swagger, use `SWAGGER_UI_VERSION=xxx make update-swagger` to choose a version other than latest - curl --output /tmp/swagger.tar.gz -sSfL "https://github.com/swagger-api/swagger-ui/archive/refs/tags/${SWAGGER_UI_VERSION}.tar.gz" - tar xf /tmp/swagger.tar.gz --transform 's;swagger-ui-.*/dist/;api/swagger/;g' - rm -f api/swagger/.npmrc - sed -i 's/url: ".*"/url: "\/swagger.json",\n validatorUrl: "none"/' api/swagger/swagger-initializer.js - -clean: clean_swagger ## Remove generated files - find api -name '*.pb.go' -print -delete - find api -name '*.pb.gw.go' -print -delete - find api -name '*.validate.go' -print -delete - - SPECS="\ - api/agentlocal/v1 \ - api/server/v1 \ - api/user/v1 \ - api/inventory/v1 \ - api/management/v1 \ - api/actions/v1 \ - api/alerting/v1 \ - api/advisors/v1 \ - api/backup/v1 \ - api/dump/v1beta1 \ - api/accesscontrol/v1beta1 \ - api/qan/v1 \ - api/platform/v1"; \ - for API in $$SPECS; do \ - rm -fr $$API/json/client $$API/json/models $$API/json/$$(basename $$API).json ; \ - done - rm -f api/swagger/swagger.json api/swagger/swagger-dev.json - test-common: ## Run tests from API (and other shared) packages only (i.e it ignores directories that are explicitly listed) go test $(shell go list ./... | grep -v -e admin -e agent -e managed -e api-tests -e qan-api2 -e update) @@ -182,14 +65,10 @@ check-all: check-license check ## Run golangci linter to check for changes ag FILES = $(shell find . -type f -name '*.go') format: ## Format source code + make -C api format bin/gofumpt -l -w $(FILES) bin/goimports -local github.com/percona/pmm -l -w $(FILES) bin/gci write --section Standard --section Default --section "Prefix(github.com/percona/pmm)" $(FILES) - bin/buf format api -w serve: ## Serve API documentation with nginx nginx -p . -c api/nginx/nginx.conf - -descriptors: ## Update API compatibility descriptors - #./prototool break descriptor-set . -o api/api.descriptor - bin/buf build -o descriptor.bin --as-file-descriptor-set api diff --git a/api/Makefile b/api/Makefile new file mode 100644 index 0000000000..922866a671 --- /dev/null +++ b/api/Makefile @@ -0,0 +1,127 @@ +# vim: ts=8:sw=8:ft=make:noai:noet +SWAGGER_UI_VERSION?=latest + +gen: ## Generate PMM API + # generated by descriptors target + ../bin/buf breaking --against descriptor.bin . + + ../bin/buf generate -v . + + SPECS="\ + agentlocal/v1 \ + server/v1 \ + user/v1 \ + inventory/v1 \ + management/v1 \ + management/v1/service \ + actions/v1 \ + advisors/v1 \ + alerting/v1 \ + backup/v1 \ + dump/v1beta1 \ + accesscontrol/v1beta1 \ + qan/v1 \ + platform/v1"; \ + for API in $$SPECS; do \ + set -x ; \ + ../bin/swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \ + ../bin/swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \ + ../bin/swagger validate $$API/json/$$(basename $$API).json ; \ + ../bin/swagger generate client --with-flatten=expand --with-flatten=remove-unused --spec=$$API/json/$$(basename $$API).json --target=$$API/json \ + --additional-initialism=aws \ + --additional-initialism=db \ + --additional-initialism=ok \ + --additional-initialism=pmm \ + --additional-initialism=psmdb \ + --additional-initialism=pxc \ + --additional-initialism=pt \ + --additional-initialism=qan \ + --additional-initialism=rds \ + --additional-initialism=sql \ + --additional-initialism=ha ; \ + done + + # generate public API spec, omit agentlocal and inventory (always private), + # as well as a number of protos that are in beta (not v1 yet, they all go to a similar call below) + ../bin/swagger mixin --output=swagger/swagger.json \ + swagger/header.json \ + server/v1/json/v1.json \ + user/v1/json/v1.json \ + inventory/v1/json/v1.json \ + management/v1/json/v1.json \ + actions/v1/json/v1.json \ + alerting/v1/json/v1.json \ + advisors/v1/json/v1.json \ + backup/v1/json/v1.json \ + qan/v1/json/v1.json \ + platform/v1/json/v1.json + ../bin/swagger validate swagger/swagger.json + + ../bin/swagger-order --output=swagger/swagger.json swagger/swagger.json + + # generate API spec with all PMM Server APIs (omit agentlocal) + ../bin/swagger mixin --output=swagger/swagger-dev.json \ + swagger/header-dev.json \ + server/v1/json/v1.json \ + user/v1/json/v1.json \ + inventory/v1/json/v1.json \ + management/v1/json/v1.json \ + actions/v1/json/v1.json \ + alerting/v1/json/v1.json \ + advisors/v1/json/v1.json \ + backup/v1/json/v1.json \ + dump/v1beta1/json/v1beta1.json \ + accesscontrol/v1beta1/json/v1beta1.json \ + qan/v1/json/v1.json \ + platform/v1/json/v1.json + + ../bin/swagger validate swagger/swagger-dev.json + + ../bin/swagger-order --output=swagger/swagger-dev.json swagger/swagger-dev.json + +format: ## Format API definitions + ../bin/buf format . -w + +clean-swagger: + find . -name '*.swagger.json' -print -delete + + +ifeq ($(shell test "${SWAGGER_UI_VERSION}" = "latest" && echo 1 || echo 0), 1) +get_swagger_version: +override SWAGGER_UI_VERSION = $(shell curl --silent --head https://github.com/swagger-api/swagger-ui/releases/latest | grep -E '^[lL]ocation' | sed 's;^.*/;;') +else +get_swagger_version: +endif + +update-swagger: get_swagger_version ## Update Swagger UI in swagger, use `SWAGGER_UI_VERSION=xxx make update-swagger` to choose a version other than latest + curl --output /tmp/swagger.tar.gz -sSfL "https://github.com/swagger-api/swagger-ui/archive/refs/tags/${SWAGGER_UI_VERSION}.tar.gz" + tar xf /tmp/swagger.tar.gz --transform 's;swagger-ui-.*/dist/;swagger/;g' + rm -f swagger/.npmrc + sed -i 's/url: ".*"/url: "\/swagger.json",\n validatorUrl: "none"/' swagger/swagger-initializer.js + +clean: clean-swagger ## Remove generated files + find . -name '*.pb.go' -print -delete + find . -name '*.pb.gw.go' -print -delete + find . -name '*.validate.go' -print -delete + + SPECS="\ + agentlocal/v1 \ + server/v1 \ + user/v1 \ + inventory/v1 \ + management/v1 \ + actions/v1 \ + alerting/v1 \ + advisors/v1 \ + backup/v1 \ + dump/v1beta1 \ + accesscontrol/v1beta1 \ + qan/v1 \ + platform/v1"; \ + for API in $$SPECS; do \ + rm -fr $$API/json/client $$API/json/models $$API/json/$$(basename $$API).json ; \ + done + rm -f swagger/swagger.json swagger/swagger-dev.json + +descriptors: ## Update API compatibility descriptors + ../bin/buf build -o descriptor.bin --as-file-descriptor-set . \ No newline at end of file diff --git a/api/buf.gen.yaml b/api/buf.gen.yaml new file mode 100644 index 0000000000..b412cc9f32 --- /dev/null +++ b/api/buf.gen.yaml @@ -0,0 +1,34 @@ +version: v2 +managed: + enabled: true + disable: + - file_option: go_package + module: buf.build/googleapis/googleapis + - file_option: go_package + module: buf.build/grpc-ecosystem/grpc-gateway + - file_option: go_package + module: buf.build/envoyproxy/protoc-gen-validate + override: + - file_option: go_package_prefix + value: github.com/percona/pmm/api +plugins: + - local: ../bin/protoc-gen-go + out: . + opt: paths=source_relative + - local: ../bin/protoc-gen-go-grpc + out: . + opt: paths=source_relative + - local: ../bin/protoc-gen-grpc-gateway + out: . + opt: paths=source_relative + - local: ../bin/protoc-gen-validate + out: . + opt: + - paths=source_relative + - lang=go + - local: ../bin/protoc-gen-openapiv2 + out: . + opt: + - simple_operation_ids=true + - json_names_for_fields=false + - proto3_optional_nullable=true diff --git a/api/buf.lock b/api/buf.lock index 66753d27f3..0e9cebedcf 100644 --- a/api/buf.lock +++ b/api/buf.lock @@ -1,18 +1,12 @@ # Generated by buf. DO NOT EDIT. -version: v1 +version: v2 deps: - - remote: buf.build - owner: envoyproxy - repository: protoc-gen-validate + - name: buf.build/envoyproxy/protoc-gen-validate commit: eac44469a7af47e7839a7f1f3d7ac004 - digest: shake256:0feabcde01b6b11e3c75a5e3f807968d5995626546f39c37e5d4205892b3a59cced0ed83b35a2eb9e6dddd3309660ad46b737c9dcd224b425de0a6654ce04417 - - remote: buf.build - owner: googleapis - repository: googleapis + digest: b5:2ef7b25d9671fde82d58278e94d209788fb6a42936cc2d60e42303b67dcdb72b2d0e121238ba44caf1870a34edab55518df5d3f1ee1e62b8d9b6cf98817eae6e + - name: buf.build/googleapis/googleapis commit: 28151c0d0a1641bf938a7672c500e01d - digest: shake256:49215edf8ef57f7863004539deff8834cfb2195113f0b890dd1f67815d9353e28e668019165b9d872395871eeafcbab3ccfdb2b5f11734d3cca95be9e8d139de - - remote: buf.build - owner: grpc-ecosystem - repository: grpc-gateway + digest: b5:93b70089baa4fc05a92d3e52db91a4b7812db3b57b9664f6cb301733938cb630e377a938e8a56779388171c749c1d42a2e9a6c6230f2ff45f127a8102a6a27d0 + - name: buf.build/grpc-ecosystem/grpc-gateway commit: 3f42134f4c564983838425bc43c7a65f - digest: shake256:3d11d4c0fe5e05fda0131afefbce233940e27f0c31c5d4e385686aea58ccd30f72053f61af432fa83f1fc11cda57f5f18ca3da26a29064f73c5a0d076bba8d92 + digest: b5:291b947d8ac09492517557e4e72e294788cb8201afc7d0df7bda80fa10931adb60d4d669208a7696bf24f1ecb2a33a16d4c1e766e6f31809248b00343119569b diff --git a/api/buf.yaml b/api/buf.yaml index 84e841ca3e..b130c4ee02 100644 --- a/api/buf.yaml +++ b/api/buf.yaml @@ -1,21 +1,28 @@ -version: v1 -breaking: - use: - - FILE +version: v2 +modules: + - path: . +deps: + - buf.build/envoyproxy/protoc-gen-validate + - buf.build/googleapis/googleapis + - buf.build/grpc-ecosystem/grpc-gateway lint: use: - DEFAULT + except: + - FIELD_NOT_REQUIRED + - PACKAGE_NO_IMPORT_CYCLE ignore_only: + PACKAGE_VERSION_SUFFIX: + - api/common/common.proto + - api/common/metrics_resolutions.proto RPC_REQUEST_STANDARD_NAME: - - agent/v1/agent.proto # We want our naming in this file to be different + - api/agent/v1/agent.proto RPC_RESPONSE_STANDARD_NAME: - - agent/v1/agent.proto # We want our naming in this file to be different - PACKAGE_VERSION_SUFFIX: - - common/common.proto # We don't want to version this file - - common/metrics_resolutions.proto # We don't want to version this file - - -deps: - - buf.build/googleapis/googleapis - - buf.build/grpc-ecosystem/grpc-gateway - - buf.build/envoyproxy/protoc-gen-validate + - api/agent/v1/agent.proto + disallow_comment_ignores: true +breaking: + use: + - FILE + except: + - EXTENSION_NO_DELETE + - FIELD_SAME_DEFAULT diff --git a/descriptor.bin b/api/descriptor.bin similarity index 100% rename from descriptor.bin rename to api/descriptor.bin diff --git a/buf.gen.yaml b/buf.gen.yaml deleted file mode 100644 index 53d088dcf4..0000000000 --- a/buf.gen.yaml +++ /dev/null @@ -1,44 +0,0 @@ -version: v1 -managed: - enabled: true - go_package_prefix: - default: github.com/percona/pmm/api - except: - - buf.build/googleapis/googleapis - - buf.build/grpc-ecosystem/grpc-gateway - - buf.build/envoyproxy/protoc-gen-validate - -plugins: - - name: go - path: bin/protoc-gen-go - out: api - opt: - - paths=source_relative - - - name: go-grpc - path: bin/protoc-gen-go-grpc - out: api - opt: - - paths=source_relative - - - name: grpc-gateway - path: bin/protoc-gen-grpc-gateway - out: api - opt: - - paths=source_relative - - - name: validate - path: bin/protoc-gen-validate - out: api - opt: - - paths=source_relative - - lang=go - - - name: openapiv2 - out: api - path: bin/protoc-gen-openapiv2 - opt: - - simple_operation_ids=true - - json_names_for_fields=false - - proto3_optional_nullable=true - # - visibility_restriction_selectors=INTERNAL # Uncomment to show elements marked up with `INTERNAL` diff --git a/buf.work.yaml b/buf.work.yaml deleted file mode 100644 index a2fa5f5176..0000000000 --- a/buf.work.yaml +++ /dev/null @@ -1,3 +0,0 @@ -version: v1 -directories: - - api From 442d8f53011c67e12a14dfe7d9b4d5e51ecd5770 Mon Sep 17 00:00:00 2001 From: Artem Gavrilov Date: Mon, 26 Aug 2024 15:39:05 +0200 Subject: [PATCH 2/2] Hit CI