From f5312e550a40900697632f61478b5b62af7e56e1 Mon Sep 17 00:00:00 2001 From: Adrian Arumugam Date: Wed, 29 Sep 2021 17:45:21 +0800 Subject: [PATCH] Update kafka exporter makefile and build documentation --- Makefile | 20 +++++++++++++------- README.md | 54 ++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 53 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index d452fa1c..4b41e8a1 100644 --- a/Makefile +++ b/Makefile @@ -4,12 +4,12 @@ pkgs = $(shell $(GO) list ./... | grep -v /vendor/) PREFIX ?= $(shell pwd) BIN_DIR ?= $(shell pwd) -DOCKER_IMAGE_NAME ?= ghcr.io/mr-yum/kafka-exporter -DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD)) +GHCR_IMAGE_NAME ?= ghcr.io/mr-yum/kafka-exporter +GHCR_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD | md5sum | cut -d " " -f1)) TAG := $(shell echo `if [ "$(TRAVIS_BRANCH)" = "master" ] || [ "$(TRAVIS_BRANCH)" = "" ] ; then echo "latest"; else echo $(TRAVIS_BRANCH) ; fi`) PUSHTAG ?= type=registry,push=true -DOCKER_PLATFORMS ?= linux/amd64,linux/s390x,linux/arm64,linux/ppc64le +PLATFORMS ?= linux/amd64,linux/s390x,linux/arm64,linux/ppc64le all: format build test @@ -45,14 +45,20 @@ tarball: promu docker: build @echo ">> building docker image" - @docker build -t "$(DOCKER_IMAGE_NAME):latest" --build-arg BIN_DIR=. . + @docker build -t "$(GHCR_IMAGE_NAME):$(GHCR_IMAGE_TAG)" --build-arg BIN_DIR=. . + +# Before running this make sure you have a Github Container Registry Personal Access Token (CR_PAT) exported +# locally which at a minimum has RW packages permissions and have logged into ghcr.io via docker login. +# https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry#authenticating-to-the-container-registry push: crossbuild - @echo ">> building and pushing multi-arch docker images, mr-yum,$(DOCKER_IMAGE_NAME),latest" + @echo ">> building and pushing multi-arch docker images, mr-yum,$(GHCR_IMAGE_NAME):$(GHCR_IMAGE_TAG),latest" @docker buildx create --use - @docker buildx build -t "$(DOCKER_IMAGE_NAME):latest" \ + @docker buildx build \ + --tag "$(GHCR_IMAGE_NAME):$(GHCR_IMAGE_TAG)" \ + --tag "$(GHCR_IMAGE_NAME):latest" \ --output "$(PUSHTAG)" \ - --platform "$(DOCKER_PLATFORMS)" \ + --platform "$(PLATFORMS)" \ . release: promu github-release diff --git a/README.md b/README.md index f9f4ef72..5b4d545d 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,12 @@ kafka_exporter ============== -[![Build Status](https://app.travis-ci.com/danielqsj/kafka_exporter.svg?branch=master)](https://app.travis-ci.com/danielqsj/kafka_exporter)[![Docker Pulls](https://img.shields.io/docker/pulls/danielqsj/kafka-exporter.svg)](https://hub.docker.com/r/danielqsj/kafka-exporter)[![Go Report Card](https://goreportcard.com/badge/github.com/danielqsj/kafka_exporter)](https://goreportcard.com/report/github.com/danielqsj/kafka_exporter)[![Language](https://img.shields.io/badge/language-Go-red.svg)](https://github.com/danielqsj/kafka-exporter)[![GitHub release](https://img.shields.io/badge/release-1.4.2-green.svg)](https://github.com/alibaba/derrick/releases)[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) +Kafka exporter for Prometheus - patched by Mr Yum Eng. -Kafka exporter for Prometheus. For other metrics from Kafka, have a look at the [JMX exporter](https://github.com/prometheus/jmx_exporter). +Mr Yum forked and patched this exporter to temporarily overcome an issue with the consumergroupMembers metric which was intermittently causing the kafka_exporter +to fail metric exports. + +This repository will only temporarily exist while we work on a more robust patch and submit that upstream to `danielqsj/kafka_exporter`. Table of Contents ----------------- @@ -12,8 +15,10 @@ Table of Contents - [Dependency](#dependency) - [Download](#download) - [Compile](#compile) - - [Build Binary](#build-binary) - - [Build Docker Image](#build-docker-image) + - [Build Binary](#build-binary-for-testing-locally) + - [Build Docker Image](#build-docker-image-for-testing-locally) + - [Build & Push Multi-Arch Docker Image to ghcr.io](#build-docker-image-and-push-to-github-container-registry) +- [Github Container Registry Image](#github-container-registry-image) - [Run](#run) - [Run Binary](#run-binary) - [Run Docker Image](#run-docker-image) @@ -40,34 +45,55 @@ Dependency - [Sarama](https://shopify.github.io/sarama) - [Golang](https://golang.org) -Download --------- - -Binary can be downloaded from [Releases](https://github.com/danielqsj/kafka_exporter/releases) page. - Compile ------- -### Build Binary +### Build Binary - for testing locally ```shell make ``` -### Build Docker Image +### Build Docker Image - for testing locally ```shell make docker ``` +### Build Docker Image and push to Github Container Registry + +**As this is a temporary patch, only Mr Yum SRE team has permissions to upload new packages to ghcr.io for this particular container path.** + +>**NOTE:** If you haven't set up a personal access token with read/write permissions for +Github packages then please follow the instructions [here](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry#authenticating-to-the-container-registry) +before moving on. + +1. Export your CR_PAT (Github Container Registry Personal Access Token) and login via docker. + +```shell +export CR_PAT="replace-with-your-cr-pat" +export GH_USERNAME="replace-with-your-github-username" +echo ${CR_PAT?} | docker login ghcr.io -u ${GH_USERNAME?} --password-stdin + ``` + +2. Once you've received the 'Login Succeeded' message you can run the multi-arch build. This will + build various target_arch containers and push them to ghcr.io. + +```shell +make push +``` + +>**NOTE:**: This will push to `ghcr.io/mr-yum/kafka-exporter` and assign two tags. +The first tag will be the versioning tag which is a md5 hash of the build branch (`ghcr.io/mr-yum/kafka-exporter:$md5sum`). +The second tag with be the latest tag (`ghcr.io/mr-yum/kafka-exporter:latest`). -Docker Hub Image +Github Container Registry Image ---------------- ```shell -docker pull danielqsj/kafka-exporter:latest +docker pull ghcr.io/mr-yum/kafka-exporter:latest ``` -It can be used directly instead of having to build the image yourself. ([Docker Hub danielqsj/kafka-exporter](https://hub.docker.com/r/danielqsj/kafka-exporter)\) +It can be used directly instead of having to build the image yourself. ([ghcr.io/mr-yum/kafka-exporter](https://github.com/orgs/mr-yum/packages/container/package/kafka-exporter)) Run ---