Merge remote-tracking branch 'origin/main' into feature/logger

# Conflicts:
#	go.mod
#	go.sum
#	internal/app.go
#	internal/config/application.go

Author: ldmi3i

.pre-commit-config.yaml

@@ -9,11 +9,11 @@ repos:
       - id: check-yaml
       - id: check-added-large-files
   - repo: https://github.com/golangci/golangci-lint
-    rev: v1.47.3
+    rev: v1.48.0
     hooks:
       - id: golangci-lint
   - repo: https://github.com/zricethezav/gitleaks
-    rev: v8.9.0
+    rev: v8.10.2
     hooks:
       - id: gitleaks
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.18-alpine as builder
+FROM golang:1.19.0-alpine as builder
 
 RUN apk update && apk upgrade && \
     apk add --no-cache make bash

docker/docker-compose-dev.yml

@@ -13,6 +13,8 @@ services:
     build:
       context: ../.
       dockerfile: Dockerfile
+    environment:
+      QUEUE_URL: amqp://guest:guest@rabbit:5672/
     restart: unless-stopped
     depends_on:
       - rabbit

go.mod

@@ -4,8 +4,9 @@ go 1.18
 
 require (
 	github.com/golang/mock v1.6.0
+	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/rabbitmq/amqp091-go v1.4.0
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.8.0
 	github.com/wagslane/go-rabbitmq v0.10.0
 	go.uber.org/zap v1.21.0
 )
@@ -16,4 +17,5 @@ require (
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,10 +1,12 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
+github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
+github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -18,8 +20,11 @@ github.com/rabbitmq/amqp091-go v1.4.0 h1:T2G+J9W9OY4p64Di23J6yH7tOkMocgnESvYeBju
 github.com/rabbitmq/amqp091-go v1.4.0/go.mod h1:JsV0ofX5f1nwOGafb8L5rBItt9GyhfQfcJj+oyz0dGg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/wagslane/go-rabbitmq v0.10.0 h1:y9Bw8Q/9gOvsHfjMOGQjCW3033aYTKabxDm8eyjUGjs=
 github.com/wagslane/go-rabbitmq v0.10.0/go.mod h1:u6xM1V7OO4D0szUy/F6Bya/9r0lLae/2FXBijkAQmn0=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
@@ -66,3 +71,5 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/app.go

@@ -14,7 +14,11 @@ func Start() int {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	conf := config.ReadConfig() //read configuration from file & env
+	conf, err := config.ReadConfig() //read configuration from env
+	if err != nil {
+		fmt.Println("error while reading configuration")
+		return 1
+	}
 	log, err := logger.NewLogger(&conf)
 	if err != nil {
 		fmt.Println("error creating logger: ", err)
@@ -23,23 +27,23 @@ func Start() int {
 	//initialize publisher connection to the queue
 	//this library assumes using one publisher and one consumer per application
 	//https://github.com/wagslane/go-rabbitmq/issues/79
-	pub, err := publisher.NewPublisher(conf.QueueConfig, log)
+	pub, err := publisher.NewPublisher(conf.Queue, log)
 	if err != nil {
 		log.Error("error while starting publisher: ", err)
 		return 1
 	}
 	defer publisher.ClosePublisher(pub, log)
 	//initialize consumer connection to the queue
-	consumer, err := queue.NewConsumer(conf.QueueConfig, log)
+	consumer, err := queue.NewConsumer(conf.Queue, log)
 	if err != nil {
 		log.Error("error while connecting to the queue: ", err)
 		return 1
 	}
 	defer queue.CloseConsumer(consumer, log)
 
-	handl := handler.NewApiSpecDocHandler(pub, conf.QueueConfig, log)
+	handl := handler.NewApiSpecDocHandler(pub, conf.Queue, log)
 	listener := queue.NewListener()
-	err = listener.Start(consumer, &conf.QueueConfig, handl)
+	err = listener.Start(consumer, &conf.Queue, handl)
 	if err != nil {
 		log.Error("error while listening queue ", err)
 		return 1

internal/config/application.go

@@ -1,22 +1,22 @@
 package config
 
+import (
+	"fmt"
+	"github.com/kelseyhightower/envconfig"
+)
+
 type ApplicationConfig struct {
 	Env         Environment `default:"dev"`
 	Logger      LoggerConfig
-	QueueConfig QueueConfig
+	Queue QueueConfig
 }
 
-func ReadConfig() ApplicationConfig {
-	//Stub this method before the configuration task is not resolved
-	//https://github.com/rog-golang-buddies/api-hub_data-scraping-service/issues/10
-	//TODO implement with the method to read configuration from file and env
-	return ApplicationConfig{
-		QueueConfig: QueueConfig{
-			UrlRequestQueue:     "data-scraping-asd",
-			ScrapingResultQueue: "storage-update-asd",
-			NotificationQueue:   "gateway-scrape_notifications",
-			Url:                 "amqp://guest:guest@rabbit:5672/",
-			Concurrency:         10,
-		},
+//ReadConfig reads configuration from the environment and populates the structure with it
+func ReadConfig() (*ApplicationConfig, error) {
+	var conf ApplicationConfig
+	if err := envconfig.Process("", &conf); err != nil {
+		return nil, err
 	}
+	fmt.Printf("conf: %+v\n", conf)
+	return &conf, nil
 }

internal/config/application_test.go

@@ -0,0 +1,38 @@
+package config
+
+import (
+	"github.com/stretchr/testify/assert"
+	"os"
+	"strconv"
+	"testing"
+)
+
+func TestReadConfig(t *testing.T) {
+	expUrlRequestQueue := "url_request_queue"
+	err := os.Setenv("QUEUE_URL_REQUEST_QUEUE", expUrlRequestQueue)
+	assert.Nil(t, err)
+
+	expScrResQueue := "scraping_res_queue"
+	err = os.Setenv("QUEUE_SCRAPING_RESULT_QUEUE", expScrResQueue)
+	assert.Nil(t, err)
+
+	expNotificationQueue := "test_notifications"
+	err = os.Setenv("QUEUE_NOTIFICATION_QUEUE", expNotificationQueue)
+	assert.Nil(t, err)
+
+	expQueueUrl := "test_url"
+	err = os.Setenv("QUEUE_URL", expQueueUrl)
+	assert.Nil(t, err)
+
+	expQueueConcurrency := 50
+	err = os.Setenv("QUEUE_CONCURRENCY", strconv.Itoa(expQueueConcurrency))
+	assert.Nil(t, err)
+
+	conf, err := ReadConfig()
+	assert.Nil(t, err)
+	assert.Equal(t, expUrlRequestQueue, conf.Queue.UrlRequestQueue)
+	assert.Equal(t, expScrResQueue, conf.Queue.ScrapingResultQueue)
+	assert.Equal(t, expNotificationQueue, conf.Queue.NotificationQueue)
+	assert.Equal(t, expQueueUrl, conf.Queue.Url)
+	assert.Equal(t, expQueueConcurrency, conf.Queue.Concurrency)
+}

internal/config/queue.go

@@ -2,9 +2,14 @@ package config
 
 //QueueConfig queue configuration
 type QueueConfig struct {
-	UrlRequestQueue     string //UrlRequestQueue name to listen to the new events
-	ScrapingResultQueue string //Queue name to send processed ApiSpecDoc
-	NotificationQueue   string //Queue name to notify a user about error or success (if required)
-	Url                 string //RabbitMQ url
-	Concurrency         int    //Number of parallel handlers
+	//UrlRequestQueue name to listen to the new events
+	UrlRequestQueue string `default:"data-scraping-asd" envconfig:"URL_REQUEST_QUEUE"`
+	//ScrapingResultQueue represents a queue name to send processed ApiSpecDoc
+	ScrapingResultQueue string `default:"storage-update-asd" envconfig:"SCRAPING_RESULT_QUEUE"`
+	//NotificationQueue represents a queue name to notify a user about an error or success (if required)
+	NotificationQueue string `default:"gateway-scrape-notifications" envconfig:"NOTIFICATION_QUEUE"`
+	//Url is a RabbitMQ url
+	Url string `default:"amqp://guest:guest@localhost:5672/"`
+	//Concurrency represents number of parallel handlers
+	Concurrency int `default:"30"`
 }