Merge branch 'f_ctf_config' of github.com:smartcontractkit/chainlink-…

…testing-framework into f_ctf_config

config/logging.go

@@ -156,24 +156,42 @@ func (l *LokiConfig) ApplyOverrides(from *LokiConfig) error {
 }
 
 type GrafanaConfig struct {
-	Url *string `toml:"url"`
+	BaseUrl      *string `toml:"base_url"`
+	DashboardUrl *string `toml:"dashboard_url"`
+	BearerToken  *string `toml:"bearer_token"`
 }
 
 func (c *GrafanaConfig) ApplyOverrides(from *GrafanaConfig) error {
 	if from == nil {
 		return nil
 	}
-	if from.Url != nil {
-		c.Url = from.Url
+	if from.BaseUrl != nil {
+		c.BaseUrl = from.BaseUrl
+	}
+	if from.DashboardUrl != nil {
+		c.DashboardUrl = from.DashboardUrl
+	}
+	if from.BearerToken != nil {
+		c.BearerToken = from.BearerToken
 	}
 
 	return nil
 }
 
 func (c *GrafanaConfig) Validate() error {
-	if c.Url != nil {
-		if !net.IsValidURL(*c.Url) {
-			return fmt.Errorf("invalid grafana url %s", *c.Url)
+	if c.BaseUrl != nil {
+		if !net.IsValidURL(*c.BaseUrl) {
+			return fmt.Errorf("invalid grafana url %s", *c.BaseUrl)
+		}
+	}
+	if c.DashboardUrl != nil {
+		if *c.DashboardUrl == "" {
+			return errors.New("if set, grafana dashboard url cannot be an empty string")
+		}
+	}
+	if c.BearerToken != nil {
+		if *c.BearerToken == "" {
+			return errors.New("if set, grafana Bearer token cannot be an empty string")
 		}
 	}
 

config/tomls/example.toml

@@ -19,9 +19,13 @@ endpoint="https://loki.url/api/v3/push"
 # currently only needed when using public instance
 basic_auth="loki-basic-auth"
 
+# LogStream will try to shorten Grafana URLs by default (if all 3 variables are set)
 [Logging.Grafana]
-# url of your grafana dashboard
-url="http://grafana.url/your-dashboard"
+# grafana url (trailing "/" will be stripped)
+base_url="http://grafana.url"
+# url of your grafana dashboard (prefix and suffix "/" are stirpped), example: /d/ad61652-2712-1722/my-dashboard
+dashboard_url="/d/your-dashboard"
+bearer_token="my-awesome-token"
 
 # if you want to use polygon_mumbial
 [Network]

logstream/grafana.go

@@ -0,0 +1,47 @@
+package logstream
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+)
+
+const (
+	ShorteningFailedErr = "failed to shorten Grafana URL"
+)
+
+func ShortenUrl(grafanaUrl, urlToShorten, bearerToken string) (string, error) {
+	jsonBody := []byte(`{"path":"` + urlToShorten + `"}`)
+	bodyReader := bytes.NewReader(jsonBody)
+
+	var responseObject struct {
+		Uid string `json:"uid"`
+		Url string `json:"url"`
+	}
+
+	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%sapi/short-urls", grafanaUrl), bodyReader)
+	if err != nil {
+		return "", fmt.Errorf("%s: %w", ShorteningFailedErr, err)
+	}
+
+	req.Header.Add("Authorization", "Bearer "+bearerToken)
+	req.Header.Add("Content-Type", "application/json")
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("%s: %w", ShorteningFailedErr, err)
+	}
+
+	if res.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("%s: status code: %s", ShorteningFailedErr, res.Status)
+	}
+
+	defer res.Body.Close()
+	err = json.NewDecoder(res.Body).Decode(&responseObject)
+	if err != nil {
+		return "", fmt.Errorf("%s: %w", ShorteningFailedErr, err)
+	}
+
+	return responseObject.Url, nil
+}

logstream/logstream.go

@@ -386,8 +386,12 @@ func (m *LogStream) SaveLogTargetsLocations(writer LogWriter) {
 		name := string(handler.GetTarget())
 		location, err := handler.GetLogLocation(m.consumers)
 		if err != nil {
-			m.log.Error().Str("Handler", name).Err(err).Msg("Failed to get log location")
-			continue
+			if strings.Contains(err.Error(), ShorteningFailedErr) {
+				m.log.Warn().Str("Handler", name).Err(err).Msg("Failed to shorten Grafana URL, will use original one")
+			} else {
+				m.log.Error().Str("Handler", name).Err(err).Msg("Failed to get log location")
+				continue
+			}
 		}
 
 		if err := writer(m.testName, name, location); err != nil {

logstream/logstream_handlers.go

@@ -164,17 +164,26 @@ func (h *LokiLogHandler) GetLogLocation(consumers map[string]*ContainerLogConsum
 	}
 
 	// if no Grafana URL has been set let's at least print query parameters that can be manually added to the dashboard url
-	grafanaDashboardUrl := ""
-	if h.loggingConfig.Grafana.Url != nil {
-		grafanaDashboardUrl = *h.loggingConfig.Grafana.Url
-		grafanaDashboardUrl = strings.TrimSuffix(grafanaDashboardUrl, "/")
+	baseUrl := ""
+	if h.loggingConfig.Grafana != nil && h.loggingConfig.Grafana.BaseUrl != nil {
+		baseUrl = *h.loggingConfig.Grafana.BaseUrl
+		baseUrl = strings.TrimSuffix(baseUrl, "/")
+		baseUrl = baseUrl + "/"
+	}
+
+	dabshoardUrl := ""
+	if h.loggingConfig.Grafana != nil && h.loggingConfig.Grafana.DashboardUrl != nil {
+		dabshoardUrl = *h.loggingConfig.Grafana.DashboardUrl
+		dabshoardUrl = strings.TrimSuffix(dabshoardUrl, "/")
+		dabshoardUrl = strings.TrimPrefix(dabshoardUrl, "/")
 	}
 
 	rangeFrom := time.Now()
 	rangeTo := time.Now().Add(time.Minute) //just to make sure we get the last message
 
 	var sb strings.Builder
-	sb.WriteString(grafanaDashboardUrl)
+	sb.WriteString(dabshoardUrl)
+	sb.WriteString("?orgId=1&")
 	sb.WriteString(fmt.Sprintf("&var-run_id=%s", *h.loggingConfig.RunId))
 
 	var testName string
@@ -196,8 +205,19 @@ func (h *LokiLogHandler) GetLogLocation(consumers map[string]*ContainerLogConsum
 	if testName != "" {
 		sb.WriteString(fmt.Sprintf("&var-test=%s", testName))
 	}
+
+	relativeUrl := sb.String()
+	sb.WriteString(baseUrl)
 	h.grafanaUrl = sb.String()
 
+	// try to shorten the URL only if we have all the required configuration parameters
+	if baseUrl != "" && dabshoardUrl != "" && h.loggingConfig.Grafana.BearerToken != nil {
+		shortened, err := ShortenUrl(baseUrl, relativeUrl, *h.loggingConfig.Grafana.BearerToken)
+		if err == nil {
+			h.grafanaUrl = shortened
+		}
+	}
+
 	return h.grafanaUrl, nil
 }
 

logstream/logstream_user_loki_test.go

@@ -53,12 +53,33 @@ func TestExampleLokiStreaming(t *testing.T) {
 				LogProducerTimeout:    &blockchain.StrDuration{Duration: 10 * time.Second},
 				LogProducerRetryLimit: ptr.Ptr(uint(10)),
 			}
+			loggingConfig.Loki = &config.LokiConfig{
+				TenantId: ptr.Ptr("CHANGE-ME"),
+				Endpoint: ptr.Ptr("CHANGE-ME"),
+			}
+			loggingConfig.Grafana = &config.GrafanaConfig{
+				BaseUrl:      ptr.Ptr("CHANGE-ME"),
+				DashboardUrl: ptr.Ptr("CHANGE-ME"),
+				BearerToken:  ptr.Ptr("CHANGE-ME"),
+			}
 
 			lw, err := logstream.NewLogStream(t, &loggingConfig)
 			require.NoError(t, err)
-			err = d.ConnectLogs(lw)
-			require.NoError(t, err)
+			for _, c := range d.containers {
+				err = lw.ConnectContainer(ctx, c, "")
+				require.NoError(t, err)
+			}
 			time.Sleep(5 * time.Second)
+
+			// we don't want them to keep logging after we have stopped log stream by flushing logs
+			for _, c := range d.containers {
+				err = lw.DisconnectContainer(c)
+				require.NoError(t, err)
+			}
+
+			err = lw.FlushLogsToTargets()
+			require.NoError(t, err)
+			lw.PrintLogTargetsLocations()
 		})
 	}
 }