From 8ff9e15721f5794a630093b1f9f242072e78f756 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 11 Jan 2024 18:33:58 -0300 Subject: [PATCH] [TT-755] grafana url shortener (#811) * shorten grafana urls * do not expect dashboard url to have any query params --- config/logging.go | 30 +++++++++++++---- config/tomls/example.toml | 8 +++-- logstream/grafana.go | 47 +++++++++++++++++++++++++++ logstream/logstream.go | 8 +++-- logstream/logstream_handlers.go | 30 ++++++++++++++--- logstream/logstream_user_loki_test.go | 25 ++++++++++++-- 6 files changed, 131 insertions(+), 17 deletions(-) create mode 100644 logstream/grafana.go diff --git a/config/logging.go b/config/logging.go index f3527d3a3..2f6c24fd3 100644 --- a/config/logging.go +++ b/config/logging.go @@ -155,24 +155,42 @@ func (l *LokiConfig) ApplyOverrides(from *LokiConfig) error { } type GrafanaConfig struct { - Url *string `toml:"url"` + BaseUrl *string `toml:"base_url"` + DashboardUrl *string `toml:"dashboard_url"` + BearerToken *string `toml:"bearer_token"` } func (c *GrafanaConfig) ApplyOverrides(from *GrafanaConfig) error { if from == nil { return nil } - if from.Url != nil { - c.Url = from.Url + if from.BaseUrl != nil { + c.BaseUrl = from.BaseUrl + } + if from.DashboardUrl != nil { + c.DashboardUrl = from.DashboardUrl + } + if from.BearerToken != nil { + c.BearerToken = from.BearerToken } return nil } func (c *GrafanaConfig) Validate() error { - if c.Url != nil { - if !net.IsValidURL(*c.Url) { - return errors.Errorf("invalid grafana url %s", *c.Url) + if c.BaseUrl != nil { + if !net.IsValidURL(*c.BaseUrl) { + return errors.Errorf("invalid grafana url %s", *c.BaseUrl) + } + } + if c.DashboardUrl != nil { + if *c.DashboardUrl == "" { + return errors.Errorf("if set, grafana dashboard url cannot be an empty string") + } + } + if c.BearerToken != nil { + if *c.BearerToken == "" { + return errors.Errorf("if set, grafana Bearer token cannot be an empty string") } } diff --git a/config/tomls/example.toml b/config/tomls/example.toml index defa99697..2ddd97752 100644 --- a/config/tomls/example.toml +++ b/config/tomls/example.toml @@ -19,9 +19,13 @@ endpoint="https://loki.url/api/v3/push" # currently only needed when using public instance basic_auth="loki-basic-auth" +# LogStream will try to shorten Grafana URLs by default (if all 3 variables are set) [Logging.Grafana] -# url of your grafana dashboard -url="http://grafana.url/your-dashboard" +# grafana url (trailing "/" will be stripped) +base_url="http://grafana.url" +# url of your grafana dashboard (prefix and suffix "/" are stirpped), example: /d/ad61652-2712-1722/my-dashboard +dashboard_url="/d/your-dashboard" +bearer_token="my-awesome-token" # if you want to use polygon_mumbial [Network] diff --git a/logstream/grafana.go b/logstream/grafana.go new file mode 100644 index 000000000..2c159846a --- /dev/null +++ b/logstream/grafana.go @@ -0,0 +1,47 @@ +package logstream + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" +) + +const ( + ShorteningFailedErr = "failed to shorten Grafana URL" +) + +func ShortenUrl(grafanaUrl, urlToShorten, bearerToken string) (string, error) { + jsonBody := []byte(`{"path":"` + urlToShorten + `"}`) + bodyReader := bytes.NewReader(jsonBody) + + var responseObject struct { + Uid string `json:"uid"` + Url string `json:"url"` + } + + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%sapi/short-urls", grafanaUrl), bodyReader) + if err != nil { + return "", fmt.Errorf("%s: %w", ShorteningFailedErr, err) + } + + req.Header.Add("Authorization", "Bearer "+bearerToken) + req.Header.Add("Content-Type", "application/json") + + res, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("%s: %w", ShorteningFailedErr, err) + } + + if res.StatusCode != http.StatusOK { + return "", fmt.Errorf("%s: status code: %s", ShorteningFailedErr, res.Status) + } + + defer res.Body.Close() + err = json.NewDecoder(res.Body).Decode(&responseObject) + if err != nil { + return "", fmt.Errorf("%s: %w", ShorteningFailedErr, err) + } + + return responseObject.Url, nil +} diff --git a/logstream/logstream.go b/logstream/logstream.go index 7a34c7719..441ea9b60 100644 --- a/logstream/logstream.go +++ b/logstream/logstream.go @@ -386,8 +386,12 @@ func (m *LogStream) SaveLogTargetsLocations(writer LogWriter) { name := string(handler.GetTarget()) location, err := handler.GetLogLocation(m.consumers) if err != nil { - m.log.Error().Str("Handler", name).Err(err).Msg("Failed to get log location") - continue + if strings.Contains(err.Error(), ShorteningFailedErr) { + m.log.Warn().Str("Handler", name).Err(err).Msg("Failed to shorten Grafana URL, will use original one") + } else { + m.log.Error().Str("Handler", name).Err(err).Msg("Failed to get log location") + continue + } } if err := writer(m.testName, name, location); err != nil { diff --git a/logstream/logstream_handlers.go b/logstream/logstream_handlers.go index 6c2786d6d..4c33344cc 100644 --- a/logstream/logstream_handlers.go +++ b/logstream/logstream_handlers.go @@ -164,17 +164,26 @@ func (h *LokiLogHandler) GetLogLocation(consumers map[string]*ContainerLogConsum } // if no Grafana URL has been set let's at least print query parameters that can be manually added to the dashboard url - grafanaDashboardUrl := "" - if h.loggingConfig.Grafana.Url != nil { - grafanaDashboardUrl = *h.loggingConfig.Grafana.Url - grafanaDashboardUrl = strings.TrimSuffix(grafanaDashboardUrl, "/") + baseUrl := "" + if h.loggingConfig.Grafana != nil && h.loggingConfig.Grafana.BaseUrl != nil { + baseUrl = *h.loggingConfig.Grafana.BaseUrl + baseUrl = strings.TrimSuffix(baseUrl, "/") + baseUrl = baseUrl + "/" + } + + dabshoardUrl := "" + if h.loggingConfig.Grafana != nil && h.loggingConfig.Grafana.DashboardUrl != nil { + dabshoardUrl = *h.loggingConfig.Grafana.DashboardUrl + dabshoardUrl = strings.TrimSuffix(dabshoardUrl, "/") + dabshoardUrl = strings.TrimPrefix(dabshoardUrl, "/") } rangeFrom := time.Now() rangeTo := time.Now().Add(time.Minute) //just to make sure we get the last message var sb strings.Builder - sb.WriteString(grafanaDashboardUrl) + sb.WriteString(dabshoardUrl) + sb.WriteString("?orgId=1&") sb.WriteString(fmt.Sprintf("&var-run_id=%s", *h.loggingConfig.RunId)) var testName string @@ -196,8 +205,19 @@ func (h *LokiLogHandler) GetLogLocation(consumers map[string]*ContainerLogConsum if testName != "" { sb.WriteString(fmt.Sprintf("&var-test=%s", testName)) } + + relativeUrl := sb.String() + sb.WriteString(baseUrl) h.grafanaUrl = sb.String() + // try to shorten the URL only if we have all the required configuration parameters + if baseUrl != "" && dabshoardUrl != "" && h.loggingConfig.Grafana.BearerToken != nil { + shortened, err := ShortenUrl(baseUrl, relativeUrl, *h.loggingConfig.Grafana.BearerToken) + if err == nil { + h.grafanaUrl = shortened + } + } + return h.grafanaUrl, nil } diff --git a/logstream/logstream_user_loki_test.go b/logstream/logstream_user_loki_test.go index 83fa1f39c..dd59b541a 100644 --- a/logstream/logstream_user_loki_test.go +++ b/logstream/logstream_user_loki_test.go @@ -53,12 +53,33 @@ func TestExampleLokiStreaming(t *testing.T) { LogProducerTimeout: &blockchain.StrDuration{Duration: 10 * time.Second}, LogProducerRetryLimit: ptr.Ptr(uint(10)), } + loggingConfig.Loki = &config.LokiConfig{ + TenantId: ptr.Ptr("CHANGE-ME"), + Endpoint: ptr.Ptr("CHANGE-ME"), + } + loggingConfig.Grafana = &config.GrafanaConfig{ + BaseUrl: ptr.Ptr("CHANGE-ME"), + DashboardUrl: ptr.Ptr("CHANGE-ME"), + BearerToken: ptr.Ptr("CHANGE-ME"), + } lw, err := logstream.NewLogStream(t, &loggingConfig) require.NoError(t, err) - err = d.ConnectLogs(lw) - require.NoError(t, err) + for _, c := range d.containers { + err = lw.ConnectContainer(ctx, c, "") + require.NoError(t, err) + } time.Sleep(5 * time.Second) + + // we don't want them to keep logging after we have stopped log stream by flushing logs + for _, c := range d.containers { + err = lw.DisconnectContainer(c) + require.NoError(t, err) + } + + err = lw.FlushLogsToTargets() + require.NoError(t, err) + lw.PrintLogTargetsLocations() }) } }