feat: extract the cloud API Url from the metadata (#19)

* feat: extract the cloud API Url from the metadata

* Update pkg/clients/tsclient/client.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: Clarance Liberiste Ntwari <60586899+claranceliberi@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: 3 people

.gitignore

@@ -1 +1,2 @@
 build
+coverage.*

Dockerfile

@@ -28,7 +28,6 @@ FROM alpine:3.19
 RUN apk add --no-cache \
     ca-certificates \
     tzdata \
-    procfs \
     && rm -rf /var/cache/apk/*
 
 # Create non-root user (though process metrics may require root)
@@ -54,7 +53,6 @@ RUN chmod +x sc-agent
 # Environment variables with defaults
 ENV SC_LOG_LEVEL=info
 ENV SC_COLLECTION_INTERVAL=30s
-ENV SC_INGESTOR_ENDPOINT=http://localhost:8080/ingest
 ENV SC_HTTP_TIMEOUT=30s
 ENV SC_COLLECTOR_PROCESSES=true
 

README.md

@@ -125,9 +125,6 @@ sudo ./build/sc-agent
 
 # Specify a config file
 sudo SC_AGENT_CONFIG=/path/to/your/config.yaml ./build/sc-agent
-
-# Override settings with environment variables
-sudo SC_INGESTOR_ENDPOINT=http://localhost:8080 SC_LOG_LEVEL=debug ./build/sc-agent
 ```
 
 ### Checking Version

cmd/agent/main.go

@@ -119,7 +119,6 @@ func main() {
 
 	logger.Info("Starting SC metrics agent",
 		zap.Duration("collection_interval", cfg.CollectionInterval),
-		zap.String("ingestor_endpoint", cfg.IngestorEndpoint),
 		zap.String("metadata_service_endpoint", cfg.MetadataServiceEndpoint),
 		zap.String("vm_id", cfg.VMID),
 		zap.Any("collectors", cfg.Collectors),
@@ -141,17 +140,17 @@ func main() {
 
 	metricDecorator := decorator.NewMetricDecorator(cfg.VMID, cfg.Labels, logger)
 	aggregator := aggregate.NewAggregator(logger)
+	authMgr := metadata.NewAuthManager(cfg, logger)
 
 	// Create HTTP client for metric writing
 	clientConfig := tsclient.ClientConfig{
-		Endpoint:   cfg.IngestorEndpoint,
+		AuthMgr:    authMgr, 
 		Timeout:    cfg.HTTPTimeout,
 		MaxRetries: cfg.MaxRetries,
 		RetryDelay: cfg.RetryInterval,
 	}
-	httpClient := tsclient.NewClientWithConfig(clientConfig, logger)
+	httpClient := tsclient.NewClient(clientConfig, logger)
 	metricWriter := tsclient.NewMetricWriter(httpClient, logger)
-	authMgr := metadata.NewAuthManager(cfg, logger)
 
 	// Create processing pipeline
 	pipelineProcessor := pipeline.NewProcessor(

config.example.yaml

@@ -5,8 +5,6 @@
 collection_interval: 30s
 http_timeout: 30s
 
-# Ingestor endpoint configuration
-ingestor_endpoint: "https://api.cloud.strettch.dev/resource-manager/api/v1/metrics/ingest"
 
 # Metadata service endpoint for authentication
 metadata_service_endpoint: "http://169.254.169.254/metadata/v1/auth-token"

pkg/clients/metadata/auth.go

@@ -10,12 +10,13 @@ import (
 
 // AuthManager handles periodic token refresh for metadata service authentication.
 type AuthManager struct {
-	client        *Client
-	vmID          string
-	logger        *zap.Logger
-	refreshTicker *time.Ticker
-	stopCh        chan struct{}
-	currentToken  string
+	client           *Client
+	vmID             string
+	logger           *zap.Logger
+	refreshTicker    *time.Ticker
+	stopCh           chan struct{}
+	currentToken     string
+	cloudAPIURL string
 }
 
 // NewAuthManager creates a new auth manager.
@@ -35,20 +36,28 @@ func (am *AuthManager) IsRefreshRunning() bool {
 	return am.refreshTicker != nil
 }
 
-// fetchAndStoreToken fetches a token and stores it internally.
+// fetchAndStoreToken fetches a token and CloudAPI URL and stores them internally.
 func (am *AuthManager) fetchAndStoreToken(ctx context.Context, forceFetch bool) error {
 	if forceFetch {
 		am.logger.Debug("Forcing token refresh")
 		am.client.InvalidateCache()
 	} else {
 		am.logger.Debug("Fetching token (using cache if valid)")
 	}
+	
 	token, err := am.client.GetAuthToken(ctx, am.vmID)
 	if err != nil {
 		return err
 	}
 	am.currentToken = token
-	am.logger.Debug("Token stored successfully")
+	
+	cloudAPIURL, err := am.client.GetCloudAPIURL(ctx, am.vmID)
+	if err != nil {
+		return err
+	}
+	am.cloudAPIURL = cloudAPIURL
+	
+	am.logger.Debug("Token and CloudAPI URL stored successfully")
 	return nil
 }
 
@@ -87,6 +96,11 @@ func (am *AuthManager) GetCurrentToken() string {
 	return am.currentToken
 }
 
+// GetCloudAPIURL returns the cached CloudAPI URL
+func (am *AuthManager) GetCloudAPIURL() string {
+	return am.cloudAPIURL
+}
+
 // Close stops the refresh loop.
 func (am *AuthManager) Close() {
 	if am.refreshTicker != nil {

pkg/clients/metadata/client.go

@@ -26,7 +26,8 @@ const (
 
 // TokenResponse represents the response from the metadata service
 type TokenResponse struct {
-	Token string `json:"token"`
+	Token       string `json:"token"`
+	CloudAPIUrl string `json:"cloudAPIUrl"`
 }
 
 // Client handles communication with the metadata service with token caching
@@ -35,9 +36,10 @@ type Client struct {
 	httpClient    *http.Client
 	logger        *zap.Logger
 
-	// Token caching
+	// Token and CloudAPI URL caching
 	tokenMu       sync.RWMutex
 	cachedToken   string
+	cachedAPIURL  string
 	tokenExpiry   time.Time
 	tokenLifetime time.Duration
 }
@@ -81,30 +83,31 @@ func (c *Client) GetAuthToken(ctx context.Context, vmID string) (string, error)
 		return c.cachedToken, nil
 	}
 
-	// Fetch new token
-	token, err := c.fetchAuthToken(ctx, vmID)
+	// Fetch new token and metadata
+	tokenResp, err := c.fetchAuthToken(ctx, vmID)
 	if err != nil {
 		return "", err
 	}
 
-	// Cache the token
-	c.cachedToken = token
+	// Cache the token and CloudAPI URL
+	c.cachedToken = tokenResp.Token
+	c.cachedAPIURL = tokenResp.CloudAPIUrl
 	c.tokenExpiry = time.Now().Add(c.tokenLifetime)
 
 	c.logger.Info("Successfully fetched and cached new auth token",
 		zap.Time("expires_at", c.tokenExpiry),
 		zap.Duration("lifetime", c.tokenLifetime))
 
-	return token, nil
+	return tokenResp.Token, nil
 }
 
-// fetchAuthToken fetches a new authentication token from the metadata service
-func (c *Client) fetchAuthToken(ctx context.Context, vmID string) (string, error) {
+// fetchAuthToken fetches a new authentication token and metadata
+func (c *Client) fetchAuthToken(ctx context.Context, vmID string) (*TokenResponse, error) {
 	c.logger.Debug("Fetching new auth token from metadata service", zap.String("endpoint", c.endpoint))
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.endpoint, nil)
 	if err != nil {
-		return "", fmt.Errorf("failed to create request: %w", err)
+		return nil, fmt.Errorf("failed to create request: %w", err)
 	}
 
 	req.Header.Set(HeaderAccept, AcceptJSON)
@@ -113,7 +116,7 @@ func (c *Client) fetchAuthToken(ctx context.Context, vmID string) (string, error
 	resp, err := c.httpClient.Do(req)
 	if err != nil {
 		c.logger.Error("Failed to fetch auth token", zap.Error(err))
-		return "", fmt.Errorf("failed to fetch auth token: %w", err)
+		return nil, fmt.Errorf("failed to fetch auth token: %w", err)
 	}
 	defer func() {
 		if closeErr := resp.Body.Close(); closeErr != nil {
@@ -126,25 +129,25 @@ func (c *Client) fetchAuthToken(ctx context.Context, vmID string) (string, error
 		c.logger.Error("Metadata service returned error",
 			zap.Int("status_code", resp.StatusCode),
 			zap.String("response", string(body)))
-		return "", fmt.Errorf("metadata service returned status %d: %s", resp.StatusCode, string(body))
+		return nil, fmt.Errorf("metadata service returned status %d: %s", resp.StatusCode, string(body))
 	}
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return "", fmt.Errorf("failed to read response body: %w", err)
+		return nil, fmt.Errorf("failed to read response body: %w", err)
 	}
 
 	var tokenResp TokenResponse
 	if err := json.Unmarshal(body, &tokenResp); err != nil {
-		return "", fmt.Errorf("failed to unmarshal token response: %w", err)
+		return nil, fmt.Errorf("failed to unmarshal token response: %w", err)
 	}
 
 	if tokenResp.Token == "" {
-		return "", fmt.Errorf("received empty token from metadata service")
+		return nil, fmt.Errorf("received empty token from metadata service")
 	}
 
-	c.logger.Debug("Successfully fetched auth token")
-	return tokenResp.Token, nil
+	c.logger.Debug("Successfully fetched auth token and metadata")
+	return &tokenResp, nil
 }
 
 // GetAuthTokenWithRetry fetches an auth token with retry logic similar to tsclient
@@ -191,10 +194,39 @@ func (c *Client) InvalidateCache() {
 	defer c.tokenMu.Unlock()
 
 	c.cachedToken = ""
+	c.cachedAPIURL = ""
 	c.tokenExpiry = time.Time{}
 	c.logger.Debug("Token cache invalidated")
 }
 
+// GetCloudAPIURL returns the cached CloudAPI URL
+func (c *Client) GetCloudAPIURL(ctx context.Context, vmID string) (string, error) {
+	c.tokenMu.RLock()
+	if c.cachedAPIURL != "" && time.Now().Before(c.tokenExpiry) {
+		url := c.cachedAPIURL
+		c.tokenMu.RUnlock()
+		c.logger.Debug("Using cached CloudAPI URL", zap.String("url", url))
+		return url, nil
+	}
+	c.tokenMu.RUnlock()
+
+	// Need to fetch new metadata to get the URL
+	_, err := c.GetAuthToken(ctx, vmID)
+	if err != nil {
+		return "", fmt.Errorf("failed to get CloudAPI URL: %w", err)
+	}
+
+	c.tokenMu.RLock()
+	url := c.cachedAPIURL
+	c.tokenMu.RUnlock()
+
+	if url == "" {
+		return "", fmt.Errorf("CloudAPI URL not available in metadata response")
+	}
+
+	return url, nil
+}
+
 // Close closes the HTTP client
 func (c *Client) Close() error {
 	c.httpClient.CloseIdleConnections()