Merge pull request #294 from USACE/chore/survey123-telemetry-proxy

chore(api, telemetry): Update Survey123 to use telemetry proxy

Author: dennisgsmith

api/internal/config/api.go

@@ -1,7 +1,6 @@
 package config
 
 import (
-	"net/url"
 	"os"
 	"time"
 
@@ -22,7 +21,6 @@ type ApiConfig struct {
 	AlertEventBatchSize    int           `env:"ALERT_EVENT_BATCH_SIZE" envDefault:"100"`
 	AlertEventBatchTimeout time.Duration `env:"ALERT_EVENT_TIMEOUT" envDefault:"30s"`
 	AlertEventFlushWorkers int           `env:"ALERT_EVENT_FLUSH_WORKERS" envDefault:"4"`
-	Survey123IPWhitelist   []url.URL     `env:"SURVEY123_IP_WHITELIST"`
 	IrisFdsnProxyURL       string        `env:"IRIS_FDSN_PROXY_URL" envDefault:"https://service.iris.edu/fdsnws/station/1/query"`
 	CwmsProxyURL           string        `env:"CWMS_PROXY_URL" envDefault:"https://cwms-data.usace.army.mil/cwms-data/"`
 }

api/internal/db/models.go

@@ -171,7 +171,6 @@ type SaaMeasurement struct {
 type Survey123EquivalencyTableField struct {
 	FieldName    string     `json:"field_name"`
 	DisplayName  string     `json:"display_name"`
-	InstrumentID *uuid.UUID `json:"instrument_id"`
 	TimeseriesID *uuid.UUID `json:"timeseries_id"`
 }
 

api/internal/db/overrides.go

@@ -1,6 +1,8 @@
 package dto
 
 import (
+	"encoding/json"
+
 	"github.com/google/uuid"
 )
 
@@ -33,3 +35,8 @@ type Survey123ApplyEditsDTO struct {
 	ObjectID   any            `json:"objectId,omitempty" required:"false"`
 	Geometry   any            `json:"geometry,omitempty" required:"false"`
 }
+
+type Survey123TelemetryDTO struct {
+	Key     string
+	Payload map[string]json.RawMessage
+}

api/internal/db/querier.go

@@ -134,8 +134,7 @@ type apiMiddlewares struct {
 	ProjectAdmin,
 	ProjectMember,
 	AppAdmin,
-	InternalApp,
-	Survey123IPWhitelist huma.Middlewares
+	InternalApp huma.Middlewares
 }
 
 type security []map[string][]string
@@ -202,7 +201,6 @@ func NewApiRouter(ctx context.Context, h *ApiHandler) *Router {
 	h.ProjectMember = append(h.ProjectMember, mw.IsProjectMember)
 	h.Cac = huma.Middlewares{mw.JWT, mw.AttachClaims, mw.RequireClaims}
 	h.InternalApp = huma.Middlewares{mw.AppKeyAuth}
-	h.Survey123IPWhitelist = huma.Middlewares{mw.NewWhitelistXFFMiddleware(h.Config.Survey123IPWhitelist)}
 
 	humaCfg := huma.DefaultConfig("MIDAS API", API_VERSION)
 	humaCfg.Servers = append(humaCfg.Servers, &huma.Server{URL: h.Config.ServerBaseUrl + API_VERSION_PREFIX})

api/internal/db/survey123.sql_gen.go

@@ -2,14 +2,18 @@ package handler
 
 import (
 	"context"
+	"database/sql"
 	"encoding/json"
+	"errors"
 	"net/http"
 	"time"
 
 	"github.com/USACE/instrumentation-api/api/v4/internal/ctxkey"
 	"github.com/USACE/instrumentation-api/api/v4/internal/db"
 	"github.com/USACE/instrumentation-api/api/v4/internal/dto"
 	"github.com/USACE/instrumentation-api/api/v4/internal/httperr"
+	"github.com/USACE/instrumentation-api/api/v4/internal/password"
+	"github.com/USACE/instrumentation-api/api/v4/internal/service"
 	"github.com/danielgtaylor/huma/v2"
 )
 
@@ -50,6 +54,9 @@ func (h *ApiHandler) RegisterSurvey123(api huma.API) {
 	}) (*Response[db.Survey123Preview], error) {
 		a, err := h.DBService.Survey123PreviewGet(ctx, input.Survey123ID.UUID)
 		if err != nil {
+			if errors.Is(err, sql.ErrNoRows) {
+				return nil, httperr.NotFound(err)
+			}
 			return nil, httperr.InternalServerError(err)
 		}
 		return NewResponse(a), nil
@@ -67,7 +74,7 @@ func (h *ApiHandler) RegisterSurvey123(api huma.API) {
 	}, func(ctx context.Context, input *struct {
 		ProjectIDParam
 		Body dto.Survey123DTO
-	}) (*Response[ID], error) {
+	}) (*Response[service.Survey123IDWithKey], error) {
 		sv := input.Body
 		sv.ProjectID = input.ProjectID.UUID
 
@@ -82,7 +89,34 @@ func (h *ApiHandler) RegisterSurvey123(api huma.API) {
 			return nil, httperr.InternalServerError(err)
 		}
 
-		return NewResponseID(a), nil
+		return NewResponse(a), nil
+	})
+
+	huma.Register(api, huma.Operation{
+		Middlewares: h.ProjectAdmin,
+		Security:    projectAdminSecurity,
+		OperationID: "survey123-update-key",
+		Method:      http.MethodPut,
+		Path:        "/projects/{project_id}/survey123/{survey123_id}/key",
+		Description: "cycles a Survey123 key (project admin)",
+		Tags:        survey123Tags,
+	}, func(ctx context.Context, input *struct {
+		ProjectIDParam
+		Survey123IDParam
+	}) (*Response[service.Survey123IDWithKey], error) {
+		key := password.GenerateRandom(40)
+		hash := password.MustCreateHash(key, password.DefaultParams)
+
+		if err := h.DBService.Survey123HashUpdate(ctx, db.Survey123HashUpdateParams{
+			Survey123ID: input.Survey123ID.UUID,
+			Hash:        hash,
+		}); err != nil {
+			return nil, httperr.InternalServerError(err)
+		}
+		return NewResponse(service.Survey123IDWithKey{
+			ID:  input.Survey123ID.UUID,
+			Key: key,
+		}), nil
 	})
 
 	huma.Register(api, huma.Operation{
@@ -139,22 +173,32 @@ func (h *ApiHandler) RegisterSurvey123(api huma.API) {
 
 	huma.Register(api, huma.Operation{
 		Hidden:      true,
-		Middlewares: h.Survey123IPWhitelist,
+		Middlewares: h.InternalApp,
 		OperationID: "survey123-telemetry-create-or-update-measurements",
 		Method:      http.MethodPost,
 		Path:        "/webhooks/survey123/{survey123_id}/measurements",
 		Description: "webhook for creating or updating measurements for survey123 mappings from AGS04",
 		Tags:        survey123Tags,
 	}, func(ctx context.Context, input *struct {
 		Survey123IDParam
-		Body map[string]json.RawMessage
-	}) (*Response[ID], error) {
+		Body dto.Survey123TelemetryDTO
+	}) (*struct{}, error) {
+		hash, err := h.DBService.Survey123HashGet(ctx, input.Survey123ID.UUID)
+		if err != nil {
+			if errors.Is(err, sql.ErrNoRows) {
+				return nil, httperr.Unauthorized(err)
+			}
+			return nil, httperr.InternalServerError(err)
+		}
+		match, err := password.ComparePasswordAndHash(input.Body.Key, hash)
+		if err != nil || !match {
+			return nil, httperr.Unauthorized(errors.New("error validating survey123 hash"))
+		}
 
-		preview, err := json.Marshal(input.Body)
+		preview, err := json.Marshal(input.Body.Payload)
 		if err != nil {
 			return nil, httperr.BadRequest(err)
 		}
-
 		if err := h.DBService.Survey123PreviewCreateOrUpdate(ctx, db.Survey123PreviewCreateOrUpdateParams{
 			Survey123ID: input.Survey123ID.UUID,
 			Preview:     string(preview),
@@ -168,7 +212,7 @@ func (h *ApiHandler) RegisterSurvey123(api huma.API) {
 			return nil, httperr.ServerErrorOrNotFound(err)
 		}
 
-		raw := input.Body
+		raw := input.Body.Payload
 		var et string
 		if err := json.Unmarshal(raw["eventType"], &et); err != nil {
 			return nil, httperr.UnprocessableEntity(err)
@@ -185,6 +229,6 @@ func (h *ApiHandler) RegisterSurvey123(api huma.API) {
 			return nil, httperr.InternalServerError(err)
 		}
 
-		return NewResponseID(input.Survey123ID.UUID), nil
+		return nil, nil
 	})
 }

api/internal/dto/survey123.go

@@ -0,0 +1,92 @@
+package handler
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/USACE/instrumentation-api/api/v4/internal/dto"
+	"github.com/USACE/instrumentation-api/api/v4/internal/httperr"
+	"github.com/USACE/instrumentation-api/api/v4/internal/util"
+	"github.com/danielgtaylor/huma/v2"
+)
+
+func (h *TelemetryHandler) RegisterSurvey123Proxy(api huma.API) {
+	huma.Register(api, huma.Operation{
+		Hidden:       true,
+		Path:         "/telemetry/survey123/{survey123_id}",
+		OperationID:  "survey123-telemetry-proxy",
+		Method:       http.MethodPost,
+		Description:  "survey123 webhook telemetry ingress",
+		MaxBodyBytes: maxBodyBytes,
+	}, func(_ context.Context, input *struct {
+		Key string `query:"key"`
+		Survey123IDParam
+		RawBody map[string]json.RawMessage
+	}) (*ResponseWithStatus[any], error) {
+		ctx := context.Background()
+		body, err := json.Marshal(dto.Survey123TelemetryDTO{
+			Key:     input.Key,
+			Payload: input.RawBody,
+		})
+		if err != nil {
+			return nil, httperr.BadRequest(err)
+		}
+		req, err := http.NewRequestWithContext(
+			ctx,
+			http.MethodPost,
+			fmt.Sprintf("%s/webhooks/survey123/%s/measurements?key=%s", h.Config.ApiHost, input.Survey123IDParam, h.Config.ApplicationKey),
+			bytes.NewReader(body),
+		)
+		if err != nil {
+			return nil, httperr.BadRequest(errors.New("could not construct request with given parameters"))
+		}
+		defer func() {
+			if err := req.Body.Close(); err != nil {
+				h.Logger.Error(ctx, "error closing request body", "error", err)
+			}
+		}()
+
+		res, err := h.Client.Do(req)
+		if err != nil {
+			var urlErr *url.Error
+			if errors.As(err, &urlErr) {
+				u, err := url.Parse(urlErr.URL)
+				if err != nil {
+					return nil, httperr.InternalServerError(errors.New("error occured while attempting to parse the url of another error"))
+				}
+				util.RedactQueryParams(u, "key")
+				urlErr.URL = u.String()
+				return nil, httperr.InternalServerError(fmt.Errorf("urlErr %w", urlErr))
+			}
+			return nil, httperr.InternalServerError(fmt.Errorf("expected error to be *url.Error type, got %T", err))
+		}
+		defer func() {
+			if err := res.Body.Close(); err != nil {
+				h.Logger.Error(ctx, "error closing response body", "error", err)
+			}
+		}()
+		resBody, err := io.ReadAll(res.Body)
+		if err != nil {
+			return nil, httperr.InternalServerError(fmt.Errorf("failed to read response body %w", err))
+		}
+
+		h.Logger.Debug(ctx, "received response from api server", "code", res.StatusCode, "response_body", json.RawMessage(resBody))
+
+		switch res.StatusCode {
+		case http.StatusCreated:
+			return NewResponseWithStatus(http.StatusCreated, any(resBody)), nil
+		case http.StatusNotFound:
+			// Issue with upstream server
+			return nil, httperr.Message(http.StatusBadGateway, "invalid response from upstream proxy")
+		default:
+			return NewResponseWithStatus(http.StatusInternalServerError, any(json.RawMessage(resBody))), nil
+		}
+
+	})
+}

api/internal/handler/api.go

@@ -76,11 +76,16 @@ func newTelemetryRouter(h *TelemetryHandler) *Router {
 	api := humaecho.NewWithGroup(e, g, humaCfg)
 
 	humaCfg.Components.SecuritySchemes = map[string]*huma.SecurityScheme{
-		keyAuth: {
+		"apiKey-header": {
 			Name: "X-Api-Key",
 			Type: "apiKey",
 			In:   "header",
 		},
+		"apiKey-query": {
+			Name: "key",
+			Type: "apiKey",
+			In:   "query",
+		},
 	}
 
 	huma.AutoRegister(api, h)