[3.3.3 backport] CBG-5116: Diagnostic API Improvements (#7999)

Co-authored-by: Ben Brooks <[email protected]>
Co-authored-by: Copilot <[email protected]>

Author: 3 people

base/audit_events.go

@@ -30,9 +30,10 @@ const (
 	AuditIDSyncGatewayStartup AuditID = 53260
 
 	// API events
-	AuditIDPublicHTTPAPIRequest  AuditID = 53270
-	AuditIDAdminHTTPAPIRequest   AuditID = 53271
-	AuditIDMetricsHTTPAPIRequest AuditID = 53272
+	AuditIDPublicHTTPAPIRequest      AuditID = 53270
+	AuditIDAdminHTTPAPIRequest       AuditID = 53271
+	AuditIDMetricsHTTPAPIRequest     AuditID = 53272
+	AuditIDDiagnosticsHTTPAPIRequest AuditID = 53273
 
 	// Auth (public) events
 	AuditIDPublicUserAuthenticated        AuditID = 53280
@@ -266,6 +267,29 @@ var AuditEvents = events{
 		EventType:          eventTypeAdmin,
 		IsGlobalEvent:      true,
 	},
+	AuditIDDiagnosticsHTTPAPIRequest: {
+		Name:        "Diagnostic HTTP API request",
+		Description: "Diagnostic HTTP API request was made",
+		MandatoryFields: AuditFields{
+			AuditFieldHTTPMethod: "GET, POST, etc.",
+			AuditFieldHTTPPath:   "request_path",
+			AuditFieldHTTPStatus: 200,
+		},
+		OptionalFields: AuditFields{
+			AuditFieldRequestBody: "request_body",
+			AuditFieldRealUserID: map[string]any{
+				AuditFieldRealUserIDDomain: "user domain",
+				AuditFieldRealUserIDUser:   "user name",
+			},
+		},
+		mandatoryFieldGroups: []fieldGroup{
+			fieldGroupRequest,
+		},
+		EnabledByDefault:   false,
+		FilteringPermitted: true,
+		EventType:          eventTypeAdmin,
+		IsGlobalEvent:      true,
+	},
 	AuditIDPublicUserAuthenticated: {
 		Name:        "Public API user authenticated",
 		Description: "Public API user successfully authenticated",

base/error.go

@@ -361,3 +361,53 @@ func (me *MultiError) ErrorOrNil() error {
 	}
 	return me
 }
+
+const syncFnDryRunErrorPrefix = "Error returned from Sync Function"
+
+// SyncFnDryRunError is returned when the sync function dry run returns an error.
+// It wraps the original error for errors.Is and the type supports errors.As
+type SyncFnDryRunError struct {
+	Err error
+}
+
+func (e *SyncFnDryRunError) Error() string {
+	if e == nil {
+		return syncFnDryRunErrorPrefix
+	}
+	if e.Err == nil {
+		return syncFnDryRunErrorPrefix
+	}
+	return syncFnDryRunErrorPrefix + ": " + e.Err.Error()
+}
+
+func (e *SyncFnDryRunError) Unwrap() error {
+	if e == nil {
+		return nil
+	}
+	return e.Err
+}
+
+const importFilterErrorPrefix = "Error returned from Import Filter"
+
+// ImportFilterError is returned when the import filter dry run returns an error.
+// It wraps the original error for errors.Is and the type supports errors.As
+type ImportFilterDryRunError struct {
+	Err error
+}
+
+func (e *ImportFilterDryRunError) Error() string {
+	if e == nil {
+		return importFilterErrorPrefix
+	}
+	if e.Err == nil {
+		return importFilterErrorPrefix
+	}
+	return importFilterErrorPrefix + ": " + e.Err.Error()
+}
+
+func (e *ImportFilterDryRunError) Unwrap() error {
+	if e == nil {
+		return nil
+	}
+	return e.Err
+}

channels/sync_runner.go

@@ -116,12 +116,10 @@ type SyncRunner struct {
 	expiry            *uint32             // document expiry (in seconds) specified via expiry() callback
 }
 
-func NewSyncRunner(ctx context.Context, funcSource string, timeout time.Duration) (*SyncRunner, error) {
+func NewSyncRunnerWithLogging(ctx context.Context, funcSource string, timeout time.Duration, errorLogFunc, infoLogFunc func(string)) (*SyncRunner, error) {
 	funcSource = wrappedFuncSource(funcSource)
 	runner := &SyncRunner{}
-	err := runner.InitWithLogging(funcSource, timeout,
-		func(s string) { base.ErrorfCtx(ctx, base.KeyJavascript.String()+": Sync %s", base.UD(s)) },
-		func(s string) { base.InfofCtx(ctx, base.KeyJavascript, "Sync %s", base.UD(s)) })
+	err := runner.InitWithLogging(funcSource, timeout, errorLogFunc, infoLogFunc)
 	if err != nil {
 		return nil, err
 	}
@@ -212,6 +210,12 @@ func NewSyncRunner(ctx context.Context, funcSource string, timeout time.Duration
 	return runner, nil
 }
 
+func NewSyncRunner(ctx context.Context, funcSource string, timeout time.Duration) (*SyncRunner, error) {
+	errorLogFunc := func(s string) { base.ErrorfCtx(ctx, base.KeyJavascript.String()+": Sync %s", base.UD(s)) }
+	infoLogFunc := func(s string) { base.InfofCtx(ctx, base.KeyJavascript, "Sync %s", base.UD(s)) }
+	return NewSyncRunnerWithLogging(ctx, funcSource, timeout, errorLogFunc, infoLogFunc)
+}
+
 func (runner *SyncRunner) SetFunction(funcSource string) (bool, error) {
 	funcSource = wrappedFuncSource(funcSource)
 	return runner.JSRunner.SetFunction(funcSource)

db/change_cache.go

@@ -376,7 +376,7 @@ func (c *changeCache) DocChanged(event sgbucket.FeedEvent, docType DocumentType)
 	}
 
 	// First unmarshal the doc (just its metadata, to save time/memory):
-	syncData, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(docJSON, event.DataType, collection.userXattrKey(), false)
+	syncData, rawBody, rawXattrs, err := UnmarshalDocumentSyncDataFromFeed(docJSON, event.DataType, collection.UserXattrKey(), false)
 	if err != nil {
 		// Avoid log noise related to failed unmarshaling of binary documents.
 		if event.DataType != base.MemcachedDataTypeRaw {
@@ -389,7 +389,7 @@ func (c *changeCache) DocChanged(event sgbucket.FeedEvent, docType DocumentType)
 	}
 
 	// If using xattrs and this isn't an SG write, we shouldn't attempt to cache.
-	rawUserXattr := rawXattrs[collection.userXattrKey()]
+	rawUserXattr := rawXattrs[collection.UserXattrKey()]
 	if collection.UseXattrs() {
 		if syncData == nil {
 			return

db/crud.go

@@ -958,7 +958,7 @@ func (db *DatabaseCollectionWithUser) Put(ctx context.Context, docid string, bod
 		}
 
 		// Make up a new _rev, and add it to the history:
-		bodyWithoutInternalProps, wasStripped := stripInternalProperties(body)
+		bodyWithoutInternalProps, wasStripped := StripInternalProperties(body)
 		canonicalBytesForRevID, err := base.JSONMarshalCanonical(bodyWithoutInternalProps)
 		if err != nil {
 			return nil, nil, false, nil, err
@@ -1174,87 +1174,47 @@ func (db *DatabaseCollectionWithUser) PutExistingRevWithBody(ctx context.Context
 
 }
 
-// SyncFnDryrun Runs a document through the sync function and returns expiry, channels doc was placed in, access map for users, roles, handler errors and sync fn exceptions
-func (db *DatabaseCollectionWithUser) SyncFnDryrun(ctx context.Context, body Body, docID string) (*channels.ChannelMapperOutput, error, error) {
-	doc := &Document{
-		ID:    docID,
-		_body: body,
-	}
-	oldDoc := doc
-	if docID != "" {
-		if docInBucket, err := db.GetDocument(ctx, docID, DocUnmarshalAll); err == nil {
-			oldDoc = docInBucket
-			if doc._body == nil {
-				body = oldDoc.Body(ctx)
-				doc._body = body
-				// If no body is given, use doc in bucket as doc with no old doc
-				oldDoc._body = nil
-			}
-			doc._body[BodyRev] = oldDoc.SyncData.CurrentRev
-		} else {
-			return nil, err, nil
-		}
-	} else {
-		oldDoc._body = nil
-	}
-
-	delete(body, BodyId)
-
-	// Get the revision ID to match, and the new generation number:
-	matchRev, _ := body[BodyRev].(string)
-	generation, _ := ParseRevID(ctx, matchRev)
-	if generation < 0 {
-		return nil, base.HTTPErrorf(http.StatusBadRequest, "Invalid revision ID"), nil
+// SyncFnDryrun Runs the given document body through a sync function and returns expiry, channels doc was placed in,
+// access map for users, roles, handler errors and sync fn exceptions.
+// If syncFn is provided, it will be used instead of the one configured on the database.
+func (db *DatabaseCollectionWithUser) SyncFnDryrun(ctx context.Context, newDoc, oldDoc *Document, userMeta, syncOptions map[string]any, syncFn string, errorLogFunc, infoLogFunc func(string)) (*channels.ChannelMapperOutput, error) {
+	mutableBody, metaMap, _, err := db.prepareSyncFn(oldDoc, newDoc)
+	if err != nil {
+		base.InfofCtx(ctx, base.KeyDiagnostic, "Failed to prepare to run sync function: %v", err)
+		return nil, err
 	}
-	generation++
 
-	// Create newDoc which will be used to pass around Body
-	newDoc := &Document{
-		ID: docID,
+	if userMeta != nil {
+		metaMap = userMeta
 	}
-	// Pull out attachments
-	newDoc.DocAttachments = GetBodyAttachments(body)
-	delete(body, BodyAttachments)
 
-	delete(body, BodyRevisions)
-
-	err := validateAPIDocUpdate(body)
-	if err != nil {
-		return nil, err, nil
-	}
-	bodyWithoutInternalProps, wasStripped := stripInternalProperties(body)
-	canonicalBytesForRevID, err := base.JSONMarshalCanonical(bodyWithoutInternalProps)
-	if err != nil {
-		return nil, err, nil
+	if syncOptions == nil {
+		syncOptions = MakeUserCtx(db.user, db.ScopeName, db.Name)
 	}
 
-	// We needed to keep _deleted around in the body until we generated a rev ID, but now we can ditch it.
-	_, isDeleted := body[BodyDeleted]
-	if isDeleted {
-		delete(body, BodyDeleted)
+	// fetch configured sync function if one is not provided
+	if syncFn == "" {
+		if db.ChannelMapper != nil {
+			syncFn = db.ChannelMapper.Function()
+		} else {
+			scopeAndCollectionName := db.ScopeAndCollectionName()
+			syncFn = channels.GetDefaultSyncFunction(scopeAndCollectionName.Scope, scopeAndCollectionName.Collection)
+		}
 	}
 
-	// and now we can finally update the newDoc body to be without any special properties
-	newDoc.UpdateBody(body)
-
-	// If no special properties were stripped and document wasn't deleted, the canonical bytes represent the current
-	// body.  In this scenario, store canonical bytes as newDoc._rawBody
-	if !wasStripped && !isDeleted {
-		newDoc._rawBody = canonicalBytesForRevID
+	// create new sync runner instance for this dry run
+	jsTimeout := time.Duration(base.DefaultJavascriptTimeoutSecs) * time.Second
+	syncRunner, err := channels.NewSyncRunnerWithLogging(ctx, syncFn, jsTimeout, errorLogFunc, infoLogFunc)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create sync runner: %v", err)
 	}
 
-	newRev := CreateRevIDWithBytes(generation, matchRev, canonicalBytesForRevID)
-	newDoc.RevID = newRev
-	mutableBody, metaMap, _, err := db.prepareSyncFn(oldDoc, newDoc)
+	jsOutput, err := syncRunner.Call(ctx, mutableBody, sgbucket.JSONString(oldDoc._rawBody), metaMap, syncOptions)
 	if err != nil {
-		base.InfofCtx(ctx, base.KeyDiagnostic, "Failed to prepare to run sync function: %v", err)
-		return nil, err, nil
+		return nil, &base.SyncFnDryRunError{Err: err}
 	}
 
-	output, err := db.ChannelMapper.MapToChannelsAndAccess(ctx, mutableBody, string(oldDoc._rawBody), metaMap,
-		MakeUserCtx(db.user, db.ScopeName, db.Name))
-
-	return output, nil, err
+	return jsOutput.(*channels.ChannelMapperOutput), nil
 }
 
 // resolveConflict runs the conflictResolverFunction with doc and newDoc.  doc and newDoc's bodies and revision trees
@@ -1596,7 +1556,7 @@ func (db *DatabaseCollectionWithUser) storeOldBodyInRevTreeAndUpdateCurrent(ctx
 
 func (db *DatabaseCollectionWithUser) prepareSyncFn(doc *Document, newDoc *Document) (mutableBody Body, metaMap map[string]interface{}, newRevID string, err error) {
 	// Marshal raw user xattrs for use in Sync Fn. If this fails we can bail out so we should do early as possible.
-	metaMap, err = doc.GetMetaMap(db.userXattrKey())
+	metaMap, err = doc.GetMetaMap(db.UserXattrKey())
 	if err != nil {
 		return
 	}

db/database.go

@@ -1679,7 +1679,7 @@ func (db *DatabaseCollectionWithUser) getResyncedDocument(ctx context.Context, d
 			base.WarnfCtx(ctx, "Error unmarshalling body %s/%s for sync function %s", base.UD(docid), rev.ID, err)
 			return
 		}
-		metaMap, err := doc.GetMetaMap(db.userXattrKey())
+		metaMap, err := doc.GetMetaMap(db.UserXattrKey())
 		if err != nil {
 			return
 		}

db/database_collection.go

@@ -235,7 +235,7 @@ func (c *DatabaseCollection) unsupportedOptions() *UnsupportedOptions {
 // syncAndUserXattrKeys returns the xattr keys for the user and sync xattrs.
 func (c *DatabaseCollection) syncAndUserXattrKeys() []string {
 	xattrKeys := []string{base.SyncXattrName}
-	userXattrKey := c.userXattrKey()
+	userXattrKey := c.UserXattrKey()
 	if userXattrKey != "" {
 		xattrKeys = append(xattrKeys, userXattrKey)
 	}
@@ -248,15 +248,15 @@ func (c *DatabaseCollection) syncMouAndUserXattrKeys() []string {
 	if c.useMou() {
 		xattrKeys = append(xattrKeys, base.MouXattrName)
 	}
-	userXattrKey := c.userXattrKey()
+	userXattrKey := c.UserXattrKey()
 	if userXattrKey != "" {
 		xattrKeys = append(xattrKeys, userXattrKey)
 	}
 	return xattrKeys
 }
 
 // Returns the xattr key that will be accessible from the sync function. This is controlled at a database level.
-func (c *DatabaseCollection) userXattrKey() string {
+func (c *DatabaseCollection) UserXattrKey() string {
 	return c.dbCtx.Options.UserXattrKey
 }
 

db/document.go

@@ -391,7 +391,7 @@ func unmarshalDocument(docid string, data []byte) (*Document, error) {
 
 // unmarshalDocumentWithXattrs populates individual xattrs on unmarshalDocumentWithXattrs from a provided xattrs map
 func (db *DatabaseCollection) unmarshalDocumentWithXattrs(ctx context.Context, docid string, data []byte, xattrs map[string][]byte, cas uint64, unmarshalLevel DocumentUnmarshalLevel) (doc *Document, err error) {
-	return unmarshalDocumentWithXattrs(ctx, docid, data, xattrs[base.SyncXattrName], xattrs[base.MouXattrName], xattrs[db.userXattrKey()], cas, unmarshalLevel)
+	return unmarshalDocumentWithXattrs(ctx, docid, data, xattrs[base.SyncXattrName], xattrs[base.MouXattrName], xattrs[db.UserXattrKey()], cas, unmarshalLevel)
 
 }