CBG-4412: Add ability to pass custom import filter into diagnostic API

base/error.go

@@ -386,3 +386,28 @@ func (e *SyncFnDryRunError) Unwrap() error {
 	}
 	return e.Err
 }
+
+const importFilterErrorPrefix = "Error returned from Import Filter"
+
+// ImportFilterError is returned when the import filter dry run returns an error.
+// It wraps the original error for errors.Is and the type supports errors.As
+type ImportFilterDryRunError struct {
+	Err error
+}
+
+func (e *ImportFilterDryRunError) Error() string {
+	if e == nil {
+		return importFilterErrorPrefix
+	}
+	if e.Err == nil {
+		return importFilterErrorPrefix
+	}
+	return importFilterErrorPrefix + ": " + e.Err.Error()
+}
+
+func (e *ImportFilterDryRunError) Unwrap() error {
+	if e == nil {
+		return nil
+	}
+	return e.Err
+}

db/import.go

@@ -489,11 +489,9 @@ func (db *DatabaseCollectionWithUser) backupPreImportRevision(ctx context.Contex
 // ////// Import Filter Function
 
 // Compiles a JavaScript event function to a jsImportFilterRunner object.
-func newImportFilterRunner(ctx context.Context, funcSource string, timeout time.Duration) (sgbucket.JSServerTask, error) {
+func newImportFilterRunnerWithLogging(ctx context.Context, funcSource string, timeout time.Duration, errorLogFunc, infoLogFunc func(string)) (sgbucket.JSServerTask, error) {
 	importFilterRunner := &jsEventTask{}
-	err := importFilterRunner.InitWithLogging(funcSource, timeout,
-		func(s string) { base.ErrorfCtx(ctx, base.KeyJavascript.String()+": Import %s", base.UD(s)) },
-		func(s string) { base.InfofCtx(ctx, base.KeyJavascript, "Import %s", base.UD(s)) })
+	err := importFilterRunner.InitWithLogging(funcSource, timeout, errorLogFunc, infoLogFunc)
 	if err != nil {
 		return nil, err
 	}
@@ -506,6 +504,12 @@ func newImportFilterRunner(ctx context.Context, funcSource string, timeout time.
 	return importFilterRunner, nil
 }
 
+func newImportFilterRunner(ctx context.Context, funcSource string, timeout time.Duration) (sgbucket.JSServerTask, error) {
+	errLogFunc := func(s string) { base.ErrorfCtx(ctx, base.KeyJavascript.String()+": Import %s", base.UD(s)) }
+	infoLogFunc := func(s string) { base.InfofCtx(ctx, base.KeyJavascript, "Import %s", base.UD(s)) }
+	return newImportFilterRunnerWithLogging(ctx, funcSource, timeout, errLogFunc, infoLogFunc)
+}
+
 type ImportFilterFunction struct {
 	*sgbucket.JSServer
 }
@@ -545,20 +549,43 @@ func (i *ImportFilterFunction) EvaluateFunction(ctx context.Context, doc Body, d
 		return false, errors.New("Import filter function returned non-boolean value.")
 	}
 }
-func (db *DatabaseCollectionWithUser) ImportFilterDryRun(ctx context.Context, doc Body, docid string) (bool, error) {
-
-	importFilter := db.importFilter()
-	if docid != "" {
-		docInBucket, err := db.GetDocument(ctx, docid, DocUnmarshalAll)
-		if err == nil {
-			if doc == nil {
-				doc = docInBucket.Body(ctx)
-			}
-		} else {
+
+// ImportFilterDryRun Runs a document through the import filter and returns a boolean and error
+func (db *DatabaseCollectionWithUser) ImportFilterDryRun(ctx context.Context, doc Body, importFn string, errorLogFunc, infoLogFunc func(string)) (bool, error) {
+
+	var shouldImport bool
+
+	// fetch configured import filter if one is not specified
+	if importFn == "" {
+		importFilter := db.importFilter()
+		if importFilter == nil {
+			return true, nil
+		}
+		importFn = importFilter.Function()
+	}
+
+	// create new import filter runner for this dry run
+	jsTimeout := time.Duration(base.DefaultJavascriptTimeoutSecs) * time.Second
+	importRunner, err := newImportFilterRunnerWithLogging(ctx, importFn, jsTimeout, errorLogFunc, infoLogFunc)
+	if err != nil {
+		return false, errors.New("failed to create import filter runner: " + err.Error())
+	}
+	importOutput, err := importRunner.Call(ctx, doc)
+	if err != nil {
+		return false, &base.ImportFilterDryRunError{Err: err}
+	}
+	switch result := importOutput.(type) {
+	case bool:
+		shouldImport = result
+	case string:
+		boolResult, err := strconv.ParseBool(result)
+		if err != nil {
 			return false, err
 		}
+		shouldImport = boolResult
+	default:
+		return false, &base.ImportFilterDryRunError{Err: err}
 	}
-	shouldImport, err := importFilter.EvaluateFunction(ctx, doc, true)
 
-	return shouldImport, err
+	return shouldImport, nil
 }

docs/api/paths/diagnostic/keyspace-import_filter.yaml

@@ -7,16 +7,31 @@
 # the file licenses/APL2.txt.
 parameters:
   - $ref: ../../components/parameters.yaml#/keyspace
-get:
+  - $ref: ../../components/parameters.yaml#/doc_id
+post:
   summary: Run a doc body through the Import filter and return results.
   description: |-
-    Run a document body through the import filter and return whether its imported or not, and any error messages.
+    Runs a document body through the import filter and return whether its
+    imported or not, and any error messages. If no custom import filter is
+    provided in the request body, the default or user-defined import filter
+    is used.
+
     * Sync Gateway Application Read Only
   requestBody:
     content:
       application/json:
         schema:
-          $ref: ../../components/schemas.yaml#/Document
+          type: object
+          properties:
+            import_filter:
+              description: |-
+                A JavaScript function that all imported documents in the
+                default scope and collection are ran through in order to filter
+                out what to import and what not to import.
+              type: string
+              example: 'function(doc) { if (doc.type != ''mobile'') { return false; } return true; }'
+            doc:
+              $ref: ../../components/schemas.yaml#/Document
   responses:
     '200':
       description: Document Processed by import filter successfully

rest/diagnostic_doc_api.go

@@ -12,7 +12,6 @@ package rest
 
 import (
 	"errors"
-	"fmt"
 	"net/http"
 
 	"github.com/couchbase/sync_gateway/auth"
@@ -21,30 +20,36 @@ import (
 	"github.com/couchbase/sync_gateway/db"
 )
 
-type SyncFnDryRunLogging struct {
+type DryRunLogging struct {
 	Errors []string `json:"errors"`
 	Info   []string `json:"info"`
 }
 
 type SyncFnDryRun struct {
-	Channels  base.Set            `json:"channels"`
-	Access    channels.AccessMap  `json:"access"`
-	Roles     channels.AccessMap  `json:"roles"`
-	Exception string              `json:"exception,omitempty"`
-	Expiry    *uint32             `json:"expiry,omitempty"`
-	Logging   SyncFnDryRunLogging `json:"logging"`
+	Channels  base.Set           `json:"channels"`
+	Access    channels.AccessMap `json:"access"`
+	Roles     channels.AccessMap `json:"roles"`
+	Exception string             `json:"exception,omitempty"`
+	Expiry    *uint32            `json:"expiry,omitempty"`
+	Logging   DryRunLogging      `json:"logging"`
 }
 
 type ImportFilterDryRun struct {
-	ShouldImport bool   `json:"shouldImport"`
-	Error        string `json:"error"`
+	ShouldImport bool          `json:"shouldImport"`
+	Error        string        `json:"error"`
+	Logging      DryRunLogging `json:"logging"`
 }
 
 type SyncFnDryRunPayload struct {
 	Function string  `json:"sync_function"`
 	Doc      db.Body `json:"doc,omitempty"`
 }
 
+type ImportFilterDryRunPayload struct {
+	Function string  `json:"import_filter"`
+	Doc      db.Body `json:"doc,omitempty"`
+}
+
 func populateDocChannelInfo(doc db.Document) map[string][]auth.GrantHistorySequencePair {
 	resp := make(map[string][]auth.GrantHistorySequencePair, len(doc.Channels))
 
@@ -169,7 +174,7 @@ func (h *handler) handleSyncFnDryRun() error {
 		errMsg := syncFnDryRunErr.Error()
 		resp := SyncFnDryRun{
 			Exception: errMsg,
-			Logging:   SyncFnDryRunLogging{Errors: logErrors, Info: logInfo},
+			Logging:   DryRunLogging{Errors: logErrors, Info: logInfo},
 		}
 		h.writeJSON(resp)
 		return nil
@@ -185,7 +190,7 @@ func (h *handler) handleSyncFnDryRun() error {
 		output.Roles,
 		errorMsg,
 		output.Expiry,
-		SyncFnDryRunLogging{Errors: logErrors, Info: logInfo},
+		DryRunLogging{Errors: logErrors, Info: logInfo},
 	}
 	h.writeJSON(resp)
 	return nil
@@ -195,24 +200,49 @@ func (h *handler) handleSyncFnDryRun() error {
 func (h *handler) handleImportFilterDryRun() error {
 	docid := h.getQuery("doc_id")
 
-	body, err := h.readDocument()
+	var importFilterPayload ImportFilterDryRunPayload
+	err := h.readJSONInto(&importFilterPayload)
 	if err != nil {
-		if docid == "" {
-			return fmt.Errorf("Error reading body: %s, no doc id provided for dry run", err)
-		}
+		return base.HTTPErrorf(http.StatusBadRequest, "Error reading import filter payload: %v", err)
 	}
 
-	if docid != "" && body != nil {
+	// Cannot pass both doc_id and body in the request body
+	if len(importFilterPayload.Doc) > 0 && docid != "" {
 		return base.HTTPErrorf(http.StatusBadRequest, "doc body and doc id provided. Please provide either the body or a doc id for the import filter dry run")
 	}
-	shouldImport, err := h.collection.ImportFilterDryRun(h.ctx(), body, docid)
+
+	var doc db.Body
+	if docid != "" {
+		docInBucket, err := h.collection.GetDocument(h.ctx(), docid, db.DocUnmarshalSync)
+		if err != nil {
+			return err
+		}
+		doc = docInBucket.Body(h.ctx())
+	} else {
+		doc = importFilterPayload.Doc
+	}
+
+	logErrors := make([]string, 0)
+	logInfo := make([]string, 0)
+	errorLogFn := func(s string) {
+		logErrors = append(logErrors, s)
+	}
+	infoLogFn := func(s string) {
+		logInfo = append(logInfo, s)
+	}
+	shouldImport, err := h.collection.ImportFilterDryRun(h.ctx(), doc, importFilterPayload.Function, errorLogFn, infoLogFn)
 	errorMsg := ""
 	if err != nil {
+		var importFilterDryRunErr *base.ImportFilterDryRunError
+		if !errors.As(err, &importFilterDryRunErr) {
+			return err
+		}
 		errorMsg = err.Error()
 	}
 	resp := ImportFilterDryRun{
 		shouldImport,
 		errorMsg,
+		DryRunLogging{Errors: logErrors, Info: logInfo},
 	}
 	h.writeJSON(resp)
 	return nil