diff --git a/base/error.go b/base/error.go index 1290d57707..070d0e77c8 100644 --- a/base/error.go +++ b/base/error.go @@ -386,3 +386,28 @@ func (e *SyncFnDryRunError) Unwrap() error { } return e.Err } + +const importFilterErrorPrefix = "Error returned from Import Filter" + +// ImportFilterError is returned when the import filter dry run returns an error. +// It wraps the original error for errors.Is and the type supports errors.As +type ImportFilterDryRunError struct { + Err error +} + +func (e *ImportFilterDryRunError) Error() string { + if e == nil { + return importFilterErrorPrefix + } + if e.Err == nil { + return importFilterErrorPrefix + } + return importFilterErrorPrefix + ": " + e.Err.Error() +} + +func (e *ImportFilterDryRunError) Unwrap() error { + if e == nil { + return nil + } + return e.Err +} diff --git a/db/import.go b/db/import.go index a1501c0d47..59346ef109 100644 --- a/db/import.go +++ b/db/import.go @@ -252,13 +252,13 @@ func (db *DatabaseCollectionWithUser) importDoc(ctx context.Context, docid strin if isDelete && body == nil { deleteBody := Body{BodyDeleted: true} - shouldImport, err = importFilter.EvaluateFunction(ctx, deleteBody, false) + shouldImport, err = importFilter.EvaluateFunction(ctx, deleteBody) } else if isDelete && body != nil { deleteBody := body.ShallowCopy() deleteBody[BodyDeleted] = true - shouldImport, err = importFilter.EvaluateFunction(ctx, deleteBody, false) + shouldImport, err = importFilter.EvaluateFunction(ctx, deleteBody) } else { - shouldImport, err = importFilter.EvaluateFunction(ctx, body, false) + shouldImport, err = importFilter.EvaluateFunction(ctx, body) } if err != nil { @@ -489,11 +489,9 @@ func (db *DatabaseCollectionWithUser) backupPreImportRevision(ctx context.Contex // ////// Import Filter Function // Compiles a JavaScript event function to a jsImportFilterRunner object. -func newImportFilterRunner(ctx context.Context, funcSource string, timeout time.Duration) (sgbucket.JSServerTask, error) { +func newImportFilterRunnerWithLogging(ctx context.Context, funcSource string, timeout time.Duration, errorLogFunc, infoLogFunc func(string)) (sgbucket.JSServerTask, error) { importFilterRunner := &jsEventTask{} - err := importFilterRunner.InitWithLogging(funcSource, timeout, - func(s string) { base.ErrorfCtx(ctx, base.KeyJavascript.String()+": Import %s", base.UD(s)) }, - func(s string) { base.InfofCtx(ctx, base.KeyJavascript, "Import %s", base.UD(s)) }) + err := importFilterRunner.InitWithLogging(funcSource, timeout, errorLogFunc, infoLogFunc) if err != nil { return nil, err } @@ -506,6 +504,12 @@ func newImportFilterRunner(ctx context.Context, funcSource string, timeout time. return importFilterRunner, nil } +func newImportFilterRunner(ctx context.Context, funcSource string, timeout time.Duration) (sgbucket.JSServerTask, error) { + errLogFunc := func(s string) { base.ErrorfCtx(ctx, base.KeyJavascript.String()+": Import %s", base.UD(s)) } + infoLogFunc := func(s string) { base.InfofCtx(ctx, base.KeyJavascript, "Import %s", base.UD(s)) } + return newImportFilterRunnerWithLogging(ctx, funcSource, timeout, errLogFunc, infoLogFunc) +} + type ImportFilterFunction struct { *sgbucket.JSServer } @@ -520,16 +524,43 @@ func NewImportFilterFunction(ctx context.Context, fnSource string, timeout time. } // Calls a jsEventFunction returning an interface{} -func (i *ImportFilterFunction) EvaluateFunction(ctx context.Context, doc Body, dryRun bool) (bool, error) { +func (i *ImportFilterFunction) EvaluateFunction(ctx context.Context, doc Body) (bool, error) { result, err := i.Call(ctx, doc) if err != nil { - if !dryRun { - base.WarnfCtx(ctx, "Unexpected error invoking import filter for document %s - processing aborted, document will not be imported. Error: %v", base.UD(doc), err) - } return false, err } - switch result := result.(type) { + return parseImportFilterOutput(result) +} + +// ImportFilterDryRun Runs a document through the import filter and returns a boolean and error +func (db *DatabaseCollectionWithUser) ImportFilterDryRun(ctx context.Context, doc Body, importFn string, errorLogFunc, infoLogFunc func(string)) (bool, error) { + + // fetch configured import filter if one is not specified + if importFn == "" { + importFilter := db.importFilter() + if importFilter == nil { + return true, nil + } + importFn = importFilter.Function() + } + + // create new import filter runner for this dry run + jsTimeout := time.Duration(base.DefaultJavascriptTimeoutSecs) * time.Second + importRunner, err := newImportFilterRunnerWithLogging(ctx, importFn, jsTimeout, errorLogFunc, infoLogFunc) + if err != nil { + return false, errors.New("failed to create import filter runner: " + err.Error()) + } + importOutput, err := importRunner.Call(ctx, doc) + if err != nil { + return false, &base.ImportFilterDryRunError{Err: err} + } + + return parseImportFilterOutput(importOutput) +} + +func parseImportFilterOutput(output interface{}) (bool, error) { + switch result := output.(type) { case bool: return result, nil case string: @@ -539,26 +570,7 @@ func (i *ImportFilterFunction) EvaluateFunction(ctx context.Context, doc Body, d } return boolResult, nil default: - if !dryRun { - base.WarnfCtx(ctx, "Import filter function returned non-boolean result %v Type: %T", result, result) - } return false, errors.New("Import filter function returned non-boolean value.") } -} -func (db *DatabaseCollectionWithUser) ImportFilterDryRun(ctx context.Context, doc Body, docid string) (bool, error) { - - importFilter := db.importFilter() - if docid != "" { - docInBucket, err := db.GetDocument(ctx, docid, DocUnmarshalAll) - if err == nil { - if doc == nil { - doc = docInBucket.Body(ctx) - } - } else { - return false, err - } - } - shouldImport, err := importFilter.EvaluateFunction(ctx, doc, true) - return shouldImport, err } diff --git a/db/import_test.go b/db/import_test.go index 10fb837b2f..ddbd26189b 100644 --- a/db/import_test.go +++ b/db/import_test.go @@ -771,7 +771,7 @@ func TestEvaluateFunction(t *testing.T) { body := Body{"key": "value", "version": "1a"} source := "illegal function(doc) {}" importFilterFunc := NewImportFilterFunction(base.TestCtx(t), source, 0) - result, err := importFilterFunc.EvaluateFunction(base.TestCtx(t), body, false) + result, err := importFilterFunc.EvaluateFunction(base.TestCtx(t), body) assert.Error(t, err, "Unexpected token function error") assert.False(t, result, "Function evaluation result should be false") @@ -779,7 +779,7 @@ func TestEvaluateFunction(t *testing.T) { body = Body{"key": "value", "version": "2a"} source = `function(doc) { if (doc.version == "2a") { return true; } else { return false; }}` importFilterFunc = NewImportFilterFunction(base.TestCtx(t), source, 0) - result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body, false) + result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body) assert.NoError(t, err, "Import filter function shouldn't throw any error") assert.True(t, result, "Import filter function should return boolean value true") @@ -787,7 +787,7 @@ func TestEvaluateFunction(t *testing.T) { body = Body{"key": "value", "version": "2b"} source = `function(doc) { if (doc.version == "2b") { return 1.01; } else { return 0.01; }}` importFilterFunc = NewImportFilterFunction(base.TestCtx(t), source, 0) - result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body, false) + result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body) assert.Error(t, err, "Import filter function returned non-boolean value") assert.False(t, result, "Import filter function evaluation result should be false") @@ -795,7 +795,7 @@ func TestEvaluateFunction(t *testing.T) { body = Body{"key": "value", "version": "1a"} source = `function(doc) { if (doc.version == "1a") { return "true"; } else { return "false"; }}` importFilterFunc = NewImportFilterFunction(base.TestCtx(t), source, 0) - result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body, false) + result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body) assert.NoError(t, err, "Import filter function shouldn't throw any error") assert.True(t, result, "Import filter function should return true") @@ -803,7 +803,7 @@ func TestEvaluateFunction(t *testing.T) { body = Body{"key": "value", "version": "2a"} source = `function(doc) { if (doc.version == "1a") { return "true"; } else { return "false"; }}` importFilterFunc = NewImportFilterFunction(base.TestCtx(t), source, 0) - result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body, false) + result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body) assert.NoError(t, err, "Import filter function shouldn't throw any error") assert.False(t, result, "Import filter function should return false") @@ -811,7 +811,7 @@ func TestEvaluateFunction(t *testing.T) { body = Body{"key": "value", "version": "1a"} source = `function(doc) { if (doc.version == "1a") { return "TruE"; } else { return "FaLsE"; }}` importFilterFunc = NewImportFilterFunction(base.TestCtx(t), source, 0) - result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body, false) + result, err = importFilterFunc.EvaluateFunction(base.TestCtx(t), body) assert.Error(t, err, `strconv.ParseBool: parsing "TruE": invalid syntax`) assert.False(t, result, "Import filter function should return true") } diff --git a/docs/api/paths/diagnostic/keyspace-import_filter.yaml b/docs/api/paths/diagnostic/keyspace-import_filter.yaml index 54f8a4a570..ae78581b81 100644 --- a/docs/api/paths/diagnostic/keyspace-import_filter.yaml +++ b/docs/api/paths/diagnostic/keyspace-import_filter.yaml @@ -7,16 +7,31 @@ # the file licenses/APL2.txt. parameters: - $ref: ../../components/parameters.yaml#/keyspace -get: + - $ref: ../../components/parameters.yaml#/doc_id +post: summary: Run a doc body through the Import filter and return results. description: |- - Run a document body through the import filter and return whether its imported or not, and any error messages. + Runs a document body through the import filter and return whether its + imported or not, and any error messages. If no custom import filter is + provided in the request body, the default or user-defined import filter + is used. + * Sync Gateway Application Read Only requestBody: content: application/json: schema: - $ref: ../../components/schemas.yaml#/Document + type: object + properties: + import_filter: + description: |- + A JavaScript function that all imported documents in the + default scope and collection are ran through in order to filter + out what to import and what not to import. + type: string + example: 'function(doc) { if (doc.type != ''mobile'') { return false; } return true; }' + doc: + $ref: ../../components/schemas.yaml#/Document responses: '200': description: Document Processed by import filter successfully diff --git a/rest/diagnostic_doc_api.go b/rest/diagnostic_doc_api.go index 95d39f45c2..3f85899555 100644 --- a/rest/diagnostic_doc_api.go +++ b/rest/diagnostic_doc_api.go @@ -12,7 +12,6 @@ package rest import ( "errors" - "fmt" "net/http" "github.com/couchbase/sync_gateway/auth" @@ -21,23 +20,24 @@ import ( "github.com/couchbase/sync_gateway/db" ) -type SyncFnDryRunLogging struct { +type DryRunLogging struct { Errors []string `json:"errors"` Info []string `json:"info"` } type SyncFnDryRun struct { - Channels base.Set `json:"channels"` - Access channels.AccessMap `json:"access"` - Roles channels.AccessMap `json:"roles"` - Exception string `json:"exception,omitempty"` - Expiry *uint32 `json:"expiry,omitempty"` - Logging SyncFnDryRunLogging `json:"logging"` + Channels base.Set `json:"channels"` + Access channels.AccessMap `json:"access"` + Roles channels.AccessMap `json:"roles"` + Exception string `json:"exception,omitempty"` + Expiry *uint32 `json:"expiry,omitempty"` + Logging DryRunLogging `json:"logging"` } type ImportFilterDryRun struct { - ShouldImport bool `json:"shouldImport"` - Error string `json:"error"` + ShouldImport bool `json:"shouldImport"` + Error string `json:"error"` + Logging DryRunLogging `json:"logging"` } type SyncFnDryRunPayload struct { @@ -45,6 +45,11 @@ type SyncFnDryRunPayload struct { Doc db.Body `json:"doc,omitempty"` } +type ImportFilterDryRunPayload struct { + Function string `json:"import_filter"` + Doc db.Body `json:"doc,omitempty"` +} + func populateDocChannelInfo(doc db.Document) map[string][]auth.GrantHistorySequencePair { resp := make(map[string][]auth.GrantHistorySequencePair, len(doc.Channels)) @@ -169,7 +174,7 @@ func (h *handler) handleSyncFnDryRun() error { errMsg := syncFnDryRunErr.Error() resp := SyncFnDryRun{ Exception: errMsg, - Logging: SyncFnDryRunLogging{Errors: logErrors, Info: logInfo}, + Logging: DryRunLogging{Errors: logErrors, Info: logInfo}, } h.writeJSON(resp) return nil @@ -185,7 +190,7 @@ func (h *handler) handleSyncFnDryRun() error { output.Roles, errorMsg, output.Expiry, - SyncFnDryRunLogging{Errors: logErrors, Info: logInfo}, + DryRunLogging{Errors: logErrors, Info: logInfo}, } h.writeJSON(resp) return nil @@ -195,24 +200,53 @@ func (h *handler) handleSyncFnDryRun() error { func (h *handler) handleImportFilterDryRun() error { docid := h.getQuery("doc_id") - body, err := h.readDocument() + var importFilterPayload ImportFilterDryRunPayload + err := h.readJSONInto(&importFilterPayload) if err != nil { - if docid == "" { - return fmt.Errorf("Error reading body: %s, no doc id provided for dry run", err) - } + return base.HTTPErrorf(http.StatusBadRequest, "Error reading import filter payload: %v", err) } - if docid != "" && body != nil { + // Cannot pass both doc_id and body in the request body + if len(importFilterPayload.Doc) > 0 && docid != "" { return base.HTTPErrorf(http.StatusBadRequest, "doc body and doc id provided. Please provide either the body or a doc id for the import filter dry run") } - shouldImport, err := h.collection.ImportFilterDryRun(h.ctx(), body, docid) + + if len(importFilterPayload.Doc) == 0 && docid == "" { + return base.HTTPErrorf(http.StatusBadRequest, "no doc body and doc id provided. Please provide either the body or a doc id for the import filter dry run") + } + + var doc db.Body + if docid != "" { + docInBucket, err := h.collection.GetDocument(h.ctx(), docid, db.DocUnmarshalAll) + if err != nil { + return err + } + doc = docInBucket.Body(h.ctx()) + } else { + doc = importFilterPayload.Doc + } + + logErrors := make([]string, 0) + logInfo := make([]string, 0) + errorLogFn := func(s string) { + logErrors = append(logErrors, s) + } + infoLogFn := func(s string) { + logInfo = append(logInfo, s) + } + shouldImport, err := h.collection.ImportFilterDryRun(h.ctx(), doc, importFilterPayload.Function, errorLogFn, infoLogFn) errorMsg := "" if err != nil { + var importFilterDryRunErr *base.ImportFilterDryRunError + if !errors.As(err, &importFilterDryRunErr) { + return err + } errorMsg = err.Error() } resp := ImportFilterDryRun{ shouldImport, errorMsg, + DryRunLogging{Errors: logErrors, Info: logInfo}, } h.writeJSON(resp) return nil diff --git a/rest/diagnostic_doc_api_test.go b/rest/diagnostic_doc_api_test.go index 9c6cce1080..42245b5653 100644 --- a/rest/diagnostic_doc_api_test.go +++ b/rest/diagnostic_doc_api_test.go @@ -64,76 +64,6 @@ func TestGetAlldocChannels(t *testing.T) { } -func TestGetDocDryRuns(t *testing.T) { - base.LongRunningTest(t) - - base.SkipImportTestsIfNotEnabled(t) - rt := NewRestTester(t, &RestTesterConfig{PersistentConfig: true}) - defer rt.Close() - bucket := rt.Bucket().GetName() - ImportFilter := `"function(doc) { if (doc.user.num) { return true; } else { return false; } }"` - newSyncFn := `"function(doc,oldDoc){if (doc.user.num >= 100) {channel(doc.channel);} else {throw({forbidden: 'user num too low'});}if (oldDoc){ console.log(oldDoc); if (oldDoc.user.num > doc.user.num) { access(oldDoc.user.name, doc.channel);} else {access(doc.user.name[0], doc.channel);}}}"` - resp := rt.SendAdminRequest("PUT", "/db/", fmt.Sprintf( - `{"bucket":"%s", "num_index_replicas": 0, "enable_shared_bucket_access": %t, "sync":%s, "import_filter":%s}`, - bucket, base.TestUseXattrs(), newSyncFn, ImportFilter)) - RequireStatus(t, resp, http.StatusCreated) - - // Import filter import=false and type error - response := rt.SendDiagnosticRequest("GET", "/{{.keyspace}}/_import_filter", `{"accessUser": "user"}`) - RequireStatus(t, response, http.StatusOK) - - var respMap2 ImportFilterDryRun - err := json.Unmarshal(response.BodyBytes(), &respMap2) - assert.NoError(t, err) - assert.Equal(t, respMap2.Error, "TypeError: Cannot access member 'num' of undefined") - assert.False(t, respMap2.ShouldImport) - - // Import filter import=true and no error - response = rt.SendDiagnosticRequest("GET", "/{{.keyspace}}/_import_filter", `{"user":{"num":23}}`) - RequireStatus(t, response, http.StatusOK) - - err = json.Unmarshal(response.BodyBytes(), &respMap2) - assert.NoError(t, err) - assert.Equal(t, respMap2.Error, "") - assert.True(t, respMap2.ShouldImport) - - // Import filter import=true and no error - response = rt.SendDiagnosticRequest("GET", "/{{.keyspace}}/_import_filter", `{"user":23}`) - RequireStatus(t, response, http.StatusOK) - - err = json.Unmarshal(response.BodyBytes(), &respMap2) - assert.NoError(t, err) - assert.Equal(t, respMap2.Error, "") - assert.False(t, respMap2.ShouldImport) - - _ = rt.PutDoc("doc2", `{"user":{"num":125}}`) - // Import filter get doc from bucket with no body - response = rt.SendDiagnosticRequest("GET", "/{{.keyspace}}/_import_filter?doc_id=doc2", ``) - RequireStatus(t, response, http.StatusOK) - - err = json.Unmarshal(response.BodyBytes(), &respMap2) - assert.NoError(t, err) - assert.Equal(t, respMap2.Error, "") - assert.True(t, respMap2.ShouldImport) - - // Import filter get doc from bucket error doc not found - response = rt.SendDiagnosticRequest("GET", "/{{.keyspace}}/_import_filter?doc_id=doc404", ``) - RequireStatus(t, response, http.StatusOK) - err = json.Unmarshal(response.BodyBytes(), &respMap2) - assert.NoError(t, err) - if base.UnitTestUrlIsWalrus() { - assert.Equal(t, respMap2.Error, `key "doc404" missing`) - } else { - assert.Contains(t, respMap2.Error, "doc404: Not Found") - } - assert.False(t, respMap2.ShouldImport) - - // Import filter get doc from bucket error body provided - response = rt.SendDiagnosticRequest("GET", "/{{.keyspace}}/_import_filter?doc_id=doc2", `{"user":{"num":23}}`) - RequireStatus(t, response, http.StatusBadRequest) - -} - func TestGetUserDocAccessSpan(t *testing.T) { tests := []struct { name string @@ -1057,7 +987,7 @@ func TestSyncFuncDryRun(t *testing.T) { Access: channels.AccessMap{"user": channels.BaseSetOf(t, "dynamicChan5412")}, Roles: channels.AccessMap{}, Expiry: base.Ptr(uint32(10)), - Logging: SyncFnDryRunLogging{ + Logging: DryRunLogging{ Errors: []string{}, Info: []string{}, }, @@ -1079,7 +1009,7 @@ func TestSyncFuncDryRun(t *testing.T) { Access: channels.AccessMap{"user": channels.BaseSetOf(t, "dynamicChan5412")}, Roles: channels.AccessMap{}, Expiry: base.Ptr(uint32(10)), - Logging: SyncFnDryRunLogging{ + Logging: DryRunLogging{ Errors: []string{}, Info: []string{}, }, @@ -1107,7 +1037,7 @@ func TestSyncFuncDryRun(t *testing.T) { Access: channels.AccessMap{"user": channels.BaseSetOf(t, "dynamicChan5412")}, Roles: channels.AccessMap{}, Expiry: base.Ptr(uint32(10)), - Logging: SyncFnDryRunLogging{ + Logging: DryRunLogging{ Errors: []string{}, Info: []string{}, }, @@ -1131,7 +1061,7 @@ func TestSyncFuncDryRun(t *testing.T) { Access: channels.AccessMap{"user": channels.BaseSetOf(t, "dynamicChan5412")}, Roles: channels.AccessMap{}, Expiry: base.Ptr(uint32(10)), - Logging: SyncFnDryRunLogging{ + Logging: DryRunLogging{ Errors: []string{}, Info: []string{}, }, @@ -1153,7 +1083,7 @@ func TestSyncFuncDryRun(t *testing.T) { Channels: base.SetFromArray([]string{"channel_from_request_sync_func"}), Access: channels.AccessMap{}, Roles: channels.AccessMap{}, - Logging: SyncFnDryRunLogging{ + Logging: DryRunLogging{ Errors: []string{}, Info: []string{}, }, @@ -1179,7 +1109,7 @@ func TestSyncFuncDryRun(t *testing.T) { Channels: base.SetOf("newdoc_channel", "olddoc_channel"), Access: channels.AccessMap{}, Roles: channels.AccessMap{}, - Logging: SyncFnDryRunLogging{ + Logging: DryRunLogging{ Errors: []string{}, Info: []string{}, }, @@ -1203,7 +1133,7 @@ func TestSyncFuncDryRun(t *testing.T) { Access: channels.AccessMap{}, Roles: channels.AccessMap{}, Exception: "403 user num too low", - Logging: SyncFnDryRunLogging{ + Logging: DryRunLogging{ Errors: []string{}, Info: []string{}, }, @@ -1231,7 +1161,7 @@ func TestSyncFuncDryRun(t *testing.T) { existingDocBody: `{"user":{"num":123, "name":["user1"]}, "channel":"channel1"}`, expectedOutput: SyncFnDryRun{ Exception: "Error returned from Sync Function: TypeError: Cannot access member '0' of undefined", - Logging: SyncFnDryRunLogging{ + Logging: DryRunLogging{ Errors: []string{}, Info: []string{`got oldDoc`}, }, @@ -1247,7 +1177,7 @@ func TestSyncFuncDryRun(t *testing.T) { Channels: base.SetOf(defaultChannelName), Access: channels.AccessMap{}, Roles: channels.AccessMap{}, - Logging: SyncFnDryRunLogging{ + Logging: DryRunLogging{ Errors: []string{}, Info: []string{}, }, @@ -1278,7 +1208,7 @@ func TestSyncFuncDryRun(t *testing.T) { Channels: base.SetFromArray([]string{"chanLog"}), Access: channels.AccessMap{}, Roles: channels.AccessMap{}, - Logging: SyncFnDryRunLogging{ + Logging: DryRunLogging{ Errors: []string{"This is a console.error log from doc.logerror"}, Info: []string{"This is a console.log log from doc.loginfo", "one more info for good measure..."}, }, @@ -1307,7 +1237,7 @@ func TestSyncFuncDryRun(t *testing.T) { Channels: base.SetFromArray([]string{"chanLog"}), Access: channels.AccessMap{}, Roles: channels.AccessMap{}, - Logging: SyncFnDryRunLogging{ + Logging: DryRunLogging{ Errors: []string{"This is a console.error log from doc.logerror"}, Info: []string{"This is a console.log log from doc.loginfo", "one more info for good measure..."}, }, @@ -1361,6 +1291,456 @@ func TestSyncFuncDryRunErrors(t *testing.T) { RequireStatus(t, rt.SendDiagnosticRequest(http.MethodPost, "/{{.keyspace}}/_sync", `{"doc": "invalid_doc"}`), http.StatusBadRequest) // invalid doc body type RequireStatus(t, rt.SendDiagnosticRequest(http.MethodPost, "/{{.keyspace}}/_sync", `{"doc": {"_sync":"this is a forbidden field"}}`), http.StatusBadRequest) - // invalid javascript function syntax - RequireStatus(t, rt.SendDiagnosticRequest(http.MethodPost, "/{{.keyspace}}/_sync", `{"sync_function": "this isn't valid javascript'"}`), http.StatusBadRequest) +} + +func TestImportFilterDryRun(t *testing.T) { + + base.SkipImportTestsIfNotEnabled(t) + base.SetUpTestLogging(t, base.LevelDebug, base.KeyAll) + + rt := NewRestTester(t, &RestTesterConfig{ + PersistentConfig: true, + }) + defer rt.Close() + + RequireStatus(t, rt.CreateDatabase("db", rt.NewDbConfig()), http.StatusCreated) + + tests := []struct { + name string + dbImportFilter string + request ImportFilterDryRunPayload + requestDocID bool + existingDocBody string + expectedOutput ImportFilterDryRun + expectedStatus int + }{ + { + name: "db import filter", + dbImportFilter: `function(doc) { + if (doc.user.num) { + return true; + } else { + return false; + } + }`, + request: ImportFilterDryRunPayload{ + Function: "", + Doc: db.Body{ + "user": map[string]interface{}{"num": 23}, + }, + }, + expectedOutput: ImportFilterDryRun{ + ShouldImport: true, + Logging: DryRunLogging{ + Errors: []string{}, + Info: []string{}, + }, + }, + expectedStatus: http.StatusOK, + }, + { + name: "request db import filter", + request: ImportFilterDryRunPayload{ + Function: `function(doc) { + if (doc.user.num) { + return true; + } else { + return false; + } + }`, + Doc: db.Body{ + "user": map[string]interface{}{"num": 23}, + }, + }, + expectedOutput: ImportFilterDryRun{ + ShouldImport: true, + Logging: DryRunLogging{ + Errors: []string{}, + Info: []string{}, + }, + }, + expectedStatus: http.StatusOK, + }, + { + name: "db and request import filter", + dbImportFilter: `function(doc) { + if (doc.user.num) { + return true; + } else { + return false; + } + }`, + request: ImportFilterDryRunPayload{ + Function: `function(doc) { + if (doc.user.num) { + return true; + } else { + return false; + } + }`, + Doc: db.Body{ + "user": map[string]interface{}{"num": 23}, + }, + }, + expectedOutput: ImportFilterDryRun{ + ShouldImport: true, + Logging: DryRunLogging{ + Errors: []string{}, + Info: []string{}, + }, + }, + expectedStatus: http.StatusOK, + }, + { + name: "db import filter type error", + dbImportFilter: `function(doc) { + // This will cause a Type error since the document tested will not contain doc.user.num + if (doc.user.num) { + return true; + } else { + return false; + } + }`, + request: ImportFilterDryRunPayload{ + Doc: db.Body{ + "accessUser": "user", + }, + }, + expectedOutput: ImportFilterDryRun{ + Error: "Error returned from Import Filter: TypeError: Cannot access member 'num' of undefined", + Logging: DryRunLogging{ + Errors: []string{}, + Info: []string{}, + }, + }, + expectedStatus: http.StatusOK, + }, + { + name: "request import filter type error", + request: ImportFilterDryRunPayload{ + Function: `function(doc) { + // This will cause a Type error since the document tested will not contain doc.user.num + if (doc.user.num) { + return true; + } else { + return false; + } + }`, + Doc: db.Body{ + "accessUser": "user", + }, + }, + expectedOutput: ImportFilterDryRun{ + Error: "Error returned from Import Filter: TypeError: Cannot access member 'num' of undefined", + Logging: DryRunLogging{ + Errors: []string{}, + Info: []string{}, + }, + }, + expectedStatus: http.StatusOK, + }, + { + name: "db and request import filter type error", + dbImportFilter: `function(doc) { + // This will cause a Type error since the document tested will not contain doc.user.num + if (doc.user.num) { + return true; + } else { + return false; + } + }`, + request: ImportFilterDryRunPayload{ + Function: `function(doc) { + // This will cause a Type error since the document tested will not contain doc.user.num + if (doc.user.num) { + return true; + } else { + return false; + } + }`, + Doc: db.Body{ + "accessUser": "user", + }, + }, + expectedOutput: ImportFilterDryRun{ + Error: "Error returned from Import Filter: TypeError: Cannot access member 'num' of undefined", + Logging: DryRunLogging{ + Errors: []string{}, + Info: []string{}, + }, + }, + expectedStatus: http.StatusOK, + }, + { + name: "db import filter does not import doc", + dbImportFilter: `function(doc) { + if (doc.user.num) { + return true; + } else { + return false; + } + }`, + request: ImportFilterDryRunPayload{ + Doc: db.Body{ + "user": 23, + }, + }, + expectedOutput: ImportFilterDryRun{ + Logging: DryRunLogging{ + Errors: []string{}, + Info: []string{}, + }, + }, + expectedStatus: http.StatusOK, + }, + { + name: "request import filter does not import doc", + request: ImportFilterDryRunPayload{ + Function: `function(doc) { + if (doc.user.num) { + return true; + } else { + return false; + } + }`, + Doc: db.Body{ + "user": 23, + }, + }, + expectedOutput: ImportFilterDryRun{ + Logging: DryRunLogging{ + Errors: []string{}, + Info: []string{}, + }, + }, + expectedStatus: http.StatusOK, + }, + { + name: "request and db import filter does not import doc", + dbImportFilter: `function(doc) { + if (doc.user.num) { + return true; + } else { + return false; + } + }`, + request: ImportFilterDryRunPayload{ + Function: `function(doc) { + if (doc.user.num) { + return true; + } else { + return false; + } + }`, + Doc: db.Body{ + "user": 23, + }, + }, + expectedOutput: ImportFilterDryRun{ + Logging: DryRunLogging{ + Errors: []string{}, + Info: []string{}, + }, + }, + expectedStatus: http.StatusOK, + }, + { + name: "db import filter with existing doc", + dbImportFilter: `function(doc) { + if (doc.user.num) { + return true; + } else { + return false; + } + }`, + requestDocID: true, + existingDocBody: `{"user":{"num":125}}`, + expectedOutput: ImportFilterDryRun{ + ShouldImport: true, + Logging: DryRunLogging{ + Errors: []string{}, + Info: []string{}, + }, + }, + expectedStatus: http.StatusOK, + }, + { + name: "request import filter with existing doc", + request: ImportFilterDryRunPayload{ + Function: `function(doc) { + if (doc.user.num) { + return true; + } else { + return false; + } + }`, + }, + requestDocID: true, + existingDocBody: `{"user":{"num":125}}`, + expectedOutput: ImportFilterDryRun{ + ShouldImport: true, + Logging: DryRunLogging{ + Errors: []string{}, + Info: []string{}, + }, + }, + expectedStatus: http.StatusOK, + }, + { + name: "request and db import filter with existing doc", + dbImportFilter: `function(doc) { + if (doc.user.num) { + return true; + } else { + return false; + } + }`, + request: ImportFilterDryRunPayload{ + Function: `function(doc) { + if (doc.user.num) { + return true; + } else { + return false; + } + }`, + }, + requestDocID: true, + existingDocBody: `{"user":{"num":125}}`, + expectedOutput: ImportFilterDryRun{ + ShouldImport: true, + Logging: DryRunLogging{ + Errors: []string{}, + Info: []string{}, + }, + }, + expectedStatus: http.StatusOK, + }, + { + name: "no import filter with existing doc", + requestDocID: true, + existingDocBody: `{"user":{"num":125}}`, + expectedOutput: ImportFilterDryRun{ + ShouldImport: true, + Logging: DryRunLogging{ + Errors: []string{}, + Info: []string{}, + }, + }, + expectedStatus: http.StatusOK, + }, + { + name: "no import filter with request doc", + request: ImportFilterDryRunPayload{ + Doc: db.Body{ + "user": 23, + }, + }, + expectedOutput: ImportFilterDryRun{ + ShouldImport: true, + Logging: DryRunLogging{ + Errors: []string{}, + Info: []string{}, + }, + }, + expectedStatus: http.StatusOK, + }, + { + name: "logging with db import filter", + dbImportFilter: `function(doc) { + if (doc.logerror) { + console.error("This is a console.error log from doc.logerror"); + } else { + console.log("This is a console.log log from doc.logerror"); + } + if (doc.loginfo) { + console.log("This is a console.log log from doc.loginfo"); + } else { + console.error("This is a console.error log from doc.loginfo"); + } + console.log("one more info for good measure..."); + return true + }`, + requestDocID: true, + existingDocBody: `{ "logerror": true, "loginfo": true}`, + expectedOutput: ImportFilterDryRun{ + ShouldImport: true, + Logging: DryRunLogging{ + Errors: []string{"This is a console.error log from doc.logerror"}, + Info: []string{"This is a console.log log from doc.loginfo", "one more info for good measure..."}, + }, + }, + expectedStatus: http.StatusOK, + }, + { + name: "logging with request import filter", + request: ImportFilterDryRunPayload{ + Function: `function(doc) { + if (doc.logerror) { + console.error("This is a console.error log from doc.logerror"); + } else { + console.log("This is a console.log log from doc.logerror"); + } + if (doc.loginfo) { + console.log("This is a console.log log from doc.loginfo"); + } else { + console.error("This is a console.error log from doc.loginfo"); + } + console.log("one more info for good measure..."); + return true + }`, + }, + requestDocID: true, + existingDocBody: `{ "logerror": true, "loginfo": true}`, + expectedOutput: ImportFilterDryRun{ + ShouldImport: true, + Logging: DryRunLogging{ + Errors: []string{"This is a console.error log from doc.logerror"}, + Info: []string{"This is a console.log log from doc.loginfo", "one more info for good measure..."}, + }, + }, + expectedStatus: http.StatusOK, + }, + } + + for _, test := range tests { + rt.Run(test.name, func(t *testing.T) { + + RequireStatus(t, rt.SendAdminRequest("PUT", "/{{.keyspace}}/_config/import_filter", test.dbImportFilter), http.StatusOK) + + if test.existingDocBody != "" { + rt.PutDoc(test.name, test.existingDocBody) + } + + bodyBytes, err := json.Marshal(test.request) + require.NoError(t, err) + + url := "/{{.keyspace}}/_import_filter" + if test.requestDocID { + url += "?doc_id=" + test.name + } + resp := rt.SendDiagnosticRequest("POST", url, string(bodyBytes)) + RequireStatus(t, resp, test.expectedStatus) + + var output ImportFilterDryRun + err = json.Unmarshal(resp.Body.Bytes(), &output) + assert.NoError(t, err) + assert.Equal(t, test.expectedOutput, output) + }) + } +} + +func TestImportFilterDryRunErrors(t *testing.T) { + rt := NewRestTester(t, nil) + defer rt.Close() + + // doc ID not found + RequireStatus(t, rt.SendDiagnosticRequest(http.MethodPost, "/{{.keyspace}}/_import_filter?doc_id=missing", `{}`), http.StatusNotFound) + // invalid request + RequireStatus(t, rt.SendDiagnosticRequest(http.MethodPost, "/{{.keyspace}}/_import_filter?doc_id=doc", `{"doc": { "user" : {"num": 23 }}}`), http.StatusBadRequest) + // invalid request json + RequireStatus(t, rt.SendDiagnosticRequest(http.MethodPost, "/{{.keyspace}}/_import_filter", `{"doc": {"invalid_json"}`), http.StatusBadRequest) + // invalid doc body type + RequireStatus(t, rt.SendDiagnosticRequest(http.MethodPost, "/{{.keyspace}}/_import_filter", `{"doc": "invalid_doc"}`), http.StatusBadRequest) + // no docID and no body + RequireStatus(t, rt.SendDiagnosticRequest(http.MethodPost, "/{{.keyspace}}/_import_filter", `{}`), http.StatusBadRequest) } diff --git a/rest/routing.go b/rest/routing.go index f93be27c1f..8021f7f681 100644 --- a/rest/routing.go +++ b/rest/routing.go @@ -413,7 +413,7 @@ func createDiagnosticRouter(sc *ServerContext) *mux.Router { keyspace.Handle("/{docid:"+docRegex+"}/_all_channels", makeHandler(sc, adminPrivs, []Permission{PermReadAppData}, nil, (*handler).handleGetDocChannels)).Methods("GET") keyspace.Handle("/_user/{name}", makeHandler(sc, adminPrivs, []Permission{PermReadAppData}, nil, (*handler).handleGetUserDocAccessSpan)).Methods("GET") keyspace.Handle("/_sync", makeHandler(sc, adminPrivs, []Permission{PermReadAppData}, nil, (*handler).handleSyncFnDryRun)).Methods("POST") - keyspace.Handle("/_import_filter", makeHandler(sc, adminPrivs, []Permission{PermReadAppData}, nil, (*handler).handleImportFilterDryRun)).Methods("GET") + keyspace.Handle("/_import_filter", makeHandler(sc, adminPrivs, []Permission{PermReadAppData}, nil, (*handler).handleImportFilterDryRun)).Methods("POST") dbr.Handle("/_user/{name}/_all_channels", makeHandler(sc, adminPrivs, []Permission{PermReadPrincipal}, nil, (*handler).handleGetAllChannels)).Methods("GET")