From 531f06151471a3d2a515368c866576ba3f0521f0 Mon Sep 17 00:00:00 2001 From: Abhinav Dangeti Date: Thu, 24 Feb 2022 00:58:06 +0000 Subject: [PATCH 1/4] DOC-9450: Update SEARCH_META(..) instructions and examples (#2468) * DOC-9450: Update SEARCH_META(..) instructions and examples * Remove references to fts-demonstration-indexes * Fix reference to fts-supported-queries (cherry picked from commit 59f5d2e38547a2ba28d4aeaa543cf79a64997f46) --- modules/fts/pages/fts-quickstart-guide.adoc | 5 +- .../fts/pages/fts-searching-from-N1QL.adoc | 130 +++++++++++++++--- ...ts-supported-queries-geopoint-spatial.adoc | 57 +++++++- 3 files changed, 167 insertions(+), 25 deletions(-) diff --git a/modules/fts/pages/fts-quickstart-guide.adoc b/modules/fts/pages/fts-quickstart-guide.adoc index 1661037aee..6cf13c58a2 100644 --- a/modules/fts/pages/fts-quickstart-guide.adoc +++ b/modules/fts/pages/fts-quickstart-guide.adoc @@ -43,6 +43,5 @@ Refer to xref:n1ql:n1ql-language-reference/searchfun.adoc[Search Functions] for [#establishing-demonstration-indexes] == Accessing the Search service via the Java SDK -The Java SDK code-example provided in xref:java-sdk:howtos:full-text-searching-with-sdk.adoc[Searching from the Java SDK] contains multiple demonstration calls — each featuring a different query-combination — and makes use of three different index-definitions, related to the `travel-sample` bucket: for the code example to run successfully, the three indexes must be appropriately pre-established. -//The definitions are provided in xref:fts-demonstration-indexes.adoc[Demonstration Indexes]. -Instructions on how to use the Couchbase REST API to establish the definitions refer to xref:fts-creating-index-with-rest-api.adoc[Index Creation with REST API]. +The Java SDK code-example provided in xref:java-sdk:howtos:full-text-searching-with-sdk.adoc[Searching from the SDK] contains multiple demonstration calls — each featuring a different query-combination — and makes use of three different index-definitions, related to the `travel-sample` bucket: for the code example to run successfully, the three indexes must be appropriately pre-established. +Instructions on how to use the Couchbase REST API to establish the definitions refer to xref:fts-creating-index-with-rest-api.adoc[Creating Index with REST API]. diff --git a/modules/fts/pages/fts-searching-from-N1QL.adoc b/modules/fts/pages/fts-searching-from-N1QL.adoc index 24c64a1738..e8916ca5e9 100644 --- a/modules/fts/pages/fts-searching-from-N1QL.adoc +++ b/modules/fts/pages/fts-searching-from-N1QL.adoc @@ -5,11 +5,57 @@ Search functions enable you to use full text search queries directly within a {s == Prerequisites To use any of the search functions, the Search service must be available on the cluster. -It is also recommended, but not required, that you should create suitable full text indexes for the searches that you need to perform. +It is required for you to create suitable full text indexes for the searches that you want to perform. [NOTE] -- -The examples in this page all assume that demonstration full text indexes have been created. +The examples in this page assume that you have a full text search index created with this definition, unless specified otherwise .. +[source,json] +---- +{ + "name": "travel-sample-inventory-hotels", + "type": "fulltext-index", + "params": { + "doc_config": { + "mode": "scope.collection.type_field", + "type_field": "type" + }, + "mapping": { + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": false + }, + "default_type": "_default", + "docvalues_dynamic": false, + "index_dynamic": true, + "store_dynamic": false, + "type_field": "_type", + "types": { + "inventory.hotel": { + "dynamic": true, + "enabled": true + } + } + }, + "store": { + "indexType": "scorch", + "segmentVersion": 15 + } + }, + "sourceType": "gocbcore", + "sourceName": "travel-sample", + "sourceUUID": "", + "sourceParams": {}, + "planParams": { + "indexPartitions": 1 + }, + "uuid": "" +} +---- + -- === Authorization @@ -203,14 +249,14 @@ The following queries are equivalent: [source,sqlpp] ---- SELECT META(t1).id -FROM `travel-sample`.inventory.airline AS t1 +FROM `travel-sample`.inventory.hotel AS t1 WHERE SEARCH(t1.country, "+United +States"); ---- [source,sqlpp] ---- SELECT META(t1).id -FROM `travel-sample`.inventory.airline AS t1 +FROM `travel-sample`.inventory.hotel AS t1 WHERE SEARCH(t1, "country:\"United States\""); ---- @@ -218,18 +264,14 @@ WHERE SEARCH(t1, "country:\"United States\""); [source,json] ---- [ - - { - "id": "airline_10" - }, { - "id": "airline_10123" + "id": "hotel_26215" }, { - "id": "airline_10226" + "id": "hotel_7396" }, { - "id": "airline_10748" + "id": "hotel_32177" }, ... ] @@ -350,8 +392,51 @@ WHERE t1.type = "hotel" AND SEARCH(t1.description, "amazing"); ---- If the full text search index being queried has its default mapping disabled and has a custom type mapping defined, the query needs to specify the type explicitly. - -//The above query uses the demonstration index xref:fts:fts-demonstration-indexes.adoc#travel-sample-index-hotel-description[travel-sample-index-hotel-description], which has the custom type mapping "hotel". +The above query uses the following index definition .. +[source,json] +---- +{ + "name": "travel-sample-hotels", + "type": "fulltext-index", + "params": { + "doc_config": { + "mode": "type_field", + "type_field": "type" + }, + "mapping": { + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": false + }, + "default_type": "_default", + "index_dynamic": true, + "store_dynamic": true, + "type_field": "type", + "types": { + "hotel": { + "dynamic": true, + "enabled": true + } + } + }, + "store": { + "indexType": "scorch", + "segmentVersion": 15 + } + }, + "sourceType": "gocbcore", + "sourceName": "travel-sample", + "sourceUUID": "", + "sourceParams": {}, + "planParams": { + "indexPartitions": 1 + }, + "uuid": "" +} +---- For more information on defining custom type mappings within the full text search index, refer to xref:fts:fts-type-mappings.adoc[Type Mappings]. Note that for {sqlpp} queries, only full text search indexes with one type mapping are searchable. @@ -506,6 +591,10 @@ AND SEARCH(t1, { ---- [ { + "meta": { + "id": "hotel_17598", + "score": 2.1256278997816835 + }, "name": "Marina del Rey Marriott" } ] @@ -569,16 +658,19 @@ LIMIT 3; ---- [ { - "description": "3 Star Hotel next to the Mountain Railway terminus and set in 30 acres of grounds which include Dolbadarn Castle", - "name": "The Royal Victoria Hotel" + "description": "370 guest rooms offering both water and mountain view.", + "name": "Marina del Rey Marriott", + "score": 2.1256278997816835 }, { - "description": "370 guest rooms offering both water and mountain view.", - "name": "Marina del Rey Marriott" + "description": "Log cabin glamping in a rural setting with panoramic views toward the Clwydian Mountain Range.", + "name": "Clwydian Holidays", + "score": 1.6956645086702617 }, { - "description": "This small family run hotel captures the spirit of Mull and is a perfect rural holiday retreat. The mountain and sea blend together to give fantastic, panoramic views from the hotel which is in an elevated position on the shoreline. Panoramic views are also available from the bar and restaurant which serves local produce 7 days a week.", - "name": "The Glenforsa Hotel" + "description": "3 Star Hotel next to the Mountain Railway terminus and set in 30 acres of grounds which include Dolbadarn Castle", + "name": "The Royal Victoria Hotel", + "score": 1.5030458987111712 } ] ---- diff --git a/modules/fts/pages/fts-supported-queries-geopoint-spatial.adoc b/modules/fts/pages/fts-supported-queries-geopoint-spatial.adoc index 075cfd5bb1..131d67d491 100644 --- a/modules/fts/pages/fts-supported-queries-geopoint-spatial.adoc +++ b/modules/fts/pages/fts-supported-queries-geopoint-spatial.adoc @@ -232,10 +232,61 @@ Detailed instructions for setting up indexes, and specifying type mappings, are For initial experimentation with geospatial geopoint querying (based on the type geopoint), the `geo` field of documents within the `travel-sample` bucket can be specified as a child field of the `default` type mapping (keyspace `travel-sample._default._default` as follows: -include::partial$fts-creating-geopoint-common.adoc[] +The following query examples in this page will require the following index definition .. -The index once created can also be accessed by means of the Search REST API -see xref:fts-searching-with-curl-http-requests.adoc[Searching with the REST API]. Furthermore the index could have been created in the first place via the Search REST API see xref:fts-creating-index-with-rest-api.adoc[Index Creation with REST API] for more information on using the Search REST API syntax. +[source,json] +---- +{ + "type": "fulltext-index", + "name": "geoIndex", + "uuid": "", + "sourceType": "gocbcore", + "sourceName": "travel-sample", + "sourceUUID": "", + "planParams": { + "indexPartitions": 1 + }, + "params": { + "doc_config": { + "mode": "type_field", + "type_field": "type" + }, + "mapping": { + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": true, + "properties": { + "geo": { + "dynamic": false, + "enabled": true, + "fields": [ + { + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "geo", + "store": true, + "type": "geopoint" + } + ] + } + } + }, + "default_type": "_default", + "index_dynamic": true, + "store_dynamic": false + }, + "store": { + "indexType": "scorch", + "segmentVersion": 15 + } + }, + "sourceParams": {} +} +---- [#creating_geospatial_rest_query_radius_based] [#creating_geopoint_rest_query_radius_based] From 4866bb71d3fde5cc4ae6638bc191aca2cad70f79 Mon Sep 17 00:00:00 2001 From: Jon Strabala <54073900+jon-strabala@users.noreply.github.com> Date: Thu, 24 Feb 2022 18:43:25 +0000 Subject: [PATCH 2/4] minor xref fixes for fts (#2493) * fix syntax error in xref * fix xref to fts-cluster-options.adoc * resolve dead fts link to bleveMaxResultWindow update dead reference from xref:fts:fts-response-object-schema.adoc#request[maximum number of full text search results] to xref:fts:fts-advanced-settings-bleveMaxResultWindow.adoc[bleveMaxResultWindow] * informational update As per Abhinav corrections (cherry picked from commit 1f26a9aab1b6afe99a6fba487ee958ce670fc565) --- .../fts/pages/fts-supported-queries-geo-point-distance.adoc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/fts/pages/fts-supported-queries-geo-point-distance.adoc b/modules/fts/pages/fts-supported-queries-geo-point-distance.adoc index 5c6158525a..a018b66a12 100644 --- a/modules/fts/pages/fts-supported-queries-geo-point-distance.adoc +++ b/modules/fts/pages/fts-supported-queries-geo-point-distance.adoc @@ -1,8 +1,7 @@ = Creating a Query: Radius-Based -Note that a detailed example for Geopoint index creation and also executing queries can be found at xref:fts-supported-queries-geopoint-spatial.adoc#creating_a_geospatial_geopoint_index[Geopoint Index Creation] and running queries xref:fts-supported-queries-geopoint-spatial.adoc#creating_geopoint_rest_query_radius_based[Geopoint Radius Queries]. - -In addition detailed information on performing queries with the Search REST API can be found in xref:fts-searching-with-curl-http-requests.adoc[Searching with the REST API]; which shows how to use the full `curl` command and how to incorporate query-bodies into your cURL requests. +This section and those following, provide examples of the query-bodies required to make geospatial queries with the Couchbase REST API. +Note that more detailed information on performing queries with the Couchbase REST API can be found in xref:fts-searching-with-curl-http-requests.adoc[Searching with the REST API]; which shows how to use the full `curl` command and how to incorporate query-bodies into it. The following query-body specifies a longitude of `-2.235143` and a latitude of `53.482358`. The target-field `geo` is specified, as is a `distance` of `100` miles: this is the radius within which target-locations must reside for their documents to be returned. From 1d51f7aad85638fcf77361cc0c943847ecf5907e Mon Sep 17 00:00:00 2001 From: Abhinav Dangeti Date: Thu, 24 Feb 2022 17:35:11 +0000 Subject: [PATCH 3/4] DOC-9574: Add a note that N1QL can select any qualifying FTS index (#2490) (cherry picked from commit c4eb1e2fc7cef5d769dabcb7f61674e42a00f6a5) --- modules/fts/pages/fts-searching-from-N1QL.adoc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/fts/pages/fts-searching-from-N1QL.adoc b/modules/fts/pages/fts-searching-from-N1QL.adoc index e8916ca5e9..93f44de961 100644 --- a/modules/fts/pages/fts-searching-from-N1QL.adoc +++ b/modules/fts/pages/fts-searching-from-N1QL.adoc @@ -240,6 +240,11 @@ Order pushdown is possible only if query ORDER BY has only <> on t Offset and Limit pushdown is possible if the query only has a SEARCH() predicate, using a single search index -- no IntersectScan or OrderIntersectScan. Group aggregates and projection are not pushed. +[NOTE] +-- +If the "index" setting isn't specified within the options argument of the _SEARCH_ function, N1QL can pick any FTS index available in the system that qualifies for the query. If a number of FTS indexes qualify, then one that is defined most precise will be chosen. +-- + === Examples .Search using a query string From ce86174fe197a5209becad9da620d50646a5d971 Mon Sep 17 00:00:00 2001 From: Thejas-bhat <35959007+thejas-bhat@users.noreply.github.com> Date: Wed, 30 Mar 2022 23:19:48 +0100 Subject: [PATCH 4/4] DOC-9722: FTS redundant pages cleanup (#2531) * fts redundant pages cleanup and fixed dead links * Update fts-creating-index-from-UI-classic-editor.adoc * minor update * remove extra space * spelling in title * minor updates * minor update Co-authored-by: Jon Strabala <54073900+jon-strabala@users.noreply.github.com> (cherry picked from commit 3ddff1879f93b825f5e6f4f98c1f4210a41fc989) --- modules/fts/pages/fts-advanced.adoc | 47 ---- .../pages/fts-analyzers-search-functions.adoc | 52 ---- modules/fts/pages/fts-clone-index.adoc | 11 - ...creating-index-from-UI-classic-editor.adoc | 51 ++++ ...ng-index-specifying-advanced-settings.adoc | 30 --- ...ex-using-the-index-definition-preview.adoc | 11 - modules/fts/pages/fts-custom-filters.adoc | 6 +- modules/fts/pages/fts-delete-index.adoc | 11 - modules/fts/pages/fts-edit-index.adoc | 13 - ...fts-editing-cloning-full-text-indexes.adoc | 20 -- modules/fts/pages/fts-flex.adoc | 97 ------- ...-for-search-high-availability-indexes.adoc | 8 - ...r-search-zero-downtime-index-upgrades.adoc | 3 - modules/fts/pages/fts-highlighting.adoc | 67 ----- modules/fts/pages/fts-index-analyzers.adoc | 4 +- modules/fts/pages/fts-introduction.adoc | 11 +- .../pages/fts-multi-collection-behaviour.adoc | 8 +- modules/fts/pages/fts-pagination.adoc | 107 -------- modules/fts/pages/fts-quick-index.adoc | 74 ------ modules/fts/pages/fts-quickstart-guide.adoc | 4 +- modules/fts/pages/fts-scoring.adoc | 124 --------- modules/fts/pages/fts-search-facets.adoc | 192 -------------- modules/fts/pages/fts-search-request.adoc | 2 +- ...fts-search-response-error-information.adoc | 104 -------- .../fts/pages/fts-search-response-facets.adoc | 249 ------------------ .../fts/pages/fts-search-response-hits.adoc | 77 ------ ...-search-response-search-status-expect.adoc | 86 ------ .../fts/pages/fts-search-response-took.adoc | 12 - modules/fts/pages/fts-search-response.adoc | 29 +- .../fts/pages/fts-searching-from-N1QL.adoc | 2 +- modules/fts/pages/fts-sorting.adoc | 239 ----------------- .../pages/fts-supported-queries-match.adoc | 2 +- ...-supported-queries-non-analytic-query.adoc | 2 + ...-supported-queries-query-string-query.adoc | 128 ++++++++- modules/fts/pages/fts-system-indexes.adoc | 13 - modules/fts/pages/fts-troubleshooting.adoc | 2 +- .../fts-type-mapping-specifying-fields.adoc | 27 -- .../fts-type-mappings-Docid-with-regexp.adoc | 66 ----- ...ype-mappings-add-child-field-analyzer.adoc | 9 - ...pe-mappings-add-child-field-docvalues.adoc | 13 - ...e-mappings-add-child-field-field-name.adoc | 8 - ...s-add-child-field-field-searchable-as.adoc | 8 - ...e-mappings-add-child-field-field-type.adoc | 11 - ...-add-child-field-include-in-all-field.adoc | 16 -- ...-add-child-field-include-term-vectors.adoc | 14 - ...s-type-mappings-add-child-field-index.adoc | 10 - ...s-type-mappings-add-child-field-store.adoc | 22 -- .../fts-type-mappings-add-child-field.adoc | 42 --- .../fts-type-mappings-add-child-mappings.adoc | 29 -- modules/fts/pages/fts-type-mappings.adoc | 21 +- 50 files changed, 229 insertions(+), 1965 deletions(-) delete mode 100644 modules/fts/pages/fts-advanced.adoc delete mode 100644 modules/fts/pages/fts-analyzers-search-functions.adoc delete mode 100644 modules/fts/pages/fts-clone-index.adoc delete mode 100644 modules/fts/pages/fts-creating-index-specifying-advanced-settings.adoc delete mode 100644 modules/fts/pages/fts-creating-index-using-the-index-definition-preview.adoc delete mode 100644 modules/fts/pages/fts-delete-index.adoc delete mode 100644 modules/fts/pages/fts-edit-index.adoc delete mode 100644 modules/fts/pages/fts-editing-cloning-full-text-indexes.adoc delete mode 100644 modules/fts/pages/fts-flex.adoc delete mode 100644 modules/fts/pages/fts-high-availability-for-search-high-availability-indexes.adoc delete mode 100644 modules/fts/pages/fts-high-availability-for-search-zero-downtime-index-upgrades.adoc delete mode 100644 modules/fts/pages/fts-highlighting.adoc delete mode 100644 modules/fts/pages/fts-pagination.adoc delete mode 100644 modules/fts/pages/fts-quick-index.adoc delete mode 100644 modules/fts/pages/fts-scoring.adoc delete mode 100644 modules/fts/pages/fts-search-facets.adoc delete mode 100644 modules/fts/pages/fts-search-response-error-information.adoc delete mode 100644 modules/fts/pages/fts-search-response-facets.adoc delete mode 100644 modules/fts/pages/fts-search-response-hits.adoc delete mode 100644 modules/fts/pages/fts-search-response-search-status-expect.adoc delete mode 100644 modules/fts/pages/fts-search-response-took.adoc delete mode 100644 modules/fts/pages/fts-sorting.adoc delete mode 100644 modules/fts/pages/fts-system-indexes.adoc delete mode 100644 modules/fts/pages/fts-type-mapping-specifying-fields.adoc delete mode 100644 modules/fts/pages/fts-type-mappings-Docid-with-regexp.adoc delete mode 100644 modules/fts/pages/fts-type-mappings-add-child-field-analyzer.adoc delete mode 100644 modules/fts/pages/fts-type-mappings-add-child-field-docvalues.adoc delete mode 100644 modules/fts/pages/fts-type-mappings-add-child-field-field-name.adoc delete mode 100644 modules/fts/pages/fts-type-mappings-add-child-field-field-searchable-as.adoc delete mode 100644 modules/fts/pages/fts-type-mappings-add-child-field-field-type.adoc delete mode 100644 modules/fts/pages/fts-type-mappings-add-child-field-include-in-all-field.adoc delete mode 100644 modules/fts/pages/fts-type-mappings-add-child-field-include-term-vectors.adoc delete mode 100644 modules/fts/pages/fts-type-mappings-add-child-field-index.adoc delete mode 100644 modules/fts/pages/fts-type-mappings-add-child-field-store.adoc delete mode 100644 modules/fts/pages/fts-type-mappings-add-child-field.adoc delete mode 100644 modules/fts/pages/fts-type-mappings-add-child-mappings.adoc diff --git a/modules/fts/pages/fts-advanced.adoc b/modules/fts/pages/fts-advanced.adoc deleted file mode 100644 index 8c1c2c26b1..0000000000 --- a/modules/fts/pages/fts-advanced.adoc +++ /dev/null @@ -1,47 +0,0 @@ - -= Advanced Settings - -Advanced settings can be specified in the *Advanced* panel. When opened, the Advanced panel appears as follows: - -[#fts_advanced_panel] -image::fts-advanced-panel.png[,420,align=left] - -The Advanced panel provides the following options: - -== Default Type - -The default type for documents in the selected bucket or scope and collection. The default value for this field is `default`. - -== Default Analyzer - -This is the default analyzer to be used. The default value is `standard`. - -The default analyzer is applicable to all the text fields across type mappings unless explicitly overridden. - -It is the _standard_ analyzer in which analysis is done by the means of the Unicode tokenizer, the to_lower token filter, and the stop token filter. - -== Default Date/Time Parser - -This is the default date/time parser to be used. - -The default datetime parser is applicable to all the datetime fields across the type mappings unless explicitly overridden. - -The default value is `dateTimeOptional`. - -== Default Field - -Indexed fields need to have this option selected to support `include in _all`, where _all is the composite field. - -The default value is `_all`. - -== Store Dynamic Fields - -This option, when selected, ensures the inclusion of field content in returned results. Otherwise, the field content is not included in the result. - -== Index Dynamic Fields - -This option, When selected, ensures that the dynamic fields are indexed. Otherwise, the dynamic fields are not indexed. - -== DocValues for Dynamic Fields - -This option, When selected, ensures that the values of the dynamic fields are included in the index. Otherwise, the dynamic field values are not included in the index. \ No newline at end of file diff --git a/modules/fts/pages/fts-analyzers-search-functions.adoc b/modules/fts/pages/fts-analyzers-search-functions.adoc deleted file mode 100644 index 4452c07838..0000000000 --- a/modules/fts/pages/fts-analyzers-search-functions.adoc +++ /dev/null @@ -1,52 +0,0 @@ -= Analyzers - Search Functions - -xref:n1ql:n1ql-language-reference/searchfun.adoc[Search functions] allow users to execute full text search requests within a {sqlpp} query. - -In the context of {sqlpp} queries, a full text search index can be described as one of the following : - -* xref:n1ql:n1ql-language-reference/covering-indexes.adoc[Covering index] - -* Non-covering index - -This characterization depends on the extent to which it could answer all aspects of the SELECT predicate and the WHERE clauses of a {sqlpp} query. -A {sqlpp} query against a non-covering index will go through a "Verification phase". In this phase documents are fetched from the query service based on the results of the search index, and the documents are validated as per the clauses defined in the query. - -For example, an index with only the field `field1` configured is considered a non-covering index for a query `field1=abc` and `field2=xyz`. - -== Use case - -Consider a use case where a user has defined a special analyzer for a field in their full text search index. The following can be expected: - -. If the query does not use the same analyzer as specified in the full text search index, the query will not be allowed to run. - -. By default, the analyzer used for indexing the field (as per the index definition) will be picked up if no analyzer is specified in the analytic query. - -. If the index is a non-covering index for an analytic query and the user has not specified an explicit analyzer to be used, the verification phase might drop documents that should have been returned as results due to lack of query context. - -The user can explicitly specify the search query context in the following three ways: - -. Explicitly specify the analyzer to use in the query (to match with that specified in the index). -+ -Example 1 -+ -.... -SEARCH(keyspace, {"match": "xyz", "field": "abc", "analyzer": "en"}) -.... - -. Specify index name within the options argument of the SEARCH function, so this index’s mapping is picked up during the verification process -+ -Example 2 -+ -.... -SEARCH(keyspace, {"match": "xyz", "field": "abc"}, {"index": "fts-index-1"}) -.... - -. Specify the index mapping itself as a JSON object within the options argument of the SEARCH function, which is used directly for the verification process -+ -Example 3 -+ -.... -SEARCH(keyspace, {"match": "xyz", "field": "abc"}, {"index": {.......}) -.... - -NOTE: If users fail to provide this query context for non-covering queries, they may see incorrect results, including dropped documents, especially while using non-standard and custom analyzers. \ No newline at end of file diff --git a/modules/fts/pages/fts-clone-index.adoc b/modules/fts/pages/fts-clone-index.adoc deleted file mode 100644 index 4c42981bac..0000000000 --- a/modules/fts/pages/fts-clone-index.adoc +++ /dev/null @@ -1,11 +0,0 @@ -= Clone Index - -You can access the Full Text Indexes from the *Search* tab. Left-click on this to display the *Full Text Search* panel, which contains a tabular presentation of currently existing indexes, with a row for each index. -(See xref:fts-searching-from-the-UI.adoc[Searching from the UI] for a full illustration.) - -To clone an index, left-click on its row. The row expands, as follows: - -[#fts_index_management_ui] -image::fts-index-management-ui.png[,820,align=left] - -[.ui]*Clone* button click brings up the *Clone Index* screen, which allows a copy of the current index to be modified as appropriate and required, and saved under a new name. \ No newline at end of file diff --git a/modules/fts/pages/fts-creating-index-from-UI-classic-editor.adoc b/modules/fts/pages/fts-creating-index-from-UI-classic-editor.adoc index 30cbe7debc..7936ebc32e 100644 --- a/modules/fts/pages/fts-creating-index-from-UI-classic-editor.adoc +++ b/modules/fts/pages/fts-creating-index-from-UI-classic-editor.adoc @@ -9,3 +9,54 @@ include::partial$fts-user-prerequisites-common.adoc[] == Quickstart via the Classic Editor include::partial$fts-creating-indexes-common.adoc[] + +image::fts-index-name-and-bucket.png[,450,align=left] + +This is all you need to specify in order to create a basic index for test and development. No further configuration is required. Note, however, that such default indexing is not recommended for production environments since it creates indexes that may be unnecessarily large, and therefore insufficiently performant. + +To save your index, left-click on the *Create Index* button near the bottom of the screen: + +At this point, you are returned to the Full Text Search screen. In the Full Text Indexes panel, a row now appears for the index you have created. When left-clicked on, the row opens as follows: + +image::fts-new-index-progress.png[,750,align=left] + +NOTE: The percentage figure appears under the indexing progress column and is incremented in correspondence with the build-progress of the index. When 100% is reached, the index build is complete. However, search queries will be allowed as soon as the index is created, meaning partial results can be expected until the index build is complete. + +Once the new index has been built, it supports Full Text Searches performed by all available means: the Console UI, the Couchbase REST API, and the Couchbase SDK. + +The indexing progress is determined as `index_doc_count` / `source_doc_count`. + +The `index_doc_count` is retrieved from the search endpoint. The `source_doc_count` is retrieved from a KV endpoint. + +NOTE: If one or more of the nodes in the cluster running data service goes down and/or are failed over, indexing progress may show a value > 100% as the source_doc_count for the bucket would be missing some active partitions. + +[#using-non-default-scope-collections] + +== Using Non-Default Scope/Collection + +Search indexes can be created on non-default scopes and collections, providing the ability to make an index more personal or use case specific and also resulting in a lower index size. + +image::fts-select-non-default-scope-collections.png[,250,align=left] + +Select this checkbox if you want the index to stream data from a non-default scope and/or non-default collection(s) on the source bucket. + +To specify the non-default scope, click the scope drop-down list and select the required scope. + +image::fts-non-default-scope-collections1.png[,400,align=left] + +The Search service allows you to index as many collections as you want in a scope by using Type Mappings. For further details on creating an index on non-default collection(s), see xref::fts-type-mappings.adoc#Specifying-Type-Mapping-for-Collection[Specifying Type Mapping for Collection] +to review the wide range of available options for creating indexes appropriate for production environments, see xref:fts-creating-indexes.adoc[Creating Indexes]. + +NOTE: An index can be created only on a single scope. However, within a scope any number of collections can be indexed. + +[#using-the-index-definition-preview] +== Using the Index Definition Preview + +The _Index Definition Preview_ appears to the right-hand side of the *Edit Index* screen. +Following index-definition, the upper portion may appear as follows: + +[#fts_index_definition_preview] +image::fts-index-definition-preview.png[,300,align=left] + +The preview consists of the JSON document that describes the current index configuration, as created by means of the user interface. +By left-clicking on the [.ui]*copy to clipboard* tab, the definition can be saved. diff --git a/modules/fts/pages/fts-creating-index-specifying-advanced-settings.adoc b/modules/fts/pages/fts-creating-index-specifying-advanced-settings.adoc deleted file mode 100644 index db3ed708e9..0000000000 --- a/modules/fts/pages/fts-creating-index-specifying-advanced-settings.adoc +++ /dev/null @@ -1,30 +0,0 @@ -[#specifying-advanced-settings] -== Specifying Advanced Settings - -Advanced settings can be specified by means of the *Advanced* panel. -When opened, this appears as follows: - -[#fts_advanced_panel] -image::fts-advanced-panel.png[,420,align=left] - -The following, interactive fields are displayed: - -* *Default Type*: The default type for documents in this bucket. -The default value is `_default`. - -* *Default Analyzer*: The default analyzer to be used for this bucket. -The default value is `standard`. -A list of available options can be displayed and selected from, by means of the pull-down menu at the right-hand side of the field. - -* *Default Date/Time Parser*: The default date/time parser to be used for this bucket. -The default value is `dateTimeOptional`. -A list of available options can be displayed and selected from, by means of the pull-down menu at the right-hand side of the field. - -* *Default Field*: The default field for this bucket. -the default value is `_all`. - -* *Store Dynamic Fields*: When checked, ensures inclusion of field-content in returned results. -When unchecked, no such inclusion occurs. - -* *Index Dynamic Fields*: When checked, ensures dynamic fields are indexed. -When unchecked, they are not indexed. \ No newline at end of file diff --git a/modules/fts/pages/fts-creating-index-using-the-index-definition-preview.adoc b/modules/fts/pages/fts-creating-index-using-the-index-definition-preview.adoc deleted file mode 100644 index 0bfa922c80..0000000000 --- a/modules/fts/pages/fts-creating-index-using-the-index-definition-preview.adoc +++ /dev/null @@ -1,11 +0,0 @@ -[#using-the-index-definition-preview] -= Using the Index Definition Preview - -The _Index Definition Preview_ appears to the right-hand side of the *Edit Index* screen. -Following index-definition, the upper portion may appear as follows: - -[#fts_index_definition_preview] -image::fts-index-definition-preview.png[,300,align=left] - -The preview consists of the JSON document that describes the current index configuration, as created by means of the user interface. -By left-clicking on the [.ui]*copy to clipboard* tab, the definition can be saved. diff --git a/modules/fts/pages/fts-custom-filters.adoc b/modules/fts/pages/fts-custom-filters.adoc index 8bba8b666a..21af708ee9 100644 --- a/modules/fts/pages/fts-custom-filters.adoc +++ b/modules/fts/pages/fts-custom-filters.adoc @@ -89,7 +89,7 @@ The following interactive fields are provided: * *Name*: A suitable, user-defined name for the new token filter. -* *Type*: The type of post-processing to be provided by the new token filter. The default is `length`, which creates tokens whose minimum number of characters is specified by the integer provided in the *Min* field and whose maximum by the integer provided in the *Max*. +* *Type*: The type of post-processing to be provided by the new token filter. The default is `length`, which creates tokens with a minimum number of characters is specified by the integer provided in the *Min* field and with a maximum by the integer provided in the *Max*. Additional post-processing types can be selected from the pull-down menu at the right of the field: + [#fts_custom_filters_token_filter_types] @@ -97,7 +97,7 @@ image::fts-custom-filters-token-filter-types.png[,420,align=left] + NOTE: The type-selection determines which interactive fields appear in the *Custom Token Filter* dialog, following *Name* and *Type*. The pull-down menu displays a list of available types. -For descriptions, see the section xref:fts-analyzers.adoc#Token-Filters[Token Filters], on the page xref:fts-analyzers.adoc#Understanding-Analyzers[Understanding Analyzers]. +For descriptions, see the section xref:fts-index-analyzers.adoc#Token-Filters[Token Filters], on the page xref:fts-index-analyzers.adoc#Understanding-Analyzers[Understanding Analyzers]. * *Min*: The minimum length of the token, in characters. Note that this interactive field is displayed for the `length` type, and may not appear, or be replaced, when other types are specified. @@ -142,4 +142,4 @@ To remove a word, select the word within the *Words* panel and left-click on the To save, left-click on [.ui]*Save*. The new word list is displayed on its own row, with options for further editing and deleting: [#fts_custom_filters_panel_new_word_list] -image::fts-custom-filters-panel-new-word-list.png[,700,align=left] \ No newline at end of file +image::fts-custom-filters-panel-new-word-list.png[,700,align=left] diff --git a/modules/fts/pages/fts-delete-index.adoc b/modules/fts/pages/fts-delete-index.adoc deleted file mode 100644 index c095eb383c..0000000000 --- a/modules/fts/pages/fts-delete-index.adoc +++ /dev/null @@ -1,11 +0,0 @@ -= Delete Index - -You can access the Full Text Indexes from the *Search* tab. Left-click on this to display the *Full Text Search* panel, which contains a tabular presentation of currently existing indexes, with a row for each index. -(See xref:fts-searching-from-the-UI.adoc[Searching from the UI] for a full illustration.) - -To delete an index, left-click on its row. The row expands, as follows: - -[#fts_index_management_ui] -image::fts-index-management-ui.png[,820,align=left] - -* [.ui]*Delete* causes the current index to be deleted. \ No newline at end of file diff --git a/modules/fts/pages/fts-edit-index.adoc b/modules/fts/pages/fts-edit-index.adoc deleted file mode 100644 index 8d1c396604..0000000000 --- a/modules/fts/pages/fts-edit-index.adoc +++ /dev/null @@ -1,13 +0,0 @@ -= Edit Index - -You can access the Full Text Indexes from the *Search* tab. Left-click on this to display the *Full Text Search* panel, which contains a tabular presentation of currently existing indexes, with a row for each index. -(See xref:fts-searching-from-the-UI.adoc[Searching from the UI] for a full illustration.) - -To edit an index, left-click on its row. The row expands, as follows: - -[#fts_index_management_ui] -image::fts-index-management-ui.png[,820,align=left] - -* [.ui]*Edit* brings up the *Edit Index* screen, which allows the index to be modified. Saving modifications cause the index to be rebuilt. - -"Quick Edit" that goes to the quick editor for an index definition also results in the same functionalities. \ No newline at end of file diff --git a/modules/fts/pages/fts-editing-cloning-full-text-indexes.adoc b/modules/fts/pages/fts-editing-cloning-full-text-indexes.adoc deleted file mode 100644 index 9fd38d6b55..0000000000 --- a/modules/fts/pages/fts-editing-cloning-full-text-indexes.adoc +++ /dev/null @@ -1,20 +0,0 @@ -[#Editing/Cloning-Full-Text-Indexes] -= Editing/ Cloning Full Text Indexes - -Full Text Indexes, once created can be cloned, edited and/or deleted. They are accessed from the *Search* tab: left-click on this to display the *Full Text Search* panel, which contains a tabular presentation of currently existing indexes, with a row for each index. - -(See xref:fts-searching-from-the-UI.adoc[Searching from the UI] for a full illustration.) - -To manage an index, left-click on its row. The row expands, as follows: - -[#fts_index_management_ui] -image::fts-index-management-ui.png[,820,align=left] - -== Editing Full Text Indexes -[.ui]*Edit* button click, brings up the *Edit Index* screen, which allows the same index to be modified and saved. - -"Quick Edit" that goes to the quick editor for an index definition also results in same functionalities. - -== Cloning Full Text Indexes - -[.ui]*Clone* button click, brings up the *Clone Index* screen, which allows a copy of the current index to be modified as appropriate and required, and saved under a new name. \ No newline at end of file diff --git a/modules/fts/pages/fts-flex.adoc b/modules/fts/pages/fts-flex.adoc deleted file mode 100644 index f749f437bf..0000000000 --- a/modules/fts/pages/fts-flex.adoc +++ /dev/null @@ -1,97 +0,0 @@ -= FTS FLEX (FTS + {sqlpp} Extended Support For Collections) - -FTS is capable of supporting multiple collections within a single index definition. - -Pre Couchbase Server 7.0 index definitions will continue to be supported with 7.0 FTS. - -If the user wants to set up an index definition to subscribe to just a few collections within a single scope, they will be able to do so by toggling the "doc_config.mode" to either of ["scope.collection.type_field", "scope.collection.docid_prefix"]. - -The type mappings will now take the form of either "scope_name.collection_name" (to index all documents within that scope.collection) or "scope_name.collection_name.type_name" (to index only those documents within that scope.collection that match "type" = "type_name") . We will refer to FTS index definitions in this mode as collection-aware FTS indexes. - -NOTE: The type expression check within {sqlpp} queries becomes unnecessary with collection-aware FTS indexes. - -== Example - -When you set up an FTS index definition to stream from 2 collections: landmark, hotel such as: - ----- -{ - "type": "fulltext-index", - "name": "travel", - "sourceType": "gocbcore", - "sourceName": "travel-sample", - "params": { - "doc_config": { - "mode": "scope.collection.type_field", - "type_field": "type" - }, - "mapping": { - "analysis": {}, - "default_analyzer": "standard", - "default_mapping": { - "dynamic": true, - "enabled": false - }, - "types": { - "inventory.hotel": { - "enabled": true, - "properties": { - "reviews": { - "enabled": true, - "properties": { - "content": { - "enabled": true, - "fields": [ - { - "analyzer": "keyword", - "index": true, - "name": "content", - "type": "text" - } - ] - } - } - } - } - }, - "inventory.landmark": { - "enabled": true, - "properties": { - "content": { - "enabled": true, - "fields": [ - { - "analyzer": "keyword", - "index": true, - "name": "content", - "type": "text" - } - ] - } - } - } - } - } - } -} ----- - -Below are some {sqlpp} queries targeting the above index definition. - ----- -SELECT META().id -FROM `travel-sample`.`inventory`.`landmark` t USE INDEX(USING FTS) -WHERE content LIKE "%travel%"; ----- - ----- -SELECT META().id -FROM `travel-sample`.`inventory`.`hotel` t USE INDEX(USING FTS) -WHERE reviews.content LIKE "%travel%"; ----- - ----- -SELECT META().id -FROM `travel-sample`.`inventory`.`hotel` t USE INDEX(USING FTS) -WHERE content LIKE "%travel%"; ----- diff --git a/modules/fts/pages/fts-high-availability-for-search-high-availability-indexes.adoc b/modules/fts/pages/fts-high-availability-for-search-high-availability-indexes.adoc deleted file mode 100644 index 59d07d003b..0000000000 --- a/modules/fts/pages/fts-high-availability-for-search-high-availability-indexes.adoc +++ /dev/null @@ -1,8 +0,0 @@ -= High Availability Indexes - -High availability indexes help users prevent downtime caused by unplanned incidents and enhance the overall system availability. It is achieved through Replica and Failover mechanisms with Search Service. - -Replica for the index is not automatically enabled. The user must explicitly configure the required number of replicas for the index partitions according to their availability needs during the index creation step. Refer xref:fts-index-replicas.adoc[Index Replicas] to the index replica section during index creation. -They could also update the replica count without rebuilding the primary partitions or live traffic for an existing index. - -Server Groups/Rack zone awareness - a Server Group is a logical group of nodes. It can be based on their physical location in the network. Server Group awareness ensures that active and replica partitions are automatically assigned to different server groups(racks). So that, if a physical rack fails, the replica partition remains safe and available in another rack. Starting with Couchbase Server release 7.0.2, the Search service supports Server Groups/Rack zone aware replica placements. \ No newline at end of file diff --git a/modules/fts/pages/fts-high-availability-for-search-zero-downtime-index-upgrades.adoc b/modules/fts/pages/fts-high-availability-for-search-zero-downtime-index-upgrades.adoc deleted file mode 100644 index e5ae565b4d..0000000000 --- a/modules/fts/pages/fts-high-availability-for-search-zero-downtime-index-upgrades.adoc +++ /dev/null @@ -1,3 +0,0 @@ -= Zero Downtime Index Upgrades - -Users can update/recreate/rebuild the indexes without interfering with the live systems by leveraging the index-alias feature. The index aliases permit indirection in the index naming, whereby applications refer to an alias-name that never changes, leaving administrators free to change the identity of the real index pointed to by the alias. This may be particularly useful when an index needs to be updated: to avoid downtime, while the current index remains in service, a clone of the current index can be created, modified, and tested. Then, when the clone is ready, the existing alias can be retargeted so that the clone becomes the current index; and the (now) previous index can be removed. \ No newline at end of file diff --git a/modules/fts/pages/fts-highlighting.adoc b/modules/fts/pages/fts-highlighting.adoc deleted file mode 100644 index ec2ab29bd9..0000000000 --- a/modules/fts/pages/fts-highlighting.adoc +++ /dev/null @@ -1,67 +0,0 @@ -= Highlighting - -The `Highlight` object indicates whether highlighting was requested. - -The pre-requisite includes term vectors and store options to be enabled at the field level to support Highlighting. - -The highlight object contains the following fields: - -* *style* - (Optional) Specifies the name of the highlighter. For example, "html"or "ansi". - -* *fields* - Specifies an array of field names to which Highlighting is restricted. - -== Example 1 - -As per the following example, when you search the content in the index, the matched content in the `address` field is highlighted in the search response. - -[source,console] ----- -curl -u username:password -XPOST -H "Content-Type: application/json" \ -http://localhost:8094/api/index/travel-sample-index/query \ --d '{ - "explain": true, - "fields": [ - "*" - ], - "highlight": { - "style":"html", - "fields": ["address"] - }, - "query": { - "query": "address:farm" - } -}' ----- - -=== Result - -[#fts_highlighting_in_address_field] -image::fts-highlighting-in-address-field.png[,520,align=left] - -== Example 2 - -As per the following example, when you search the content in the index, the matched content in the `description` field is highlighted in the search response. - -[source,console] ----- -curl -u username:password -XPOST -H "Content-Type: application/json" \ -http://localhost:8094/api/index/travel-sample-index/query \ --d '{ - "explain": true, - "fields": [ - "*" - ], - "highlight": { - "style":"html", - "fields": ["description"] - }, - "query": { - "query": "description:complementary breakfast" - } -}' ----- - -=== Result - -[#fts_highlighting_in_description_field] -image::fts-highlighting-in-description-field.png[,520,align=left] \ No newline at end of file diff --git a/modules/fts/pages/fts-index-analyzers.adoc b/modules/fts/pages/fts-index-analyzers.adoc index ca2a5cc843..698556349c 100644 --- a/modules/fts/pages/fts-index-analyzers.adoc +++ b/modules/fts/pages/fts-index-analyzers.adoc @@ -260,8 +260,8 @@ Analyzers increase search-awareness by transforming input text into token-stream An analyzer consists of modules, each of which performs a particular role in the transformation (for example, removing undesirable characters, transforming standard words into stemmed or otherwise modified forms, referred to as tokens, and performing miscellaneous post-processing activities). -For more information on analyzers, see -xref:fts-analyzers.adoc[Understanding Analyzers]. +For more information on analyzers, see +xref:Understanding-Analyzers[Understanding Analyzers]. A default selection of analyzers is made available from the pull-down menu provided by the *Type Mappings* interface discussed above. Additional analyzers can be custom-created, by means of the *Analyzers* panel, which appears as follows: diff --git a/modules/fts/pages/fts-introduction.adoc b/modules/fts/pages/fts-introduction.adoc index ebb1e8fa01..d586cbd982 100644 --- a/modules/fts/pages/fts-introduction.adoc +++ b/modules/fts/pages/fts-introduction.adoc @@ -70,18 +70,23 @@ For each matched document, the hits field shows the document id, the score, the "end": 173, "array_positions": null }, -... + ] + } + } + }, + ... + ] ---- Examples of natural language support include: * _Language-aware_ searching; allowing users to search for, say, the word `traveling`, and additionally obtain results for `travel` and `traveler`. -* xref:fts-scoring.adoc[_Scoring_] of results, according to relevancy; allowing users to obtain result-sets that only contain documents awarded the highest scores. +* xref:fts-search-response.adoc#scoring[_Scoring_] of results, according to relevancy; allowing users to obtain result-sets that only contain documents awarded the highest scores. This keeps result-sets manageably small, even when the total number of documents returned is extremely large. == Stages of Full text search query -A Full Text Search query , once built at the client, can be targeted to any server in the Couchbase cluster hosting the search service. +A Full Text Search query, once built at the client, can be targeted to any server in the Couchbase cluster hosting the search service. Here are the stages it goes through: diff --git a/modules/fts/pages/fts-multi-collection-behaviour.adoc b/modules/fts/pages/fts-multi-collection-behaviour.adoc index f228e54f07..373990474a 100644 --- a/modules/fts/pages/fts-multi-collection-behaviour.adoc +++ b/modules/fts/pages/fts-multi-collection-behaviour.adoc @@ -1,10 +1,10 @@ -= Multi-Collection Behaviour += Multi-Collection Behavior Couchbase's FTS service is the only service that can create indexes that span collections. -Multi-Collection Index: A user can search multi-collection indexes in the same way as that of a bucket-based index. Since a multi-collection index contains data from multiple source collections, it is helpful to know the source collection of every document xref:fts-search-response-hits.adoc[hit] in the search result. +Multi-Collection Index: A user can search multi-collection indexes in the same way as that of a bucket-based index. Since a multi-collection index contains data from multiple source collections, it is helpful to know the source collection of every document xref:fts-search-response.adoc#Hits[hit] in the search result. -* Users can see the source collection names in the fields section of each document xref:fts-search-response-hits.adoc[hit] under the key _$c. See the image below for an example. +* Users can see the source collection names in the fields section of each document xref:fts-search-response.adoc#Hits[hit] under the key _$c. See the image below for an example. image::fts-multi-collection-behaviour.png[,750,align=left] @@ -217,4 +217,4 @@ NOTE: With multi-collection indexes, the user must have search reader roles for == Data lifecycle impact -Multi-collection indexes are deleted when any of the corresponding source collections are deleted. Therefore, multi-collection indexes are best suited for collections with similar data lifespans. \ No newline at end of file +Multi-collection indexes are deleted when any of the corresponding source collections are deleted. Therefore, multi-collection indexes are best suited for collections with similar data lifespans. diff --git a/modules/fts/pages/fts-pagination.adoc b/modules/fts/pages/fts-pagination.adoc deleted file mode 100644 index 696635715e..0000000000 --- a/modules/fts/pages/fts-pagination.adoc +++ /dev/null @@ -1,107 +0,0 @@ -[#pagination] -= Pagination - -The number of results obtained for a Full Text Search request can be large. Pagination of these results becomes essential for sorting and displaying a subset of these results. - -There are multiple ways to achieve pagination with settings within a search request. Pagination will fetch a deterministic set of results when the results are sorted in a certain fashion. - -Pagination provides the following options: - -== Size/from or offset/limit - -This pagination settings can be used to obtain a subset of results and works deterministically when combined with a certain sort order. - -Using `size/limit` and `offset/from` would fetch at least `size + from` ordered results from a partition and then return the `size` number of results starting at offset `from`. - -Deep pagination can therefore get pretty expensive when using `size + from` on a sharded index due to each shard having to possibly return large resultsets (at least `size + from`) over the network for merging at the coordinating node before returning the `size` number of results starting at offset `from`. - -The default sort order is based on _score_ (relevance) where the results are ordered from the highest to the lowest score. - -=== Example - -Here's an example query that fetches results from the 11th onwards to the 15th that have been ordered by _score_. - ----- -{ - "query": { - "match": "California", - "field": "state" - }, - "size": 5, - "from": 10 -} ----- - -//---- -//{ -// "query": { -// "match": "California", -// "field": "state" -// }, -// "offset": 5, -// "limit": 10 -//} -//---- - -== search_after, search_before - -For an efficient pagination, you can use the `search_after/search_before` settings. - -`search_after` is designed to fetch the `size` number of results after the key specified and `search_before` is designed to fetch the `size` number of results before the key specified. - -These settings allow for the client to maintain state while paginating - the sort key of the last result (for search_after) or the first result (for search_before) in the current page. - -Both the attributes accept an array of strings (sort keys) - the length of this array will need to be the same length of the "sort" array within the search request. - -NOTE: You cannot use both `search_after` and `search_before` in the same search request. - -=== Example - -Here are some examples using `search_after/search_before` over sort key "_id" (an internal field that carries the document ID). - ----- -{ - "query": { - "match": "California", - "field": "state" - }, - "sort": ["_id"], - "search_after": ["hotel_10180"], - "size": 3 -} ----- - ----- -{ - "query": { - "match": "California", - "field": "state" - }, - "sort": ["_id"], - "search_before": ["hotel_17595"], - "size": 4 -} ----- - -NOTE: A Full Text Search request that doesn't carry any pagination settings will return the first 10 results (`"size: 10", "from": 0`) ordered by _score_ sequentially from the highest to lowest. - -== Pagination tips and recommendations - -The pagination of search results can be done using the 'from' and 'size' parameters in the search request. But as the search gets into deeper pages, it starts consuming more resources. - -To safeguard against any arbitrary higher memory requirements, FTS provides a configurable limit bleveMaxResultWindow (10000 default) on the maximum allowable page offsets. However, bumping this limit to higher levels is not a scalable solution. - -To circumvent this problem, the concept of key set pagination in FTS, is introduced. - -Instead of providing _from_ as a number of search results to skip, the user will provide the sort value of a previously seen search result (usually, the last result shown on the current page). The idea is that to show the next page of the results, we just want the top N results of that sort after the last result from the previous page. - -This solution requires a few preconditions be met: - -* The search request must specify a sort order. -NOTE: The sort order must impose a total order on the results. Without this, any results which share the same sort value might be left out when handling the page navigation boundaries. - -A common solution to this is to always include the document ID as the final sort criteria. - -For example, if you want to sort by [“name”, “-age”], instead of sort by [“name”, “-age”, “_id”]. - -With `search_after`/`search_before` paginations, the heap memory requirement of deeper page searches is made proportional to the requested page size alone. So it reduces the heap memory requirement of deeper page searches significantly down from the offset+from values. \ No newline at end of file diff --git a/modules/fts/pages/fts-quick-index.adoc b/modules/fts/pages/fts-quick-index.adoc deleted file mode 100644 index 431fddaac3..0000000000 --- a/modules/fts/pages/fts-quick-index.adoc +++ /dev/null @@ -1,74 +0,0 @@ -= Quick Index - -To create a quick index, left-click on the *QUICK INDEX* button, towards the right-hand side: - - -The QUICK INDEX screen appears: - -image::fts-quick-index-screen.png[,750,align=left] - -To define a basic index on which Full Text Search can be performed, begin by entering a unique name for the index into the Index Name field, at the upper-left: for example, travel-sample-index. (Note that only alphanumeric characters, hyphens, and underscores are allowed for index names. Note also that the first character of the name must be an alphabetic character.) Then, use the pull-down menu provided for the Keyspace field, at the upper-right, to specify as follows: - -bucket: `travel-sample` - -scope: `inventory` - -collection: `hotel` - -image::fts-quick-index-name-and-bucket.png[,750,align=left] - -The user can continue to randomly pick documents until they find a document of their intended type/schema. It is also possible to have multi-schema documents within a collection. - -image::fts-quick-index-json.png[,750,align=left] - -Select the required field from the document, which is needed to be mapped to this index. Once the field is selected, the configuration panel is displayed at the right. - -image::fts-quick-index-json-configuration.png[,750,align=left] - -Select the related type of the field from the *Type* dropdown. - -Select *Index this field as an identifier* to index the identifier values precisely without any transformation; for this case, language selection is disabled. - -After that, select the required language for the chosen field. - -Additionally, select from the following configuration options corresponding to the selected language: - -* *Include in search results*: Select this option to include the field in the search result. -* *Support highlighting*: Select this option to highlight the matched field. For this option, you must select the *Include in search result* option. -* *Support phrase matching*: Select this option to match the phrases in the index. -* *Support sorting and faceting*: Select this option to allow sorting and faceting the index. - -NOTE: Selecting configuration options requires additional storage and makes the index size larger. - -== Document Refresh/Reselection option - -The 'Refresh' option will randomly select a document from the given Keyspace (bucket.scope.collection). - -image::fts-quick-index-refresh.png[,750,align=left] - -Include In search results, Support phrase matching, and Support sorting and faceting. Searchable As field allows you to modify searchable input for the selected field. - -image::fts-quick-index-searchable-input.png[,750,align=left] - -Once the configuration is completed for the selected fields, click Add. Mapped fields will display the updated columns. - -image::fts-quick-index-json-mapping.png[,750,align=left] - -This is all you need to specify in order to create a basic index for test and development. No further configuration is required. - -Note, however, that such default indexing is not recommended for production environments since it creates indexes that may be unnecessarily large, and therefore insufficiently performant. To review the wide range of available options for creating indexes appropriate for production environments, see Creating Indexes. - -To save your index, - -Left-click on the *Create Index* button near the bottom of the screen: - - -At this point, you are returned to the Full Text Search screen. A row now appears, in the Full Text Indexes panel, for the quick index you have created. When left-clicked on, the row opens as follows: - -image::fts-new-quick-index-progress.png[,900,align=left] - -NOTE: The percentage figure: this appears under the indexing progress column, and is incremented in correspondence with the build-progress of the index. When 100% is reached, the index build is said to be complete. Search queries will, however, be allowed as soon as the index is created, meaning partial results can be expected until the index build is complete. - -Once the new index has been built, it supports Full Text Searches performed by all available means: the Console UI, the Couchbase REST API, and the Couchbase SDK. - -In the event where one or more of the nodes in the cluster running data service go down and/or are failed over, indexing progress may show a value > 100%. \ No newline at end of file diff --git a/modules/fts/pages/fts-quickstart-guide.adoc b/modules/fts/pages/fts-quickstart-guide.adoc index 6cf13c58a2..b164746ef9 100644 --- a/modules/fts/pages/fts-quickstart-guide.adoc +++ b/modules/fts/pages/fts-quickstart-guide.adoc @@ -17,8 +17,8 @@ For a more detailed explanation of the available Query options, refer to xref:ft NOTE: During index creation, in support of most query-types, you can select (or create) and use an _analyzer_. This is optional: if you do not specify an analyzer, a default analyzer is provided. -Analyzers can be created by means of the Couchbase Web Console, during index creation, as described in xref:fts-creating-indexes.adoc[Creating Search Indexes]. -Their functionality and inner components are described in detail in xref:fts-analyzers.adoc[Understanding Analyzers]. +Analyzers can be created by means of the Couchbase Web Console, during index creation, as described in xref:fts-creating-indexes.adoc[Creating Indexes]. +Their functionality and inner components are described in detail in xref:fts-index-analyzers.adoc[Understanding Analyzers]. [#performing-full-text-searches] == Methods to Access the Search service diff --git a/modules/fts/pages/fts-scoring.adoc b/modules/fts/pages/fts-scoring.adoc deleted file mode 100644 index fc15fa2d5a..0000000000 --- a/modules/fts/pages/fts-scoring.adoc +++ /dev/null @@ -1,124 +0,0 @@ -[#scoring] -= Scoring - -Search result scoring occurs at a query time. The result of the search request is ordered by *score* (relevance), with the descending sort order unless explicitly set not to do so. - -Couchbase uses a slightly modified version of the standard *tf-idf* algorithm. This deviation is to normalize the score and is based on *tf-idf* algorithm. - -For more details on tf-idf, refer xref:#scoring-td-idf[tf-idf] - -By selecting the `explain score` option within the search request, you can obtain the explanation of how the score was calculated for a result alongside it. - -image::fts-td-idf-explain-scoring-enabled.png[,850,align=left] - -Search query scores all the qualified documents for relevance and applies relevant filters. - -In a search request, you can set `score` to `none` to disable scoring by. See xref:#scoring-option-none[Score:none] - -== Example - -The following sample query response shows the *score* field for each document retrieved for the query request: - -[source,json] ----- - "hits": [ - { - "index": "DemoIndex_76059e8b3887351c_4c1c5584", - "id": "hotel_10064", - "score": 10.033205341869529, - "sort": [ - "_score" - ], - "fields": { - "_$c": "hotel" - } - }, - { - "index": "DemoIndex_76059e8b3887351c_4c1c5584", - "id": "hotel_10063", - "score": 10.033205341869529, - "sort": [ - "_score" - ], - "fields": { - "_$c": "hotel" - } - } - ], - "total_hits": 2, - "max_score": 10.033205341869529, - "took": 284614211, - "facets": null -} ----- - -[#scoring-td-idf] -== tf-idf - -`tf-idf`, a short form of *term frequency-inverse document frequency*, is a numerical statistical value that is used to reflect how important a word is to a document in collection or scope. - -`tf-idf` is used as a weighting factor in a search for information retrieval and text mining. The `tf–idf` value increases proportionally to the number of times a word appears in the document, and it is offset by the number of documents in the collection or scope that contains the word. - -Search engines often use the variations of `tf-idf` weighting scheme as a tool in scoring and ranking a document's relevance for a given query. The tf-idf scoring for a document relevancy is done on the basis of per-partition index, which means that documents across different partitions may have different scores. - -When bleve scores a document, it sums a set of sub scores to reach the final score. The scores across different searches are not directly comparable as the scores are directly dependent on the search criteria. So, changing the search criteria, like terms, boost factor etc. can vary the score. - -The more conjuncts/disjuncts/sub clauses in a query can influence the scoring. Also, the score of a particular search result is not absolute, which means you can only use the score as a comparison to the highest score from the same search result. - -FTS does not provide any predefined range for valid scores. - -In Couchbase application, you get an option to explore the score computations during any search in FTS. - -[#fts_explain_scoring_option] -image::fts-td-idf-explain-scoring.png[,850,align=left] - -On the Search page, you can search for a term in any index. The search result displays the search records along with the option *Explain Scoring* to view the score deriving details for search hits and which are determined by using the `tf-idf` algorithm. - -[#fts_explain_scoring_option_enabled-old] -image::fts-td-idf-explain-scoring-enabled.png[,850,align=left] - -[#scoring-option-none] -== Score:none - -You can disable the scoring by setting `score` to `none` in the search request. This is recommended in a situation where scoring (document relevancy) is not needed by the application. - -NOTE: Using `"score": "none"` is expected to boost query performance in certain situations. - -=== Example - -[source, json] ----- -{ - "query": { - "match": "California", - "field": "state" - }, - "score": "none", - "size": 100 -} ----- - -== Scoring Tips and Recommendations - -For a select term, FTS calculates the relevancy score. So, the documents having a higher relevancy score automatically appear at the top in the result. - -It is often observed that users are using Full-Text Search for the exact match queries with a bit of fuzziness or other search-specific capabilities like geo. - -Text relevancy score does not matter when the user is looking for exact or more targeted searches with many predicates or when the dataset size is small. - -In such a case, FTS unnecessarily uses more resources in calculating the relevancy score. Users can, however, optimize the query performance by skipping the scoring. Users may skip the scoring by passing a “score”: “none” option in the search request. - -=== Example - -[source,json] ----- -{ - - "query": {}, - "score": "none", - "size": 10, - "from": 0 -} ----- - -This improves the search query performance significantly in many cases, especially for composite queries with many child search clauses. \ No newline at end of file diff --git a/modules/fts/pages/fts-search-facets.adoc b/modules/fts/pages/fts-search-facets.adoc deleted file mode 100644 index 6bf4ee904e..0000000000 --- a/modules/fts/pages/fts-search-facets.adoc +++ /dev/null @@ -1,192 +0,0 @@ -= Search Facets - -Facets are aggregate information collected on a particular result set. -For any search, the user can collect additional facet information along with it. - -All the facet examples below, are for the query "[.code]``water``" on the beer-sample dataset. -FTS supports the following types of facets: - -[#term-facet] -== Term Facet - -A term facet counts how many matching documents have a particular term for a specific field. - -NOTE: When building a term facet, use the keyword analyzer. Otherwise, multi-term values get tokenized, and the user gets unexpected results. - -=== Example - -* Term Facet - computes facet on the type field which has two values: `beer` and `brewery`. -+ ----- -curl -X POST -H "Content-Type: application/json" \ -http://localhost:8094/api/index/bix/query -d \ -'{ - "size": 10, - "query": { - "boost": 1, - "query": "water" - }, - "facets": { - "type": { - "size": 5, - "field": "type" - } - } -}' ----- -+ -The result snippet below only shows the facet section for clarity. -Run the curl command to see the HTTP response containing the full results. -+ -[source,json] ----- -"facets": { - "type": { - "field": "type", - "total": 91, - "missing": 0, - "other": 0, - "terms": [ - { - "term": "beer", - "count": 70 - }, - { - "term": "brewery", - "count": 21 - } - ] - } -} ----- - -[#numeric-range-facet] -== Numeric Range Facet - -A numeric range facet works by the users defining their own buckets (numeric ranges). - -The facet then counts how many of the matching documents fall into a particular bucket for a particular field. - -=== Example - -* Numeric Range Facet - computes facet on the `abv` field with 2 buckets describing `high` (greater than 7) and `low` (less than 7). -+ ----- -curl -X POST -H "Content-Type: application/json" \ -http://localhost:8094/api/index/bix/query -d \ -'{ - "size": 10, - "query": { - "boost": 1, - "query": "water" - }, - "facets": { - "abv": { - "size": 5, - "field": "abv", - "numeric_ranges": [ - { - "name": "high", - "min": 7 - }, - { - "name": "low", - "max": 7 - } - ] - } - } -}' ----- -+ -Results: -+ -[source,json] ----- -facets": { - "abv": { - "field": "abv", - "total": 70, - "missing": 21, - "other": 0, - "numeric_ranges": [ - { - "name": "high", - "min": 7, - "count": 13 - }, - { - "name": "low", - "max": 7, - "count": 57 - } - ] - } -} ----- - -[#date-range-facet] -== Date Range Facet - -The Date Range facet is same as numeric facet, but on dates instead of numbers. - -Full text search and Bleve expect dates to be in the format specified by https://www.ietf.org/rfc/rfc3339.txt[RFC-3339^], which is a specific profile of ISO-8601 that is more restrictive. - -== Example - -* Date Range Facet - computes facet on the ‘updated’ field that has 2 values old and new. -+ - ----- -curl -XPOST -H "Content-Type: application/json" -uAdministrator:asdasd http://:8094/api/index/bix/query -d '{ -"ctl": {"timeout": 0}, -"from": 0, -"size": 0, -"query": { - "field": "country", - "term": "united" -}, - "facets": { - "types": { - "size": 10, - "field": "updated", - "date_ranges": [ - { - "name": "old", - "end": "2010-08-01" - }, - { - "name": "new", - "start": "2010-08-01" - } -] -} -} -}' ----- -+ -Results -+ -[source,json] ----- - "facets": { - "types": { - "field": "updated", - "total": 954, - "missing": 0, - "other": 0, - "date_ranges": [ - { - "name": "old", - "end": "2010-08-01T00:00:00Z", - "count": 934 - }, - { - "name": "new", - "start": "2010-08-01T00:00:00Z", - "count": 20 - } - ] - } - } ----- \ No newline at end of file diff --git a/modules/fts/pages/fts-search-request.adoc b/modules/fts/pages/fts-search-request.adoc index 983f2e5fc7..760a7e1877 100644 --- a/modules/fts/pages/fts-search-request.adoc +++ b/modules/fts/pages/fts-search-request.adoc @@ -66,7 +66,7 @@ If index-maintenance is running behind, out-of-date results may be returned. The consistency vectors supporting the consistency mechanism in Couchbase contain the mapping of the vBucket and sequence number of the data stored in the vBucket. -For more information about consistency mechanism, see xref:fts-consistency.adoc[Consistency] +For more information about consistency mechanism, see xref:#Consistency[Consistency] ==== Example [source, JSON] diff --git a/modules/fts/pages/fts-search-response-error-information.adoc b/modules/fts/pages/fts-search-response-error-information.adoc deleted file mode 100644 index 525bde7aae..0000000000 --- a/modules/fts/pages/fts-search-response-error-information.adoc +++ /dev/null @@ -1,104 +0,0 @@ -= Error Information - -NOTE: For all errors listed in the table below, the "status" in the search response always show "fail". - -.Search Query Response Codes -|=== -| Error/Reason | Description | Response Status | Response Description - -| success -| query was processed successfully. -| 200 -| OK - -| query request rejected -| high memory consumption, throttler steps in -| 429 -| Too many requests - -| authentication failure -| incorrect auth credentials or no permissions -| 403 -| Forbidden - -| malformed query request -| unrecognized/bad query -| 400 -| Bad request - -| page not found -| endpoint is invalid -| 404 -| Not found - -| pre-condition failed -| pre-condition not met - consistency error -| 412 -| Precondition failed - -| .* -| an error that the server can potentially recover from -| 500 -| Internal server error - -|=== - -== Partial Errors - -It is possible that when some of the index partitions have reported failures, the "status" in the response does not show fail. This is when the user will see a partial result set for their query. - -Below is the sample response - -[source,json] ----- -{ - "status": { - "total": 6, - "failed": 2, - "successful": 4, - "errors": { - "pindex_name_1": "xyz", - "pindex_name_2": "xyz" - } - }, - "request": { - "query": { - ... - }, - "size": 10, - "from": 0, - "facets": {}, - "sort": [ - "-_score" - ] - }, - "hits": [...], - "total_hits": ..., - "max_score": ..., - "took": ..., - "facets": {} -} ----- - -If one or more of the index partitions failed to cater to the request, the user can see partial results. In such a case, the response status is shown as 200 , and the errors object in the response will be a non-zero length value. - -.Partial Error Details -|=== -| Partial Error | Description - -| context deadline exceeded -| request wasn’t processed/responded-to by the partition in the requested time period - -| no planPIndexes for indexName -| FTS node in the process of a rebalance, partitions are being moved - -| bleve: pindex_consistency mismatched partition -| RYOW failure - received data from a vbucket with a different UUID while waiting on a sequence number - possibly due to KV rebalance/failover - -| pindex not available -| one or more primary index partitions (that do not have replicas) have been failed over (need to rebalance to set them up again) - -| cannot perform operation on empty alias -| one or more index partitions are in the process of being set up during a rebalance - -|=== \ No newline at end of file diff --git a/modules/fts/pages/fts-search-response-facets.adoc b/modules/fts/pages/fts-search-response-facets.adoc deleted file mode 100644 index 63383b7ff3..0000000000 --- a/modules/fts/pages/fts-search-response-facets.adoc +++ /dev/null @@ -1,249 +0,0 @@ -[#Facets] -= Facets - -[abstract] -Facets are aggregate information collected on a particular result set. - -In Facets, you already have a search in mind, and you want to collect additional facet information along with it. - -Facet-query results may not equal the total number of documents across all buckets if: - -1. There is more than one pindex. -2. Facets_size is less than the possible values for the field. - -== Facets Results - -For each facet that you build, a `FacetResult` is returned containing the following: - -* *Field*: The name of the field the facet was built on. - -* *Total*: The total number of values encountered (if each document had one term, this should match the total number of documents in the search result) - -* *Missing*: The number of documents which do not have any value for this field - -* *Other*: The number of documents for which a value exists, but it was not in the top N number of facet buckets requested - -* *Array of Facets*: Each Facet contains the count indicating the number of items in this facet range/bucket: - -** *Term*: Terms Facets include the name of the term. - -** *Numeric Range*: Numeric Range Facets include the range for this bucket. - -** *DateTime Range*: DateTime Range Facets include the datetime range for this bucket. - -All of the facet examples given in this topic are for the query "[.code]``water``" on the beer-sample dataset. - -FTS supports the following types of facets: - -* *Term Facet* - A term facet counts up how many of the matching documents have a particular term in a particular field. -+ -Most of the time, this only makes sense for relatively low cardinality fields, like a type or tags. -+ -It would not make sense to use it on a unique field like an ID. - -* *Field*: The field over which you want to gather the facet information. - -* *Size*: The number of top categories per partition to be considered for the facet results. -+ -For example, size - 3 => facets results returns the top 3 categories across all partitions and merges them as the final result. -+ -Varying size value varies the count value of each facet and the “others” value as well. This is due to the fact that when the size is varied, some of the categories fall out of the top “n” and into the “others” category. -+ -NOTE: It is recommended to keep the size reasonably large, close to the number of unique terms to get consistent results. - -* *Numeric Range Facet*: A numeric range facet works by the user defining their own buckets (numeric ranges). -+ -The facet then counts how many of the matching documents fall into a particular bucket for a particular field. -+ -Along with the two fields from term facet, “numeric_ranges” field has to include all the numeric ranges for the faceted field. -+ -“Numeric_ranges” could possibly be an array of ranges and each entry of it must specify either min, max or both for the range. - -** *Name*: Name for the facet. - -** *Min*: The lower bound value of this range. - -** *Max*: The upper bound value of this range. - -* *Date Range Facet*: The Date Range facet is same as numeric facet, but on dates instead of numbers. -Full text search and Bleve expect dates to be in the format specified by https://www.ietf.org/rfc/rfc3339.txt[RFC-3339^], which is a specific profile of ISO-8601 that is more restrictive. -+ -Along with the two fields from term facet, “date_ranges” field has to include all the numeric ranges for the faceted field. -+ -The facet ranges go under a field named “date_ranges”. -+ -“date_ranges” could possibly be an array of ranges and each entry of it must specify either start, end or both for the range. - -** *Name*: Name for the facet. - -** *Start*: Start date for this range. - -** *End*: End date for this range. - -NOTE: Most of the time, when building a term facet, you must use the keyword analyzer. Otherwise, multi-term values are tokenized, which might cause unexpected results. - -== Example - -=== Term Facet -Computes facet on the type field which has 2 values: `beer` and `brewery`. - -[source, console] ----- -curl -X POST -H "Content-Type: application/json" \ -http://localhost:8094/api/index/bix/query -d \ -'{ - "size": 10, - "query": { - "boost": 1, - "query": "water" - }, - "facets": { - "type": { - "size": 5, - "field": "type" - } - } -}' ----- - -Result: - -The result snippet below, only shows the facet section for clarity. -Run the curl command to see the HTTP response containing the full results. - -[source,json] ----- -"facets": { - "type": { - "field": "type", - "total": 91, - "missing": 0, - "other": 0, - "terms": [ - { - "term": "beer", - "count": 70 - }, - { - "term": "brewery", - "count": 21 - } - ] - } -} ----- -=== Numeric Range Facet -Computes facet on the `abv` field with two buckets describing `high` (greater than 7) and `low` (less than 7). - -[source, console] ----- -curl -X POST -H "Content-Type: application/json" \ -http://localhost:8094/api/index/bix/query -d \ -'{ - "size": 10, - "query": { - "boost": 1, - "query": "water" - }, - "facets": { - "abv": { - "size": 5, - "field": "abv", - "numeric_ranges": [ - { - "name": "high", - "min": 7 - }, - { - "name": "low", - "max": 7 - } - ] - } - } -}' ----- - -Results: - -[source,json] ----- -facets": { - "abv": { - "field": "abv", - "total": 70, - "missing": 21, - "other": 0, - "numeric_ranges": [ - { - "name": "high", - "min": 7, - "count": 13 - }, - { - "name": "low", - "max": 7, - "count": 57 - } - ] - } -} ----- - -=== Date Range Facet -Computes facet on the ‘updated’ field that has 2 values old and new - -[source, consle] ----- -curl -XPOST -H "Content-Type: application/json" -u username:password http://:8094/api/index/bix/query -d '{ - "ctl": {"timeout": 0}, - "from": 0, - "size": 0, - "query": { - "field": "country", - "term": "united" - }, - "facets": { - "types": { - "size": 10, - "field": "updated", - "date_ranges": [ - { - "name": "old", - "end": "2010-08-01" - }, - { - "name": "new", - "start": "2010-08-01" - } - ] - } - } -}' ----- - -Results: - -[source,json] ----- -"facets": { - "types": { - "field": "updated", - "total": 954, - "missing": 0, - "other": 0, - "date_ranges": [ - { - "name": "old", - "end": "2010-08-01T00:00:00Z", - "count": 934 - }, - { - "name": "new", - "start": "2010-08-01T00:00:00Z", - "count": 20 - } - ] - } -} ----- \ No newline at end of file diff --git a/modules/fts/pages/fts-search-response-hits.adoc b/modules/fts/pages/fts-search-response-hits.adoc deleted file mode 100644 index ede05bbac7..0000000000 --- a/modules/fts/pages/fts-search-response-hits.adoc +++ /dev/null @@ -1,77 +0,0 @@ -= Hits - -[abstract] -Hits return an array containing the matches for the executed query. - -The length of the array is equal to or less than the size specified in the request. - -Index:: -The unique ID of the pindex. -The index name always begins with a string. - -ID:: The document ID that matched. - -Score:: The document score. - -Locations:: -This object contains field names where matches were found. -The "Locations" object depends on the term vectors being stored; if term vectors are not stored, locations are not returned in the result object. -+ -*{Field Name}* -+ -Lists the field names where the match was found. -These fields are scoped so that "description: american" searches for "american" scoped to the "description" field. -In the example below, there are two fields named "description" and "name". -+ -*{Term Found}* -+ -A name value pair whose name is the name of the term that was found and whose value is an array on objects representing the vector information that describes the position of the matched term in the field. -This value is only present if the term vectors are calculated. -For each match, the object contains the position ([.out]`pos`), start, end, and array positions ([.out]`array_positions`). -+ -.Sample Locations Fragment - -[source,json] ----- -"locations": { - "reviews.content": { - "light": [ - { - "pos": 277, - "start": 1451, - "end": 1456, - "array_positions": [ - 0 - ] - }, - { - "pos": 247, - "start": 1321, - "end": 1326, - "array_positions": [ - 3 - ] - } - ] - } -}, ----- - -Fragments:: -These objects, also known as snippets, contain field names that contain an array of one or more text strings. -The text strings contain the "[.code]````" tags surrounding the term that was matched in order to render highlighting. - -Fields:: -This object returns the value of the field that was matched. -However, unlike the Fragments field, this does not have any tags to render highlighting. - -Sort:: -This field contains an array of one or more values that were used to sort the search results. -Documents that don't have a value in the index for a particular field used in sorting will return a series of non-printable Unicode characters: `\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd`. - -== Total_hits - -Total hits represent the total number of matches for this result. -It can be any integer starting from 0. - -// === Example \ No newline at end of file diff --git a/modules/fts/pages/fts-search-response-search-status-expect.adoc b/modules/fts/pages/fts-search-response-search-status-expect.adoc deleted file mode 100644 index e00e59c621..0000000000 --- a/modules/fts/pages/fts-search-response-search-status-expect.adoc +++ /dev/null @@ -1,86 +0,0 @@ -= Search Status Expect - -[abstract] -A Full Text Search _Response Object_ is composed of multiple child objects that provides important information. - -== Status - -The status object includes the number of successful and failed partition indexes queries. - -We recommend that the status object be checked for failures as a preferred alternative to relying on HTTP response codes alone. -For example, FTS returns an HTTP 200 response in case of partition index failures or timeouts (not consistency timeouts). This is done so that you can choose to accept partial results in an application. -However, this also means FTS returns an HTTP 200 response even when ALL partition indexes fail. - -Refer to the following table for more information about the possible status messages. - -.Response Object Status Values -|=== -| Status | Description - -| HTTP 400 -| Returned when an error is detected when validating the inputs. -The text error message accompanying the status describes the problem. - -| HTTP 412 -| Returned when the consistency requirements are not satisfied within the specified timeout. -The JSON structure accompanying the status provides information about the current consistency levels. - -| HTTP 200 -| Returned when an error is detected during query execution. -The errors might be contained in the JSON section named "status". -|=== - -Total:: -The `total` field returns an integer representing the total number of partition indexes queried. -This value varies depending on how your index is configured, the platform you are running on, and whether you are querying an index or an index alias. -Each index has a setting for the number of vBuckets per pindex. -For example, when running on a cluster with 1024 vBuckets and with an index setting of 171 vBuckets per pindex, the total number of partition indexes is 6. -When you query an index alias, `total` is the sum of all the partition indexes in each index that is included in the definition of the index alias. - -Successful:: -This field returns the total number of partition indexes successfully queried. -This integer value can range from 0 to the total number of partition indexes. -+ -.Example Status: Successful - -[source,json] ----- - "status": { - "total": 6, - "failed": 0, - "successful": 6 -}, ----- - -Failed:: -This field returns the total number of partition indexes with failed queries. -This integer value can range from 0 to the total number of partition indexes. -If the number is greater than 0, the response object contains an `errors` array. - -Errors:: -This field returns an array of error messages as reported by individual partition indexes. -This array only appears if the field "failed" is greater than 0. -+ -.Example Status: Failure -[source,json] ----- -"status": { - "total":6, - "failed":3, - "successful":6, - "errors":{ - "IndexClient - http://127.0.0.1:9201/api/pindex/beer-search_447fd6df2d6f4b54_0a44bddb":"context deadline exceeded", - "IndexClient - http://127.0.0.1:9201/api/pindex/beer-search_447fd6df2d6f4b54_24e7ea2d":"context deadline exceeded", - ... - } -} ----- - -== Search Response Score - -Scoring of results, according to relevancy, allows users to obtain result-sets that only contain documents awarded the highest scores. This keeps result-sets manageably small, even when the total number of documents returned is extremely large. The response returns the score of the search query. - -Scores are directly dependent on the search criteria. Changing the search criteria can vary the score, and therefore, scores are not comparable across searches. - -For more details on scores, follow the link -xref:fts-scoring.adoc[Scoring] \ No newline at end of file diff --git a/modules/fts/pages/fts-search-response-took.adoc b/modules/fts/pages/fts-search-response-took.adoc deleted file mode 100644 index fdd4cc00c2..0000000000 --- a/modules/fts/pages/fts-search-response-took.adoc +++ /dev/null @@ -1,12 +0,0 @@ -= Took - -The time taken (in nanoseconds) to complete the execution of a query. - -== Example - -[source,json] ----- -"total_hits": 56, -"max_score": 0.8041158525040355, -"took": 1449005, ----- \ No newline at end of file diff --git a/modules/fts/pages/fts-search-response.adoc b/modules/fts/pages/fts-search-response.adoc index 2ae40a159d..d600574b7d 100644 --- a/modules/fts/pages/fts-search-response.adoc +++ b/modules/fts/pages/fts-search-response.adoc @@ -3,7 +3,7 @@ [abstract] {description} -Prior to the search, a _facet_ can be add to the response object, and information on _aggregations_ thereby returned. +Prior to the search, a _facet_ can be added to the response object, and information on _aggregations_ thereby returned. [#response-object-structure] == Response-Object Structure @@ -22,11 +22,11 @@ Pagination provides the following options: === Size/from or offset/limit -This pagination settings can be used to obtain a subset of results and works deterministically when combined with a certain sort order. +These pagination settings can be used to obtain a subset of results and works deterministically when combined with a certain sort order. Using `size/limit` and `offset/from` would fetch at least `size + from` ordered results from a partition and then return the `size` number of results starting at offset `from`. -Deep pagination can therefore get pretty expensive when using `size + from` on a sharded index due to each shard having to possibly return large resultsets (at least `size + from`) over the network for merging at the coordinating node before returning the `size` number of results starting at offset `from`. +Deep pagination can therefore get pretty expensive when using `size + from` on a sharded index due to each shard having to possibly return large result sets (at least `size + from`) over the network for merging at the coordinating node before returning the `size` number of results starting at offset `from`. The default sort order is based on _score_ (relevance) where the results are ordered from the highest to the lowest score. @@ -135,6 +135,7 @@ multi-Collection index search results in Couchbase 7.0 You can also narrow your full-text search requests to only specific Collection(s) within the multi-Collection index. This focus speeds up your searches on a large index. +[#sorting-query-results] == Sorting Query Results The FTS results are returned as objects. FTS query includes options to order the results. @@ -166,7 +167,7 @@ The default sort-order is _ascending_. If a field-name is prefixed with the `-` character, that field's results are sorted in _descending_ order. * `_id`: Refers to the document identifier. -Whenever encountered in the array, causes sorting to occur by document identifer. +Whenever encountered in the array, causes sorting to occur by document identifier. * `_score`: Refers to the score assigned the document in the result-set. Whenever encountered in the array, causes sorting to occur by score. @@ -503,7 +504,7 @@ This improves the search query performance significantly in many cases, especial [#Consistency] == Consistency -A mechanism to ensure that the Full Text Search (FTS) index can obtain the most up-to-date version of the document written to a collection or a bucket. +A selected mode to ensure that the Full Text Search (FTS) index can obtain the most up-to-date version of the document written to a collection or a bucket. The consistency mechanism provides xref:#consistency-vectors[Consistency Vectors] as objects in the search query that ensures FTS index searches all your last data written to the vBucket. @@ -547,9 +548,9 @@ If index-maintenance is running behind, out-of-date results may be returned. [#consistency-vectors] === Consistency Vectors -The consistency vectors supporting the consistency mechanism in Couchbase contain the mapping of the vbucket and sequence number of the data stored in the vBucket. +The consistency vectors supporting the consistency mechanism in Couchbase contain the mapping of the vBucket and sequence number of the data stored in the vBucket. -For more information about consistency mechanism, see xref:fts-consistency.adoc[Consistency] +For more information about consistency mechanism, see xref:#Consistency[Consistency] ==== Example [source, JSON] @@ -736,7 +737,7 @@ Varying size value varies the count value of each facet and the “others” val + NOTE: It is recommended to keep the size reasonably large, close to the number of unique terms to get consistent results. -* *Numeric Range Facet*: A numeric range facet works by the user defining their own buckets (numeric ranges). +* *Numeric Range Facet*: A numeric range facet works by the user defining buckets (numeric ranges). + The facet then counts how many of the matching documents fall into a particular bucket for a particular field. + @@ -853,7 +854,7 @@ Results: [source,json] ---- -facets": { +"facets": { "abv": { "field": "abv", "total": 70, @@ -941,7 +942,7 @@ The pre-requisite includes term vectors and store options to be enabled at the f The highlight object contains the following fields: -* *style* - (Optional) Specifies the name of the highlighter. For example, "html"or "ansi". +* *style* - (Optional) Specifies the name of the highlighter. For example, "html" or "ansi". * *fields* - Specifies an array of field names to which Highlighting is restricted. @@ -1013,7 +1014,7 @@ The time taken (in nanoseconds) to complete the execution of a query. "max_score": 0.8041158525040355, "took": 1449005, ---- - +[#Hits] == Hits Hits return an array containing the matches for the executed query. @@ -1091,7 +1092,7 @@ It can be any integer starting from 0. == Error Information -NOTE: For all errors listed in the table below, the "status" in the search response always show "fail". +NOTE: For all errors listed in the table below, the "status" in the search response always shows "fail". .Search Query Response Codes |=== @@ -1196,7 +1197,7 @@ If one or more of the index partitions failed to cater to the request, the user == Search Status Expect -A Full Text Search _Response Object_ is composed of multiple child objects that provides important information. +A Full Text Search _Response Object_ is composed of multiple child objects that provide important information. === Status @@ -1278,7 +1279,7 @@ Scoring of results, according to relevancy, allows users to obtain result-sets t Scores are directly dependent on the search criteria. Changing the search criteria can vary the score, and therefore, scores are not comparable across searches. For more details on scores, follow the link -xref:fts-scoring.adoc[Scoring] +xref:#scoring[Scoring] == Array Positions diff --git a/modules/fts/pages/fts-searching-from-N1QL.adoc b/modules/fts/pages/fts-searching-from-N1QL.adoc index 93f44de961..f5226a109e 100644 --- a/modules/fts/pages/fts-searching-from-N1QL.adoc +++ b/modules/fts/pages/fts-searching-from-N1QL.adoc @@ -130,7 +130,7 @@ For more details, refer to xref:fts:fts-supported-queries.adoc[Supported queries | object | A complete full text search request, including sort and pagination options, and so on. -For more details, refer to xref:fts:fts-sorting.adoc[Sorting Query Results]. +For more details, refer to xref:fts:fts-search-response.adoc#sorting-query-results[Sorting Query Results]. [NOTE] ==== diff --git a/modules/fts/pages/fts-sorting.adoc b/modules/fts/pages/fts-sorting.adoc deleted file mode 100644 index 3b5e92f140..0000000000 --- a/modules/fts/pages/fts-sorting.adoc +++ /dev/null @@ -1,239 +0,0 @@ -= Sorting Query Results - -[abstract] -The FTS results are returned as objects. FTS query includes options to order the results. - -== Sorting Result Data - -FTS sorting is sorted by descending order of relevance. It can , however, be customized to sort by different fields, depending on the application. - -On query-completion, _sorting_ allows specified members of the result-set to be displayed prior to others: this facilitates a review of the most significant data. - -Within a JSON query object, the required sort-type is specified by using the `sort` field. - -This takes an array of either _strings_, _objects_, or _numeric_ as its value. - -== Sorting with Strings - -You can specify the value of the `sort` field as an array of strings. -These can be of three types: - -* _field name_: Specifies the name of a field. -+ -If multiple fields are included in the array, the sorting of documents begins according to their values for the field whose name is first in the array. -+ -If any number of these values are identical, their documents are sorted again, this time according to their values for the field whose name is second; then, if any number of these values are identical, their documents are sorted a third time, this time according to their values for the field whose name is third; and so on. -+ -Any document-field may be specified to hold the value on which sorting is to be based, provided that the field has been indexed in some way, whether dynamically or specifically. -+ -The default sort-order is _ascending_. -If a field-name is prefixed with the `-` character, that field's results are sorted in _descending_ order. - -* `_id`: Refers to the document identifier. -Whenever encountered in the array, causes sorting to occur by document identifer. - -* `_score`: Refers to the score assigned the document in the result-set. -Whenever encountered in the array, causes sorting to occur by score. - -== Example -// #Need full example here# ----- -"sort": ["country", "state", "city","-_score"] ----- - -This `sort` statement specifies that results will first be sorted by `country`. - -If some documents are then found to have the same value in their `country` fields, they are re-sorted by `state`. - -Next, if some of these documents are found to have the same value in their `state` fields, they are re-sorted by `city`. - -Finally, if some of these documents are found to have the same value in their `city` fields, they are re-sorted by `score`, in _descending_ order. - -The following JSON query demonstrates how and where the `sort` property can be specified: - -[source,json] ----- -{ - "explain": false, - "fields": [ - "title" - ], - "highlight": {}, - "sort": ["country", "-_score","-_id"], - "query":{ - "query": "beautiful pool" - } -} ----- - -The following example shows how the `sort` field accepts _combinations_ of strings and objects as its value. - -[source,json] ----- -{ - ... - "sort": [ - "country", - { - "by" : "field", - "field" : "reviews.ratings.Overall", - "mode" : "max", - "missing" : "last", - "type": "number" - }, - { - "by" : "field", - "field" : "reviews.ratings.Location", - "mode" : "max", - "missing" : "last", - "type": "number" - }, - "-_score" - ] -} ----- - -== Sorting with Objects - -Fine-grained control over sort-procedure can be achieved by specifying _objects_ as array-values in the `sort` field. - -Each object can have the following fields: - -* `by`: Sorts results on `id`, `score`, or a specified `field` in the Full Text Index. - -* `field`: Specifies the name of a field on which to sort. -Used only if `field` has been specified as the value for the `by` field; otherwise ignored. - -* `missing`: Specifies the sort-procedure for documents with a missing value in a field specified for sorting. -The value of `missing` can be `first`, in which case results with missing values appear _before_ other results; or `last` (the default), in which case they appear _after_. - -* `mode`: Specifies the search-order for index-fields that contain multiple values (in consequence of arrays or multi-token analyzer-output). -The `default` order is undefined but deterministic, allowing the paging of results from `from (_offset_)`, with reliable ordering. -To sort using the minimum or maximum value, the value of `mode` should be set to either `min` or `max`. - -* `type`: Specifies the type of the search-order field value. -For example, `string` for text fields, `date` for DateTime fields, or `number` for numeric/geo fields. - -To fetch more accurate sort results, we strongly recommend specifying the `type` of the sort fields in the sort section of the search request. - -== Example - -The example below shows how to specify the object-sort. - -[source, json] ----- -{ - "explain": false, - "fields": [ - "*" - ], - "highlight": {}, - "query": { - "match": "bathrobes", - "field": "reviews.content", - "analyzer": "standard" - }, - "size" : 10, - "sort": [ - { - "by" : "field", - "field" : "reviews.ratings.Overall", - "mode" : "max", - "missing" : "last", - "type": "number" - } - ] -} ----- - -NOTE: The above sample assumes that the `travel-sample` bucket has been loaded, and a default index has been created on it. - -For information on loading sample buckets, see xref:manage:manage-settings/install-sample-buckets.adoc[Sample Buckets]. For instructions on creating a default Full Text Index by means of the Couchbase Web Console, see xref:fts-creating-index-from-UI-classic-editor.adoc[Creating Index from UI]. - -This query sorts search-results based on `reviews.ratings.Overall` — a field that is normally multi-valued because it contains an array of different users' ratings. - -When there are multiple values, the highest `Overall` ratings are used for sorting. - -Hotels with no `Overall` rating are placed at the end. - -The following example shows how the `sort` field accepts _combinations_ of strings and objects as its value. - -[source,json] ----- -{ - - "sort": [ - "country", - { - "by" : "field", - "field" : "reviews.ratings.Overall", - "mode" : "max", - "missing" : "last", - "type": "number" - }, - { - "by" : "field", - "field" : "reviews.ratings.Location", - "mode" : "max", - "missing" : "last", - "type": "number" - }, - "-_score" - ] -} ----- - -== Sorting with Numeric - -You can specify the value of the `sort` field as a numeric type. You can use the `type` field in the object that you specify with the sort. - -With `type` field, you can specify the type of the search order to numeric, string, or DateTime. - -== Example - -The example below shows how to specify the object-sort with type field as `number`. - -[source,json] ----- -{ - "explain": false, - "fields": [ - "*" - ], - "highlight": {}, - "query": { - "match": "bathrobes", - "field": "reviews.content", - "analyzer": "standard" - }, - "size" : 10, - "sort": [ - { - "by" : "field", - "field" : "reviews.ratings.Overall", - "mode" : "max", - "missing" : "last", - "type": "number" - } - ] -} ----- - -== Tips for Sorting with fields - -When you sort results on a field that is not indexed, or when a particular document is missing a value for that field, you will see the following series of Unicode non-printable characters appear in the sort field: - -`\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd` - -The same characters may render differently when using a graphic tool or command line tools like `jq`. - -[source,json] ----- - "sort": [ - "����������", - "hotel_9723", - "_score" - ] ----- - -Check your index definition to confirm that you are indexing all the fields you intend to sort by. You can control the sort behavior for missing attributes using the missing field. \ No newline at end of file diff --git a/modules/fts/pages/fts-supported-queries-match.adoc b/modules/fts/pages/fts-supported-queries-match.adoc index 04f9dbc4b1..88ed8653ab 100644 --- a/modules/fts/pages/fts-supported-queries-match.adoc +++ b/modules/fts/pages/fts-supported-queries-match.adoc @@ -7,7 +7,7 @@ For example, `pool` performs match query for the term `pool`. A match query _analyzes_ input text and uses the results to query an index. Options include specifying an analyzer, performing a _fuzzy match_, and performing a _prefix match_. -By default, the analyzer used for the search text is what was set for the specified field during index creation. For information on analyzers, see xref:fts-analyzers.adoc[Understanding Analyzers]. +By default, the analyzer used for the search text is what was set for the specified field during index creation. For information on analyzers, see xref:fts-index-analyzers.adoc[Understanding Analyzers]. NOTE: If the field is not specified, the match query will target the `_all` field within the index. Including content within the `_all` field is a setting during index creation. diff --git a/modules/fts/pages/fts-supported-queries-non-analytic-query.adoc b/modules/fts/pages/fts-supported-queries-non-analytic-query.adoc index ea1c1d1731..f6a379f670 100644 --- a/modules/fts/pages/fts-supported-queries-non-analytic-query.adoc +++ b/modules/fts/pages/fts-supported-queries-non-analytic-query.adoc @@ -12,4 +12,6 @@ The following queries are non-Analytic queries: * xref:fts-supported-queries-fuzzy.adoc[Fuzzy] * xref:fts-supported-queries-wildcard.adoc[Wildcard] +In most cases, given the benefits of using analyzers, use of match and match phrase queries is preferable to that of term and phrase. For information on analyzers, see xref:fts-index-analyzers.adoc[Understanding Analyzers]. + diff --git a/modules/fts/pages/fts-supported-queries-query-string-query.adoc b/modules/fts/pages/fts-supported-queries-query-string-query.adoc index 69fc2f1479..829788aa34 100644 --- a/modules/fts/pages/fts-supported-queries-query-string-query.adoc +++ b/modules/fts/pages/fts-supported-queries-query-string-query.adoc @@ -15,4 +15,130 @@ NOTE: The Full Text Searches conducted with the Couchbase Web Console themselves Certain queries supported by FTS are not yet supported by the query string syntax. These include wildcards and regular expressions. -More detailed information is provided in xref:fts-query-string-syntax.adoc[Query String Syntax] +Query strings enable you to describe complex queries using a simple syntax. Following subsections provide the ways in which a user can modify the query string to represent a query with complex logic. + +[#Boosting] +== Boosting + +When you specify multiple query-clauses, you can specify the relative importance to a given clause by suffixing it with the `^` operator, followed by a number or by specifying the `boost` parameter with the number to boost the search. + +=== Example + +[source, json] +---- +description:pool name:pool^5 +---- + +The above syntax performs Match Queries for *pool* in both the `name` and `description` fields, but documents having the term in the `name` field score higher. + +[source, json] +---- +"query": { + ​​​​​  "disjuncts": [ +      { + ​​​​​"match": "glossop", + "field": "city", + "boost": 10 + }​​​​​, +      { + ​​​​​"match": "glossop", + "field": "title" + }​​​​​     + ]   +}​​​​​ +---- + +The above syntax performs Match Queries for a city *glossop* in both the `city` and `title` fields, but documents having the term in the `city` field score higher. + +[#Date-Range] +== Date Range + +You can perform date range searches by using the `>`, `>=`, `<`, and `\<=` operators, followed by a date value in quotes. + +For example, `created:>"2016-09-21"` will perform a xref:fts-supported-queries-date-range.adoc[date range query] on the `created` field for values after September 21, 2016. + +[#Escaping] +== Escaping + +The following quoted-string enumerates the characters which may be escaped: + +---- +"+-=&|>`, `>=`, `<`, and `\<=` operators, each followed by a numeric value. + +=== Example + +`reviews.ratings.Cleanliness:>4` performs a xref:fts-supported-queries-numeric-range.adoc[numeric range query] on the `reviews.ratings.Cleanliness` field, for values greater than 4. \ No newline at end of file diff --git a/modules/fts/pages/fts-system-indexes.adoc b/modules/fts/pages/fts-system-indexes.adoc deleted file mode 100644 index 2375388c29..0000000000 --- a/modules/fts/pages/fts-system-indexes.adoc +++ /dev/null @@ -1,13 +0,0 @@ -= System:indexes - FTS indexes which are eligible to be queried from {sqlpp} - -Use the following command to find all the FTS indexes in the system table that can be queried from {sqlpp}. - ----- -SELECT * FROM system:indexes ----- - -An additional link describes various scenarios in which the FTS Index becomes ineligible to be queried by {sqlpp}. - -xref:n1ql/pages/n1ql-language-reference/searchfun.adoc#limitations[Scenarios where FTS Index becomes ineligible to be queried by {sqlpp}] - -NOTE: Querying system:indexes only returns indexes on non-system keyspaces. To return all indexes, including indexes on system keyspaces, use the query system:all_indexes. diff --git a/modules/fts/pages/fts-troubleshooting.adoc b/modules/fts/pages/fts-troubleshooting.adoc index 39e155bc50..6773ed6cbd 100644 --- a/modules/fts/pages/fts-troubleshooting.adoc +++ b/modules/fts/pages/fts-troubleshooting.adoc @@ -134,7 +134,7 @@ The same characters may render differently when using a graphic tool or command ---- Check your index definition to confirm that you're indexing all the fields you intend to sort by. -You can control the sort behavior for missing attributes using the `missing` field. See xref:fts:fts-sorting.adoc[Sorting Query Results]. +You can control the sort behavior for missing attributes using the `missing` field. See xref:fts:fts-search-response.adoc#sorting-query-results[Sorting Query Results]. Also remember, documents that have the same value for every field you specified in the `sort` field will be sorted non-deterministically. Try adding `_id`, which is guaranteed unique. diff --git a/modules/fts/pages/fts-type-mapping-specifying-fields.adoc b/modules/fts/pages/fts-type-mapping-specifying-fields.adoc deleted file mode 100644 index 82ebb967aa..0000000000 --- a/modules/fts/pages/fts-type-mapping-specifying-fields.adoc +++ /dev/null @@ -1,27 +0,0 @@ -[#specifying-fields-for-type-mapping] -= Specifying fields for Type Mapping - -A Full Text Index can be defined not only to include (or exclude) documents of a certain type but also to include (or exclude) specified fields within each of the typed documents. - -To specify one or more fields, hover with the mouse cursor over a row in the Type Mappings panel that contains an enabled type mapping. Buttons labeled *edit* and *+* appear: - -image::fts-type-mappings-ui-fields-buttons.png[,700,align=left] - -Left-clicking on the *edit* button displays the following interface: - -image::fts-type-mappings-ui-edit.png[,500,align=left] - -This allows the mapping to be deleted or associated with a different analyzer. - -NOTE: FTS Indexing does not work for fields having a dot (. or period) in the field name. Users must avoid adding dot (. or period) in the field name. Like using `field.name` or `country.name` is not supported. For example, `{ "database.name": "couchbase"}` - -If the *only index specified fields* checkbox is checked, only fields specified by the user are included in the index. - -Left-clicking on the *+* button displays a pop-up that features two options: - -image::fts-type-mappings-ui-field-options.png[,700,align=left] - -These options are described in the following sections. - -* xref:fts-type-mappings-add-child-mappings.adoc[Add Child Mapping] -* xref:fts-type-mappings-add-child-field.adoc[Add Child Field] \ No newline at end of file diff --git a/modules/fts/pages/fts-type-mappings-Docid-with-regexp.adoc b/modules/fts/pages/fts-type-mappings-Docid-with-regexp.adoc deleted file mode 100644 index 67b19f2031..0000000000 --- a/modules/fts/pages/fts-type-mappings-Docid-with-regexp.adoc +++ /dev/null @@ -1,66 +0,0 @@ -= DocID with regexp in Type Mappings - -“Doc ID with regexp” is another way the search service allows the user to extract “type identifiers” for indexing. - -* Set up a valid regular expression within docid_regexp. Remember this will be applied on the document IDs. -* Choose a type mapping name that is considered a match for the regexp. -* The type mapping name CANNOT be a regexp. - -For example, while working with the `travel-sample` bucket, set up docid_regexp to `air[a-z]{4}` and use the following type mappings. -* airline -* airport - -Below is a full index definition using it. -{ - "name": "airline-airport-index", - "type": "fulltext-index", - "params": { - "doc_config": { - "docid_prefix_delim": "", - "docid_regexp": "air[a-z]{4}", - "mode": "docid_regexp", - "type_field": "type" - }, - "mapping": { - "default_analyzer": "standard", - "default_datetime_parser": "dateTimeOptional", - "default_field": "_all", - "default_mapping": { - "dynamic": true, - "enabled": false - }, - "default_type": "_default", - "docvalues_dynamic": false, - "index_dynamic": true, - "store_dynamic": false, - "type_field": "_type", - "types": { - "airline": { - "dynamic": true, - "enabled": true - }, - "airport": { - "dynamic": true, - "enabled": true - } - } - }, - "store": { - "indexType": "scorch", - "segmentVersion": 15 - } - }, - "sourceType": "gocbcore", - "sourceName": "travel-sample", - "sourceParams": {}, - "planParams": { - "indexPartitions": 1 - } -} - -So setting this as the index definition would index all attributes of documents with “airline” or "airport" in its document IDs. - -image::fts-type-mapping-regexp-with-docid.png[,750,align=left] - -Note: The golang regexp support is based on -xref:https://github.com/google/re2/wiki/Syntax[Access the github link] diff --git a/modules/fts/pages/fts-type-mappings-add-child-field-analyzer.adoc b/modules/fts/pages/fts-type-mappings-add-child-field-analyzer.adoc deleted file mode 100644 index a2eaba9ba3..0000000000 --- a/modules/fts/pages/fts-type-mappings-add-child-field-analyzer.adoc +++ /dev/null @@ -1,9 +0,0 @@ -= Child Field Analyzer - -An analyzer optionally to be used for the field. -The list of available analyzers can be displayed by means of the field's pull-down menu, and can be selected from. - -== Example - -image::fts-type-mappings-child-field-analysers.png[,200,align=left] - diff --git a/modules/fts/pages/fts-type-mappings-add-child-field-docvalues.adoc b/modules/fts/pages/fts-type-mappings-add-child-field-docvalues.adoc deleted file mode 100644 index babf2aca70..0000000000 --- a/modules/fts/pages/fts-type-mappings-add-child-field-docvalues.adoc +++ /dev/null @@ -1,13 +0,0 @@ -= Child Field DocValues - -To include the value for each instance of the field in the index, the docvalues checkbox must be checked. This is essential for xref:fts-search-response-facets.adoc[Facets]. - -For sorting of search results based on field values: see xref:fts-sorting.adoc[Sorting Query Results]. - -By default, this checkbox is selected. If it is _unchecked_, the values are _not_ added to the index; and in consequence, neither Search Facets nor value-based result-sorting is supported. - -== Example - -image::fts-type-mappings-child-field-docvalues.png[,750,align=left] - -NOTE: When this checkbox is checked, the resulting index will increase proportionately in size. \ No newline at end of file diff --git a/modules/fts/pages/fts-type-mappings-add-child-field-field-name.adoc b/modules/fts/pages/fts-type-mappings-add-child-field-field-name.adoc deleted file mode 100644 index bd3f7f2e29..0000000000 --- a/modules/fts/pages/fts-type-mappings-add-child-field-field-name.adoc +++ /dev/null @@ -1,8 +0,0 @@ -= Child Field Name - -The name of any field within the document that contains a single value or an array, rather than a JSON object. - - -== Example - -image::fts-type-mappings-child-field-field-name.png[,750,align=left] \ No newline at end of file diff --git a/modules/fts/pages/fts-type-mappings-add-child-field-field-searchable-as.adoc b/modules/fts/pages/fts-type-mappings-add-child-field-field-searchable-as.adoc deleted file mode 100644 index 795301304c..0000000000 --- a/modules/fts/pages/fts-type-mappings-add-child-field-field-searchable-as.adoc +++ /dev/null @@ -1,8 +0,0 @@ -= Child Field Searchable As - -Typically identical to the [.ui]*field* (and dynamically supplied during text-input of the [.ui]*field*-value). -This can be modified, to indicate an alternative field-name, whose associated value thereby becomes included in the indexed content, rather than that associated with the field-name specified in *field*. - -== Example - -image::fts-type-mappings-child-field-field-searchable-as.png[,750,align=left] \ No newline at end of file diff --git a/modules/fts/pages/fts-type-mappings-add-child-field-field-type.adoc b/modules/fts/pages/fts-type-mappings-add-child-field-field-type.adoc deleted file mode 100644 index 01ad86e447..0000000000 --- a/modules/fts/pages/fts-type-mappings-add-child-field-field-type.adoc +++ /dev/null @@ -1,11 +0,0 @@ -= Child Field Type - -The _data-type_ of the value of the field. -This can be `text`, `number`, `datetime`, `boolean`, `disabled`, or `geopoint`; and can be selected from the field's pull-down menu, as follows: - -[#fts_type_mappings_ui_select_data_type] -image::fts-type-mappings-ui-select-data-type.png[,300,align=left] - -== Example - -image::fts-type-mappings-child-field-type.png[,750,align=left] \ No newline at end of file diff --git a/modules/fts/pages/fts-type-mappings-add-child-field-include-in-all-field.adoc b/modules/fts/pages/fts-type-mappings-add-child-field-include-in-all-field.adoc deleted file mode 100644 index 1de8838469..0000000000 --- a/modules/fts/pages/fts-type-mappings-add-child-field-include-in-all-field.adoc +++ /dev/null @@ -1,16 +0,0 @@ -= Child Field - Include in_all field: - -When checked, the field is included in the definition of [.ui]*_all*, which is the field specified by default in the [.ui]*Advanced* panel. -When unchecked, the field is not included. - -Inclusion means when _query strings_ are used to specify searches, the text in the current field is searchable without the field name requiring a prefix. -For Example, a search on description:modern can be accomplished simply by specifying the word ‘modern’. This is applicable for all query types and not just limited to query string query type. - - -== Example - -image::fts-type-mappings-child-field-include-in-all.png[,750,align=left] - -NOTE: "include in _all" will write a copy of the tokens generated for a particular field to the "_all" composite field. When this checkbox is checked, the resulting index will proportionately increase in size. - -Enabling this option results in larger indexes, so disable this option to always use field scoped queries in the search requests. diff --git a/modules/fts/pages/fts-type-mappings-add-child-field-include-term-vectors.adoc b/modules/fts/pages/fts-type-mappings-add-child-field-include-term-vectors.adoc deleted file mode 100644 index 1f4283d11a..0000000000 --- a/modules/fts/pages/fts-type-mappings-add-child-field-include-term-vectors.adoc +++ /dev/null @@ -1,14 +0,0 @@ -= Child Field - Include term vectors - -When checked, term vectors are included. -When unchecked, term vectors are not included. - -Term vectors are the locations of terms in a particular field. -Certain kinds of functionality (such as highlighting, and phrase search) require term vectors. -Inclusion of term vectors results in larger indexes and correspondingly slower index build-times. - -== Example - -image::fts-type-mappings-child-field-termvectors.png[,750,align=left] - -NOTE: "include term vectors" indexes the array positions (locations) of the terms within the field (needed for phrase searching and highlighting). When this checkbox is checked, the resulting index will proportionately increase in size. diff --git a/modules/fts/pages/fts-type-mappings-add-child-field-index.adoc b/modules/fts/pages/fts-type-mappings-add-child-field-index.adoc deleted file mode 100644 index 8595dcb93c..0000000000 --- a/modules/fts/pages/fts-type-mappings-add-child-field-index.adoc +++ /dev/null @@ -1,10 +0,0 @@ -= Child Field Index - -When checked, the field is indexed; when unchecked, the field is not indexed. -This may be used, therefore, to explicitly remove an already-defined field from the index. - -== Example - -image::fts-type-mappings-child-field-index.png[,750,align=left] - -NOTE: When this checkbox is checked, the resulting index will proportionately increase in size. \ No newline at end of file diff --git a/modules/fts/pages/fts-type-mappings-add-child-field-store.adoc b/modules/fts/pages/fts-type-mappings-add-child-field-store.adoc deleted file mode 100644 index c171f9feeb..0000000000 --- a/modules/fts/pages/fts-type-mappings-add-child-field-store.adoc +++ /dev/null @@ -1,22 +0,0 @@ -= Child Field Store - -When the child field 'store' option is checked, the original field content is included in the FTS index, enabling the retrieval of stored field values during a search operation. - -When unchecked, the original field content is not included in the FTS index. Storing the field within the index is necessary to support highlighting, which also needs "term vectors” for the field to be indexed. - -== Example -image::fts-type-mappings-child-field-store.png[,700,align=left] - -Ideally, enabling this 'Child Field Store' option has a sizing aspect to the index definition. This option also permits highlighting of search texts in the returned results, so that matched expressions can be easily seen. However, enabling this option also results in larger indexes and slightly longer indexing times. -The field content will show up in queries (when the index has the store option checked) only when requested. There is a ‘fields’ section in the query for it. - ----- -{ -"query": {...}, -"fields": ["store_field_name"] -} -Setting "fields" to ["*"] will include the contents of all stored fields in the index. ----- - -NOTE: "store" - writes a copy of the field content into the index. When this checkbox is checked, the resulting index will proportionately increase in size. - diff --git a/modules/fts/pages/fts-type-mappings-add-child-field.adoc b/modules/fts/pages/fts-type-mappings-add-child-field.adoc deleted file mode 100644 index 57730dd77c..0000000000 --- a/modules/fts/pages/fts-type-mappings-add-child-field.adoc +++ /dev/null @@ -1,42 +0,0 @@ -= Add Child Field - -The option [.ui]*insert child field* allows a field to be individually included for (or excluded from) indexing, provided that it contains a single value or an array rather than a JSON object. -Selecting this option displays the following: - -[#fts_type_mappings_child_field_dialog] -image::fts-type-mappings-child-field-dialog.png[,700,align=left] - -The interactive fields and checkboxes are: - -** xref:fts-type-mappings-add-child-field-field-name.adoc[Field Name] - -** xref:fts-type-mappings-add-child-field-field-type.adoc[Field Type] - -** xref:fts-type-mappings-add-child-field-field-searchable-as.adoc[Field Searchable As] - -** xref:fts-type-mappings-add-child-field-analyzer.adoc[Analyzer] - -** xref:fts-type-mappings-add-child-field-index.adoc[Index] - -** xref:fts-type-mappings-add-child-field-store.adoc[Store] - -** xref:fts-type-mappings-add-child-field-include-term-vectors.adoc[Include term vectors] - -** xref:fts-type-mappings-add-child-field-include-in-all-field.adoc[Include in _all field] - -** xref:fts-type-mappings-add-child-field-docvalues.adoc[DocValues] - - - -The dialog, when completed, might look as follows: - -[#fts_type-mappings_child_field_dialog_complete] -image::fts-type-mappings-child-field-dialog-complete.png[,700,align=left] - -Left-click on [.ui]*OK*. -The field is saved, and its principal attributes displayed on a new row: - -[#fts_type-mappings_child_field_saved] -image::fts-type-mappings-child-field-saved.png[,700,align=left] - -Note that when this row is hovered over with the mouse, an *Edit* button appears, whereby updates to the definition can be made. diff --git a/modules/fts/pages/fts-type-mappings-add-child-mappings.adoc b/modules/fts/pages/fts-type-mappings-add-child-mappings.adoc deleted file mode 100644 index 260b7a04d1..0000000000 --- a/modules/fts/pages/fts-type-mappings-add-child-mappings.adoc +++ /dev/null @@ -1,29 +0,0 @@ -[#inserting-a-child-mapping] -= Add Child Mapping - -The option [.ui]*insert child mapping* specifies a document-field whose value is a JSON object. -Selecting this option displays the following: - -[#fts_type_mappings_child_mapping_dialog] -image::fts-type-mappings-child-mapping-dialog.png[,700,align=left] - -The following interactive field and checkbox are displayed: - -* [.ui]*{}*: The name of a field whose value is a JSON object. -Note that an analyzer for the field, by means of the pull-down menu. -* [.ui]*only index specified fields*: When checked, only fields explicitly specified are added to the index. -Note that the JSON object specified as the value for [.ui]*{}* has multiple fields of its own. -Checking this box ensures that all or a subset of these can be selected for indexing. - -When completed, this panel might look as follows (note that `reviews` is a field within the `hotel`-type documents of the `travel-sample` bucket whose value is a JSON object): - -[#fts_type_mappings_child_mapping_dialog_complete] -image::fts-type-mappings-child-mapping-dialog-complete.png[,700,align=left] - -Save by left-clicking *OK*. -The field is now displayed as part of the `hotel` type mapping. -Note that by hovering over the `reviews` row with the mouse, the [.ui]*Edit* and [.ui]*+* buttons are revealed: the [.ui]*+* button is present because `reviews` is an object that contains child-fields; which can now themselves be individually indexed. -Left-click on this, and a child-field, such as `content`, can be specified: - -[#fts_type_mappings_child_mapping_add_field] -image::fts-type-mappings-child-mapping-add-field.png[,700,align=left] \ No newline at end of file diff --git a/modules/fts/pages/fts-type-mappings.adoc b/modules/fts/pages/fts-type-mappings.adoc index b2214e170d..786b6be011 100644 --- a/modules/fts/pages/fts-type-mappings.adoc +++ b/modules/fts/pages/fts-type-mappings.adoc @@ -34,9 +34,9 @@ For information on how values are data-typed when dynamic mapping is specified, To specify a type mapping, type an appropriate string (for example, `hotel`) into the interactive text field. Note the [.ui]*only index specified fields* checkbox: if this is checked, only user-specified fields from the document are included in the index. -(For an example, see xref:fts-type-mapping-specifying-fields.adoc[Specifying Fields], below.) +(For an example, see xref:specifying-fields-for-type-mapping[Specifying Fields], below.) -Optionally, an _analyzer_ can be specified for the type mapping: for all queries that do indeed support the use of an analyzer, the specified analyzer will be applied, rather than the default analyzer (which is itself specified in the *Advanced* pane, as described below, in xref:fts-creating-index-specifying-advanced-settings.adoc[Specifying Advanced Settings]). +Optionally, an _analyzer_ can be specified for the type mapping: for all queries that support the use of an analyzer, the specified analyzer will be applied, rather than the default analyzer (which is itself specified in the *Advanced* pane, as described below, in xref:fts-default-settings.adoc[Default Settings]). A list of available analyzers can be accessed and selected from, by means of the pull-down menu to the right of the interactive text-field: @@ -44,10 +44,10 @@ A list of available analyzers can be accessed and selected from, by means of the image::fts-type-mappings-ui-analyzers-menu.png[,620,align=left] The default value, `inherit`, means that the type mapping inherits the default analyzer. -Note that custom analyzers can be created and stored for the index that is being defined using the [.ui]*Analyzers* panel, described below in xref:fts-analyzers.adoc#Creating-Analyzers[Creating Analyzers]. +Note that custom analyzers can be created and stored for the index that is being defined using the [.ui]*Analyzers* panel, described below in xref:fts-index-analyzers.adoc#Creating-Analyzers[Creating Analyzers]. On creation, all custom analyzers are available for association with a type mapping, and so appear in the pull-down menu shown above. -Additional information on analyzers can also be found on the page xref:fts-analyzers.adoc#Understanding-Analyzers[Understanding Analyzers]. +Additional information on analyzers can also be found on the page xref:fts-index-analyzers.adoc#Understanding-Analyzers[Understanding Analyzers]. The [.ui]*Type Mappings* panel now appears as follows: @@ -68,11 +68,12 @@ Note also that should you wish to ensure that all documents in the bucket are in [#fts_type_mappings_ui_addition_default_checked] image::fts-type-mappings-ui-addition-default-checked.png[,750,align=left] +[#Specifying-Type-Mapping-for-Collection] == Specifying Type Mapping for Collection Type Mapping will allow you to search for documents from the selected scope, selected collections from the scope, and for a specific document type from the selected scope and collections. -For using non-default scope/collections, please refer: xref:fts-creating-index-from-UI.adoc#using-non-default-scope-collections[Using Non-Default Scope/Collections]. +For using non-default scope/collections, please refer: xref:fts-creating-index-from-UI-classic-editor.adoc#using-non-default-scope-collections[Using Non-Default Scope/Collections]. ** Left click on the *+ Add Type Mapping* button. The display now appears as follows: @@ -211,8 +212,8 @@ image::fts-type-mappings-ui-field-options.png[,700,align=left] These options are described in the following sections. -* xref:fts-type-mappings-add-child-mappings.adoc[Add Child Mapping] -* xref:fts-type-mappings-add-child-field.adoc[Add Child Field] +* xref:inserting-a-child-mapping[Add Child Mapping] +* xref:inserting-a-child-field[Add Child Field] [#inserting-a-child-mapping] == Add Child Mapping @@ -244,7 +245,7 @@ Left-click on this, and a child-field, such as `content`, can be specified: [#fts_type_mappings_child_mapping_add_field] image::fts-type-mappings-child-mapping-add-field.png[,700,align=left] - +[#inserting-a-child-field] == Add Child Field The option [.ui]*insert child field* allows a field to be individually included for (or excluded from) indexing, provided that it contains a single value or an array rather than a JSON object. @@ -369,9 +370,9 @@ Enabling this option results in larger indexes, so disable this option to always === DocValues -To include the value for each instance of the field in the index, the docvalues checkbox must be checked. This is essential for xref:fts-search-response-facets.adoc[Facets]. +To include the value for each instance of the field in the index, the docvalues checkbox must be checked. This is essential for xref:fts-search-response.adoc#Facets[Facets]. -For sorting of search results based on field values: see xref:fts-sorting.adoc[Sorting Query Results]. +For sorting of search results based on field values: see xref:fts-search-response.adoc#sorting-query-results[Sorting Query Results]. By default, this checkbox is selected. If it is _unchecked_, the values are _not_ added to the index; and in consequence, neither Search Facets nor value-based result-sorting is supported.