WIP: Introduce typesense search

source/_static/js/myscript-v1.js

@@ -2,6 +2,7 @@ $(document).ready(function () {
     // Function to set the custom theme attribute based on the current theme
     function setCustomTheme(theme) {
         $('body').attr('data-custom-theme', theme);
+        $('html').attr('data-theme', theme);
     }
 
     // Check for a manually set theme

source/_static/scss/partials/_sidebar.scss

@@ -50,7 +50,13 @@
     }
 }
 
+.DocSearch-Container {
+    margin-top: 55px;
+}
+
 .sidebar-search {
+    display: none;
+
     border-radius: 4px;
     border: 1px solid rgba(63, 67, 80, 0.16);
     background: #FFF;
@@ -69,6 +75,7 @@ body:not([data-custom-theme="light"]) {
 }
 
 .sidebar-search-container:before {
+    height: 0px;
     left: 32px;
     z-index: 20;
     background-color: currentColor;

source/_templates/custom-nav.html

@@ -1,3 +1,26 @@
+<script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
+<script>
+	window.addEventListener('load', () => {
+		const options = location.hostname === 'localhost' ? {
+			node: { host: 'localhost', port: '8108', protocol: 'http' },
+			apiKey: 'test_api_key',
+		} : {
+			node: { host: 'h0agqxfpir543j9lp-1.a1.typesense.net', port: '443', protocol: 'https' },
+			apiKey: 'ZOLa3xKupe9e7DRPhkv56g8VUoCygd00',
+		};
+
+		docsearch({
+			container: '.sidebar-search-container',
+			typesenseCollectionName: 'mm_product_docs',
+			typesenseServerConfig: {
+				nodes: [options.node],
+				apiKey: options.apiKey
+			},
+			typesenseSearchParams: {}
+		});
+	});
+</script>
+
 <div data-swiftype-index=false class="notification-bar sticky-top">
     <div class="notification-bar__content">
         <a class="notification-bar__close" data-ol-has-click-handler="">
@@ -69,7 +92,7 @@
 											<a href="https://mattermost.com/channels/">
 												Channels
 											</a>
-										</li>	
+										</li>
 										<li class="sub-menu__links--single">
 											<a href="https://mattermost.com/playbooks/">
 												Playbooks
@@ -127,7 +150,7 @@
 											<a href="https://mattermost.com/enterprise/cloud/">
 											Cloud
 											</a>
-										</li>	
+										</li>
 									</ul>
 								</div>
 							</div>
@@ -173,7 +196,7 @@
 											<a href="https://mattermost.com/solutions/use-cases/integrated-security-operations/">
 											Integrated Security Operations
 											</a>
-										</li>	
+										</li>
 										<li class="sub-menu__links--single">
 											<a href="https://mattermost.com/solutions/use-cases/out-of-band-incident-response/">
 											Out-of-Band Incident Response
@@ -384,7 +407,7 @@
 									<div class="sub-menu__dev-link-col">
 										<p class="sub-menu__link-col-header sub-menu__link-col-header-space">Documentation</p>
 										<ul>
-											
+
 											<li class="sub-menu__links--single">
 												<a href="https://academy.mattermost.com/">
 												Academy
@@ -429,7 +452,7 @@
 												<a href="https://mattermost.com/community/">
 													Join Community
 												</a>
-											</li>	
+											</li>
 											<li class="sub-menu__links--single">
 												<a href="https://mattermost.com/contribute/">
 													Contribute

source/conf.py

@@ -4072,7 +4072,8 @@ def setup(_: Sphinx):
 html_css_files = [
     "css/mattermost-global.css",
     "css/homepage-v1.css",
-    "css/compass-icons.css"
+    "css/compass-icons.css",
+    "https://cdn.jsdelivr.net/npm/[email protected]",
 ]
 
 # A list of JavaScript filenames. The entry must be a filename string or a tuple containing the filename string and the

typesense/Dockerfile

@@ -0,0 +1,16 @@
+from python:3
+
+RUN apt-get update
+
+WORKDIR /app
+
+RUN pip3 install pipenv
+
+COPY ./Pipfile .
+COPY ./Pipfile.lock .
+
+RUN pipenv install --dev
+
+RUN mkdir -p build/html
+
+COPY . .

typesense/README.md

@@ -0,0 +1,34 @@
+# Using Typesense for documentation search
+
+We'll have 3 terminals open, and run the following in the first two:
+- `make livehtml` - Run Sphinx docs server
+- `cd typesense && docker compose up typesense` - Run Typesense server
+
+After those are running, in the third terminal:
+- `cd typesense && docker compose up scraper` - Run scraper to populate Typesense. The process will exit once complete
+
+
+If you'd like to re-index the docs, you can run:
+
+```sh
+cd typesense
+./scripts/reset-typesense-collection.sh # This will delete all existing documents in the collection. Typesense will de-duplicate docs naturally, but this forces it to remove metadata from previous runs that we may want to remove as we change the schema/filters.
+docker compose up scraper
+```
+
+To export the index into a jsonl file, run:
+
+```sh
+cd typesense
+./scripts/download-typesense-collection.sh
+```
+
+---
+
+The scripts mentioned above support the following environment variables for configuration:
+
+- `TYPESENSE_API_KEY` - Defaults to `test_api_key`
+- `TYPESENSE_HOST` - Defaults to `http://localhost:8108`
+- `TYPESENSE_HOSTNAME` - Defaults to `localhost`
+- `TYPESENSE_PORT` - Defaults to `8108`
+- `TYPESENSE_PROTOCOL` - Defaults to `http`

typesense/config.json

@@ -0,0 +1,39 @@
+{
+  "index_name": "mm_product_docs",
+  "allowed_domains": [
+    "localhost",
+    "mattermost-docs-preview-pulls.s3-website-us-east-1.amazonaws.com"
+  ],
+  "start_urls": [
+    {
+      "url": "http://localhost:8000",
+      "tags": []
+    }
+  ],
+  "sitemap_urls": [
+    "http://localhost:8000/sitemap.xml"
+  ],
+  "selectors": {
+    "default": {
+      "lvl0": "article h1",
+      "lvl1": "article h2",
+      "lvl2": "article h3",
+      "lvl3": "article h4",
+      "lvl4": "article h5",
+      "lvl5": "article h6",
+      "text": "article p, article li"
+    }
+  },
+  "custom_settings": {
+    "token_separators": [
+      "-"
+    ],
+    "symbols_to_index": [
+      "@"
+    ]
+  },
+  "strip_chars": " .,;:#",
+  "stop_urls": [],
+  "scrape_start_urls": false,
+  "nb_hits": 64205
+}

typesense/docker-compose.yml

@@ -0,0 +1,28 @@
+services:
+  typesense:
+    image: typesense/typesense:0.24.0
+    environment:
+      - TYPESENSE_API_KEY=test_api_key
+      - TYPESENSE_DATA_DIR=/data
+      - TYPESENSE_ENABLE_CORS=true
+    ports:
+      - "8108:8108"
+    volumes:
+      - typesense-data:/data
+
+  scraper:
+    image: typesense/docsearch-scraper
+    volumes:
+      - ./config.json:/app/config.json
+    network_mode: "host"
+    environment:
+      - CONFIG=/app/config.json
+      - TYPESENSE_DATA_DIR=/data
+      - TYPESENSE_ENABLE_CORS=true
+      - TYPESENSE_API_KEY=${TYPESENSE_API_KEY:-test_api_key}
+      - TYPESENSE_HOST=${TYPESENSE_HOSTNAME:-localhost}
+      - TYPESENSE_PORT=${TYPESENSE_PORT:-8108}
+      - TYPESENSE_PROTOCOL=${TYPESENSE_PROTOCOL:-http}
+
+volumes:
+  typesense-data:

typesense/scripts/create-typesense-collection.sh

@@ -0,0 +1,15 @@
+TYPESENSE_HOST="${TYPESENSE_HOST:-http://localhost:8108}"
+TYPESENSE_API_KEY="${TYPESENSE_API_KEY:-test_api_key}"
+
+curl "${TYPESENSE_HOST}/collections" \
+       -X POST \
+       -H "Content-Type: application/json" \
+       -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" \
+       -d '{
+         "name": "mm_product_docs",
+         "fields": [
+           {"name": "category", "type": "string" },
+           {"name": "weight", "type": "int32" }
+         ],
+         "default_sorting_field": "weight"
+       }'

typesense/scripts/download-typesense-collection.sh

@@ -0,0 +1,7 @@
+TYPESENSE_HOST="${TYPESENSE_HOST:-http://localhost:8108}"
+TYPESENSE_API_KEY="${TYPESENSE_API_KEY:-test_api_key}"
+
+curl "${TYPESENSE_HOST}/collections/mm_product_docs/documents/export" \
+       -X GET \
+       -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}" \
+       -o documents.jsonl

typesense/scripts/reset-typesense-collection.sh

@@ -0,0 +1,7 @@
+TYPESENSE_HOST="${TYPESENSE_HOST:-http://localhost:8108}"
+TYPESENSE_API_KEY="${TYPESENSE_API_KEY:-test_api_key}"
+TYPESENSE_COLLECTION_NAME="mm_product_docs"
+
+curl -X DELETE \
+  "${TYPESENSE_HOST}/collections/${TYPESENSE_COLLECTION_NAME}/documents?truncate=true&filter_by=anchor:!=none" \
+  -H "X-TYPESENSE-API-KEY: ${TYPESENSE_API_KEY}"