diff --git a/.yarn/cache/axios-npm-0.24.0-39e5c1e79e-468cf496c0.zip b/.yarn/cache/axios-npm-0.24.0-39e5c1e79e-468cf496c0.zip deleted file mode 100644 index 5d8e81385b..0000000000 Binary files a/.yarn/cache/axios-npm-0.24.0-39e5c1e79e-468cf496c0.zip and /dev/null differ diff --git a/.yarn/cache/axios-npm-1.6.7-d7b9974d1b-87d4d42992.zip b/.yarn/cache/axios-npm-1.6.7-d7b9974d1b-87d4d42992.zip new file mode 100644 index 0000000000..9e10f0432d Binary files /dev/null and b/.yarn/cache/axios-npm-1.6.7-d7b9974d1b-87d4d42992.zip differ diff --git a/.yarn/cache/follow-redirects-npm-1.15.5-9d14db76ca-5ca49b5ce6.zip b/.yarn/cache/follow-redirects-npm-1.15.5-9d14db76ca-5ca49b5ce6.zip new file mode 100644 index 0000000000..2c1e9bdadf Binary files /dev/null and b/.yarn/cache/follow-redirects-npm-1.15.5-9d14db76ca-5ca49b5ce6.zip differ diff --git a/.yarn/cache/form-data-npm-4.0.0-916facec2d-01135bf867.zip b/.yarn/cache/form-data-npm-4.0.0-916facec2d-01135bf867.zip new file mode 100644 index 0000000000..8ae5189b00 Binary files /dev/null and b/.yarn/cache/form-data-npm-4.0.0-916facec2d-01135bf867.zip differ diff --git a/.yarn/cache/proxy-from-env-npm-1.1.0-c13d07f26b-ed7fcc2ba0.zip b/.yarn/cache/proxy-from-env-npm-1.1.0-c13d07f26b-ed7fcc2ba0.zip new file mode 100644 index 0000000000..a58e6bf3e4 Binary files /dev/null and b/.yarn/cache/proxy-from-env-npm-1.1.0-c13d07f26b-ed7fcc2ba0.zip differ diff --git a/devguide.md b/devguide.md index 3c8ef38548..c2e092d791 100644 --- a/devguide.md +++ b/devguide.md @@ -87,3 +87,13 @@ The current breadcrumb is currently determined based on the `page.path` and the ### Searching We're using Algolia, which uses `algolia.js` and `algolia.css`. The index is updated as part of the build process on Netlify.. +### Gainsight +We use federated search with Gainsight, which requires a similar indexing operation to Algolia. Here's a quick process overview: + +1. The current Gainsight indexing implementation reuses part of the `src/sitemap.xml` template to generate an array of objects in a JSON file - `src/gainsight-pages.json`. This file gets generated during the Jekyll build process. +2. After a successful Netlify production deploy, a small Netlify build plugin (`netlify-plugins/gainsight`) reads the generated JSON file and sends a request to the Gainsight API to index the pages. +3. Configuration in `netlify.toml` ensures that the build plugin only runs after production deploys. + +Gainsight requires the following environment variables to be configured in Netlify to work properly: +- `GAINSIGHT_CLIENT_ID` +- `GAINSIGHT_CLIENT_SECRET` diff --git a/netlify-plugins/gainsight/index.js b/netlify-plugins/gainsight/index.js new file mode 100644 index 0000000000..a9998d07b0 --- /dev/null +++ b/netlify-plugins/gainsight/index.js @@ -0,0 +1,59 @@ +const fs = require('fs') +const axios = require('axios') +const { chunk } = require('lodash') + +// Limit number of URL objects per request +const PAGE_SIZE = 50 + +// Delay between requests to avoid rate limiting +const REQUEST_DELAY = 50 + +// Generate an OAuth2 token +const generateToken = async () => { + const response = await axios.post( + 'https://api2-us-west-2.insided.com/oauth2/token', + + // Axios automatically sends URLSearchParams as form data + new URLSearchParams({ + grant_type: 'client_credentials', + client_id: process.env.GAINSIGHT_CLIENT_ID, + client_secret: process.env.GAINSIGHT_CLIENT_SECRET, + scope: 'write' + }) + ) + return response.data.access_token +} + +// Send a POST request to the Gainsight API to index the pages +const index = async (pages) => { + const token = await generateToken() + const pageChunks = chunk(pages, PAGE_SIZE) + let currentPage = 0 + console.log(`Gainsight: indexing ${pages.length} pages in ${pageChunks.length} chunks`) + for (const pageChunk of pageChunks) { + currentPage++ + console.log(`Indexing page ${currentPage} of ${pageChunks.length}...`) + await axios.post( + 'https://api2-us-west-2.insided.com/external-content/index', + { batch: pageChunk }, + { + headers: { + 'Authorization': `Bearer ${token}`, + 'Content-Type': 'application/json', + } + } + ) + await new Promise(resolve => setTimeout(resolve, REQUEST_DELAY)) + } +} + +// The `onSuccess` hook runs after the deploy is live. This matters because Gainsight will +// access the URLs we upload, so the updated content has to be live before we index it. +// +// Copied file handling approach from Netlify's Brand Guardian plugin: +// https://github.com/tzmanics/netlify-plugin-brand-guardian/blob/33f90f745086a2bc9ed9273b15340002960afdfa/index.js +export const onSuccess = async ({ constants }) => { + const pagesJson = fs.readFileSync(`${constants.PUBLISH_DIR}/gainsight-pages.json`) + const pages = JSON.parse(pagesJson) + await index(pages) +} \ No newline at end of file diff --git a/netlify-plugins/gainsight/manifest.yaml b/netlify-plugins/gainsight/manifest.yaml new file mode 100644 index 0000000000..cb2309c25e --- /dev/null +++ b/netlify-plugins/gainsight/manifest.yaml @@ -0,0 +1 @@ +name: gainsight \ No newline at end of file diff --git a/netlify.toml b/netlify.toml index 76d7ade33b..43e2385569 100644 --- a/netlify.toml +++ b/netlify.toml @@ -5,6 +5,10 @@ # about the available build targets (develop, develop-inc, build) command = "jekyll algolia && yarn build" +[[context.production.plugins]] + # Run the gainsight index script after a successful production deploy + package = "/netlify-plugins/gainsight" + [context.deploy-preview] # For deploy previews, use the testing Jekyll environment, the develop build target calls command = "yarn develop" diff --git a/package.json b/package.json index 394ca7c3b9..622122b32e 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "@babel/plugin-proposal-object-rest-spread": "^7.15.6", "@babel/plugin-transform-runtime": "^7.15.0", "@babel/preset-env": "^7.15.6", - "axios": "^0.24.0", + "axios": "^1.6.7", "babel-loader": "^8.3.0", "concurrently": "^6.2.1", "front-matter": "^4.0.2", @@ -58,6 +58,7 @@ "globby": "11.0.4", "handlebars": "^4.7.7", "locate-path": "^7.1.0", + "lodash": "^4.17.21", "ms": "^2.1.3", "ora": "5.4.1", "p-locate": "5.0.0", diff --git a/src/gainsight-pages.json b/src/gainsight-pages.json new file mode 100644 index 0000000000..b8c0553f31 --- /dev/null +++ b/src/gainsight-pages.json @@ -0,0 +1,15 @@ +--- +layout: null +--- +[ + {% assign pages = site.html_pages | where_exp:'doc','doc.sitemap != false' | where_exp:'doc','doc.url != "/404.html"' %} + {% for page in pages %} + {% unless page.hidden %} + { + "title": "{{ page.title | xml_escape }}", + "content": "{{ page.title | xml_escape }}", + "url": "{{ page.url | replace:'/index.html','/' | absolute_url | xml_escape }}" + }{% if forloop.last != true %},{% endif %} + {% endunless %} + {% endfor %} +] diff --git a/temp-test.js b/temp-test.js new file mode 100644 index 0000000000..c27c15998e --- /dev/null +++ b/temp-test.js @@ -0,0 +1,62 @@ +require('dotenv').config() +const fs = require('fs') +const axios = require('axios') +const { chunk } = require('lodash') + +// Limit number of url objects per request +const PAGE_SIZE = 50 + +// Delay between requests to avoid rate limiting +const REQUEST_DELAY = 50 + +// Generate an OAuth2 token +const generateToken = async () => { + const response = await axios.post( + 'https://api2-us-west-2.insided.com/oauth2/token', + + // Axios automatically sends URLSearchParams as form data + new URLSearchParams({ + grant_type: 'client_credentials', + client_id: process.env.GAINSIGHT_CLIENT_ID, + client_secret: process.env.GAINSIGHT_CLIENT_SECRET, + scope: 'write' + }) + ) + return response.data.access_token +} + +// Send a POST request to the Gainsight API to index the pages +const index = async (pages) => { + const token = await generateToken() + const pageChunks = chunk(pages, PAGE_SIZE) + console.log(`Gainsight: indexing ${pages.length} pages in ${pageChunks.length} chunks`) + let currentPage = 0 + for (const pageChunk of pageChunks) { + currentPage++ + console.log(`Indexing page ${currentPage} of ${pageChunks.length}...`) + await axios.post( + 'https://api2-us-west-2.insided.com/external-content/index', + { batch: pageChunk }, + { + headers: { + 'Authorization': `Bearer ${token}`, + 'Content-Type': 'application/json', + } + } + ) + await new Promise(resolve => setTimeout(resolve, REQUEST_DELAY)) + } +} + +// The `onSuccess` hook runs after the deploy is live. This matters because Gainsight will +// access the URLs we upload, so the updated content has to be live before we index it. +// +// Copied file handling approach from Netlify's Brand Guardian plugin: +// https://github.com/tzmanics/netlify-plugin-brand-guardian/blob/33f90f745086a2bc9ed9273b15340002960afdfa/index.js +const onSuccess = async ({ constants }) => { + const pagesJson = fs.readFileSync(`${constants.PUBLISH_DIR}/gainsight-pages.json`) + const pages = JSON.parse(pagesJson) + await index(pages) +} + +onSuccess({ constants: { PUBLISH_DIR: './_site' } }) \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 1ebf13f8a3..65431fef9e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2353,12 +2353,14 @@ __metadata: languageName: node linkType: hard -"axios@npm:^0.24.0": - version: 0.24.0 - resolution: "axios@npm:0.24.0" +"axios@npm:^1.6.7": + version: 1.6.7 + resolution: "axios@npm:1.6.7" dependencies: - follow-redirects: ^1.14.4 - checksum: 468cf496c08a6aadfb7e699bebdac02851e3043d4e7d282350804ea8900e30d368daa6e3cd4ab83b8ddb5a3b1e17a5a21ada13fc9cebd27b74828f47a4236316 + follow-redirects: ^1.15.4 + form-data: ^4.0.0 + proxy-from-env: ^1.1.0 + checksum: 87d4d429927d09942771f3b3a6c13580c183e31d7be0ee12f09be6d5655304996bb033d85e54be81606f4e89684df43be7bf52d14becb73a12727bf33298a082 languageName: node linkType: hard @@ -3885,7 +3887,7 @@ __metadata: languageName: node linkType: hard -"follow-redirects@npm:^1.0.0, follow-redirects@npm:^1.14.0, follow-redirects@npm:^1.14.4": +"follow-redirects@npm:^1.0.0, follow-redirects@npm:^1.14.0": version: 1.14.7 resolution: "follow-redirects@npm:1.14.7" peerDependenciesMeta: @@ -3895,6 +3897,16 @@ __metadata: languageName: node linkType: hard +"follow-redirects@npm:^1.15.4": + version: 1.15.5 + resolution: "follow-redirects@npm:1.15.5" + peerDependenciesMeta: + debug: + optional: true + checksum: 5ca49b5ce6f44338cbfc3546823357e7a70813cecc9b7b768158a1d32c1e62e7407c944402a918ea8c38ae2e78266312d617dc68783fac502cbb55e1047b34ec + languageName: node + linkType: hard + "form-data@npm:^3.0.0": version: 3.0.1 resolution: "form-data@npm:3.0.1" @@ -3906,6 +3918,17 @@ __metadata: languageName: node linkType: hard +"form-data@npm:^4.0.0": + version: 4.0.0 + resolution: "form-data@npm:4.0.0" + dependencies: + asynckit: ^0.4.0 + combined-stream: ^1.0.8 + mime-types: ^2.1.12 + checksum: 01135bf8675f9d5c61ff18e2e2932f719ca4de964e3be90ef4c36aacfc7b9cb2fceb5eca0b7e0190e3383fe51c5b37f4cb80b62ca06a99aaabfcfd6ac7c9328c + languageName: node + linkType: hard + "format@npm:^0.2.0": version: 0.2.2 resolution: "format@npm:0.2.2" @@ -6385,6 +6408,13 @@ __metadata: languageName: node linkType: hard +"proxy-from-env@npm:^1.1.0": + version: 1.1.0 + resolution: "proxy-from-env@npm:1.1.0" + checksum: ed7fcc2ba0a33404958e34d95d18638249a68c430e30fcb6c478497d72739ba64ce9810a24f53a7d921d0c065e5b78e3822759800698167256b04659366ca4d4 + languageName: node + linkType: hard + "pump@npm:^3.0.0": version: 3.0.0 resolution: "pump@npm:3.0.0" @@ -7685,7 +7715,7 @@ __metadata: ajv: ^6.10.2 algoliasearch: ^4.10.5 ansi-regex: ^6.0.1 - axios: ^0.24.0 + axios: ^1.6.7 babel-loader: ^8.3.0 browser-sync: ^2.29.1 check-links: ^1.1.8 @@ -7703,6 +7733,7 @@ __metadata: handlebars: ^4.7.7 js-yaml: ^4.1.0 locate-path: ^7.1.0 + lodash: ^4.17.21 ms: ^2.1.3 ora: 5.4.1 p-locate: 5.0.0