feat(notion): add polling system

Author: kerwanp

examples/node-script-notion/package.json

@@ -2,10 +2,11 @@
   "name": "node-script-notion-example",
   "private": true,
   "scripts": {
-    "start": "contentlayer build && node --experimental-json-modules my-script.mjs"
+    "start": "contentlayer build && node --experimental-json-modules my-script.mjs",
+    "dev": "contentlayer dev"
   },
   "dependencies": {
     "@notionhq/client": "^2.2.3",
     "contentlayer": "latest"
   }
-}
+}

packages/@contentlayer/source-notion/package.json

@@ -1,6 +1,6 @@
 {
   "name": "contentlayer-source-notion",
-  "version": "0.0.1-alpha.25",
+  "version": "0.0.1-alpha.26",
   "type": "module",
   "exports": "./dist/index.js",
   "types": "./dist/index.d.ts",

packages/@contentlayer/source-notion/src/fetchData/fetchAllDocuments.ts

@@ -23,25 +23,35 @@ export const fetchAllDocuments = ({ databaseTypeDefs, previousCache, schemaDef,
         S.chain((pages) =>
           pipe(
             S.effect(
-              T.forEachParN_(pages, os.cpus().length, (page) =>
-                makeCacheItem({
-                  page,
-                  documentTypeDef: schemaDef.documentTypeDefMap[databaseTypeDef.name]!,
-                  databaseTypeDef,
-                  previousCache,
-                  options,
-                }),
+              pipe(
+                T.forEachParN_(pages, os.cpus().length, (page) =>
+                  makeCacheItem({
+                    page,
+                    documentTypeDef: schemaDef.documentTypeDefMap[databaseTypeDef.name]!,
+                    databaseTypeDef,
+                    previousCache,
+                    options,
+                  }),
+                ),
               ),
             ),
             OT.withStreamSpan('@contentlayer/source-notion/fetchData:makeCacheItems'),
           ),
         ),
         S.runCollect,
-        T.chain((chunks) => T.reduce_(chunks, [] as DataCache.CacheItem[], (z, a) => T.succeed([...z, ...a]))),
+        T.chain((chunks) =>
+          T.reduce_(chunks, [] as { fromCache: boolean; cacheItem: DataCache.CacheItem }[], (z, a) =>
+            T.succeed([...z, ...a]),
+          ),
+        ),
       ),
     ),
-    T.map((chunks) => Chunk.reduce_(chunks, [] as DataCache.CacheItem[], (z, a) => [...z, ...a])),
-    T.map((documents) => ({ cacheItemsMap: Object.fromEntries(documents.map((_) => [_.document._id, _])) })),
+    T.map((chunks) =>
+      Chunk.reduce_(chunks, [] as { fromCache: boolean; cacheItem: DataCache.CacheItem }[], (z, a) => [...z, ...a]),
+    ),
+    T.map((documents) => ({
+      cacheItemsMap: Object.fromEntries(documents.map((_) => [_.cacheItem.document._id, _.cacheItem])),
+    })),
     OT.withSpan('@contentlayer/source-notion/fetchData:fetchAllDocuments'),
     T.mapError((error) => new core.SourceFetchDataError({ error, alreadyHandled: false })),
   )

packages/@contentlayer/source-notion/src/fetchData/makeCacheItem.ts

@@ -1,5 +1,4 @@
 import type * as core from '@contentlayer/core'
-import { hashObject } from '@contentlayer/utils'
 import { OT, pipe, T } from '@contentlayer/utils/effect'
 import type { PageObjectResponse } from '@notionhq/client/build/src/api-endpoints'
 
@@ -27,7 +26,7 @@ export const makeCacheItem = ({ databaseTypeDef, documentTypeDef, previousCache,
         previousCache.cacheItemsMap[page.id]!.hasWarnings === false
       ) {
         const cacheItem = previousCache.cacheItemsMap[page.id]!
-        return cacheItem
+        return { cacheItem, fromCache: true }
       }
 
       const document = yield* $(makeDocument({ documentTypeDef, databaseTypeDef, page, options }))
@@ -38,10 +37,13 @@ export const makeCacheItem = ({ databaseTypeDef, documentTypeDef, previousCache,
       })
 
       return {
-        document,
-        documentHash,
-        hasWarnings: false,
-        documentTypeName: documentTypeDef.name,
+        cacheItem: {
+          document,
+          documentHash,
+          hasWarnings: false,
+          documentTypeName: documentTypeDef.name,
+        },
+        fromCache: false,
       }
     }),
     OT.withSpan('@contentlayer/source-notion/fetchData:makeCacheItem'),

packages/@contentlayer/source-notion/src/index.ts

@@ -1,6 +1,6 @@
 import type * as core from '@contentlayer/core'
 import { processArgs } from '@contentlayer/core'
-import { pipe, S, T } from '@contentlayer/utils/effect'
+import { pipe, S, SC, T } from '@contentlayer/utils/effect'
 import { NotionRenderer } from '@notion-render/client'
 import * as notion from '@notionhq/client'
 
@@ -17,13 +17,15 @@ export const makeSource: core.MakeSourcePlugin<PluginOptions & core.PartialArgs>
   const {
     options,
     extensions,
-    restArgs: { databaseTypes, ...rest },
+    restArgs: { databaseTypes, dev, ...rest },
   } = await processArgs(args, sourceKey)
 
   const databaseTypeDefs = (Array.isArray(databaseTypes) ? databaseTypes : Object.values(databaseTypes)).map((_) =>
     _.def(),
   )
 
+  const polling = dev && dev.polling === false ? false : dev?.polling ?? 5_000
+
   const client =
     rest.client instanceof notion.Client
       ? rest.client
@@ -59,6 +61,7 @@ export const makeSource: core.MakeSourcePlugin<PluginOptions & core.PartialArgs>
             T.provideService(NotionRendererTag)(renderer),
           ),
         ),
+        S.repeatSchedule<unknown, void | number>(polling ? SC.spaced(polling) : SC.stop),
       ),
   }
 }

packages/@contentlayer/source-notion/src/types.ts

@@ -8,6 +8,9 @@ export type PluginOptions = {
   client?: ConstructorParameters<typeof notion.Client>[0] | notion.Client
   renderer?: ConstructorParameters<typeof NotionRenderer>[0] | NotionRenderer
   databaseTypes: DatabaseTypes
+  dev?: {
+    polling: false | number
+  }
 }
 
 export type FieldDef = core.FieldDef & { propertyKey?: string }