Skip to content

Commit 45f4fda

Browse files
kerwanpkerwanp
committed
feat(notion): setup @contentlayer/source-notion base code
1 parent ad462a4 commit 45f4fda

29 files changed

+736
-386
lines changed

‎examples/node-script-notion/contentlayer.config.ts

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import { makeSource, defineDatabase } from '@contentlayer/source-notion';
2+
3+
const Post = defineDatabase(() => ({
4+
name: 'Post',
5+
databaseId: 'fe26b972ec3f4b32a1882230915fe111'
6+
}))
7+
8+
export default makeSource({
9+
internalIntegrationToken: process.env.NOTION_TOKEN as string,
10+
databaseTypes: [Post]
11+
})

‎examples/node-script-notion/my-script.mjs

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import { allPosts } from './.contentlayer/generated/index.mjs'
2+
3+
const postUrls = allPosts.map(post => post.url)
4+
5+
console.log(`Found ${postUrls.length} posts:`);
6+
console.log(postUrls)

‎examples/node-script-notion/package.json

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"name": "node-script-notion-example",
3+
"private": true,
4+
"scripts": {
5+
"start": "contentlayer build && node --experimental-json-modules my-script.mjs"
6+
},
7+
"dependencies": {
8+
"contentlayer": "latest"
9+
}
10+
}

‎packages/@contentlayer/core/src/plugin.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import type { DataCache } from './DataCache.js'
1010
import type { SourceFetchDataError, SourceProvideSchemaError } from './errors.js'
1111
import type { SchemaDef, StackbitExtension } from './schema/index.js'
1212

13-
export type SourcePluginType = LiteralUnion<'local' | 'contentful' | 'sanity', string>
13+
export type SourcePluginType = LiteralUnion<'local' | 'contentful' | 'notion' | 'sanity', string>
1414

1515
export type PluginExtensions = {
1616
// TODO decentralized extension definitions + logic

‎packages/@contentlayer/source-files/tsconfig.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,6 @@
66
"outDir": "./dist",
77
"tsBuildInfoFile": "./dist/.tsbuildinfo.json"
88
},
9-
"include": ["./src"],
9+
"include": ["./src", "../source-contentful/src/fetchData/types"],
1010
"references": [{ "path": "../utils" }, { "path": "../core" }]
1111
}

‎packages/@contentlayer/source-notion/package.json

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"name": "@contentlayer/source-notion",
3+
"version": "0.3.0",
4+
"type": "module",
5+
"exports": "./dist/index.js",
6+
"types": "./dist/index.d.ts",
7+
"files": [
8+
"./dist/**/*.{js,ts,map}",
9+
"./src",
10+
"./package.json"
11+
],
12+
"scripts": {
13+
"test": "echo No tests yet"
14+
},
15+
"devDependencies": {
16+
"@types/node": "^18.13.0"
17+
},
18+
"dependencies": {
19+
"@contentlayer/core": "workspace:*",
20+
"@contentlayer/utils": "workspace:*",
21+
"@notionhq/client": "^2.2.3",
22+
"contentful-management": "7.22.4",
23+
"slugify": "^1.6.5"
24+
}
25+
}

‎packages/@contentlayer/source-notion/src/errors.ts

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import { errorToString } from "@contentlayer/utils";
2+
import { Tagged } from "@contentlayer/utils/effect";
3+
4+
export class UnknownNotionError extends Tagged('UnknownNotionError')<{ readonly error: unknown }> {
5+
toString = () => `UnknownContentfulError: ${errorToString(this.error)}`
6+
}

‎packages/@contentlayer/source-notion/src/fetchData/index.ts

+77
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
import * as os from 'node:os'
2+
3+
import * as core from '@contentlayer/core'
4+
import type { HasConsole } from "@contentlayer/utils/effect";
5+
import { Chunk, OT, pipe, T } from "@contentlayer/utils/effect";
6+
import type * as notion from '@notionhq/client';
7+
import type { PageObjectResponse } from '@notionhq/client/build/src/api-endpoints.js';
8+
9+
import { UnknownNotionError } from '../errors.js';
10+
import type * as LocalSchema from '../schema/defs/index.js'
11+
import { makeCacheItem } from './page.js';
12+
13+
type Page = PageObjectResponse
14+
15+
export const fetchAllDocuments = ({
16+
client,
17+
schemaDef,
18+
databaseTypeDefs,
19+
options
20+
}: {
21+
client: notion.Client,
22+
databaseTypeDefs: LocalSchema.DatabaseTypeDef[],
23+
schemaDef: core.SchemaDef,
24+
options: core.PluginOptions
25+
}): T.Effect<OT.HasTracer & HasConsole, core.SourceFetchDataError, core.DataCache.Cache> => pipe(
26+
T.gen(function* ($) {
27+
const pages: Page[] = [];
28+
29+
for (const databaseDef of databaseTypeDefs) {
30+
const result = yield* $(fetchDatabasePages({ client, databaseDef }));;
31+
pages.push(...result);
32+
}
33+
34+
35+
const documentEntriesWithDocumentTypeDef = Object.values(schemaDef.documentTypeDefMap).flatMap(
36+
(documentTypeDef) => pages.map((page) => ({ page, documentTypeDef }))
37+
);
38+
39+
const concurrencyLimit = os.cpus().length
40+
41+
const documents = yield* $(
42+
pipe(
43+
documentEntriesWithDocumentTypeDef,
44+
T.forEachParN(concurrencyLimit, ({ page, documentTypeDef }) =>
45+
makeCacheItem({
46+
page,
47+
documentTypeDef,
48+
options
49+
})
50+
),
51+
OT.withSpan('@contentlayer/source-notion/fetchData:makeCacheItems', {
52+
attributes: { count: documentEntriesWithDocumentTypeDef.length },
53+
}),
54+
)
55+
)
56+
57+
const cacheItemsMap = Object.fromEntries(Chunk.map_(documents, (_) => [_.document._id, _]))
58+
59+
return { cacheItemsMap }
60+
}),
61+
OT.withSpan('@contentlayer/source-notion/fetchData:fetchAllDocuments', {
62+
attributes: { schemadef: JSON.stringify(schemaDef) },
63+
}),
64+
T.mapError((error) => new core.SourceFetchDataError({ error, alreadyHandled: false }))
65+
)
66+
67+
const fetchDatabasePages = ({
68+
client,
69+
databaseDef
70+
}: {
71+
client: notion.Client,
72+
databaseDef: LocalSchema.DatabaseTypeDef
73+
}): T.Effect<OT.HasTracer, UnknownNotionError, Page[]> => pipe(
74+
T.tryPromise(() => client.databases.query({ database_id: databaseDef.databaseId }).then(res => res.results as Page[])),
75+
OT.withSpan('@contentlayer/source-contentlayer/fetchData:getAllEntries'),
76+
T.mapError((error) => new UnknownNotionError({ error })),
77+
)

‎packages/@contentlayer/source-notion/src/fetchData/page.ts

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
import type * as core from '@contentlayer/core'
2+
import { HashError, hashObject } from '@contentlayer/utils';
3+
import type { HasConsole, OT } from "@contentlayer/utils/effect";
4+
import { pipe, T } from "@contentlayer/utils/effect";
5+
import type { PageObjectResponse } from "@notionhq/client/build/src/api-endpoints"
6+
7+
import { getFieldFunctions } from '../mapping/index.js';
8+
import type { FieldDef, PageProperties } from '../types';
9+
10+
type MakeDocumentError = core.UnexpectedMarkdownError | core.UnexpectedMDXError | HashError
11+
12+
export const makeCacheItem = ({
13+
page,
14+
documentTypeDef,
15+
options
16+
}: {
17+
page: PageObjectResponse,
18+
documentTypeDef: core.DocumentTypeDef,
19+
options: core.PluginOptions
20+
}): T.Effect<OT.HasTracer & HasConsole, MakeDocumentError, core.DataCache.CacheItem> =>
21+
T.gen(function* ($) {
22+
const { typeFieldName } = options.fieldOptions
23+
24+
const docValues = yield* $(
25+
T.forEachParDict_(documentTypeDef.fieldDefs as FieldDef[], { // TODO : Using workaround to use own typing to get property path (id)
26+
mapValue: (fieldDef: FieldDef) => getDataForFieldDef({
27+
fieldDef: fieldDef as FieldDef,
28+
property: page.properties[fieldDef.path] as PageProperties,
29+
options
30+
}),
31+
mapKey: (fieldDef) => T.succeed(fieldDef.name)
32+
})
33+
);
34+
35+
const document: core.Document = {
36+
...docValues,
37+
[typeFieldName]: documentTypeDef.name,
38+
_id: page.id,
39+
_raw: {},
40+
}
41+
42+
const documentHash = yield* $(hashObject(document));
43+
44+
return { document, documentHash, hasWarnings: false, documentTypeName: documentTypeDef.name }
45+
})
46+
47+
const getDataForFieldDef = ({
48+
fieldDef,
49+
property,
50+
options
51+
}: {
52+
fieldDef: FieldDef
53+
property: PageProperties,
54+
options: core.PluginOptions
55+
}): T.Effect<OT.HasTracer, MakeDocumentError, any> =>
56+
pipe(
57+
T.gen(function* ($) {
58+
const functions = getFieldFunctions(property.type);
59+
if (!functions) return;
60+
61+
return functions.getFieldData({ property, fieldDef, options })
62+
}),
63+
T.mapError((error) => new HashError({ error }))
64+
)

‎packages/@contentlayer/source-notion/src/index.ts

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
import type * as core from '@contentlayer/core'
2+
import { processArgs } from '@contentlayer/core'
3+
import { pipe, S, T } from '@contentlayer/utils/effect'
4+
import * as notion from '@notionhq/client';
5+
6+
import { fetchAllDocuments } from './fetchData/index.js'
7+
import type * as LocalSchema from './schema/defs/index.js'
8+
import { provideSchema } from './schema/provideSchema.js'
9+
import type { PluginOptions } from "./types.js"
10+
11+
export * from './schema/defs/index.js'
12+
13+
export type Args = {
14+
internalIntegrationToken: string,
15+
databaseTypes: LocalSchema.DatabaseTypes
16+
}
17+
18+
export const makeSource: core.MakeSourcePlugin<Args & PluginOptions> = async (args) => {
19+
const {
20+
options,
21+
extensions,
22+
restArgs: {
23+
internalIntegrationToken,
24+
databaseTypes
25+
}
26+
} = await processArgs(args);
27+
28+
const client = new notion.Client({ auth: internalIntegrationToken });
29+
30+
const databaseTypeDefs = (Array.isArray(databaseTypes) ? databaseTypes : Object.values(databaseTypes)).map(
31+
(_) => _.def()
32+
)
33+
34+
return {
35+
type: 'notion',
36+
extensions,
37+
options,
38+
provideSchema: () => provideSchema({ client, options, databaseTypeDefs }),
39+
fetchData: ({ schemaDef }) => pipe(
40+
S.fromEffect(
41+
pipe(
42+
fetchAllDocuments({ client, databaseTypeDefs, schemaDef, options }),
43+
T.either
44+
)
45+
),
46+
// S.repeatSchedule(SC.spaced(5_000))
47+
)
48+
}
49+
}

‎packages/@contentlayer/source-notion/src/mapping/field-checkbox.ts

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import type { FieldFunctions } from ".";
2+
3+
export const fieldCheckbox: FieldFunctions<'checkbox'> = {
4+
getFieldDef: () => {
5+
return {
6+
type: 'boolean',
7+
isRequired: false
8+
}
9+
},
10+
getFieldData: ({ property }) => {
11+
return property.checkbox;
12+
}
13+
}

‎packages/@contentlayer/source-notion/src/mapping/field-created-time.ts

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import type { FieldFunctions } from ".";
2+
3+
export const fieldCreatedTime: FieldFunctions<'created_time'> = {
4+
getFieldDef: () => {
5+
return {
6+
type: 'date',
7+
isRequired: false
8+
}
9+
},
10+
getFieldData: ({ property }) => {
11+
return new Date(property.created_time);
12+
}
13+
}

‎packages/@contentlayer/source-notion/src/mapping/field-email.ts

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import type { FieldFunctions } from ".";
2+
3+
export const fieldEmail: FieldFunctions<'email'> = {
4+
getFieldDef: () => {
5+
return {
6+
type: 'string',
7+
isRequired: false
8+
}
9+
},
10+
getFieldData: ({ property }) => {
11+
return property.email;
12+
}
13+
}

‎packages/@contentlayer/source-notion/src/mapping/field-number.ts

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import type { FieldFunctions } from ".";
2+
3+
export const fieldNumber: FieldFunctions<'number'> = {
4+
getFieldDef: () => {
5+
return {
6+
type: 'number',
7+
isRequired: false
8+
}
9+
},
10+
getFieldData: ({ property }) => {
11+
return property.number;
12+
}
13+
}

‎packages/@contentlayer/source-notion/src/mapping/field-phone-number.ts

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import type { FieldFunctions } from ".";
2+
3+
export const fieldPhoneNumber: FieldFunctions<'phone_number'> = {
4+
getFieldDef: () => {
5+
return {
6+
type: 'string',
7+
isRequired: false
8+
}
9+
},
10+
getFieldData: ({ property }) => {
11+
return property.phone_number;
12+
}
13+
}

‎packages/@contentlayer/source-notion/src/mapping/field-select.ts

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
import type { FieldFunctions } from ".";
2+
3+
export const fieldSelect: FieldFunctions<'select'> = {
4+
getFieldDef: ({ property }) => {
5+
return {
6+
type: 'enum',
7+
options: property.select.options.map(o => o.name),
8+
isRequired: false
9+
}
10+
},
11+
getFieldData: ({ property }) => {
12+
return property.select?.name;
13+
}
14+
}

‎packages/@contentlayer/source-notion/src/mapping/field-status.ts

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
import type { FieldFunctions } from ".";
2+
3+
export const fieldStatus: FieldFunctions<'status'> = {
4+
getFieldDef: ({ property }) => {
5+
return {
6+
type: 'enum',
7+
options: property.status.options.map(o => o.name),
8+
isRequired: false
9+
}
10+
},
11+
getFieldData: ({ property }) => {
12+
return property.status?.name;
13+
}
14+
}

‎packages/@contentlayer/source-notion/src/mapping/field-title.ts

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import type { FieldFunctions } from ".";
2+
3+
export const fieldTitle: FieldFunctions<'title'> = {
4+
getFieldDef: () => {
5+
return {
6+
type: 'string',
7+
isRequired: false
8+
}
9+
},
10+
getFieldData: ({ property }) => {
11+
return property.title[0]?.plain_text;
12+
}
13+
}

0 commit comments

Comments
 (0)