Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: notions export pods #3970

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
fix: notions export pods
abelfodil committed Dec 22, 2023
commit 67c3adf3d29874134447f6deed4d700f66208c1f
44 changes: 44 additions & 0 deletions packages/common-all/src/helpers.ts
Original file line number Diff line number Diff line change
@@ -29,3 +29,47 @@ export async function asyncLoopOneAtATime<T, R = any>(
export async function asyncLoop<T>(things: T[], cb: (t: T) => Promise<any>) {
return Promise.all(things.map((t) => cb(t)));
}


/**
* Retry resolving promise factory n number of times before rejecting it
* @param retriable
* @param handler
* @param n
* @returns
*/
export async function asyncRetry<T>(retriable: () => Promise<T>, handler: (arg0: any) => any, n?: number) {
n = n ?? 3;

for (let i = 0; i < n; i += 1) {
try {
/* eslint-disable no-await-in-loop */
return await retriable();
} catch (error) {
if (i < (n - 1)) {
continue;
}
return handler(error);
}
}
}

/**
* Deferrable promise
*/
export class Deferred<T> {
promise: Promise<T>;
reject!: (reason?: any) => void;
resolve!: (value: T) => void;

constructor(value?: T | null) {
this.promise = new Promise((resolve, reject) => {
this.reject = reject
this.resolve = resolve
})

if(value) {
this.resolve(value);
}
}
}
Original file line number Diff line number Diff line change
@@ -19,7 +19,7 @@ describe("GIVEN a Notion export pod", () => {
const vaultName = VaultUtils.getName(vaults[0]);
pod.createPagesInNotion = jest.fn();
pod.getAllNotionPages = jest.fn();
pod.convertMdToNotionBlock = jest.fn();
NotionExportPod.convertMdToNotionBlock = jest.fn();
await pod.execute({
engine,
vaults,
@@ -32,7 +32,7 @@ describe("GIVEN a Notion export pod", () => {
utilityMethods,
});
expect(pod.createPagesInNotion).toHaveBeenCalledTimes(1);
expect(pod.convertMdToNotionBlock).toHaveBeenCalledTimes(1);
expect(NotionExportPod.convertMdToNotionBlock).toHaveBeenCalledTimes(1);
},
{
expect,
Original file line number Diff line number Diff line change
@@ -19,6 +19,7 @@ import {
RunnableGoogleDocsV2PodConfig,
RunnableNotionV2PodConfig,
RunnableJSONV2PodConfig,
NotionExportPod,
} from "@dendronhq/pods-core";
import fs from "fs-extra";
import _ from "lodash";
@@ -471,7 +472,7 @@ describe("GIVEN a Notion Export Pod with a particular config", () => {
],
errors: [],
};
pod.convertMdToNotionBlock = jest.fn();
NotionExportPod.convertMdToNotionBlock = jest.fn();
pod.createPagesInNotion = jest.fn().mockResolvedValue(response);
const result = await pod.exportNotes([props]);
const entCreate = result.data?.created!;
4 changes: 2 additions & 2 deletions packages/pods-core/package.json
Original file line number Diff line number Diff line change
@@ -56,8 +56,8 @@
"@dendronhq/common-server": "^0.124.0",
"@dendronhq/engine-server": "^0.124.0",
"@dendronhq/unified": "^0.124.0",
"@instantish/martian": "1.0.3",
"@notionhq/client": "^0.1.9",
"@tryfabric/martian": "1.2.4",
"@notionhq/client": "^1.0.4",
"@octokit/graphql": "^4.6.4",
"@types/airtable": "^0.10.1",
"airtable": "^0.11.1",
59 changes: 39 additions & 20 deletions packages/pods-core/src/builtin/NotionPod.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
import {
asyncLoop,
asyncRetry,
Deferred,
DendronError,
ERROR_SEVERITY,
NoteProps,
} from "@dendronhq/common-all";
import { markdownToBlocks } from "@instantish/martian";
import { markdownToBlocks } from "@tryfabric/martian";
import type {
Page,
TitlePropertyValue,
@@ -53,28 +55,48 @@ export class NotionExportPod extends ExportPod<NotionExportConfig> {
* Method to create pages in Notion
*/
createPagesInNotion = (
blockPagesArray: any,
notion: Client
notes: NoteProps[],
notion: Client,
parentPageId: string,
): Promise<any[]> => {
return asyncLoop(blockPagesArray, async (block: any) => {
await limiter.removeTokens(1);
try {
await notion.pages.create(block);
} catch (error) {
throw new DendronError({
message: "Failed to export all the notes. " + JSON.stringify(error),
severity: ERROR_SEVERITY.MINOR,
const keyMap = Object.fromEntries(
notes.map(note => [
note.parent ?? "undefined",
note.parent ? new Deferred<string>() : new Deferred<string>(parentPageId)
])
);

return asyncLoop(notes, async (note: NoteProps) => {
const parentId = await keyMap[note.parent ?? "undefined"].promise;
const blockPage = NotionExportPod.convertMdToNotionBlock(note, parentId);

await asyncRetry(
async () => {
await limiter.removeTokens(1);

const page = await notion.pages.create(blockPage);

keyMap[note.id]?.resolve(page.id);

return {
notionId: page.id,
dendronId: note.id,
}
},
(error) => {
keyMap[note.id]?.reject(`Failed to export the note title:'${note.title}' - id:${note.id}.`);
throw new DendronError({
message: `Failed to export the note title:'${note.title}' - id:${note.id} - content: ${JSON.stringify(blockPage)}` + JSON.stringify(error),
severity: ERROR_SEVERITY.MINOR,
});
});
}
});
};

/**
* Method to convert markdown to Notion Block
*/
convertMdToNotionBlock = (notes: NoteProps[], pageId: string) => {
const notionBlock = notes.map((note) => {
const children = markdownToBlocks(note.body);
public static convertMdToNotionBlock = (note: NoteProps, pageId: string): any => {
return {
parent: {
page_id: pageId,
@@ -84,10 +106,8 @@ export class NotionExportPod extends ExportPod<NotionExportConfig> {
title: [{ type: "text", text: { content: note.title } }],
},
},
children,
children: markdownToBlocks(note.body),
};
});
return notionBlock;
};

/**
@@ -159,8 +179,7 @@ export class NotionExportPod extends ExportPod<NotionExportConfig> {
return { notes: [] };
}
const pageId = pagesMap[selectedPage];
const blockPagesArray = this.convertMdToNotionBlock(notes, pageId);
await this.createPagesInNotion(blockPagesArray, notion);
await this.createPagesInNotion(notes, notion, pageId);
return { notes };
}
}
75 changes: 33 additions & 42 deletions packages/pods-core/src/v2/pods/export/NotionExportPodV2.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import {
asyncLoop,
asyncRetry,
Deferred,
DendronCompositeError,
DendronError,
DEngineClient,
@@ -7,14 +10,14 @@ import {
ResponseUtil,
RespV2,
} from "@dendronhq/common-all";
import { markdownToBlocks } from "@instantish/martian";
import { JSONSchemaType } from "ajv";
import { RateLimiter } from "limiter";
import _ from "lodash";
import {
Client,
ConfigFileUtils,
ExportPodV2,
NotionExportPod,
NotionV2PodConfig,
RunnableNotionV2PodConfig,
} from "../../..";
@@ -50,8 +53,7 @@ export class NotionExportPodV2 implements ExportPodV2<NotionExportReturnType> {

async exportNotes(notes: NoteProps[]): Promise<NotionExportReturnType> {
const { parentPageId } = this._config;
const blockPagesArray = this.convertMdToNotionBlock(notes, parentPageId);
const { data, errors } = await this.createPagesInNotion(blockPagesArray);
const { data, errors } = await this.createPagesInNotion(notes, parentPageId);
const createdNotes = data.filter(
(ent): ent is NotionFields => !_.isUndefined(ent)
);
@@ -70,59 +72,48 @@ export class NotionExportPodV2 implements ExportPodV2<NotionExportReturnType> {
}
}

/**
* Method to convert markdown to Notion Block
*/
convertMdToNotionBlock = (notes: NoteProps[], parentPageId: string) => {
const notionBlock = notes.map((note) => {
const children = markdownToBlocks(note.body);
return {
dendronId: note.id,
block: {
parent: {
page_id: parentPageId,
},
properties: {
title: {
title: [{ type: "text", text: { content: note.title } }],
},
},
children,
},
};
});
return notionBlock;
};

/**
* Method to create pages in Notion
*/
createPagesInNotion = async (
blockPagesArray: any
notes: NoteProps[],
parentPageId: string,
): Promise<{
data: NotionFields[];
errors: IDendronError[];
}> => {
const notion = new Client({
auth: this._config.apiKey,
});

const keyMap = Object.fromEntries(
notes.map(note => [
note.parent ?? "undefined",
note.parent ? new Deferred<string>() : new Deferred<string>(parentPageId)
])
);

const errors: IDendronError[] = [];
const out: NotionFields[] = await Promise.all(
blockPagesArray.map(async (ent: any) => {
// @ts-ignore
await limiter.removeTokens(1);
try {
const response = await notion.pages.create(ent.block);
const out: NotionFields[] = await asyncLoop(notes, async (note: NoteProps) => {
const parentId = await keyMap[note.parent ?? "undefined"].promise;
const blockPage = NotionExportPod.convertMdToNotionBlock(note, parentId);

return asyncRetry(
async () => {
await limiter.removeTokens(1);
const page = await notion.pages.create(blockPage);
keyMap[note.id]?.resolve(page.id);
return {
notionId: response.id,
dendronId: ent.dendronId,
};
} catch (error) {
notionId: page.id,
dendronId: note.id,
}
},
(error) => {
keyMap[note.id]?.reject(`Failed to export the note title:'${note.title}' - id:${note.id}.`);
errors.push(error as DendronError);
return;
}
})
);
});
});

return {
data: out,
errors,
Loading