feat(workers): Adds publisher and author og:meta tags to Bookmark

Author: erik-nilcoast

apps/workers/crawlerWorker.ts

@@ -14,9 +14,12 @@ import { JSDOM } from "jsdom";
 import { DequeuedJob, Runner } from "liteque";
 import metascraper from "metascraper";
 import metascraperAmazon from "metascraper-amazon";
+import metascraperAuthor from "metascraper-author";
+import metascraperDate from "metascraper-date";
 import metascraperDescription from "metascraper-description";
 import metascraperImage from "metascraper-image";
 import metascraperLogo from "metascraper-logo-favicon";
+import metascraperPublisher from "metascraper-publisher";
 import metascraperReadability from "metascraper-readability";
 import metascraperTitle from "metascraper-title";
 import metascraperTwitter from "metascraper-twitter";
@@ -61,8 +64,11 @@ import {
 import { BookmarkTypes } from "@hoarder/shared/types/bookmarks";
 
 const metascraperParser = metascraper([
+  metascraperDate(),
   metascraperAmazon(),
   metascraperReadability(),
+  metascraperAuthor(),
+  metascraperPublisher(),
   metascraperTitle(),
   metascraperDescription(),
   metascraperTwitter(),
@@ -677,6 +683,10 @@ async function crawlAndParseUrl(
         htmlContent: readableContent?.content,
         crawledAt: new Date(),
         crawlStatusCode: statusCode,
+        author: meta.author,
+        publisher: meta.publisher,
+        datePublished: meta.datePublished,
+        dateModified: meta.dateModified,
       })
       .where(eq(bookmarkLinks.id, bookmarkId));
 

apps/workers/package.json

@@ -20,10 +20,13 @@
     "liteque": "^0.3.2",
     "metascraper": "^5.45.24",
     "metascraper-amazon": "^5.45.22",
+    "metascraper-author": "5.46.5",
+    "metascraper-date": "^5.46.5",
     "metascraper-description": "^5.45.22",
     "metascraper-image": "^5.45.22",
     "metascraper-logo": "^5.45.22",
     "metascraper-logo-favicon": "^5.45.22",
+    "metascraper-publisher": "5.45.22",
     "metascraper-readability": "^5.45.22",
     "metascraper-title": "^5.45.22",
     "metascraper-twitter": "^5.45.6",
@@ -66,4 +69,4 @@
     ]
   },
   "prettier": "@hoarder/prettier-config"
-}
+}

apps/workers/videoWorker.ts

@@ -71,6 +71,7 @@ function prepareYtDlpArguments(url: string, assetPath: string) {
       `best[filesize<${serverConfig.crawler.maxVideoDownloadSize}M]`,
     );
   }
+
   ytDlpArguments.push("-o", assetPath);
   ytDlpArguments.push("--no-playlist");
   return ytDlpArguments;

packages/db/drizzle/0042_flawless_whistler.sql

@@ -0,0 +1,5 @@
+ALTER TABLE `bookmarkLinks` ADD `author` text;--> statement-breakpoint
+ALTER TABLE `bookmarkLinks` ADD `publisher` text;--> statement-breakpoint
+ALTER TABLE `bookmarkLinks` ADD `datePublished` text;--> statement-breakpoint
+ALTER TABLE `bookmarkLinks` ADD `dateModified` text;--> statement-breakpoint
+ALTER TABLE `bookmarks` ADD `modifiedAt` integer;

packages/db/drizzle/meta/0041_snapshot.json packages/db/drizzle/meta/0042_snapshot.json

@@ -1,7 +1,7 @@
 {
   "version": "6",
   "dialect": "sqlite",
-  "id": "4219851a-7eed-421f-b52e-5d9cd045ff85",
+  "id": "103f5256-2aa0-4ff7-98f1-f3c897f7cc33",
   "prevId": "943b28e9-4b89-48c9-9e00-65314a09bd92",
   "tables": {
     "account": {
@@ -408,6 +408,34 @@
           "notNull": false,
           "autoincrement": false
         },
+        "author": {
+          "name": "author",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "publisher": {
+          "name": "publisher",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "datePublished": {
+          "name": "datePublished",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "dateModified": {
+          "name": "dateModified",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
         "imageUrl": {
           "name": "imageUrl",
           "type": "text",

packages/db/drizzle/meta/_journal.json

@@ -295,6 +295,13 @@
       "when": 1738424745186,
       "tag": "0041_fat_bloodstrike",
       "breakpoints": true
+    },
+    {
+      "idx": 42,
+      "version": "6",
+      "when": 1741820150134,
+      "tag": "0042_flawless_whistler",
+      "breakpoints": true
     }
   ]
 }

packages/db/schema.ts

@@ -148,6 +148,10 @@ export const bookmarkLinks = sqliteTable(
     // Crawled info
     title: text("title"),
     description: text("description"),
+    author: text("author"),
+    publisher: text("publisher"),
+    datePublished: text("datePublished"),
+    dateModified: text("dateModified"),
     imageUrl: text("imageUrl"),
     favicon: text("favicon"),
     content: text("content"),

packages/open-api/hoarder-openapi-spec.json

@@ -168,6 +168,22 @@
                   "crawledAt": {
                     "type": "string",
                     "nullable": true
+                  },
+                  "author": {
+                    "type": "string",
+                    "nullable": true
+                  },
+                  "publisher": {
+                    "type": "string",
+                    "nullable": true
+                  },
+                  "datePublished": {
+                    "type": "string",
+                    "nullable": true
+                  },
+                  "dateModified": {
+                    "type": "string",
+                    "nullable": true
                   }
                 },
                 "required": [

packages/shared/search.ts

@@ -18,6 +18,10 @@ export const zBookmarkIdxSchema = z.object({
   note: z.string().nullish(),
   summary: z.string().nullish(),
   tags: z.array(z.string()).default([]),
+  publisher: z.string().nullish(),
+  author: z.string().nullish(),
+  datePublished: z.string().nullish(),
+  dateModified: z.string().nullish(),
 });
 
 export type ZBookmarkIdx = z.infer<typeof zBookmarkIdxSchema>;

packages/shared/types/bookmarks.ts

@@ -46,6 +46,10 @@ export const zBookmarkedLinkSchema = z.object({
   favicon: z.string().nullish(),
   htmlContent: z.string().nullish(),
   crawledAt: z.date().nullish(),
+  author: z.string().nullish(),
+  publisher: z.string().nullish(),
+  datePublished: z.string().nullish(),
+  dateModified: z.string().nullish(),
 });
 export type ZBookmarkedLink = z.infer<typeof zBookmarkedLinkSchema>;
 

packages/trpc/routers/bookmarks.ts

@@ -1083,6 +1083,8 @@ export const bookmarksAppRouter = router({
 Title: ${bookmark.title ?? ""}
 Description: ${bookmark.description ?? ""}
 Content: ${bookmark.content ?? ""}
+Publisher: ${bookmark.publisher ?? ""}
+Author: ${bookmark.author ?? ""}
 `;
 
       const prompts = await ctx.db.query.customPrompts.findMany({