feat(workers): Adds publisher and author og:meta tags to Bookmark

Author: erik-nilcoast

apps/workers/crawlerWorker.ts

@@ -14,9 +14,12 @@ import { JSDOM } from "jsdom";
 import { DequeuedJob, Runner } from "liteque";
 import metascraper from "metascraper";
 import metascraperAmazon from "metascraper-amazon";
+import metascraperAuthor from "metascraper-author";
+import metascraperDate from "metascraper-date";
 import metascraperDescription from "metascraper-description";
 import metascraperImage from "metascraper-image";
 import metascraperLogo from "metascraper-logo-favicon";
+import metascraperPublisher from "metascraper-publisher";
 import metascraperReadability from "metascraper-readability";
 import metascraperTitle from "metascraper-title";
 import metascraperTwitter from "metascraper-twitter";
@@ -61,8 +64,11 @@ import {
 import { BookmarkTypes } from "@hoarder/shared/types/bookmarks";
 
 const metascraperParser = metascraper([
+  metascraperDate(),
   metascraperAmazon(),
   metascraperReadability(),
+  metascraperAuthor(),
+  metascraperPublisher(),
   metascraperTitle(),
   metascraperDescription(),
   metascraperTwitter(),
@@ -677,6 +683,10 @@ async function crawlAndParseUrl(
         htmlContent: readableContent?.content,
         crawledAt: new Date(),
         crawlStatusCode: statusCode,
+        author: meta.author,
+        publisher: meta.publisher,
+        datePublished: meta.datePublished,
+        dateModified: meta.dateModified,
       })
       .where(eq(bookmarkLinks.id, bookmarkId));
 

apps/workers/package.json

@@ -20,10 +20,13 @@
     "liteque": "^0.3.2",
     "metascraper": "^5.45.24",
     "metascraper-amazon": "^5.45.22",
+    "metascraper-author": "5.46.5",
+    "metascraper-date": "^5.46.5",
     "metascraper-description": "^5.45.22",
     "metascraper-image": "^5.45.22",
     "metascraper-logo": "^5.45.22",
     "metascraper-logo-favicon": "^5.45.22",
+    "metascraper-publisher": "5.45.22",
     "metascraper-readability": "^5.45.22",
     "metascraper-title": "^5.45.22",
     "metascraper-twitter": "^5.45.6",
@@ -66,4 +69,4 @@
     ]
   },
   "prettier": "@hoarder/prettier-config"
-}
+}

apps/workers/videoWorker.ts

@@ -71,6 +71,7 @@ function prepareYtDlpArguments(url: string, assetPath: string) {
       `best[filesize<${serverConfig.crawler.maxVideoDownloadSize}M]`,
     );
   }
+
   ytDlpArguments.push("-o", assetPath);
   ytDlpArguments.push("--no-playlist");
   return ytDlpArguments;

packages/db/drizzle/0042_right_madame_web.sql

@@ -0,0 +1,4 @@
+ALTER TABLE `bookmarkLinks` ADD `author` text;--> statement-breakpoint
+ALTER TABLE `bookmarkLinks` ADD `publisher` text;--> statement-breakpoint
+ALTER TABLE `bookmarkLinks` ADD `datePublished` text;--> statement-breakpoint
+ALTER TABLE `bookmarkLinks` ADD `dateModified` text;--> statement-breakpoint