feat(pdf): build pdf by new tsx script

Author: zoobestik

.gitignore

@@ -35,20 +35,18 @@ pdf/kotlin-reference.pdf
 pdf/tmp.html
 *.pyc
 google-credentials.json
-./package-lock.json
+package-lock.json
 .env
 pages/api/
 external/
 assets/externals
 pages/docs/tutorials/kotlin-for-py/
 _teamcity
-dist
+/dist
 libs
 generated
 .next
 .teamcity/*.iml
 
-/scripts/doindex/package-lock.json
-/search-report*
-/reports
+/reports*
 /data/page_views_map.json

.teamcity/builds/TemplateSearchIndex.kt

@@ -7,7 +7,7 @@ import jetbrains.buildServer.configs.kotlin.buildSteps.ScriptBuildStep
 import jetbrains.buildServer.configs.kotlin.triggers.schedule
 import vcsRoots.KotlinLangOrg
 
-const val SCRIPT_PATH = "scripts/doindex";
+const val SCRIPT_PATH = "scripts/dist";
 
 fun scriptDistAnalyze(block: ScriptBuildStep.() -> Unit) = ScriptBuildStep {
     id = "script-dist-analyze"
@@ -42,11 +42,7 @@ abstract class TemplateSearchIndex(init: BuildType.() -> Unit) : BuildType({
     }
 
     vcs {
-        root(
-            KotlinLangOrg, """
-                $SCRIPT_PATH
-            """.trimIndent()
-        )
+        root(KotlinLangOrg, "$SCRIPT_PATH/")
         cleanCheckout = true
         showDependenciesChanges = true
     }

.teamcity/builds/apiReferences/BuildApiPages.kt

@@ -1,6 +1,7 @@
 package builds.apiReferences
 
 import BuildParams.DOKKA_TEMPLATES_VERSION
+import builds.SCRIPT_PATH
 import builds.scriptDistAnalyze
 import jetbrains.buildServer.configs.kotlin.BuildStep
 import jetbrains.buildServer.configs.kotlin.BuildSteps
@@ -39,7 +40,7 @@ abstract class BuildApiPages(
     artifactRules = "$pagesRoot/** => pages.zip"
 
     vcs {
-        root(KotlinLangOrg, "scripts/doindex/")
+        root(KotlinLangOrg, "$SCRIPT_PATH/")
     }
 
     params {

.teamcity/builds/apiReferences/stdlib/BuildStdlibApiReference.kt

@@ -1,11 +1,13 @@
 package builds.apiReferences.stdlib
 
 import BuildParams.KOTLIN_CORE_API_BUILD_ID
+import builds.SCRIPT_PATH
 import builds.apiReferences.scriptGenerateSitemap
 import builds.apiReferences.scriptNoRobots
 import jetbrains.buildServer.configs.kotlin.AbsoluteId
 import jetbrains.buildServer.configs.kotlin.BuildType
 import jetbrains.buildServer.configs.kotlin.buildSteps.script
+import vcsRoots.KotlinLangOrg
 
 private const val PAGES_ROOT = "dist/api/core"
 
@@ -14,11 +16,7 @@ object BuildStdlibApiReference : BuildType({
     description = "Build pages for Kotlin Core API"
 
     vcs {
-        root(
-            vcsRoots.KotlinLangOrg, """
-                scripts/doindex/
-            """.trimIndent()
-        )
+        root(KotlinLangOrg, "$SCRIPT_PATH/")
     }
 
     artifactRules = """

.teamcity/builds/kotlinlang/buidTypes/PdfGenerator.kt

@@ -1,46 +1,72 @@
 package builds.kotlinlang.buidTypes
 
+import builds.SCRIPT_PATH
 import builds.kotlinlang.templates.DockerImageBuilder
-import jetbrains.buildServer.configs.kotlin.AbsoluteId
 import jetbrains.buildServer.configs.kotlin.BuildType
+import jetbrains.buildServer.configs.kotlin.FailureAction
 import jetbrains.buildServer.configs.kotlin.buildSteps.script
 
-
 object PdfGenerator : BuildType({
   name = "PDF Generator"
   description = "Build PDF reference https://kotlinlang.org/docs/"
 
   templates(DockerImageBuilder)
 
-  artifactRules = "pdf/kotlin-docs.pdf => kotlin-docs.pdf"
+  artifactRules = """
+      dist/docs/pdf.html
+      pdf/kotlin-docs.pdf
+  """.trimIndent()
+
+  requirements {
+    doesNotContain("docker.server.osType", "windows")
+  }
 
   steps {
     script {
+      id = "script-dist-pdf-html"
+      name = "Generate pdf.html"
+        //language=bash
+        scriptContent = """
+        #!/bin/sh
+        set -e
+        npm install
+        npm run generate-pdf
+      """.trimIndent()
+      dockerImage = "node:22-alpine"
+      workingDir = SCRIPT_PATH
+    }
+    script {
+      name = "Generate PDF"
+      //language=sh
       scriptContent = """
-        #!/bin/bash
-        
-        mv ./dist/docs/pdfSourceKR.html ./dist/docs/pdf.html
+        # install legacy wkhtmltopdf deps
+        apt update
+        apt install -y xfonts-75dpi xfonts-100dpi libjpeg62-turbo xfonts-base
+        wget https://deb.debian.org/debian/pool/main/o/openssl/libssl1.1_1.1.1w-0+deb11u1_amd64.deb
+        wget https://deb.debian.org/debian/pool/main/o/openssl/libssl-dev_1.1.1w-0+deb11u1_amd64.deb
+        wget https://deb.debian.org/debian/pool/main/o/openssl/openssl_1.1.1w-0+deb11u1_amd64.deb
+        dpkg -i libssl1.1_1.1.1w-0+deb11u1_amd64.deb libssl-dev_1.1.1w-0+deb11u1_amd64.deb openssl_1.1.1w-0+deb11u1_amd64.deb
+        ln -s /usr/lib/x86_64-linux-gnu/libjpeg.so.62 /usr/lib/x86_64-linux-gnu/libjpeg.so.8
+        # refresh wkhtmltopdf 
+        wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.buster_amd64.deb
+        dpkg -i wkhtmltox_0.12.6-1.buster_amd64.deb
         
         ## refresh packages
         pip install -r requirements.txt
-        
         python kotlin-website.py reference-pdf
       """.trimIndent()
-      dockerImage = "%dep.Kotlin_KotlinSites_Builds_KotlinlangOrg_BuildPythonContainer.kotlin-website-image%"
+      dockerImage = "python:3.9"
     }
   }
 
   dependencies {
-    dependency(AbsoluteId("Documentation_TransitioningProducts_KotlinReferenceWithCoroutines")) {
+    dependency(BuildReferenceDocs) {
       snapshot {
+        onDependencyFailure = FailureAction.FAIL_TO_START
+        onDependencyCancel = FailureAction.CANCEL
       }
-
       artifacts {
-        cleanDestination = true
-        artifactRules = """
-          webHelpImages.zip!** => dist/docs/images
-          pdfSourceKR.html => dist/docs/
-        """.trimIndent()
+        artifactRules = "+:docs.zip!** => dist/docs/"
       }
     }
   }

scripts/doindex/.nvmrc scripts/dist/.nvmrc

@@ -2,7 +2,8 @@
   "type": "module",
   "scripts": {
     "generate-metadata": "tsx analyzer/index.ts",
-    "build": "tsc --outDir ../pages-js --project tsconfig.json"
+    "generate-pdf": "tsx pdf/index.ts",
+    "build": "tsc --outDir ../dist --project tsconfig.json"
   },
   "dependencies": {
     "@types/fast-levenshtein": "^0.0.4",

scripts/doindex/analyzer/index.ts scripts/dist/analyzer/index.ts

@@ -0,0 +1,52 @@
+import { dirname, join } from 'node:path';
+import { EventEmitter } from 'events';
+
+import { DIST_FOLDER } from '../lib/files/index.js';
+import { appendFile, open, readFile } from 'node:fs/promises';
+import { processTocToUrls, Result } from './lib.js';
+import { newTaskExecutor } from '../lib/pool.js';
+
+console.time('Data successfully built');
+
+const TASK_PATH = import.meta.dirname + '/task';
+
+EventEmitter.defaultMaxListeners = 15;
+
+const DOCS_PATH = join(DIST_FOLDER, 'docs');
+const TOC_PATH = join(DOCS_PATH, 'HelpTOC.json');
+
+const TOC = JSON.parse(await readFile(TOC_PATH, { encoding: 'utf-8' }));
+
+const nodes = new Map(
+    (await processTocToUrls(TOC))
+        .map(id => new URL(id, 'https://kotlinlang.org/docs/'))
+        .filter(url => url.hostname === 'kotlinlang.org' && dirname(url.pathname) === '/docs')
+        .map(url => url.pathname.slice(1))
+        .map(key => [key, ''])
+);
+
+if (nodes.size === 0) throw new Error('No nodes found');
+
+async function onItem({ id, html }: Result) {
+    nodes.set(id, html);
+}
+
+const [pool, finish] = newTaskExecutor(
+    TASK_PATH, onItem
+);
+
+for (const id of nodes.keys()) {
+    pool.push(id);
+}
+
+await finish;
+
+const pdfFile = await open(join(DOCS_PATH, 'pdf.html'), 'w');
+
+for (const html of nodes.values()) {
+    await appendFile(pdfFile, html);
+}
+
+await pdfFile.close();
+
+console.timeEnd('Data successfully built');

scripts/doindex/analyzer/metadata.ts scripts/dist/analyzer/metadata.ts

@@ -0,0 +1,80 @@
+import { join } from 'node:path';
+import { readFile } from 'node:fs/promises';
+import { CheerioAPI, load } from 'cheerio';
+
+import { FileType } from '../lib/files/type.js';
+import { ROOT_DIR } from '../lib/files/index.js';
+
+export type Result = {
+    id: string
+    html: string
+}
+
+export function isHiddenProved(relativePath: string, type?: FileType) {
+    return (!type || type === 'Hidden') && (
+        relativePath.startsWith('docs/kotlin-tour-') ||
+        relativePath === 'docs/multiplatform.html'
+    );
+}
+
+const TREE_XML_PATH = join(ROOT_DIR + '/docs/kr.tree');
+
+let TREE_XML: CheerioAPI | null = null;
+
+async function readTreeXMLEntities() {
+    if (!TREE_XML)
+        TREE_XML = load(await readFile(TREE_XML_PATH, { encoding: 'utf-8' }), {
+            xml: true
+        });
+
+    return TREE_XML;
+}
+
+async function getSubtreeItems(id: string) {
+    const $tree = await readTreeXMLEntities();
+    const topic = id.replace(/\.html$/, '.md');
+
+    return $tree(`toc-element[topic="${topic}"] > toc-element[topic]`).toArray().map(node => {
+        const file = node?.attribs?.topic?.replace(/\.(md|topic)$/, '.html');
+        if (!file || !isHiddenProved(`docs/${file}`)) return null;
+        return file;
+    })
+        .filter(Boolean);
+}
+
+export async function processTocToUrls({ topLevelIds, entities }: {
+    entities: { pages: Record<string, { url?: string, pages: string[] }> },
+    topLevelIds: string[]
+}) {
+    const result: string[] = [];
+    const visited = new Set<string>();
+    const stack = [...topLevelIds.reverse()]; // Reverse to maintain order
+    const data = entities.pages;
+    while (stack.length) {
+        const id = stack.pop();
+
+        if (visited.has(id)) continue;
+        visited.add(id);
+
+        const page = data[id];
+        if (!page) continue;
+        const { url, pages } = page;
+
+        if (url) result.push(page.url);
+
+        if (isHiddenProved(`docs/${page.url}`)) {
+            const hiddenPages = await getSubtreeItems(page.url);
+            for (const id of hiddenPages) {
+                result.push(id);
+            }
+        }
+
+        if (pages?.length) {
+            for (let i = page.pages.length - 1; i >= 0; i--) {
+                stack.push(page.pages[i]);
+            }
+        }
+    }
+
+    return result;
+}

scripts/doindex/analyzer/reports/utils.ts scripts/dist/analyzer/reports/utils.ts

@@ -0,0 +1,142 @@
+import { basename, dirname, join } from 'node:path';
+import { CheerioAPI } from 'cheerio';
+import { Element } from 'domhandler';
+import { getType } from '../lib/files/type.js';
+import { DIST_FOLDER } from '../lib/files/index.js';
+import { isHiddenProved, Result } from './lib.js';
+import { replaceNode } from '../lib/html.js';
+
+function sendEvent(...args: Parameters<typeof process.send>) {
+    process.send(...args);
+}
+
+function sendWarning() {
+    console.error('process.send is not defined');
+}
+
+const send = process.send ? sendEvent : sendWarning;
+
+const HOME_HTML_CONTENT = '' +
+    '<section class = "panel__content" ><div class = "container"><article class="article">' +
+    '<h1 id = "home.xml">Kotlin Docs</h1>' +
+    '</article><div id="disqus_thread"></div ></div></section>';
+
+
+function buildAnchor(filename: string, hash: string) {
+    let result = '#' + filename.replace(/\.html$/, '.md');
+    if (hash) result += '-' + hash.substring(1);
+    return result;
+}
+
+async function fixSectionHtml($: CheerioAPI, node: Element, relativePath: string) {
+    const $node = $(node);
+
+    $node.find('.last-modified, .navigation-links._bottom').remove();
+    $node.find('img[data-gif-src]').each(function(_i, img) {
+        img.attribs.src = img.attribs['data-gif-src'];
+    });
+
+    replaceNode($node, '.code-collapse', function($node, _attrs, content) {
+        return `<div class="code-block" data-lang="${$node.attr('data-lang')}">${content}</div>`;
+    });
+
+    replaceNode($node, 'object[data]', function($node, attrs, content) {
+        const textUrl = $node.attr('data');
+        try {
+            const { hostname, pathname, searchParams } = new URL(textUrl, 'https://kotlinlang.org/');
+            if (hostname.endsWith('youtube.com') || hostname.endsWith('youtu.be')) {
+                let id = '';
+                // https://www.youtube.com/watch?v=...&feature=...
+                if (pathname === '/watch') id = searchParams.get('v');
+                // http://www.youtube.com/v/...?blabalab
+                else if (pathname.startsWith('/v/')) id = pathname.substring(3);
+                // http://www.youtube.com/embed/...?blabla
+                else if (pathname.startsWith('/embed/')) id = pathname.substring(6);
+
+                if (id)
+                    return `<figure class="video"><img src="https://img.youtube.com/vi/${id}/maxresdefault.jpg" width="560"></figure>` +
+                        `<p><a href="https://youtube.com/v/${id}">Watch video online.</a></p>`;
+            }
+        } catch (e) {
+            // skip content
+        }
+
+        return `<object ${attrs}>${content}</object>`;
+    });
+
+    $node.find('figure:not([title]) img[title]').each(function(_i, img) {
+        $(img).closest('figure').attr('title', img.attribs.title);
+    });
+
+    $node.find('.code-block').each(function(_i, node) {
+        let child = node.firstChild;
+        while (child) {
+            if (child.type === 'text') {
+                child.data = child.data.replace(/^[\n\s]+/g, '').trim();
+                if (!child.data) child = child.nextSibling;
+                else child = null;
+            }
+        }
+    });
+
+    $node.find('[id]:not(h1)').each(function(_i, node) {
+        const article = $(node).closest('.article');
+        const h1Id = article.find('h1[id$=".md"]').attr('id');
+        if (h1Id && article.length === 1) node.attribs.id = h1Id + '-' + node.attribs.id;
+    });
+    $node.find('a[href]').each(function(_i, node) {
+        let anchor = '';
+
+        const href = node.attribs.href;
+        const url = new URL(href, 'https://kotlinlang.org/docs/');
+
+        if (href[0] === '#') anchor = buildAnchor(basename(relativePath), href);
+        else if (url.hostname === 'kotlinlang.org' && dirname(url.pathname) === '/docs') {
+            const filename = basename(url.pathname);
+            if (filename.endsWith('.html')) anchor = buildAnchor(filename, url.hash);
+        }
+
+        if (anchor) {
+            node.attribs.href = anchor;
+            node.attribs['data-origin-href'] = href;
+        }
+    });
+
+    return $.html(node)
+        // sample drop
+        .replace(/\/\/sampleStart\n/g, '')
+        .replace(/\n\/\/sampleEnd/g, '');
+}
+
+async function onMessage(relativePath: string) {
+    let html: string = null;
+
+    const path = join(DIST_FOLDER, relativePath);
+    let [type, $] = await getType(relativePath, path);
+
+    if (isHiddenProved(relativePath, type))
+        type = 'Page_Documentation';
+
+    if (type === 'Page_Documentation') {
+        const sections = $('section.panel__content');
+
+        if (relativePath === 'docs/home.html') {
+            html = HOME_HTML_CONTENT;
+        } else if (sections.length > 0) {
+            html = '';
+            for (const node of sections) {
+                html += await fixSectionHtml($, node, relativePath);
+            }
+        }
+    }
+
+    const data: Result = {
+        id: relativePath,
+        html: html || ''
+    };
+
+    send({ event: 'result', data });
+}
+
+process.on('message', onMessage);
+send({ event: 'inited' });