Improve diagram rendering. (#232)

- Add support for inline or included plantuml diagrams.
- Use figure identifiers for rendered mermaid / plantuml / aavg
  diagrams, allowing deletion of stale renders.
- Collate all rendered output (including latex cache files) to a .cache/ directory.
- Fix permissions for cached/generated files.

Author: bluegate010

.gitignore

@@ -1,12 +1,4 @@
 *.pdf
 *.docx
 *.html
-/*.tex
-*.log
-*.aux
-*.lof
-*.lot
-*.toc
-*.fdb_latexmk
-*.upa
-*.upb
+.cache/*

Dockerfile

@@ -102,11 +102,14 @@ RUN apt update && apt install -y fontconfig && \
 
 RUN apt install -y \
     bash \
+    default-jre \
     moreutils \
     nodejs \
     npm \
+    rsync \
     sed \
-    software-properties-common
+    software-properties-common \
+    wget
 
 # Install Chromium via custom repo
 # https://askubuntu.com/questions/1204571/how-to-install-chromium-without-snap/1511695#1511695
@@ -119,6 +122,8 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
 
 RUN npm install --global --unsafe-perm puppeteer@23.2.1 imgur@2.4.2 @mermaid-js/mermaid-cli@11.1.1 typescript@5.5.4 pandiff@0.6.0
 
+RUN wget -O /usr/share/plantuml.jar https://github.com/plantuml/plantuml/releases/download/v1.2025.2/plantuml-asl-1.2025.2.jar
+
 # Important: /usr/local/texlive/bin/ paths come before other paths. We want to use the texlive we
 # built above, not any that happen to have come along with our base image.
 ENV PATH="/usr/local/texlive/bin/aarch64-linux:/usr/local/texlive/bin/x86_64-linux:${PATH}"
@@ -205,7 +210,6 @@ RUN apt install -y \
     libsecret-1-0 \
     libxss1 \
     openbox \
-    wget \
     xorg \
     xvfb
 

build.sh

@@ -264,6 +264,11 @@ if [[ -d /extra_resources ]]; then
 fi
 cd "${BUILD_DIR}"
 
+readonly src_uidgid=$(stat -c "%u:%g" "${SOURCE_DIR}")
+
+# Directory to hold rendered images and other cache files.
+mkdir -p ".cache/"
+
 # Let git work
 git config --global --add safe.directory "${BUILD_DIR}"
 
@@ -711,20 +716,17 @@ analyze_latex_logs() {
 	fi
 }
 
-# Copy generated files (if any) back to the source directory so they can be cached and speed up future runs.
+# Copy file(s), assuming ownership of SOURCE_DIR.
+cp_chown() {
+	local src=$1
+	local dst=$2
+
+	rsync --archive --mkpath --chown="${src_uidgid}" "${src}" "${dst}"
+}
+
+# Sync generated files (if any) back to the source directory so they can be cached and speed up future runs.
 cache_generated_files() {
-	find . -type f \( \
-		-name "*.aux" -o \
-		-name "*.lof" -o \
-		-name "*.lot" -o \
-		-name "*.toc" -o \
-		-name "*.upa" -o \
-		-name "*.upb" -o \
-		-name "*.convert.pdf" -o \
-		-name "*.mermaid.pdf" -o \
-		-name "*.mermaid.svg" -o \
-		-name "*.aasvg.pdf" \
-	\) -exec cp --parents {} "${SOURCE_DIR}" \; 2>/dev/null
+	cp_chown .cache "${SOURCE_DIR}"
 }
 
 # Takes Markdown input and writes LaTeX output using pandoc.
@@ -744,11 +746,11 @@ do_latex() {
 		--standalone
 		--no-highlight
 		--template=${TEMPLATE_PDF}
-		--lua-filter=mermaid-filter.lua
-		--lua-filter=aasvg-filter.lua
-		--lua-filter=informative-sections.lua
+		--lua-filter=convert-diagrams.lua
+		--lua-filter=convert-aasvg.lua
 		--lua-filter=convert-images.lua
 		--lua-filter=center-images.lua
+		--lua-filter=informative-sections.lua
 		--lua-filter=parse-html.lua
 		--lua-filter=apply-classes-to-tables.lua
 		--lua-filter=landscape-pages.lua
@@ -786,6 +788,10 @@ do_latex() {
 	fi
 	local end=$(date +%s)
 	echo "Elapsed time: $(($end-$start)) seconds"
+
+	# Cache generated files now so they don't need to be
+	# re-rendered if the build later fails.
+	cache_generated_files
 }
 
 # Takes LaTeX input and writes PDF output and logs using the PDF engine of choice.
@@ -807,7 +813,7 @@ do_pdf() {
 	if [[ "${ignore_errors}" = "true" ]]; then
 		pdflatex="${pdflatex} -interaction=nonstopmode"
 	fi
-	latexmk "${input}" -pdflatex="${pdflatex}" -pdf -diagnostics > "${logfile}"
+	latexmk "${input}" -pdflatex="${pdflatex}" -pdf -diagnostics -output-directory=.cache/ > "${logfile}"
 	if [ $? -ne 0 ]; then
 		FAILED=true
 		echo "PDF output failed"
@@ -816,12 +822,11 @@ do_pdf() {
 	# Write any LaTeX errors to stderr.
 	>&2 grep -A 5 "] ! " "${logfile}"
 
-	cache_generated_files
 	echo "Elapsed time: $(($end-$start)) seconds"
 	# Write any LaTeX errors to stderr.
 	>&2 grep -A 5 "! " "${logfile}"
 	if [[ ! "${FAILED}" = "true" || "${ignore_errors}" = "true" ]]; then
-		mv "${temp_pdf_file}" "${output}"
+		cp_chown ".cache/${temp_pdf_file}" "${output}" && rm ".cache/${temp_pdf_file}"
 	fi
 	if [[ ! "${FAILED}" = "true" ]]; then
 		analyze_latex_logs "${logfile}"
@@ -852,8 +857,8 @@ do_docx() {
 	cmd=(pandoc
 		--embed-resources
 		--standalone
-		--lua-filter=mermaid-filter.lua
-		--lua-filter=aasvg-filter.lua
+		--lua-filter=convert-diagrams.lua
+		--lua-filter=convert-aasvg.lua
 		--lua-filter=convert-images.lua
 		--lua-filter=parse-html.lua
 		--lua-filter=apply-classes-to-tables.lua
@@ -876,8 +881,6 @@ do_docx() {
 	else
 		echo "DOCX output generated to file: ${output}"
 	fi
-
-	cache_generated_files
 }
 
 # Takes Markdown input and writes HTML output using pandoc.
@@ -897,8 +900,8 @@ do_html() {
 		--standalone
 		--template=${TEMPLATE_HTML}
 		${HTML_STYLESHEET_ARGS}
-		--lua-filter=mermaid-filter.lua
-		--lua-filter=aasvg-filter.lua
+		--lua-filter=convert-diagrams.lua
+		--lua-filter=convert-aasvg.lua
 		--lua-filter=parse-html.lua
 		--lua-filter=apply-classes-to-tables.lua
 		--lua-filter=landscape-pages.lua
@@ -932,7 +935,6 @@ do_html() {
 		FAILED=true
 		echo "HTML output failed"
 	else
-		cache_generated_files
 		echo "HTML output generated to file: ${output}"
 	fi
 }
@@ -945,7 +947,7 @@ if [ -n "${PDF_OUTPUT}" -o -n "${LATEX_OUTPUT}" -o -n "${DIFFPDF_OUTPUT}" -o -n
 	do_latex "${BUILD_DIR}/${INPUT_FILE}" "${TEMP_TEX_FILE}" "${CROSSREF_TYPE}" "${EXTRA_PANDOC_OPTIONS}"
 fi
 if [ -n "${LATEX_OUTPUT}" ]; then
-	cp "${TEMP_TEX_FILE}" "${SOURCE_DIR}/${LATEX_OUTPUT}"
+	cp_chown "${TEMP_TEX_FILE}" "${SOURCE_DIR}/${LATEX_OUTPUT}"
 fi
 
 # Generate the PDF output
@@ -955,8 +957,7 @@ if [ -n "${PDF_OUTPUT}" ]; then
 
 	# Copy the logs, if requested.
 	if [ -n "${PDFLOG_OUTPUT}" ]; then
-		mkdir -p "$(dirname ${SOURCE_DIR}/${PDFLOG_OUTPUT})"
-		cp "${LATEX_LOG}" "${SOURCE_DIR}/${PDFLOG_OUTPUT}"
+		cp_chown "${LATEX_LOG}" "${SOURCE_DIR}/${PDFLOG_OUTPUT}"
 	fi
 fi
 
@@ -1000,8 +1001,7 @@ if [ -n "${DIFFPDF_OUTPUT}" -o -n "${DIFFTEX_OUTPUT}" ]; then
 		else
 			do_diff_tex_fixups "${TEMP_DIFF_TEX_FILE}"
 			if [ -n "${DIFFTEX_OUTPUT}" ]; then
-				mkdir -p "$(dirname ${SOURCE_DIR}/${DIFFTEX_OUTPUT})"
-				cp "${TEMP_DIFF_TEX_FILE}" "${SOURCE_DIR}/${DIFFTEX_OUTPUT}"
+				cp_chown "${TEMP_DIFF_TEX_FILE}" "${SOURCE_DIR}/${DIFFTEX_OUTPUT}"
 			fi
 		fi
 	fi
@@ -1029,5 +1029,8 @@ if [ "${PRE_DIFFING_FAILED}" == "true" ]; then
 	exit 1
 fi
 
+# Do a final sync for any later generated files.
+cache_generated_files
+
 echo "Overall workflow succeeded"
 exit 0

filter/aasvg-filter.lua filter/convert-aasvg.luafilter/aasvg-filter.lua renamed to filter/convert-aasvg.lua

@@ -1,61 +1,40 @@
 -- Turn aasvg-classed code blocks into figures, retaining other classes on the
 -- code block as classes on the figure.
 
-function runCommandWithInput(command, input)
-    local pipe = io.popen(command, "w")
-    if not pipe then
-        return false
-    end
-    pipe:write(input)
-    pipe:flush()
-    pipe:close()
-    return true
-end
-
-function runCommandSuppressOutput(command)
-    -- N.B.: we are using io.popen so we can suppress the output of the command.
-    local pipe = io.popen(command)
-    if not pipe then
-        return false
-    end
-    pipe:flush()
-    local output = pipe:read("*all")
-    pipe:close()
-    return true
-end
-
-function getContentsHash(contents)
-    return pandoc.sha1(contents):sub(1,10)
-end
-
-function fileExists(file)
-    local f = io.open(file)
-    if f then
-        f:close()
-        return true
-    end
-    return false
-end
+-- Patch the path to include the current script's directory.
+package.path = package.path .. ";" .. debug.getinfo(1).source:match("@?(.*/)") .. "?.lua"
+utils = require "utils"
 
 function aasvgFigure(code, caption, attrs)
-    local filename_base = getContentsHash('code=' .. code .. 'caption=' .. pandoc.utils.stringify(caption) .. 'attrs=' .. pandoc.utils.stringify(attrs)) .. '.aasvg'
+    local filename_base = '.cache/' .. attrs.identifier .. '.' .. utils.getContentsHash('code=' .. code .. 'caption=' .. pandoc.utils.stringify(caption) .. 'attrs=' .. pandoc.utils.stringify(attrs)) .. '.aasvg'
     local filename_svg = filename_base .. '.svg'
     local filename_pdf = filename_base .. '.pdf'
 
-    if fileExists(filename_pdf) then
-        print(string.format('%s already exists; not re-rendering it', filename_pdf))
+    if utils.fileExists(filename_pdf) then
+        print(string.format("    not converting %s (already up-to-date as %s)", attrs.identifier, filename_pdf))
     else
         print(string.format('rendering %s using aasvg ...', filename_svg))
-        if not runCommandWithInput(string.format(
+        if not utils.runCommandWithInput(string.format(
             "aasvg --fill > %s 2>&1", filename_svg), code) then
             print(string.format('failed to convert ASCII art SVG (aasvg) diagram to %s using aasvg, falling back to letting LaTeX try to pick it up', filename_svg))
             return false
+        else
+            print(string.format("    rendered %s to %s", attrs.identifier, filename_svg))
         end
 
         print(string.format('converting %s to %s using rsvg-convert ...', filename_svg, filename_pdf))
-        if not runCommandSuppressOutput(string.format("rsvg-convert --format=pdf --keep-aspect-ratio --output %s %s 2>&1", filename_pdf, filename_svg)) then
+        if not utils.runCommandSuppressOutput(string.format("rsvg-convert --format=pdf --keep-aspect-ratio --output %s %s 2>&1", filename_pdf, filename_svg)) then
             print(string.format('failed to convert %s to %s using rsvg-convert, falling back to letting LaTeX try to pick it up', filename_svg, filename_pdf))
             return false
+        else
+            print(string.format('    converted %s to %s', filename_svg, filename_pdf))
+        end
+
+        -- Delete stale copies of the files. This makes it easier to cache only the latest converted pdfs.
+        -- Don't do this if the "keepstaleimages" variable is set.
+        if not PANDOC_WRITER_OPTIONS.variables["keepstaleimages"] then
+            utils.deleteFilesExcept('.cache/' .. attrs.identifier .. '*.aasvg.svg*', filename_svg)
+            utils.deleteFilesExcept('.cache/' .. attrs.identifier .. '*.aasvg.pdf*', filename_pdf)
         end
     end