markup: add --citeproc to pandoc converter

Adds the citeproc filter to the pandoc converter.

There are several PRs for it this feature already. However, I think
simply adding `--citeproc` is the cleanest way to enable this feature,
with the option to flesh it out later, e.g., in #7529.

Some PRs and issues attempt adding more config options to Hugo which
indirectly configure pandoc, but I think simply configuring Pandoc via
Pandoc itself is simpler, as it is already possible with two YAML
blocks -- one for Hugo, and one for Pandoc:

    ---
    title: This is the Hugo YAML block
    ---
    ---
    bibliography: assets/pandoc-yaml-block-bibliography.bib
    ...
    Document content with @citation!

There are other useful options, e.g., #4800 attempts to use `nocite`,
which works out of the box with this PR:

    ---
    title: This is the Hugo YAML block
    ---
    ---
    bibliography: assets/pandoc-yaml-block-bibliography.bib
    nocite: |
      @*
    ...
    Document content with no citations but a full bibliography:

    ## Bibliography

Other useful options are `csl: ...` and `link-citations: true`, which
set the path to a custom CSL file and create HTML links between the
references and the bibliography.

The following issues and PRs are related:

- Add support for parsing citations and Jupyter notebooks via Pandoc and/or Goldmark extension #6101
  Bundles multiple requests, this PR tackles citation parsing.

- WIP: Bibliography with Pandoc #4800
  Passes the frontmatter to Pandoc and still uses
  `--filter pandoc-citeproc` instead of `--citeproc`.
- Allow configuring Pandoc #7529
  That PR is much more extensive and might eventually supersede this PR,
  but I think --bibliography and --citeproc should be independent
  options (--bibliography should be optional and citeproc can always be
  specified).
- Pandoc - allow citeproc extension to be invoked, with bibliography. #8610
  Similar to #7529, #8610 adds a new config option to Hugo.
  I think passing --citeproc and letting the users decide on the
  metadata they want to pass to pandoc is better, albeit uglier.

docs/content/en/content-management/bibliography.md

@@ -0,0 +1,50 @@
+---
+title: Bibliographies in Markdown
+linkTitle: Bibliography
+description: Include citations and a bibliography in Markdown using LaTeX markup.
+categories: [content management]
+keywords: [latex,pandoc,citation,reference,bibliography]
+menu:
+  docs:
+    parent: content-management
+    weight: 320
+weight: 320
+toc: true
+---
+
+{{< new-in 0.144.0 />}}
+
+## Citations and Bibliographies
+
+[Pandoc](https://pandoc.org) is a universal document converter and can be used to convert markdown files.
+
+With **Pandoc >= 2.11**, you can use [citations](https://pandoc.org/MANUAL.html#extension-citations).
+One way is to employ [BibTeX files](https://en.wikibooks.org/wiki/LaTeX/Bibliography_Management#BibTeX) to cite:
+
+```
+---
+title: Citation document
+---
+---
+bibliography: assets/bibliography.bib
+...
+This is a citation: @Doe2022
+```
+
+Note that Hugo will **not** pass its metadata YAML block to Pandoc; however, it will pass the **second** meta data block, denoted with `---` and `...` to Pandoc.
+Thus, all Pandoc-specific settings should go there.
+
+You can also add all elements from a bibliography file (without citing them explicitly) using:
+
+```
+---
+title: My Publications
+---
+---
+bibliography: assets/bibliography.bib
+nocite: |
+  @*
+...
+```
+
+It is also possible to provide a custom [CSL style](https://citationstyles.org/authors/) by passing `csl: path-to-style.csl` as a Pandoc option.

docs/content/en/content-management/formats.md

@@ -111,6 +111,12 @@ Hugo passes these CLI flags when calling the Pandoc executable:
 --mathjax
 ```
 
+If your Pandoc has version 2.11 or later, it also passes this CLI flag:
+
+```text
+--citeproc
+```
+
 [Pandoc]: https://pandoc.org/
 
 ### reStructuredText

markup/pandoc/convert.go

@@ -15,10 +15,14 @@
 package pandoc
 
 import (
+	"bytes"
+	"strconv"
+	"strings"
+	"sync"
+
 	"github.com/gohugoio/hugo/common/hexec"
 	"github.com/gohugoio/hugo/htesting"
 	"github.com/gohugoio/hugo/identity"
-
 	"github.com/gohugoio/hugo/markup/converter"
 	"github.com/gohugoio/hugo/markup/internal"
 )
@@ -64,6 +68,9 @@ func (c *pandocConverter) getPandocContent(src []byte, ctx converter.DocumentCon
 		return src, nil
 	}
 	args := []string{"--mathjax"}
+	if supportsCitations(c.cfg) {
+		args = append(args[:], "--citeproc")
+	}
 	return internal.ExternallyRenderContent(c.cfg, ctx, src, binaryName, args)
 }
 
@@ -76,6 +83,69 @@ func getPandocBinaryName() string {
 	return ""
 }
 
+type pandocVersion struct {
+	major, minor int64
+}
+
+func (left pandocVersion) greaterThanOrEqual(right pandocVersion) bool {
+	return left.major > right.major || (left.major == right.major && left.minor >= right.minor)
+}
+
+var versionOnce sync.Once
+var foundPandocVersion pandocVersion
+
+// getPandocVersion parses the pandoc version output
+func getPandocVersion(cfg converter.ProviderConfig) (pandocVersion, error) {
+	var err error
+
+	versionOnce.Do(func() {
+		argsv := []any{"--version"}
+
+		var out bytes.Buffer
+		argsv = append(argsv, hexec.WithStdout(&out))
+
+		cmd, err := cfg.Exec.New(pandocBinary, argsv...)
+		if err != nil {
+			cfg.Logger.Errorf("Could not call pandoc: %v", err)
+			foundPandocVersion = pandocVersion{0, 0}
+			return
+		}
+
+		err = cmd.Run()
+		if err != nil {
+			cfg.Logger.Errorf("%s --version: %v", pandocBinary, err)
+			foundPandocVersion = pandocVersion{0, 0}
+			return
+		}
+
+		outbytes := bytes.Replace(out.Bytes(), []byte("\r"), []byte(""), -1)
+		output := strings.Split(string(outbytes), "\n")[0]
+		// Split, e.g., "pandoc 2.5" into 2 and 5 and convert them to integers
+		versionStrings := strings.Split(strings.Split(output, " ")[1], ".")
+		majorVersion, err := strconv.ParseInt(versionStrings[0], 10, 64)
+		if err != nil {
+			println(err)
+		}
+		minorVersion, err := strconv.ParseInt(versionStrings[1], 10, 64)
+		if err != nil {
+			println(err)
+		}
+		foundPandocVersion = pandocVersion{majorVersion, minorVersion}
+	})
+
+	return foundPandocVersion, err
+}
+
+// SupportsCitations returns true for pandoc versions >= 2.11, which include citeproc
+func supportsCitations(cfg converter.ProviderConfig) bool {
+	if Supports() {
+		foundPandocVersion, err := getPandocVersion(cfg)
+		supportsCitations := foundPandocVersion.greaterThanOrEqual(pandocVersion{2, 11}) && err == nil
+		return supportsCitations
+	}
+	return false
+}
+
 // Supports returns whether Pandoc is installed on this computer.
 func Supports() bool {
 	hasBin := getPandocBinaryName() != ""

markup/pandoc/convert_test.go

@@ -25,7 +25,7 @@ import (
 	qt "github.com/frankban/quicktest"
 )
 
-func TestConvert(t *testing.T) {
+func setupTestConverter(t *testing.T) (*qt.C, converter.Converter, converter.ProviderConfig) {
 	if !Supports() {
 		t.Skip("pandoc not installed")
 	}
@@ -38,7 +38,140 @@ func TestConvert(t *testing.T) {
 	c.Assert(err, qt.IsNil)
 	conv, err := p.New(converter.DocumentContext{})
 	c.Assert(err, qt.IsNil)
-	b, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+	return c, conv, cfg
+}
+
+func TestConvert(t *testing.T) {
+	c, conv, _ := setupTestConverter(t)
+	output, err := conv.Convert(converter.RenderContext{Src: []byte("testContent")})
+	c.Assert(err, qt.IsNil)
+	c.Assert(string(output.Bytes()), qt.Equals, "<p>testContent</p>\n")
+}
+
+func runCiteprocTest(t *testing.T, content string, expected string) {
+	c, conv, cfg := setupTestConverter(t)
+	if !supportsCitations(cfg) {
+		t.Skip("pandoc does not support citations")
+	}
+	output, err := conv.Convert(converter.RenderContext{Src: []byte(content)})
 	c.Assert(err, qt.IsNil)
-	c.Assert(string(b.Bytes()), qt.Equals, "<p>testContent</p>\n")
+	c.Assert(string(output.Bytes()), qt.Equals, expected)
+}
+
+func TestGetPandocVersionCallTwice(t *testing.T) {
+	c, _, cfg := setupTestConverter(t)
+
+	version1, err1 := getPandocVersion(cfg)
+	version2, err2 := getPandocVersion(cfg)
+	c.Assert(version1, qt.Equals, version2)
+	c.Assert(err1, qt.IsNil)
+	c.Assert(err2, qt.IsNil)
+}
+
+func TestPandocVersionEquality(t *testing.T) {
+	c := qt.New(t)
+	v1 := pandocVersion{1, 0}
+	v2 := pandocVersion{2, 0}
+	v3 := pandocVersion{2, 2}
+	v4 := pandocVersion{1, 2}
+	v5 := pandocVersion{2, 11}
+
+	// 1 >= 1 -> true
+	c.Assert(v1.greaterThanOrEqual(v1), qt.IsTrue)
+
+	// 1 >= 2 -> false, 2 >= 1 -> tru
+	c.Assert(v1.greaterThanOrEqual(v2), qt.IsFalse)
+	c.Assert(v2.greaterThanOrEqual(v1), qt.IsTrue)
+
+	// 2.0 >= 2.2 -> false, 2.2 >= 2.0 -> true
+	c.Assert(v2.greaterThanOrEqual(v3), qt.IsFalse)
+	c.Assert(v3.greaterThanOrEqual(v2), qt.IsTrue)
+
+	// 2.2 >= 1.2 -> true, 1.2 >= 2.2 -> false
+	c.Assert(v3.greaterThanOrEqual(v4), qt.IsTrue)
+	c.Assert(v4.greaterThanOrEqual(v3), qt.IsFalse)
+
+	// 2.11 >= 2.2 -> true, 2.2 >= 2.11 -> false
+	c.Assert(v5.greaterThanOrEqual(v3), qt.IsTrue)
+	c.Assert(v3.greaterThanOrEqual(v5), qt.IsFalse)
+}
+
+func TestCiteprocWithHugoMeta(t *testing.T) {
+	content := `
+---
+title: Test
+published: 2022-05-30
+---
+testContent
+`
+	expected := "<p>testContent</p>\n"
+	runCiteprocTest(t, content, expected)
+}
+
+func TestCiteprocWithPandocMeta(t *testing.T) {
+	content := `
+---
+---
+---
+...
+testContent
+`
+	expected := "<p>testContent</p>\n"
+	runCiteprocTest(t, content, expected)
+}
+
+func TestCiteprocWithBibliography(t *testing.T) {
+	content := `
+---
+---
+---
+bibliography: testdata/bibliography.bib
+...
+testContent
+`
+	expected := "<p>testContent</p>\n"
+	runCiteprocTest(t, content, expected)
+}
+
+func TestCiteprocWithExplicitCitation(t *testing.T) {
+	content := `
+---
+---
+---
+bibliography: testdata/bibliography.bib
+...
+@Doe2022
+`
+	expected := `<p><span class="citation" data-cites="Doe2022">Doe and Mustermann
+(2022)</span></p>
+<div id="refs" class="references csl-bib-body hanging-indent"
+role="doc-bibliography">
+<div id="ref-Doe2022" class="csl-entry" role="doc-biblioentry">
+Doe, Jane, and Max Mustermann. 2022. <span>“A Treatise on Hugo
+Tests.”</span> <em>Hugo Websites</em>.
+</div>
+</div>
+`
+	runCiteprocTest(t, content, expected)
+}
+
+func TestCiteprocWithNocite(t *testing.T) {
+	content := `
+---
+---
+---
+bibliography: testdata/bibliography.bib
+nocite: |
+  @*
+...
+`
+	expected := `<div id="refs" class="references csl-bib-body hanging-indent"
+role="doc-bibliography">
+<div id="ref-Doe2022" class="csl-entry" role="doc-biblioentry">
+Doe, Jane, and Max Mustermann. 2022. <span>“A Treatise on Hugo
+Tests.”</span> <em>Hugo Websites</em>.
+</div>
+</div>
+`
+	runCiteprocTest(t, content, expected)
 }

markup/pandoc/testdata/bibliography.bib

@@ -0,0 +1,6 @@
+@article{Doe2022,
+    author    = "Jane Doe and Max Mustermann",
+    title     = "A Treatise on Hugo Tests",
+    journal   = "Hugo Websites",
+    year      = "2022",
+}