Add CLI documentation generator

.github/workflows/go-generate-check.yml

@@ -13,6 +13,11 @@ jobs:
         with:
           go-version: "1.23"
 
+      - name: Create frontend export placeholder
+        run: |
+          mkdir -p web_ui/frontend/out
+          touch web_ui/frontend/out/placeholder
+
       - name: Run Go Generate
         run: go generate ./...
 

.github/workflows/pre-commit-linter.yml

@@ -27,7 +27,8 @@ jobs:
       - name: Generate placeholder files
         id: generate-placeholder
         run: |
-          go generate ./...
+          mkdir -p web_ui/frontend/out
+          touch web_ui/frontend/out/placeholder
 
       - name: Set up Python
         uses: actions/setup-python@v5

.github/workflows/test-linux.yml

@@ -63,6 +63,8 @@ jobs:
           ${{ runner.os }}-go-
     - name: Test
       run: |
+        mkdir -p web_ui/frontend/out
+        touch web_ui/frontend/out/placeholder
         make web-build
         # Disabling until we are able to make it more reliable -- shouldn't punish other folks for challenging tests!
         #go test      -timeout 15m -coverpkg=./director -covermode=count -coverprofile=${{ matrix.coverprofile }} -tags=${{ matrix.tags }} ./director -run TestStatMemory

.github/workflows/test-macos-windows.yml

@@ -72,6 +72,8 @@ jobs:
     - name: Test macOS
       if: runner.os == 'macOS'
       run: |
+        mkdir -p web_ui/frontend/out
+        touch web_ui/frontend/out/placeholder
         make web-build
         #go test -timeout 15m -coverpkg=./director -covermode=count -coverprofile=coverage.out ./director -run TestStatMemory
         go test -p=4 -timeout 15m -coverpkg=./... -covermode=count -coverprofile=coverage.out ./... -skip TestStatMemory
@@ -83,6 +85,8 @@ jobs:
         GOMODCACHE: D:\gomodcache
         GOTMPDIR: D:\gotmp
       run: |
+        mkdir -p web_ui/frontend/out
+        New-Item -Path web_ui/frontend/out/placeholder -ItemType File -Force
         make web-build
         go test -p=4 -timeout 15m -coverpkg=./... -covermode=count -coverprofile=coverage.out ./...
     - name: Run GoReleaser for macOS

cmd/doc_gen.go

@@ -0,0 +1,296 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/cobra/doc"
+)
+
+// copied from generate/next_generator.go
+func GenPlaceholderPathForNext() {
+	dir := "../web_ui/frontend/out"
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		log.Fatalf("error: %v", err)
+	}
+
+	filePath := filepath.Join(dir, "placeholder")
+
+	file, err := os.Create(filePath)
+	if err != nil {
+		log.Fatalf("error: %v", err)
+	}
+	file.Close()
+}
+
+// generateCLIDocs creates per-command docs under the given directory. If the path
+// is relative, it is resolved against the repository root (directory containing go.mod).
+func generateCLIDocs(outputDir string) error {
+	GenPlaceholderPathForNext()
+	resolvedDir, err := resolveOutputPath(outputDir)
+	if err != nil {
+		return err
+	}
+
+	if err := os.MkdirAll(resolvedDir, 0o755); err != nil {
+		return err
+	}
+
+	docPathRoot := filepath.Base(outputDir)
+
+	// Generate a markdown file per command, with custom file names and content wrapper
+	linkHandler := func(name string) string {
+		// Cobra passes names like "pelican_serve.md"; strip root prefix but keep underscores so we can group by tokens
+		base := strings.TrimSuffix(name, filepath.Ext(name))
+		if base == "pelican" {
+			base = ""
+		} else {
+			base = strings.TrimPrefix(base, "pelican_")
+		}
+		path := strings.ReplaceAll(base, "_", "/")
+		// Must be an absolute path from the site root
+		if path == "" {
+			return fmt.Sprintf("/%s/", docPathRoot)
+		}
+		return fmt.Sprintf("/%s/%s/", docPathRoot, path)
+	}
+
+	filePrepender := func(filename string) string {
+		// Create a minimal MDX frontmatter; derive a title from filename
+		title := filename
+		if base := filepath.Base(filename); base != "" {
+			title = strings.TrimSuffix(base, filepath.Ext(base))
+			title = strings.ReplaceAll(title, "_", " ")
+			title = strings.ReplaceAll(title, "-", " ")
+			title = strings.ToLower(title)
+		}
+		return fmt.Sprintf("---\ntitle: %s\n---\n\n", title)
+	}
+
+	// Cobra writes files directly to the destination directory (with .md extension)
+	if err := doc.GenMarkdownTreeCustom(rootCmd, resolvedDir, filePrepender, linkHandler); err != nil {
+		return err
+	}
+
+	// Rename generated .md files to .mdx so links and index work as expected
+	if err := renameMdToMdx(resolvedDir); err != nil {
+		return err
+	}
+
+	// Group by command tokens: e.g., object/get -> object/get/page.mdx
+	if err := enforceAppRouterLayout(resolvedDir); err != nil {
+		return err
+	}
+
+	if err := generateMetaFiles(resolvedDir); err != nil {
+		return err
+	}
+
+	if err := postProcessMdxFiles(resolvedDir); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// renameMdToMdx renames all .md files in dir to .mdx
+func renameMdToMdx(dir string) error {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return err
+	}
+	for _, e := range entries {
+		if e.IsDir() {
+			continue
+		}
+		name := e.Name()
+		if filepath.Ext(name) != ".md" {
+			continue
+		}
+		oldPath := filepath.Join(dir, name)
+		newPath := filepath.Join(dir, strings.TrimSuffix(name, ".md")+".mdx")
+		if err := os.Rename(oldPath, newPath); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// enforceAppRouterLayout moves each command .mdx into nested subfolders based on underscore tokens, ending with page.mdx
+func enforceAppRouterLayout(dir string) error {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return err
+	}
+	for _, e := range entries {
+		if e.IsDir() {
+			continue
+		}
+		name := e.Name()
+		if filepath.Ext(name) != ".mdx" || name == "page.mdx" {
+			continue
+		}
+		base := strings.TrimSuffix(name, ".mdx")
+		// Build nested path from underscore-delimited tokens
+		segments := strings.Split(base, "_")
+		if len(segments) > 0 && segments[0] == "pelican" {
+			segments = segments[1:]
+		}
+		// Create nested directory path
+		targetDir := filepath.Join(append([]string{dir}, segments...)...)
+		if err := os.MkdirAll(targetDir, 0o755); err != nil {
+			return err
+		}
+		src := filepath.Join(dir, name)
+		dst := filepath.Join(targetDir, "page.mdx")
+		_ = os.Remove(dst)
+		if err := os.Rename(src, dst); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// resolveOutputPath returns an absolute path for the output. If the provided path
+// is absolute, it is returned as-is. If it is relative, we attempt to resolve it
+// relative to the repository root (detected by locating go.mod). If no repo root
+// can be determined, it is resolved relative to the current working directory.
+func resolveOutputPath(path string) (string, error) {
+	if filepath.IsAbs(path) {
+		return path, nil
+	}
+
+	repoRoot, err := findRepoRoot()
+	if err == nil && repoRoot != "" {
+		return filepath.Join(repoRoot, path), nil
+	}
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(cwd, path), nil
+}
+
+// findRepoRoot walks up from the current working directory to find a directory
+// containing go.mod and returns that directory path.
+func findRepoRoot() (string, error) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		return "", err
+	}
+
+	dir := cwd
+	for {
+		if fileExists(filepath.Join(dir, "go.mod")) {
+			return dir, nil
+		}
+
+		parent := filepath.Dir(dir)
+		if parent == dir { // reached filesystem root
+			return "", errors.New("repository root not found (no go.mod)")
+		}
+		dir = parent
+	}
+}
+
+func fileExists(path string) bool {
+	info, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+	return !info.IsDir()
+}
+
+func generateMetaFiles(dir string) error {
+	return filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if !d.IsDir() {
+			return nil
+		}
+
+		entries, err := os.ReadDir(path)
+		if err != nil {
+			return err
+		}
+
+		var subdirs []string
+		for _, entry := range entries {
+			if entry.IsDir() {
+				subdirs = append(subdirs, entry.Name())
+			}
+		}
+
+		if len(subdirs) > 0 {
+			metaFilePath := filepath.Join(path, "_meta.js")
+
+			relPath, err := filepath.Rel(dir, path)
+			if err != nil {
+				return err
+			}
+
+			commandPrefix := "pelican"
+			if relPath != "." {
+				commandPrefix += " " + strings.ReplaceAll(relPath, string(filepath.Separator), " ")
+			}
+
+			content := "export default {\n"
+			for _, subdir := range subdirs {
+				title := commandPrefix + " " + subdir
+				content += fmt.Sprintf("    \"%s\": \"%s\",\n", subdir, title)
+			}
+			content += "}\n"
+
+			if err := os.WriteFile(metaFilePath, []byte(content), 0644); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
+
+func postProcessMdxFiles(dir string) error {
+	return filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if !d.IsDir() && strings.HasSuffix(d.Name(), ".mdx") {
+			content, err := os.ReadFile(path)
+			if err != nil {
+				return err
+			}
+
+			if len(content) == 0 {
+				return nil
+			}
+
+			// Trim trailing spaces on each line
+			lines := strings.Split(string(content), "\n")
+			for i, line := range lines {
+				lines[i] = strings.TrimRight(line, " \t")
+			}
+			fullContent := strings.Join(lines, "\n")
+
+			// Ensure single newline at EOF
+			fullContent = strings.TrimRight(fullContent, "\n") + "\n"
+
+			if string(content) != fullContent {
+				info, err := d.Info()
+				if err != nil {
+					return err
+				}
+				if err := os.WriteFile(path, []byte(fullContent), info.Mode()); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	})
+}

cmd/main.go

@@ -32,9 +32,17 @@ import (
 	"github.com/pelicanplatform/pelican/server_utils"
 )
 
+//go:generate go run . generate-docs
 func main() {
 	logging.SetupLogBuffering()
 	defer logging.FlushLogs(false)
+	if len(os.Args) > 1 && os.Args[1] == "generate-docs" {
+		err := generateCLIDocs("docs/app/commands-reference")
+		if err != nil {
+			os.Exit(1)
+		}
+		return
+	}
 	err := handleCLI(os.Args)
 	if err != nil {
 		os.Exit(1)

cmd/root.go

@@ -149,6 +149,7 @@ func init() {
 	rootCmd.AddCommand(config_printer.ConfigCmd)
 	preferredPrefix := config.GetPreferredPrefix()
 	rootCmd.Use = strings.ToLower(preferredPrefix.String())
+	rootCmd.DisableAutoGenTag = true
 
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.config/pelican/pelican.yaml)")
 

docs/app/_meta.js

@@ -9,5 +9,6 @@ export default {
 	"monitoring-pelican-services": "Monitoring Pelican Services",
 	"advanced-usage": "Advanced Usage",
 	"faq": "FAQs and Troubleshooting",
-	"api-docs": "API Documentation"
+	"api-docs": "API Documentation",
+	"commands-reference": "Commands Reference"
 }

docs/app/commands-reference/_meta.js

@@ -0,0 +1,15 @@
+export default {
+    "cache": "pelican cache",
+    "config": "pelican config",
+    "credentials": "pelican credentials",
+    "director": "pelican director",
+    "downtime": "pelican downtime",
+    "generate": "pelican generate",
+    "key": "pelican key",
+    "namespace": "pelican namespace",
+    "object": "pelican object",
+    "origin": "pelican origin",
+    "plugin": "pelican plugin",
+    "registry": "pelican registry",
+    "token": "pelican token",
+}

docs/app/commands-reference/cache/_meta.js

@@ -0,0 +1,3 @@
+export default {
+    "serve": "pelican cache serve",
+}