feature: add support for sparse checkouts

Author: jasonwbarnett

agent/agent_configuration.go

@@ -23,6 +23,10 @@ type AgentConfiguration struct {
 	GitCloneMirrorFlags         string
 	GitCleanFlags               string
 	GitFetchFlags               string
+	GitSparseCheckout           bool
+	GitSparseCheckoutPaths      string
+	GitCloneDepth               string
+	GitCloneFilter              string
 	GitSubmodules               bool
 	AllowedRepositories         []*regexp.Regexp
 	AllowedPlugins              []*regexp.Regexp

agent/job_runner.go

@@ -554,6 +554,10 @@ func (r *JobRunner) createEnvironment(ctx context.Context) ([]string, error) {
 	env["BUILDKITE_GIT_FETCH_FLAGS"] = r.conf.AgentConfiguration.GitFetchFlags
 	env["BUILDKITE_GIT_CLONE_MIRROR_FLAGS"] = r.conf.AgentConfiguration.GitCloneMirrorFlags
 	env["BUILDKITE_GIT_CLEAN_FLAGS"] = r.conf.AgentConfiguration.GitCleanFlags
+	env["BUILDKITE_GIT_SPARSE_CHECKOUT"] = fmt.Sprint(r.conf.AgentConfiguration.GitSparseCheckout)
+	env["BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS"] = r.conf.AgentConfiguration.GitSparseCheckoutPaths
+	env["BUILDKITE_GIT_CLONE_DEPTH"] = r.conf.AgentConfiguration.GitCloneDepth
+	env["BUILDKITE_GIT_CLONE_FILTER"] = r.conf.AgentConfiguration.GitCloneFilter
 	env["BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT"] = strconv.Itoa(r.conf.AgentConfiguration.GitMirrorsLockTimeout)
 	env["BUILDKITE_SHELL"] = r.conf.AgentConfiguration.Shell
 	env["BUILDKITE_AGENT_EXPERIMENT"] = strings.Join(experiments.Enabled(ctx), ",")

clicommand/agent_start.go

@@ -149,6 +149,10 @@ type AgentStartConfig struct {
 	GitCloneMirrorFlags   string `cli:"git-clone-mirror-flags"`
 	GitCleanFlags         string `cli:"git-clean-flags"`
 	GitFetchFlags         string `cli:"git-fetch-flags"`
+	GitSparseCheckout     bool   `cli:"git-sparse-checkout"`
+	GitSparseCheckoutPaths string `cli:"git-sparse-checkout-paths"`
+	GitCloneDepth         string `cli:"git-clone-depth"`
+	GitCloneFilter        string `cli:"git-clone-filter"`
 	GitMirrorsPath        string `cli:"git-mirrors-path" normalize:"filepath"`
 	GitMirrorsLockTimeout int    `cli:"git-mirrors-lock-timeout"`
 	GitMirrorsSkipUpdate  bool   `cli:"git-mirrors-skip-update"`
@@ -509,6 +513,29 @@ var AgentStartCommand = cli.Command{
 			Usage:  "Flags to pass to \"git fetch\" command",
 			EnvVar: "BUILDKITE_GIT_FETCH_FLAGS",
 		},
+		cli.BoolFlag{
+			Name:   "git-sparse-checkout",
+			Usage:  "Enable sparse checkout for partial clones",
+			EnvVar: "BUILDKITE_GIT_SPARSE_CHECKOUT",
+		},
+		cli.StringFlag{
+			Name:   "git-sparse-checkout-paths",
+			Value:  "",
+			Usage:  "Paths to include in sparse checkout (comma-separated)",
+			EnvVar: "BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS",
+		},
+		cli.StringFlag{
+			Name:   "git-clone-depth",
+			Value:  "",
+			Usage:  "Clone depth for shallow clones (e.g., \"200\")",
+			EnvVar: "BUILDKITE_GIT_CLONE_DEPTH",
+		},
+		cli.StringFlag{
+			Name:   "git-clone-filter",
+			Value:  "",
+			Usage:  "Filter specification for partial clones (e.g., \"tree:0\")",
+			EnvVar: "BUILDKITE_GIT_CLONE_FILTER",
+		},
 		cli.StringFlag{
 			Name:   "git-clone-mirror-flags",
 			Value:  "-v",
@@ -1019,6 +1046,10 @@ var AgentStartCommand = cli.Command{
 			GitCloneMirrorFlags:          cfg.GitCloneMirrorFlags,
 			GitCleanFlags:                cfg.GitCleanFlags,
 			GitFetchFlags:                cfg.GitFetchFlags,
+			GitSparseCheckout:            cfg.GitSparseCheckout,
+			GitSparseCheckoutPaths:       cfg.GitSparseCheckoutPaths,
+			GitCloneDepth:                cfg.GitCloneDepth,
+			GitCloneFilter:               cfg.GitCloneFilter,
 			GitSubmodules:                !cfg.NoGitSubmodules,
 			SSHKeyscan:                   !cfg.NoSSHKeyscan,
 			CommandEval:                  !cfg.NoCommandEval,

clicommand/bootstrap.go

@@ -69,6 +69,10 @@ type BootstrapConfig struct {
 	GitFetchFlags                string   `cli:"git-fetch-flags"`
 	GitCloneMirrorFlags          string   `cli:"git-clone-mirror-flags"`
 	GitCleanFlags                string   `cli:"git-clean-flags"`
+	GitSparseCheckout            bool     `cli:"git-sparse-checkout"`
+	GitSparseCheckoutPaths       string   `cli:"git-sparse-checkout-paths"`
+	GitCloneDepth                string   `cli:"git-clone-depth"`
+	GitCloneFilter               string   `cli:"git-clone-filter"`
 	GitMirrorsPath               string   `cli:"git-mirrors-path" normalize:"filepath"`
 	GitMirrorsLockTimeout        int      `cli:"git-mirrors-lock-timeout"`
 	GitMirrorsSkipUpdate         bool     `cli:"git-mirrors-skip-update"`
@@ -244,6 +248,29 @@ var BootstrapCommand = cli.Command{
 			Usage:  "Flags to pass to \"git fetch\" command",
 			EnvVar: "BUILDKITE_GIT_FETCH_FLAGS",
 		},
+		cli.BoolFlag{
+			Name:   "git-sparse-checkout",
+			Usage:  "Enable sparse checkout for partial clones",
+			EnvVar: "BUILDKITE_GIT_SPARSE_CHECKOUT",
+		},
+		cli.StringFlag{
+			Name:   "git-sparse-checkout-paths",
+			Value:  "",
+			Usage:  "Paths to include in sparse checkout (comma-separated)",
+			EnvVar: "BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS",
+		},
+		cli.StringFlag{
+			Name:   "git-clone-depth",
+			Value:  "",
+			Usage:  "Clone depth for shallow clones (e.g., \"200\")",
+			EnvVar: "BUILDKITE_GIT_CLONE_DEPTH",
+		},
+		cli.StringFlag{
+			Name:   "git-clone-filter",
+			Value:  "",
+			Usage:  "Filter specification for partial clones (e.g., \"tree:0\")",
+			EnvVar: "BUILDKITE_GIT_CLONE_FILTER",
+		},
 		cli.StringSliceFlag{
 			Name:   "git-submodule-clone-config",
 			Value:  &cli.StringSlice{},
@@ -466,6 +493,10 @@ var BootstrapCommand = cli.Command{
 			GitCloneFlags:                cfg.GitCloneFlags,
 			GitCloneMirrorFlags:          cfg.GitCloneMirrorFlags,
 			GitFetchFlags:                cfg.GitFetchFlags,
+			GitSparseCheckout:            cfg.GitSparseCheckout,
+			GitSparseCheckoutPaths:       cfg.GitSparseCheckoutPaths,
+			GitCloneDepth:                cfg.GitCloneDepth,
+			GitCloneFilter:               cfg.GitCloneFilter,
 			GitMirrorsLockTimeout:        cfg.GitMirrorsLockTimeout,
 			GitMirrorsPath:               cfg.GitMirrorsPath,
 			GitMirrorsSkipUpdate:         cfg.GitMirrorsSkipUpdate,

docs/partial-clone.md

@@ -0,0 +1,104 @@
+# Partial Clone and Sparse Checkout
+
+The Buildkite Agent supports partial clones and sparse checkouts, which can significantly reduce clone times and disk space usage for large repositories.
+
+## Overview
+
+Partial clones allow you to clone a repository without downloading all of its history or objects, while sparse checkout allows you to check out only specific directories or files from a repository.
+
+## Configuration
+
+### Environment Variables
+
+The following environment variables control partial clone behavior:
+
+- `BUILDKITE_GIT_SPARSE_CHECKOUT` - Enable sparse checkout (boolean: `true` or `false`)
+- `BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS` - Comma-separated list of paths to include in sparse checkout
+- `BUILDKITE_GIT_CLONE_DEPTH` - Clone depth for shallow clones (e.g., `200`)
+- `BUILDKITE_GIT_CLONE_FILTER` - Filter specification for partial clones (e.g., `tree:0`)
+
+### Command Line Flags
+
+When starting an agent, you can also use command line flags:
+
+```bash
+buildkite-agent start \
+  --git-sparse-checkout \
+  --git-sparse-checkout-paths "src/frontend,src/backend" \
+  --git-clone-depth 200 \
+  --git-clone-filter "tree:0"
+```
+
+## Examples
+
+### Example 1: Sparse Checkout with Multiple Directories
+
+To check out only specific directories from a monorepo:
+
+```yaml
+env:
+  BUILDKITE_GIT_SPARSE_CHECKOUT: "true"
+  BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS: "services/api,services/web,shared/utils"
+```
+
+### Example 2: Shallow Clone with Partial Objects
+
+For a shallow clone with limited history and filtered objects:
+
+```yaml
+env:
+  BUILDKITE_GIT_CLONE_DEPTH: "100"
+  BUILDKITE_GIT_CLONE_FILTER: "blob:none"
+```
+
+### Example 3: Complete Partial Clone Setup
+
+Combining all features for maximum optimization:
+
+```yaml
+env:
+  BUILDKITE_GIT_CLONE_DEPTH: "200"
+  BUILDKITE_GIT_CLONE_FILTER: "tree:0"
+  BUILDKITE_GIT_SPARSE_CHECKOUT: "true"
+  BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS: "my-service"
+```
+
+This configuration will:
+1. Clone only the last 200 commits
+2. Exclude tree objects that aren't needed (`tree:0`)
+3. Only check out the `my-service` directory
+
+## Git Clone Filters
+
+The `BUILDKITE_GIT_CLONE_FILTER` supports various filter specifications:
+
+- `blob:none` - Omit all blob objects (file contents)
+- `blob:limit=<size>` - Omit blobs larger than `<size>` bytes
+- `tree:0` - Omit all tree objects (directory listings)
+- `tree:<depth>` - Omit tree objects at specified depth
+
+## Sparse Checkout Paths
+
+The `BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS` variable accepts:
+- Single directory: `"src/frontend"`
+- Multiple directories: `"src/frontend,src/backend,docs"`
+- Paths with wildcards are not supported in cone mode (which is the default)
+
+## Performance Considerations
+
+1. **Network Usage**: Partial clones significantly reduce network bandwidth usage
+2. **Disk Space**: Sparse checkouts reduce local disk space usage
+3. **Clone Time**: Both features can dramatically reduce initial clone times
+4. **Fetch Time**: Subsequent fetches will only download required objects
+
+## Compatibility
+
+- Requires Git 2.25.0 or later for partial clone support
+- Requires Git 2.25.0 or later for cone-mode sparse checkout
+- The repository must be hosted on a server that supports partial clones
+
+## Troubleshooting
+
+1. **Missing objects**: If you encounter "object not found" errors, you may need to adjust your filter settings or disable partial clones
+2. **Sparse checkout not working**: Ensure paths are comma-separated and don't contain spaces
+3. **Performance issues**: Some operations may trigger on-demand object downloads; monitor your builds to ensure partial clones provide the expected benefits

internal/job/checkout.go

@@ -567,6 +567,18 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) error {
 		gitCloneFlags += fmt.Sprintf(" --reference %q", mirrorDir)
 	}
 
+	// Add partial clone flags if specified
+	if e.GitCloneDepth != "" {
+		gitCloneFlags += fmt.Sprintf(" --depth=%s", e.GitCloneDepth)
+	}
+	if e.GitCloneFilter != "" {
+		gitCloneFlags += fmt.Sprintf(" --filter=%s", e.GitCloneFilter)
+	}
+	// For sparse checkout, we need to add --no-checkout
+	if e.GitSparseCheckout && e.GitSparseCheckoutPaths != "" {
+		gitCloneFlags += " --no-checkout"
+	}
+
 	// Does the git directory exist?
 	existingGitDir := filepath.Join(e.shell.Getwd(), ".git")
 	if osutil.FileExists(existingGitDir) {
@@ -575,10 +587,54 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) error {
 		if _, err := e.updateRemoteURL(ctx, "", e.Repository); err != nil {
 			return fmt.Errorf("setting origin: %w", err)
 		}
+
+		// Handle sparse checkout for existing repos
+		if e.GitSparseCheckout && e.GitSparseCheckoutPaths != "" {
+			e.shell.Commentf("Configuring sparse checkout for existing repository")
+			
+			// Check if sparse checkout is already initialized
+			sparseCheckoutFile := filepath.Join(existingGitDir, "info", "sparse-checkout")
+			if !osutil.FileExists(sparseCheckoutFile) {
+				// Initialize sparse checkout
+				if err := gitSparseCheckoutInit(ctx, e.shell, true); err != nil {
+					return fmt.Errorf("initializing sparse checkout: %w", err)
+				}
+			}
+
+			// Parse comma-separated paths
+			paths := strings.Split(e.GitSparseCheckoutPaths, ",")
+			for i := range paths {
+				paths[i] = strings.TrimSpace(paths[i])
+			}
+
+			e.shell.Commentf("Setting sparse checkout paths: %v", paths)
+			if err := gitSparseCheckoutSet(ctx, e.shell, paths); err != nil {
+				return fmt.Errorf("setting sparse checkout paths: %w", err)
+			}
+		}
 	} else {
 		if err := gitClone(ctx, e.shell, gitCloneFlags, e.Repository, "."); err != nil {
 			return fmt.Errorf("cloning git repository: %w", err)
 		}
+
+		// If sparse checkout is enabled, initialize it right after cloning
+		if e.GitSparseCheckout && e.GitSparseCheckoutPaths != "" {
+			e.shell.Commentf("Initializing sparse checkout")
+			if err := gitSparseCheckoutInit(ctx, e.shell, true); err != nil {
+				return fmt.Errorf("initializing sparse checkout: %w", err)
+			}
+
+			// Parse comma-separated paths
+			paths := strings.Split(e.GitSparseCheckoutPaths, ",")
+			for i := range paths {
+				paths[i] = strings.TrimSpace(paths[i])
+			}
+
+			e.shell.Commentf("Setting sparse checkout paths: %v", paths)
+			if err := gitSparseCheckoutSet(ctx, e.shell, paths); err != nil {
+				return fmt.Errorf("setting sparse checkout paths: %w", err)
+			}
+		}
 	}
 
 	// Git clean prior to checkout, we do this even if submodules have been
@@ -594,6 +650,11 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) error {
 	}
 
 	gitFetchFlags := e.GitFetchFlags
+	
+	// Add filter flag for partial clones during fetch
+	if e.GitCloneFilter != "" {
+		gitFetchFlags += fmt.Sprintf(" --filter=%s", e.GitCloneFilter)
+	}
 
 	switch {
 	case e.RefSpec != "":

internal/job/config.go

@@ -90,6 +90,18 @@ type ExecutorConfig struct {
 	// Config key=value pairs to pass to "git" when submodule init commands are invoked
 	GitSubmoduleCloneConfig []string `env:"BUILDKITE_GIT_SUBMODULE_CLONE_CONFIG" normalize:"list"`
 
+	// Enable sparse checkout for partial clones
+	GitSparseCheckout bool `env:"BUILDKITE_GIT_SPARSE_CHECKOUT"`
+
+	// Paths to include in sparse checkout (comma-separated)
+	GitSparseCheckoutPaths string `env:"BUILDKITE_GIT_SPARSE_CHECKOUT_PATHS"`
+
+	// Clone depth for shallow clones
+	GitCloneDepth string `env:"BUILDKITE_GIT_CLONE_DEPTH"`
+
+	// Filter specification for partial clones (e.g., "tree:0")
+	GitCloneFilter string `env:"BUILDKITE_GIT_CLONE_FILTER"`
+
 	// Whether or not to run the hooks/commands in a PTY
 	RunInPty bool
 

internal/job/git.go

@@ -396,3 +396,31 @@ var gitCheckRefFormatDenyRegexp = regexp.MustCompile(strings.Join([]string{
 func gitCheckRefFormat(ref string) bool {
 	return !gitCheckRefFormatDenyRegexp.MatchString(ref)
 }
+
+func gitSparseCheckoutInit(ctx context.Context, sh *shell.Shell, cone bool) error {
+	args := []string{"sparse-checkout", "init"}
+	if cone {
+		args = append(args, "--cone")
+	}
+
+	if err := sh.Command("git", args...).Run(ctx); err != nil {
+		return &gitError{error: err, Type: gitErrorCheckout}
+	}
+
+	return nil
+}
+
+func gitSparseCheckoutSet(ctx context.Context, sh *shell.Shell, paths []string) error {
+	if len(paths) == 0 {
+		return fmt.Errorf("no paths provided for sparse checkout")
+	}
+
+	args := []string{"sparse-checkout", "set"}
+	args = append(args, paths...)
+
+	if err := sh.Command("git", args...).Run(ctx); err != nil {
+		return &gitError{error: err, Type: gitErrorCheckout}
+	}
+
+	return nil
+}