Allow for the exclusion of files from backups (#100)

* Hoist walking of files so it can be used for features other than archive creation

* Add option to ignore files from backup using glob patterns

* Use Regexp instead of glob for exclusion

* Ignore artifacts

* Add teardown to test

* Allow single Re for filtering only

* Add documentation

* Use MatchString on re, add bad input to message in case of error

Author: m90

README.md

@@ -168,6 +168,12 @@ You can populate below template according to your requirements and use it as you
 
 # BACKUP_SOURCES="/other/location"
 
+# When given, all files in BACKUP_SOURCES whose full path matches the given
+# regular expression will be excluded from the archive. Regular Expressions
+# can be used as from the Go standard library https://pkg.go.dev/regexp
+
+# BACKUP_EXCLUDE_REGEXP="\.log$"
+
 ########### BACKUP STORAGE
 
 # The name of the remote bucket that should be used for storing backups. If

cmd/backup/archive.go

@@ -11,14 +11,13 @@ import (
 	"compress/gzip"
 	"fmt"
 	"io"
-	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
 )
 
-func createArchive(inputFilePath, outputFilePath string) error {
+func createArchive(files []string, inputFilePath, outputFilePath string) error {
 	inputFilePath = stripTrailingSlashes(inputFilePath)
 	inputFilePath, outputFilePath, err := makeAbsolute(inputFilePath, outputFilePath)
 	if err != nil {
@@ -28,7 +27,7 @@ func createArchive(inputFilePath, outputFilePath string) error {
 		return fmt.Errorf("createArchive: error creating output file path: %w", err)
 	}
 
-	if err := compress(inputFilePath, outputFilePath, filepath.Dir(inputFilePath)); err != nil {
+	if err := compress(files, outputFilePath, filepath.Dir(inputFilePath)); err != nil {
 		return fmt.Errorf("createArchive: error creating archive: %w", err)
 	}
 
@@ -52,7 +51,7 @@ func makeAbsolute(inputFilePath, outputFilePath string) (string, string, error)
 	return inputFilePath, outputFilePath, err
 }
 
-func compress(inPath, outFilePath, subPath string) error {
+func compress(paths []string, outFilePath, subPath string) error {
 	file, err := os.Create(outFilePath)
 	if err != nil {
 		return fmt.Errorf("compress: error creating out file: %w", err)
@@ -62,14 +61,6 @@ func compress(inPath, outFilePath, subPath string) error {
 	gzipWriter := gzip.NewWriter(file)
 	tarWriter := tar.NewWriter(gzipWriter)
 
-	var paths []string
-	if err := filepath.WalkDir(inPath, func(path string, di fs.DirEntry, err error) error {
-		paths = append(paths, path)
-		return err
-	}); err != nil {
-		return fmt.Errorf("compress: error walking filesystem tree: %w", err)
-	}
-
 	for _, p := range paths {
 		if err := writeTarGz(p, tarWriter, prefix); err != nil {
 			return fmt.Errorf("compress error writing %s to archive: %w", p, err)

cmd/backup/config.go

@@ -3,7 +3,11 @@
 
 package main
 
-import "time"
+import (
+	"fmt"
+	"regexp"
+	"time"
+)
 
 // Config holds all configuration values that are expected to be set
 // by users.
@@ -18,6 +22,7 @@ type Config struct {
 	BackupPruningPrefix        string        `split_words:"true"`
 	BackupStopContainerLabel   string        `split_words:"true" default:"true"`
 	BackupFromSnapshot         bool          `split_words:"true"`
+	BackupExcludeRegexp        RegexpDecoder `split_words:"true"`
 	AwsS3BucketName            string        `split_words:"true"`
 	AwsS3Path                  string        `split_words:"true"`
 	AwsEndpoint                string        `split_words:"true" default:"s3.amazonaws.com"`
@@ -44,3 +49,19 @@ type Config struct {
 	ExecForwardOutput          bool          `split_words:"true"`
 	LockTimeout                time.Duration `split_words:"true" default:"60m"`
 }
+
+type RegexpDecoder struct {
+	Re *regexp.Regexp
+}
+
+func (r *RegexpDecoder) Decode(v string) error {
+	if v == "" {
+		return nil
+	}
+	re, err := regexp.Compile(v)
+	if err != nil {
+		return fmt.Errorf("config: error compiling given regexp `%s`: %w", v, err)
+	}
+	*r = RegexpDecoder{Re: re}
+	return nil
+}

cmd/backup/script.go

@@ -398,7 +398,28 @@ func (s *script) takeBackup() error {
 		s.logger.Infof("Removed tar file `%s`.", tarFile)
 		return nil
 	})
-	if err := createArchive(backupSources, tarFile); err != nil {
+
+	backupPath, err := filepath.Abs(stripTrailingSlashes(backupSources))
+	if err != nil {
+		return fmt.Errorf("takeBackup: error getting absolute path: %w", err)
+	}
+
+	var filesEligibleForBackup []string
+	if err := filepath.WalkDir(backupPath, func(path string, di fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if s.c.BackupExcludeRegexp.Re != nil && s.c.BackupExcludeRegexp.Re.MatchString(path) {
+			return nil
+		}
+		filesEligibleForBackup = append(filesEligibleForBackup, path)
+		return nil
+	}); err != nil {
+		return fmt.Errorf("compress: error walking filesystem tree: %w", err)
+	}
+
+	if err := createArchive(filesEligibleForBackup, backupSources, tarFile); err != nil {
 		return fmt.Errorf("takeBackup: error compressing backup folder: %w", err)
 	}
 

test/ignore/.gitignore

@@ -0,0 +1 @@
+local

test/ignore/docker-compose.yml

@@ -0,0 +1,15 @@
+version: '3.8'
+
+services:
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_EXCLUDE_REGEXP: '\.(me|you)$$'
+    volumes:
+      - ./local:/archive
+      - ./sources:/backup/data:ro

test/ignore/run.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+mkdir -p local
+
+docker-compose up -d
+sleep 5
+docker-compose exec backup backup
+
+docker-compose down --volumes
+
+out=$(mktemp -d)
+sudo tar --same-owner -xvf ./local/test.tar.gz -C "$out"
+
+if [ ! -f "$out/backup/data/me.txt" ]; then
+  echo "[TEST:FAIL] Expected file was not found."
+  exit 1
+fi
+echo "[TEST:PASS] Expected file was found."
+
+if [ -f "$out/backup/data/skip.me" ]; then
+  echo "[TEST:FAIL] Ignored file was found."
+  exit 1
+fi
+echo "[TEST:PASS] Ignored file was not found."