Add noop ExcludePathPatterns option

Author: pskopnik

master/medasync/syncer.go

@@ -49,6 +49,7 @@ type Config struct {
 	TemporaryDirectory  string
 	GlobalWorkDirectory string
 	NodeList            []string
+	ExcludePathPatterns []string
 
 	// Invocation dependent params
 
@@ -164,7 +165,7 @@ func (s *Syncer) prepareDatabase(ctx context.Context) error {
 }
 
 func (s *Syncer) applyPolicy() (*filelist.CloseParser, error) {
-	options := []options.PolicyOptioner{
+	policyOpts := []options.PolicyOptioner{
 		scaleadpt.PolicyOpt.SnapshotName(s.Config.SnapshotName),
 		scaleadpt.PolicyOpt.Subpath(s.Config.Subpath),
 		scaleadpt.PolicyOpt.TempDir(s.Config.TemporaryDirectory),
@@ -180,16 +181,24 @@ func (s *Syncer) applyPolicy() (*filelist.CloseParser, error) {
 		fields["global_work_directory"] = s.Config.GlobalWorkDirectory
 		fields["node_list"] = s.Config.NodeList
 
-		options = append(
-			options,
+		policyOpts = append(
+			policyOpts,
 			scaleadpt.PolicyOpt.GlobalWorkDirectory(s.Config.GlobalWorkDirectory),
 			scaleadpt.PolicyOpt.NodeList(s.Config.NodeList),
 		)
 	}
 
+	opts := []options.FilelistPolicyOptioner{
+		filelist.Opt.PolicyOpts(policyOpts...),
+	}
+
+	if len(s.Config.ExcludePathPatterns) > 0 {
+		opts = append(opts, filelist.Opt.ExcludePathPatterns(s.Config.ExcludePathPatterns))
+	}
+
 	s.fieldLogger.WithFields(fields).Info("Starting applying list policy")
 
-	parser, err := filelist.ApplyPolicy(s.Config.FileSystem, options...)
+	parser, err := filelist.ApplyPolicy(s.Config.FileSystem, opts...)
 	if err != nil {
 		return nil, errors.Wrap(err, "(*Syncer).applyPolicy: apply policy on file system")
 	}

master/medasync/syncer_config_methods.go

@@ -4,10 +4,11 @@ func (c *Config) CopyFrom(other *Config) {
 	c.SynchronisationChunkSize = other.SynchronisationChunkSize
 	c.Inserter.CopyFrom(&other.Inserter)
 
+	c.Subpath = other.Subpath
 	c.TemporaryDirectory = other.TemporaryDirectory
 	c.GlobalWorkDirectory = other.GlobalWorkDirectory
 	c.NodeList = other.NodeList
-	c.Subpath = other.Subpath
+	c.ExcludePathPatterns = other.ExcludePathPatterns
 
 	c.SnapshotName = other.SnapshotName
 	c.RunID = other.RunID
@@ -24,6 +25,9 @@ func (c *Config) Merge(other *Config) *Config {
 	}
 	c.Inserter.Merge(&other.Inserter)
 
+	if len(other.Subpath) > 0 {
+		c.Subpath = other.Subpath
+	}
 	if len(other.TemporaryDirectory) > 0 {
 		c.TemporaryDirectory = other.TemporaryDirectory
 	}
@@ -33,8 +37,8 @@ func (c *Config) Merge(other *Config) *Config {
 	if len(other.NodeList) > 0 {
 		c.NodeList = other.NodeList
 	}
-	if len(other.Subpath) > 0 {
-		c.Subpath = other.Subpath
+	if len(other.ExcludePathPatterns) > 0 {
+		c.ExcludePathPatterns = other.ExcludePathPatterns
 	}
 
 	if len(other.SnapshotName) > 0 {

scaleadpt/filelist/policy.go

@@ -53,17 +53,22 @@ func (p *CloseParser) Close() error {
 	}
 }
 
+// Opt provides simple access to FilelistPolicyOptioners methods.
+var Opt = options.FilelistPolicyOptioners{}
+
 // ApplyPolicy applies the FileListPolicy and returns a CloseParser to read the
 // matching files.
 // The CloseParser should be closed in all cases (after processing is finished
 // or if an error occurs).
-func ApplyPolicy(fs *scaleadpt.FileSystem, opts ...options.PolicyOptioner) (*CloseParser, error) {
-	// Extract TempDir from opts
-	tempDir := (&options.PolicyOptions{}).Apply(opts).TempDir
+func ApplyPolicy(fs *scaleadpt.FileSystem, opts ...options.FilelistPolicyOptioner) (*CloseParser, error) {
+	optsAggregate := (&options.FilelistPolicyOptions{}).Apply(opts)
+
+	// Extract TempDir from PolicyOptions
+	tempDir := optsAggregate.PolicyOptions.TempDir
 
 	listPath := osutils.TouchNonExistingTempFile("scaleadpt-filelist-", ".list.files", tempDir)
 
-	err := fs.ApplyListPolicy(FileListPolicy, listPath, opts...)
+	err := fs.ApplyListPolicy(FileListPolicy, listPath, &optsAggregate.PolicyOptions)
 	if err != nil {
 		return nil, err
 	}

scaleadpt/options/filelist_policy.go

@@ -0,0 +1,45 @@
+package options
+
+// FilelistPolicyOptions is the option aggregate structure for options for
+// policy apply operations in the filelist package.
+type FilelistPolicyOptions struct {
+	PolicyOptions       PolicyOptions
+	ExcludePathPatterns []string
+}
+
+func (f *FilelistPolicyOptions) Apply(opts []FilelistPolicyOptioner) *FilelistPolicyOptions {
+	for _, opt := range opts {
+		opt.apply(f)
+	}
+
+	return f
+}
+
+func (f *FilelistPolicyOptions) apply(options *FilelistPolicyOptions) {
+	*options = *f
+}
+
+type FilelistPolicyOptioner interface {
+	apply(*FilelistPolicyOptions)
+}
+
+type filelistPolicyOptionerFunc func(*FilelistPolicyOptions)
+
+func (f filelistPolicyOptionerFunc) apply(options *FilelistPolicyOptions) {
+	f(options)
+}
+
+type FilelistPolicyOptioners struct{}
+
+func (_ FilelistPolicyOptioners) PolicyOpts(opts ...PolicyOptioner) FilelistPolicyOptioner {
+	return filelistPolicyOptionerFunc(func(options *FilelistPolicyOptions) {
+		options.PolicyOptions = PolicyOptions{}
+		options.PolicyOptions.Apply(opts)
+	})
+}
+
+func (_ FilelistPolicyOptioners) ExcludePathPatterns(patterns []string) FilelistPolicyOptioner {
+	return filelistPolicyOptionerFunc(func(options *FilelistPolicyOptions) {
+		options.ExcludePathPatterns = patterns
+	})
+}

scaleadpt/options/policy.go

@@ -27,6 +27,10 @@ func (p *PolicyOptions) Apply(opts []PolicyOptioner) *PolicyOptions {
 	return p
 }
 
+func (p *PolicyOptions) apply(options *PolicyOptions) {
+	*options = *p
+}
+
 type PolicyOptioner interface {
 	apply(*PolicyOptions)
 }

scaleadpt/options/snapshot.go

@@ -14,6 +14,10 @@ func (s *SnapshotOptions) Apply(opts []SnapshotOptioner) *SnapshotOptions {
 	return s
 }
 
+func (s *SnapshotOptions) apply(options *SnapshotOptions) {
+	*options = *s
+}
+
 type SnapshotOptioner interface {
 	apply(*SnapshotOptions)
 }

tools/scaleadpt-tester/main.go

@@ -12,6 +12,7 @@ import (
 
 	"git.scc.kit.edu/sdm/lsdf-checksum/scaleadpt"
 	"git.scc.kit.edu/sdm/lsdf-checksum/scaleadpt/filelist"
+	"git.scc.kit.edu/sdm/lsdf-checksum/scaleadpt/options"
 )
 
 type Spec struct {
@@ -35,9 +36,10 @@ type SpecSnapshotDirsInfo struct {
 }
 
 type SpecListPolicy struct {
-	Subpath       string               `yaml:"subpath"`
-	Files         []SpecListPolicyFile `yaml:"files"`
-	FilesComplete bool                 `yaml:"files_complete"`
+	Subpath             string               `yaml:"subpath"`
+	Files               []SpecListPolicyFile `yaml:"files"`
+	FilesComplete       bool                 `yaml:"files_complete"`
+	ExcludePathPatterns []string             `yaml:"exclude_path_patterns"`
 }
 
 type SpecListPolicyFile struct {
@@ -404,7 +406,17 @@ func (c *Checker) checkListPolicy() {
 		specFiles[file.Path] = file
 	}
 
-	parser, err := filelist.ApplyPolicy(c.fs, scaleadpt.PolicyOpt.Subpath(c.spec.ListPolicy.Subpath))
+	options := []options.FilelistPolicyOptioner{
+		filelist.Opt.PolicyOpts(scaleadpt.PolicyOpt.Subpath(c.spec.ListPolicy.Subpath)),
+	}
+
+	if len(c.spec.ListPolicy.ExcludePathPatterns) > 0 {
+		options = append(options,
+			filelist.Opt.ExcludePathPatterns(c.spec.ListPolicy.ExcludePathPatterns),
+		)
+	}
+
+	parser, err := filelist.ApplyPolicy(c.fs, options...)
 	if err != nil {
 		f.AddRuntimeErr(err)
 		return

tools/scaleadpt-tester/spec.example.yaml

@@ -23,6 +23,10 @@ filesystems:
     list_policy:
       # Subpath in the filesystem for which to execute the list policy.
       subpath: /test-dir
+      # Optional list of SQL patterns on the filepath with matching files
+      # excluded from the output.
+      exclude_path_patterns:
+        - '%.csv'
       # Is the list of files below "complete", i.e. does it describe all files
       # in the tree?
       files_complete: false