Merge pull request #233 from moov-io/oss-skip-canceled-files-mapping

fix: skip canceled files in buildDirMapping
moov-io · Mar 29, 2024 · f6e8310 · f6e8310
2 parents 586ea83 + 025d6b7
commit f6e8310
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 3 deletions.
diff --git a/internal/pipeline/merging.go b/internal/pipeline/merging.go
@@ -336,7 +336,7 @@ func (m *filesystemMerging) WithEachMerged(ctx context.Context, f func(context.C
 	)
 
 	// Build a mapping of BatchHeader + EntryDetail from dir
-	mappings, err := m.buildDirMapping(dir)
+	mappings, err := m.buildDirMapping(dir, canceledFiles)
 	if err != nil {
 		el.Add(err)
 	}
@@ -395,9 +395,11 @@ func fileAcceptor(canceledFiles []string) func(string) ach.FileAcceptance {
 	}
 }
 
-func (m *filesystemMerging) buildDirMapping(dir string) (*treemap.TreeMap[string, string], error) {
+func (m *filesystemMerging) buildDirMapping(dir string, canceledFiles []string) (*treemap.TreeMap[string, string], error) {
 	tree := treemap.New[string, string]()
 
+	acceptor := fileAcceptor(canceledFiles)
+
 	err := fs.WalkDir(m.storage, dir, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			if strings.Contains(err.Error(), "is a directory") {
@@ -415,6 +417,11 @@ func (m *filesystemMerging) buildDirMapping(dir string) (*treemap.TreeMap[string
 			return nil
 		}
 
+		// Skip the file if merging would have skipped it
+		if acceptor(path) == ach.SkipFile {
+			return nil
+		}
+
 		fd, err := m.storage.Open(path)
 		if err != nil {
 			return fmt.Errorf("opening %s failed: %w", path, err)

diff --git a/internal/pipeline/merging_test.go b/internal/pipeline/merging_test.go
@@ -165,6 +165,12 @@ func TestMerging_mappings(t *testing.T) {
 	copyFile(t, filepath.Join("testdata", "ppd-debit4.ach"), filepath.Join(dir, "mergable", "ppd-debit4.ach"))
 	copyFile(t, filepath.Join("testdata", "duplicate-trace.ach"), filepath.Join(dir, "mergable", "duplicate-trace.ach"))
 
+	// Canceled files
+	err := os.WriteFile(filepath.Join(dir, "mergable", "foo2.ach"), nil, 0600)
+	require.NoError(t, err)
+	err = os.WriteFile(filepath.Join(dir, "mergable", "foo2.ach.canceled"), nil, 0600)
+	require.NoError(t, err)
+
 	fs, err := storage.NewFilesystem(dir)
 	require.NoError(t, err)
 
@@ -184,7 +190,8 @@ func TestMerging_mappings(t *testing.T) {
 		},
 	}
 
-	mappings, err := m.buildDirMapping(".")
+	canceledFiles := []string{"foo2.ach"}
+	mappings, err := m.buildDirMapping(".", canceledFiles)
 	require.NoError(t, err)
 
 	for it := mappings.Iterator(); it.Valid(); it.Next() {