diff --git a/.github/workflows/codetests.yml b/.github/workflows/codetests.yml index bd0de16..edf543c 100644 --- a/.github/workflows/codetests.yml +++ b/.github/workflows/codetests.yml @@ -34,9 +34,9 @@ jobs: go-version: stable - uses: actions/checkout@v4 - name: golangci-lint - uses: golangci/golangci-lint-action@v6 + uses: golangci/golangci-lint-action@v7 with: - version: v1.59 + version: v2.0 # Runs golangci-lint on linux against linux and windows. golangci-linux: strategy: @@ -52,6 +52,6 @@ jobs: go-version: stable - uses: actions/checkout@v4 - name: golangci-lint - uses: golangci/golangci-lint-action@v6 + uses: golangci/golangci-lint-action@v7 with: - version: v1.59 + version: v2.0 diff --git a/.golangci.yml b/.golangci.yml index 1a2df62..ec4be8f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,31 +1,55 @@ -run: - timeout: 1m - +version: "2" linters: - enable-all: true + default: all disable: - # deprecated - - gomnd - - execinquery # unused - nlreturn - exhaustruct - depguard + - nonamedreturns + exclusions: + rules: + - linters: + - funlen + - goconst + - wsl + - errcheck + path: .+_test.go -output: - sort-results: true + settings: + gosec: + excludes: + - G304 + gocritic: + enable-all: true + disabled-checks: + - whyNoLint + - commentedOutCode + settings: + unnamedResult: + checkExported: true + errcheck: + check-type-assertions: true + check-blank: true + disable-default-exclusions: true + exclude-functions: + - (*os.File).Close + - os.RemoveAll + - os.Remove + - (io.Closer).Close + - (*compress/gzip.Reader).Close + - (*archive/zip.ReadCloser).Close + - (*github.com/bodgit/sevenzip.ReadCloser).Close + - (*github.com/nwaples/rardecode.ReadCloser).Close issues: - # disable the default limit so we see everything - max-same-issues: 0 max-issues-per-linter: 0 - - # default enable fix where the linter supports + max-same-issues: 0 fix: true - exclude-rules: - # Exclude some linters from testing files. - - linters: - - goconst - - wsl - - funlen - path: '.+_test.go' \ No newline at end of file +formatters: + enable: + - gci + - gofmt + - gofumpt + - goimports + diff --git a/7z.go b/7z.go index d4faa9a..02dc3c9 100644 --- a/7z.go +++ b/7z.go @@ -11,7 +11,7 @@ import ( // Extract7z extracts a 7zip archive. // Volumes: https://github.com/bodgit/sevenzip/issues/54 -func Extract7z(xFile *XFile) (int64, []string, []string, error) { +func Extract7z(xFile *XFile) (size int64, filesList, archiveList []string, err error) { if len(xFile.Passwords) == 0 && xFile.Password == "" { return extract7z(xFile) } @@ -64,7 +64,7 @@ func extract7z(xFile *XFile) (int64, []string, []string, error) { size := int64(0) for _, zipFile := range sevenZip.File { - fSize, err := xFile.un7zip(zipFile) + fSize, wfile, err := xFile.un7zip(zipFile) if err != nil { lastFile := xFile.FilePath /* // https://github.com/bodgit/sevenzip/issues/54 @@ -77,36 +77,42 @@ func extract7z(xFile *XFile) (int64, []string, []string, error) { files = append(files, filepath.Join(xFile.OutputDir, zipFile.Name)) size += fSize + xFile.Debugf("Wrote archived file: %s (%d bytes), total: %d files and %d bytes", wfile, fSize, len(files), size) } return size, files, sevenZip.Volumes(), nil } -func (x *XFile) un7zip(zipFile *sevenzip.File) (int64, error) { //nolint:dupl +func (x *XFile) un7zip(zipFile *sevenzip.File) (int64, string, error) { wfile := x.clean(zipFile.Name) if !strings.HasPrefix(wfile, x.OutputDir) { // The file being written is trying to write outside of our base path. Malicious archive? - return 0, fmt.Errorf("%s: %w: %s (from: %s)", zipFile.FileInfo().Name(), ErrInvalidPath, wfile, zipFile.Name) + return 0, wfile, fmt.Errorf("%s: %w: %s (from: %s)", zipFile.FileInfo().Name(), ErrInvalidPath, wfile, zipFile.Name) } - if strings.HasSuffix(wfile, "/") || zipFile.FileInfo().IsDir() { + if zipFile.FileInfo().IsDir() { + x.Debugf("Writing archived directory: %s", wfile) + if err := os.MkdirAll(wfile, x.DirMode); err != nil { - return 0, fmt.Errorf("making zipFile dir: %w", err) + return 0, wfile, fmt.Errorf("making zipFile dir: %w", err) } - return 0, nil + return 0, wfile, nil } + x.Debugf("Writing archived file: %s (packed: %d, unpacked: %d)", + wfile, zipFile.FileInfo().Size(), zipFile.UncompressedSize) + zFile, err := zipFile.Open() if err != nil { - return 0, fmt.Errorf("zipFile.Open: %w", err) + return 0, wfile, fmt.Errorf("zipFile.Open: %w", err) } defer zFile.Close() s, err := writeFile(wfile, zFile, x.FileMode, x.DirMode) if err != nil { - return s, fmt.Errorf("%s: %w: %s (from: %s)", zipFile.FileInfo().Name(), err, wfile, zipFile.Name) + return s, wfile, fmt.Errorf("%s: %w: %s (from: %s)", zipFile.FileInfo().Name(), err, wfile, zipFile.Name) } - return s, nil + return s, wfile, nil } diff --git a/LICENSE b/LICENSE index 94ad973..63d5883 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2020-2024 Go Lift +Copyright (c) 2020-2025 Go Lift Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/ar.go b/ar.go index 99c18cb..93469f6 100644 --- a/ar.go +++ b/ar.go @@ -11,7 +11,7 @@ import ( ) // ExtractAr extracts a raw ar archive. Used by debian (.deb) packages. -func ExtractAr(xFile *XFile) (int64, []string, error) { +func ExtractAr(xFile *XFile) (size int64, filesList []string, err error) { arFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -45,6 +45,7 @@ func (x *XFile) unAr(reader io.Reader) (int64, []string, error) { } // ar format does not store directory paths. Flat list of files. + //nolint:gosec // we are not overflowing an integer with this conversion. fSize, err := writeFile(wfile, arReader, os.FileMode(header.Mode), x.DirMode) if err != nil { return size, files, err diff --git a/cpio.go b/cpio.go index 71d9d3d..fd882d0 100644 --- a/cpio.go +++ b/cpio.go @@ -13,7 +13,7 @@ import ( ) // ExtractCPIOGzip extracts a gzip-compressed cpio archive (cpgz). -func ExtractCPIOGzip(xFile *XFile) (int64, []string, error) { +func ExtractCPIOGzip(xFile *XFile) (size int64, filesList []string, err error) { compressedFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -30,7 +30,7 @@ func ExtractCPIOGzip(xFile *XFile) (int64, []string, error) { } // ExtractCPIO extracts a .cpio file. -func ExtractCPIO(xFile *XFile) (int64, []string, error) { +func ExtractCPIO(xFile *XFile) (size int64, filesList []string, err error) { fileReader, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) diff --git a/decompress.go b/decompress.go index f589ede..0526334 100644 --- a/decompress.go +++ b/decompress.go @@ -18,7 +18,7 @@ import ( ) // ExtractXZ extracts an XZ-compressed file. A single file. -func ExtractXZ(xFile *XFile) (int64, []string, error) { +func ExtractXZ(xFile *XFile) (size int64, filesList []string, err error) { compressedFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -33,7 +33,7 @@ func ExtractXZ(xFile *XFile) (int64, []string, error) { // Get the absolute path of the file being written. wfile := xFile.clean(xFile.FilePath, ".xz") - size, err := writeFile(wfile, zipReader, xFile.FileMode, xFile.DirMode) + size, err = writeFile(wfile, zipReader, xFile.FileMode, xFile.DirMode) if err != nil { return size, nil, err } @@ -42,7 +42,7 @@ func ExtractXZ(xFile *XFile) (int64, []string, error) { } // ExtractZlib extracts a zlib-compressed file. A single file. -func ExtractZlib(xFile *XFile) (int64, []string, error) { +func ExtractZlib(xFile *XFile) (size int64, filesList []string, err error) { compressedFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -58,7 +58,7 @@ func ExtractZlib(xFile *XFile) (int64, []string, error) { // Get the absolute path of the file being written. wfile := xFile.clean(xFile.FilePath, ".zz", ".zlib") - size, err := writeFile(wfile, zipReader, xFile.FileMode, xFile.DirMode) + size, err = writeFile(wfile, zipReader, xFile.FileMode, xFile.DirMode) if err != nil { return size, nil, err } @@ -67,7 +67,7 @@ func ExtractZlib(xFile *XFile) (int64, []string, error) { } // ExtractLZMA extracts an lzma-compressed file. A single file. -func ExtractLZMA(xFile *XFile) (int64, []string, error) { +func ExtractLZMA(xFile *XFile) (size int64, filesList []string, err error) { compressedFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -82,7 +82,7 @@ func ExtractLZMA(xFile *XFile) (int64, []string, error) { // Get the absolute path of the file being written. wfile := xFile.clean(xFile.FilePath, ".lzma", ".lz", ".lzip") - size, err := writeFile(wfile, zipReader, xFile.FileMode, xFile.DirMode) + size, err = writeFile(wfile, zipReader, xFile.FileMode, xFile.DirMode) if err != nil { return size, nil, err } @@ -91,7 +91,7 @@ func ExtractLZMA(xFile *XFile) (int64, []string, error) { } // ExtractLZMA2 extracts an lzma2-compressed file. A single file. -func ExtractLZMA2(xFile *XFile) (int64, []string, error) { +func ExtractLZMA2(xFile *XFile) (size int64, filesList []string, err error) { compressedFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -106,7 +106,7 @@ func ExtractLZMA2(xFile *XFile) (int64, []string, error) { // Get the absolute path of the file being written. wfile := xFile.clean(xFile.FilePath, ".lzma", ".lzma2") - size, err := writeFile(wfile, zipReader, xFile.FileMode, xFile.DirMode) + size, err = writeFile(wfile, zipReader, xFile.FileMode, xFile.DirMode) if err != nil { return size, nil, err } @@ -115,7 +115,7 @@ func ExtractLZMA2(xFile *XFile) (int64, []string, error) { } // ExtractZstandard extracts a Zstandard-compressed file. A single file. -func ExtractZstandard(xFile *XFile) (int64, []string, error) { +func ExtractZstandard(xFile *XFile) (size int64, filesList []string, err error) { compressedFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -131,7 +131,7 @@ func ExtractZstandard(xFile *XFile) (int64, []string, error) { // Get the absolute path of the file being written. wfile := xFile.clean(xFile.FilePath, ".zstd", ".zst") - size, err := writeFile(wfile, zipReader, xFile.FileMode, xFile.DirMode) + size, err = writeFile(wfile, zipReader, xFile.FileMode, xFile.DirMode) if err != nil { return size, nil, err } @@ -140,7 +140,7 @@ func ExtractZstandard(xFile *XFile) (int64, []string, error) { } // ExtractLZW extracts an LZW-compressed file. A single file. -func ExtractLZW(xFile *XFile) (int64, []string, error) { +func ExtractLZW(xFile *XFile) (size int64, filesList []string, err error) { compressedFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -155,7 +155,7 @@ func ExtractLZW(xFile *XFile) (int64, []string, error) { // Get the absolute path of the file being written. wfile := xFile.clean(xFile.FilePath, ".Z") - size, err := writeFile(wfile, zipReader, xFile.FileMode, xFile.DirMode) + size, err = writeFile(wfile, zipReader, xFile.FileMode, xFile.DirMode) if err != nil { return size, nil, err } @@ -164,7 +164,7 @@ func ExtractLZW(xFile *XFile) (int64, []string, error) { } // ExtractLZ4 extracts an LZ4-compressed file. A single file. -func ExtractLZ4(xFile *XFile) (int64, []string, error) { +func ExtractLZ4(xFile *XFile) (size int64, filesList []string, err error) { compressedFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -174,7 +174,7 @@ func ExtractLZ4(xFile *XFile) (int64, []string, error) { // Get the absolute path of the file being written. wfile := xFile.clean(xFile.FilePath, ".lz4") - size, err := writeFile(wfile, lz4.NewReader(compressedFile), xFile.FileMode, xFile.DirMode) + size, err = writeFile(wfile, lz4.NewReader(compressedFile), xFile.FileMode, xFile.DirMode) if err != nil { return size, nil, err } @@ -183,7 +183,7 @@ func ExtractLZ4(xFile *XFile) (int64, []string, error) { } // ExtractSnappy extracts a snappy-compressed file. A single file. -func ExtractSnappy(xFile *XFile) (int64, []string, error) { +func ExtractSnappy(xFile *XFile) (size int64, filesList []string, err error) { compressedFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -193,7 +193,7 @@ func ExtractSnappy(xFile *XFile) (int64, []string, error) { // Get the absolute path of the file being written. wfile := xFile.clean(xFile.FilePath, ".snappy", ".sz") - size, err := writeFile(wfile, snappy.NewReader(compressedFile), xFile.FileMode, xFile.DirMode) + size, err = writeFile(wfile, snappy.NewReader(compressedFile), xFile.FileMode, xFile.DirMode) if err != nil { return size, nil, err } @@ -202,7 +202,7 @@ func ExtractSnappy(xFile *XFile) (int64, []string, error) { } // ExtractS2 extracts a Snappy2-compressed file. A single file. -func ExtractS2(xFile *XFile) (int64, []string, error) { +func ExtractS2(xFile *XFile) (size int64, filesList []string, err error) { compressedFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -212,7 +212,7 @@ func ExtractS2(xFile *XFile) (int64, []string, error) { // Get the absolute path of the file being written. wfile := xFile.clean(xFile.FilePath, ".s2") - size, err := writeFile(wfile, s2.NewReader(compressedFile), xFile.FileMode, xFile.DirMode) + size, err = writeFile(wfile, s2.NewReader(compressedFile), xFile.FileMode, xFile.DirMode) if err != nil { return size, nil, err } @@ -221,7 +221,7 @@ func ExtractS2(xFile *XFile) (int64, []string, error) { } // ExtractBrotli extracts a Brotli-compressed file. A single file. -func ExtractBrotli(xFile *XFile) (int64, []string, error) { +func ExtractBrotli(xFile *XFile) (size int64, filesList []string, err error) { compressedFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -231,7 +231,7 @@ func ExtractBrotli(xFile *XFile) (int64, []string, error) { // Get the absolute path of the file being written. wfile := xFile.clean(xFile.FilePath, ".brotli", ".br") - size, err := writeFile(wfile, brotli.NewReader(compressedFile), xFile.FileMode, xFile.DirMode) + size, err = writeFile(wfile, brotli.NewReader(compressedFile), xFile.FileMode, xFile.DirMode) if err != nil { return size, nil, err } @@ -240,7 +240,7 @@ func ExtractBrotli(xFile *XFile) (int64, []string, error) { } // ExtractBzip extracts a bzip2-compressed file. That is, a single file. -func ExtractBzip(xFile *XFile) (int64, []string, error) { +func ExtractBzip(xFile *XFile) (size int64, filesList []string, err error) { compressedFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -250,7 +250,7 @@ func ExtractBzip(xFile *XFile) (int64, []string, error) { // Get the absolute path of the file being written. wfile := xFile.clean(xFile.FilePath, ".bz", ".bz2") - size, err := writeFile(wfile, bzip2.NewReader(compressedFile), xFile.FileMode, xFile.DirMode) + size, err = writeFile(wfile, bzip2.NewReader(compressedFile), xFile.FileMode, xFile.DirMode) if err != nil { return size, nil, err } @@ -259,7 +259,7 @@ func ExtractBzip(xFile *XFile) (int64, []string, error) { } // ExtractGzip extracts a gzip-compressed file. That is, a single file. -func ExtractGzip(xFile *XFile) (int64, []string, error) { +func ExtractGzip(xFile *XFile) (size int64, filesList []string, err error) { compressedFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -275,7 +275,7 @@ func ExtractGzip(xFile *XFile) (int64, []string, error) { // Get the absolute path of the file being written. wfile := xFile.clean(xFile.FilePath, ".gz") - size, err := writeFile(wfile, zipReader, xFile.FileMode, xFile.DirMode) + size, err = writeFile(wfile, zipReader, xFile.FileMode, xFile.DirMode) if err != nil { return size, nil, err } diff --git a/files.go b/files.go index 94a3b5b..fcd1284 100644 --- a/files.go +++ b/files.go @@ -11,7 +11,7 @@ import ( "strings" ) -// ArchiveList is the value returned when searchying for compressed files. +// ArchiveList is the value returned when searching for compressed files. // The map is directory to list of archives in that directory. type ArchiveList map[string][]string @@ -111,6 +111,8 @@ type XFile struct { Password string // (RAR/7z) Archive passwords (to try multiple). Passwords []string + // Logger allows printing debug messages. + log Logger } // Filter is the input to find compressed files. @@ -129,22 +131,38 @@ type Filter struct { // Exclude represents an exclusion list. type Exclude []string -// GetFileList returns all the files in a path. -// This is non-recursive and only returns files _in_ the base path provided. -// This is a helper method and only exposed for convenience. You do not have to call this. -func (x *Xtractr) GetFileList(path string) ([]string, error) { - if stat, err := os.Stat(path); err == nil && !stat.IsDir() { - return []string{path}, nil +// Debugf calls the debug method on the logger if it's not nil. +func (x *XFile) Debugf(format string, v ...any) { + if x.log != nil { + x.log.Debugf(format, v...) } +} - fileList, err := os.ReadDir(path) - if err != nil { - return nil, fmt.Errorf("reading path %s: %w", path, err) - } +// GetFileList returns all the files in a path or paths. +// This is non-recursive and only returns files _in_ the base paths provided. +// This is a helper method and only exposed for convenience. You do not have to call this. +func (x *Xtractr) GetFileList(paths ...string) ([]string, error) { + files := []string{} + + for _, path := range paths { + stat, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("stat: %w", err) + } + + if !stat.IsDir() { + files = append(files, path) + continue + } + + fileList, err := os.ReadDir(path) + if err != nil { + return nil, fmt.Errorf("reading path %s: %w", path, err) + } - files := make([]string, len(fileList)) - for idx, file := range fileList { - files[idx] = filepath.Join(path, file.Name()) + for _, file := range fileList { + files = append(files, filepath.Join(path, file.Name())) + } } return files, nil @@ -153,7 +171,7 @@ func (x *Xtractr) GetFileList(path string) ([]string, error) { // Difference returns all the strings that are in slice2 but not in slice1. // Used to find new files in a file list from a path. ie. those we extracted. // This is a helper method and only exposed for convenience. You do not have to call this. -func Difference(slice1 []string, slice2 []string) []string { +func Difference(slice1, slice2 []string) []string { diff := []string{} for _, s2p := range slice2 { @@ -267,7 +285,7 @@ func getCompressedFiles(path string, filter *Filter, fileList []os.FileInfo, dep files[k] = v } case strings.HasSuffix(lowerName, ".rar"): - hasParts := regexp.MustCompile(`.*\.part[0-9]+\.rar$`) + hasParts := regexp.MustCompile(`.*\.part\d+\.rar$`) partOne := regexp.MustCompile(`.*\.part0*1\.rar$`) // Some archives are named poorly. Only return part01 or part001, not all. if !hasParts.MatchString(lowerName) || partOne.MatchString(lowerName) { @@ -286,13 +304,13 @@ func getCompressedFiles(path string, filter *Filter, fileList []os.FileInfo, dep // Extract calls the correct procedure for the type of file being extracted. // Returns size of extracted data, list of extracted files, and/or error. -func (x *XFile) Extract() (int64, []string, []string, error) { +func (x *XFile) Extract() (size int64, filesList, archiveList []string, err error) { return ExtractFile(x) } // ExtractFile calls the correct procedure for the type of file being extracted. // Returns size of extracted data, list of extracted files, list of archives processed, and/or error. -func ExtractFile(xFile *XFile) (int64, []string, []string, error) { +func ExtractFile(xFile *XFile) (size int64, filesList, archiveList []string, err error) { sName := strings.ToLower(xFile.FilePath) for _, ext := range extension2function { @@ -307,7 +325,7 @@ func ExtractFile(xFile *XFile) (int64, []string, []string, error) { // MoveFiles relocates files then removes the folder they were in. // Returns the new file paths. // This is a helper method and only exposed for convenience. You do not have to call this. -func (x *Xtractr) MoveFiles(fromPath string, toPath string, overwrite bool) ([]string, error) { //nolint:cyclop +func (x *Xtractr) MoveFiles(fromPath, toPath string, overwrite bool) ([]string, error) { //nolint:cyclop var ( newFiles = []string{} keepErr error @@ -323,6 +341,8 @@ func (x *Xtractr) MoveFiles(fromPath string, toPath string, overwrite bool) ([]s toPath = strings.TrimSuffix(toPath, filepath.Ext(toPath)) } + x.config.Debugf("Moving files: %v (%d files) -> %v", fromPath, len(files), toPath) + if err := os.MkdirAll(toPath, x.config.DirMode); err != nil { return nil, fmt.Errorf("os.MkdirAll: %w", err) } @@ -405,22 +425,21 @@ func (x *Xtractr) Rename(oldpath, newpath string) error { if err != nil { return fmt.Errorf("os.Open(): %w", err) } + defer oldFile.Close() newFile, err := os.OpenFile(newpath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, x.config.FileMode) if err != nil { - oldFile.Close() return fmt.Errorf("os.OpenFile(): %w", err) } defer newFile.Close() _, err = io.Copy(newFile, oldFile) - oldFile.Close() - if err != nil { return fmt.Errorf("io.Copy(): %w", err) } // The copy was successful, so now delete the original file + _ = oldFile.Close() // Needs to be closed before delete. _ = os.Remove(oldpath) return nil @@ -485,3 +504,19 @@ func (a ArchiveList) Random() []string { return nil } + +// List returns all of the archives as a string slice. +func (a ArchiveList) List() []string { + list := []string{} + + for _, files := range a { + list = append(list, files...) + } + + return list +} + +// SetLogger sets the logger interface on an XFile. Useful when you need to debug what it's doing. +func (x *XFile) SetLogger(logger Logger) { + x.log = logger +} diff --git a/files_test.go b/files_test.go index 621461b..f95bbaf 100644 --- a/files_test.go +++ b/files_test.go @@ -38,8 +38,7 @@ func createTestPaths(t *testing.T) string { fileMode = 0o644 ) - base, err := os.MkdirTemp("", "GoTest_*_Path") - require.NoError(t, err, "cannot create temp directory for testing") + base := t.TempDir() for _, testPath := range testFileList { testPath = filepath.Join(base, testPath) @@ -56,7 +55,6 @@ func TestFindCompressedFiles(t *testing.T) { t.Parallel() base := createTestPaths(t) - defer os.RemoveAll(base) // Test 1 total := 0 diff --git a/go.mod b/go.mod index c8467b8..12ff370 100644 --- a/go.mod +++ b/go.mod @@ -1,34 +1,36 @@ module golift.io/xtractr -go 1.19 +go 1.23.0 + +toolchain go1.24.2 require ( - github.com/andybalholm/brotli v1.1.0 - github.com/bodgit/sevenzip v1.5.1 + github.com/andybalholm/brotli v1.1.1 + github.com/bodgit/sevenzip v1.6.0 github.com/cavaliergopher/cpio v1.0.1 + github.com/cavaliergopher/rpm v1.2.0 + github.com/dsnet/compress v0.0.1 github.com/kdomanski/iso9660 v0.4.0 - github.com/klauspost/compress v1.17.9 + github.com/klauspost/compress v1.18.0 github.com/nwaples/rardecode v1.1.3 github.com/peterebden/ar v0.0.0-20230524111245-4f7c7b065694 - github.com/pierrec/lz4/v4 v4.1.21 + github.com/pierrec/lz4/v4 v4.1.22 github.com/sshaman1101/dcompress v0.0.0-20200109162717-50436a6332de github.com/stretchr/testify v1.9.0 github.com/therootcompany/xz v1.0.1 + github.com/ulikunitz/xz v0.5.12 ) require ( github.com/bodgit/plumbing v1.3.0 // indirect github.com/bodgit/windows v1.0.1 // indirect - github.com/cavaliergopher/rpm v1.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dsnet/compress v0.0.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/ulikunitz/xz v0.5.12 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/text v0.24.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 9400328..6465cd4 100644 --- a/go.sum +++ b/go.sum @@ -17,12 +17,12 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= -github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= -github.com/bodgit/sevenzip v1.5.1 h1:rVj0baZsooZFy64DJN0zQogPzhPrT8BQ8TTRd1H4WHw= -github.com/bodgit/sevenzip v1.5.1/go.mod h1:Q3YMySuVWq6pyGEolyIE98828lOfEoeWg5zeH6x22rc= +github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A= +github.com/bodgit/sevenzip v1.6.0/go.mod h1:zOBh9nJUof7tcrlqJFv1koWRrhz3LbDbUNngkuZxLMc= github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/cavaliergopher/cpio v1.0.1 h1:KQFSeKmZhv0cr+kawA3a0xTQCU4QxXF1vhU7P7av2KM= @@ -85,8 +85,8 @@ github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGH github.com/kdomanski/iso9660 v0.4.0/go.mod h1:OxUSupHsO9ceI8lBLPJKWBTphLemjrCQY8LPXM7qSzU= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -97,8 +97,8 @@ github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9l github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/peterebden/ar v0.0.0-20230524111245-4f7c7b065694 h1:pDBk3JWSIjS3gNxwEk1RjGdyZLsyTW4pOHaShBs9FK8= github.com/peterebden/ar v0.0.0-20230524111245-4f7c7b065694/go.mod h1:hpFkyhCgB5Rm8FK+ISypOE+9UyrCuL6MNcjPMB1s1ec= -github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= -github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= +github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -120,6 +120,8 @@ github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0B github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -187,7 +189,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -215,8 +218,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/iso.go b/iso.go index eff7d3d..4137bde 100644 --- a/iso.go +++ b/iso.go @@ -10,7 +10,7 @@ import ( ) // ExtractISO writes an ISO's contents to disk. -func ExtractISO(xFile *XFile) (int64, []string, error) { +func ExtractISO(xFile *XFile) (size int64, filesList []string, err error) { openISO, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("failed to open iso file: %s: %w", xFile.FilePath, err) @@ -67,16 +67,19 @@ func (x *XFile) uniso(isoFile *iso9660.File, parent string) (int64, []string, er return size, files, nil } -func (x *XFile) unisofile(isoFile *iso9660.File, fileName string) (int64, []string, error) { - destFile := x.clean(fileName) +func (x *XFile) unisofile(isoFile *iso9660.File, wfile string) (int64, []string, error) { + wfile = x.clean(wfile) + //nolint:gocritic // this 1-argument filepath.Join removes a ./ prefix should there be one. - if !strings.HasPrefix(destFile, filepath.Join(x.OutputDir)) { + if !strings.HasPrefix(wfile, filepath.Join(x.OutputDir)) { // The file being written is trying to write outside of our base path. Malicious ISO? return 0, nil, fmt.Errorf("%s: %w: %s != %s (from: %s)", - x.FilePath, ErrInvalidPath, destFile, x.OutputDir, isoFile.Name()) + x.FilePath, ErrInvalidPath, wfile, x.OutputDir, isoFile.Name()) } - size, err := writeFile(destFile, isoFile.Reader(), x.FileMode, x.DirMode) + x.Debugf("Writing archived file: %s (bytes: %d)", wfile, isoFile.Size()) + + size, err := writeFile(wfile, isoFile.Reader(), x.FileMode, x.DirMode) - return size, []string{destFile}, err + return size, []string{wfile}, err } diff --git a/iso_test.go b/iso_test.go index 7dc094c..9fbf17f 100644 --- a/iso_test.go +++ b/iso_test.go @@ -1,7 +1,6 @@ package xtractr_test import ( - "fmt" "os" "path/filepath" "strings" @@ -17,7 +16,6 @@ func TestIso(t *testing.T) { t.Parallel() testFilesInfo := createTestFiles(t) - writer, err := iso9660.NewWriter() require.NoError(t, err, "failed to create writer") defer func() { @@ -25,44 +23,47 @@ func TestIso(t *testing.T) { require.NoError(t, err, "failed to cleanup writer") }() + size := int64(0) walkErr := filepath.Walk(testFilesInfo.srcFilesDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return fmt.Errorf("unexpected error: %w", err) - } + require.NoError(t, err, "unexpected") + if info.IsDir() { return nil } fileToAdd, err := os.Open(path) - if err != nil { - return fmt.Errorf("failed to open file: %w", err) - } - defer fileToAdd.Close() + require.NoError(t, err, "failed to open file") + defer safeCloser(t, fileToAdd) + + fStat, err := fileToAdd.Stat() + require.NoError(t, err, "failed to stat file") + size += fStat.Size() + + err = writer.AddFile(fileToAdd, strings.TrimPrefix(fileToAdd.Name(), testFilesInfo.srcFilesDir)) + require.NoError(t, err, "failed to add file") - err = writer.AddFile(fileToAdd, strings.TrimLeft(fileToAdd.Name(), testFilesInfo.srcFilesDir)) - if err != nil { - return fmt.Errorf("failed to add file: %w", err) - } return nil }) require.NoError(t, walkErr, "failed to walk files") isoFileName := filepath.Join(testFilesInfo.dstFilesDir, "archive.iso") + isoFile, err := os.Create(isoFileName) - defer safeCloser(t, isoFile) require.NoError(t, err, "failed to create ISO file") + defer safeCloser(t, isoFile) err = writer.WriteTo(isoFile, "test") require.NoError(t, err, "failed to write ISO") - size, files, archives, err := xtractr.ExtractFile(&xtractr.XFile{ + xSize, files, archives, err := xtractr.ExtractFile(&xtractr.XFile{ FilePath: isoFileName, OutputDir: filepath.Clean(testFilesInfo.dstFilesDir), FileMode: 0o600, DirMode: 0o700, }) require.NoError(t, err) - assert.Equal(t, int64(testFilesInfo.dataSize), size) + assert.Equal(t, testFilesInfo.dataSize, size, "data written does not match predetermined size") + assert.Equal(t, testFilesInfo.dataSize, xSize, "data extracted does not match predetermined size") assert.Len(t, files, testFilesInfo.fileCount) assert.Len(t, archives, testFilesInfo.archiveCount) } diff --git a/queue.go b/queue.go index 0f9496b..3e3b219 100644 --- a/queue.go +++ b/queue.go @@ -103,7 +103,7 @@ func (x *Xtractr) extract(ext *Xtract) { resp := &Response{ X: ext, Started: time.Now(), - Output: strings.TrimRight(ext.Filter.Path, `/\`) + x.config.Suffix, // tmp folder. + Output: strings.TrimRight(ext.Path, `/\`) + x.config.Suffix, // tmp folder. Archives: FindCompressedFiles(ext.Filter), Queued: len(x.queue), } @@ -160,7 +160,7 @@ func (x *Xtractr) decompressFolders(resp *Response) error { X: &Xtract{ Filter: Filter{ Path: subDir, - ExcludeSuffix: resp.X.Filter.ExcludeSuffix, + ExcludeSuffix: resp.X.ExcludeSuffix, }, Name: resp.X.Name, Password: resp.X.Password, @@ -327,6 +327,7 @@ func (x *Xtractr) processArchive(filename string, resp *Response) (int64, []stri DirMode: x.config.DirMode, Passwords: resp.X.Passwords, Password: resp.X.Password, + log: x.config.Logger, }) if err != nil { x.DeleteFiles(resp.Output) // clean up the mess after an error and bail. diff --git a/queue_test.go b/queue_test.go index 073db60..46b37ac 100644 --- a/queue_test.go +++ b/queue_test.go @@ -79,8 +79,8 @@ func TestWithTempFolder(t *testing.T) { // test written files here? // each directory should have its own files. - os.RemoveAll(xFile.Path) - os.RemoveAll(xFile.Path + xtractr.DefaultSuffix) + _ = os.RemoveAll(xFile.Path) + _ = os.RemoveAll(xFile.Path + xtractr.DefaultSuffix) } func TestNoTempFolder(t *testing.T) { @@ -117,16 +117,15 @@ func TestNoTempFolder(t *testing.T) { // test written files here? // each directory should have its own files. - os.RemoveAll(xFile.Path) - os.RemoveAll(xFile.Path + xtractr.DefaultSuffix) + _ = os.RemoveAll(xFile.Path) + _ = os.RemoveAll(xFile.Path + xtractr.DefaultSuffix) } // testSetupTestDir creates a temp directory with 4 copies of a rar archive in it. func testSetupTestDir(t *testing.T) string { t.Helper() - name, err := os.MkdirTemp(".", "xtractr_test_*_data") - require.NoError(t, err, "creating temp directory failed") + name := t.TempDir() testFileData, err := os.ReadFile(testFile) require.NoError(t, err, "reading test data file failed") @@ -153,7 +152,7 @@ func makeFile(t *testing.T, data []byte, fileName string) error { if err != nil { return err } - defer openFile.Close() + defer safeCloser(t, openFile) _, err = openFile.Write(data) diff --git a/rar.go b/rar.go index 7627f36..166935d 100644 --- a/rar.go +++ b/rar.go @@ -13,7 +13,8 @@ import ( "github.com/nwaples/rardecode" ) -func ExtractRAR(xFile *XFile) (int64, []string, []string, error) { +// ExtractRAR attempts to extract a file as a rar file. +func ExtractRAR(xFile *XFile) (size int64, filesList, archiveList []string, err error) { if len(xFile.Passwords) == 0 && xFile.Password == "" { return extractRAR(xFile) } @@ -54,7 +55,7 @@ func ExtractRAR(xFile *XFile) (int64, []string, []string, error) { }) } -// ExtractRAR extracts a rar file. to a destination. This wraps github.com/nwaples/rardecode. +// extractRAR extracts a rar file. to a destination. This wraps github.com/nwaples/rardecode. func extractRAR(xFile *XFile) (int64, []string, []string, error) { rarReader, err := rardecode.OpenReader(xFile.FilePath, xFile.Password) if err != nil { @@ -100,6 +101,8 @@ func (x *XFile) unrar(rarReader *rardecode.ReadCloser) (int64, []string, error) } if header.IsDir { + x.Debugf("Writing archived directory: %s", wfile) + if err = os.MkdirAll(wfile, x.DirMode); err != nil { return size, files, fmt.Errorf("os.MkdirAll: %w", err) } @@ -107,9 +110,7 @@ func (x *XFile) unrar(rarReader *rardecode.ReadCloser) (int64, []string, error) continue } - if err = os.MkdirAll(filepath.Dir(wfile), x.DirMode); err != nil { - return size, files, fmt.Errorf("os.MkdirAll: %w", err) - } + x.Debugf("Writing archived file: %s (packed: %d, unpacked: %d)", wfile, header.PackedSize, header.UnPackedSize) fSize, err := writeFile(wfile, rarReader, x.FileMode, x.DirMode) if err != nil { @@ -118,5 +119,6 @@ func (x *XFile) unrar(rarReader *rardecode.ReadCloser) (int64, []string, error) files = append(files, wfile) size += fSize + x.Debugf("Wrote archived file: %s (%d bytes), total: %d files and %d bytes", wfile, fSize, len(files), size) } } diff --git a/rar_test.go b/rar_test.go index d047726..0e6229d 100644 --- a/rar_test.go +++ b/rar_test.go @@ -1,7 +1,6 @@ package xtractr_test import ( - "os" "testing" "github.com/stretchr/testify/assert" @@ -12,16 +11,13 @@ import ( func TestExtractRAR(t *testing.T) { t.Parallel() - name, err := os.MkdirTemp(".", "xtractr_test_*_data") - require.NoError(t, err, "creating temp directory failed") - defer os.RemoveAll(name) - size, files, archives, err := xtractr.ExtractRAR(&xtractr.XFile{ FilePath: "./test_data/archive.rar", - OutputDir: name, + OutputDir: t.TempDir(), Password: "testing", // one of these is right. :) Passwords: []string{"testingmore", "some_password", "some_other"}, }) + require.NoError(t, err) assert.Equal(t, testDataSize, size) assert.Len(t, archives, 1) diff --git a/rpm.go b/rpm.go index b420f9b..ab3413b 100644 --- a/rpm.go +++ b/rpm.go @@ -14,12 +14,14 @@ import ( "github.com/ulikunitz/xz/lzma" ) +// Errors returned when decompressing RAR archives. var ( ErrUnsupportedRPMCompression = errors.New("unsupported rpm compression") ErrUnsupportedRPMArchiveFmt = errors.New("unsupported rpm archive format") ) -func ExtractRPM(xFile *XFile) (int64, []string, error) { //nolint:cyclop +// ExtractRPM extract a file as a RedHat Package Manager file. +func ExtractRPM(xFile *XFile) (size int64, filesList []string, err error) { //nolint:cyclop rpmFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -78,7 +80,7 @@ func ExtractRPM(xFile *XFile) (int64, []string, error) { //nolint:cyclop } } -func (x *XFile) unrpm(reader io.Reader, format string) (int64, []string, error) { +func (x *XFile) unrpm(reader io.Reader, format string) (size int64, filesList []string, err error) { // Check the archive format of the payload switch format { case "cpio": diff --git a/start.go b/start.go index 296293c..538821e 100644 --- a/start.go +++ b/start.go @@ -91,7 +91,7 @@ func (x *Xtractr) Start() error { x.queue = make(chan *Xtract, x.config.BuffSize) - for i := 0; i < x.config.Parallel; i++ { + for range x.config.Parallel { go x.processQueue() } @@ -137,7 +137,7 @@ func (x *Xtractr) Stop() { close(x.queue) // Wait until all running extractions are done. - for i := 0; i < x.config.Parallel; i++ { + for range x.config.Parallel { <-x.done } diff --git a/tar.go b/tar.go index 11966e5..69f42fd 100644 --- a/tar.go +++ b/tar.go @@ -16,7 +16,7 @@ import ( ) // ExtractTar extracts a raw (non-compressed) tar archive. -func ExtractTar(xFile *XFile) (int64, []string, error) { +func ExtractTar(xFile *XFile) (size int64, filesList []string, err error) { tarFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -27,7 +27,7 @@ func ExtractTar(xFile *XFile) (int64, []string, error) { } // ExtractTarBzip extracts a bzip2-compressed tar archive. -func ExtractTarBzip(xFile *XFile) (int64, []string, error) { +func ExtractTarBzip(xFile *XFile) (size int64, filesList []string, err error) { compressedFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -38,7 +38,7 @@ func ExtractTarBzip(xFile *XFile) (int64, []string, error) { } // ExtractTarXZ extracts an XZ-compressed tar archive (txz). -func ExtractTarXZ(xFile *XFile) (int64, []string, error) { +func ExtractTarXZ(xFile *XFile) (size int64, filesList []string, err error) { compressedFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -54,7 +54,7 @@ func ExtractTarXZ(xFile *XFile) (int64, []string, error) { } // ExtractTarZ extracts an LZW-compressed tar archive (tz). -func ExtractTarZ(xFile *XFile) (int64, []string, error) { +func ExtractTarZ(xFile *XFile) (size int64, filesList []string, err error) { compressedFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -70,7 +70,7 @@ func ExtractTarZ(xFile *XFile) (int64, []string, error) { } // ExtractTarGzip extracts a gzip-compressed tar archive (tgz). -func ExtractTarGzip(xFile *XFile) (int64, []string, error) { +func ExtractTarGzip(xFile *XFile) (size int64, filesList []string, err error) { compressedFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -87,7 +87,7 @@ func ExtractTarGzip(xFile *XFile) (int64, []string, error) { } // ExtractTarLzip extracts an LZIP-compressed tar archive (tlz). -func ExtractTarLzip(xFile *XFile) (int64, []string, error) { +func ExtractTarLzip(xFile *XFile) (size int64, filesList []string, err error) { compressedFile, err := os.Open(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("os.Open: %w", err) @@ -126,6 +126,8 @@ func (x *XFile) untar(reader io.Reader) (int64, []string, error) { } if header.Typeflag == tar.TypeDir { + x.Debugf("Writing archived directory: %s", wfile) + if err = os.MkdirAll(wfile, header.FileInfo().Mode()); err != nil { return size, files, fmt.Errorf("os.MkdirAll: %w", err) } @@ -133,6 +135,8 @@ func (x *XFile) untar(reader io.Reader) (int64, []string, error) { continue } + x.Debugf("Writing archived file: %s (bytes: %d)", wfile, header.FileInfo().Size()) + fSize, err := writeFile(wfile, tarReader, header.FileInfo().Mode(), x.DirMode) if err != nil { return size, files, err @@ -140,5 +144,6 @@ func (x *XFile) untar(reader io.Reader) (int64, []string, error) { files = append(files, wfile) size += fSize + x.Debugf("Wrote archived file: %s (%d bytes), total: %d files and %d bytes", wfile, fSize, len(files), size) } } diff --git a/tar_test.go b/tar_test.go index adc5f9e..bc95ade 100644 --- a/tar_test.go +++ b/tar_test.go @@ -66,7 +66,7 @@ func TestTar(t *testing.T) { DirMode: 0o700, }) require.NoError(t, err) - assert.Equal(t, int64(testFilesInfo.dataSize), size) + assert.Equal(t, testFilesInfo.dataSize, size) assert.Len(t, files, testFilesInfo.fileCount) assert.Len(t, archives, testFilesInfo.archiveCount) }) @@ -80,7 +80,7 @@ func writeTar(sourceDir string, destWriter io.Writer) error { return nil } relativePath := path[len(sourceDir):] - if len(relativePath) == 0 { + if relativePath == "" { return nil } fileReader, err := os.Open(path) @@ -109,16 +109,17 @@ func writeTar(sourceDir string, destWriter io.Writer) error { return fmt.Errorf("failed to walk source directory: %w", outErr) } -func (c *tarCompressor) Compress(t *testing.T, sourceDir string, destBase string) error { +func (c *tarCompressor) Compress(t *testing.T, sourceDir, destBase string) error { t.Helper() + tarFile, err := os.Create(destBase + ".tar") - defer safeCloser(t, tarFile) require.NoError(t, err) + defer safeCloser(t, tarFile) return writeTar(sourceDir, tarFile) } -func (c *tarZCompressor) Compress(t *testing.T, _ string, destBase string) error { +func (c *tarZCompressor) Compress(t *testing.T, _, destBase string) error { t.Helper() // No native Go library for .tar.Z and compress Unix utility is not available on @@ -126,27 +127,30 @@ func (c *tarZCompressor) Compress(t *testing.T, _ string, destBase string) error tarZFilename := destBase + ".tar.z" tarZDestFile, err := os.Create(tarZFilename) require.NoError(t, err) + defer safeCloser(t, tarZDestFile) tarZTestFile, err := os.Open("test_data/archive.tar.Z") require.NoError(t, err) + defer safeCloser(t, tarZTestFile) written, err := io.Copy(tarZDestFile, tarZTestFile) - assert.Greater(t, written, int64(0)) + assert.Positive(t, written) require.NoError(t, err) return nil } -func (c *tarBzipCompressor) Compress(t *testing.T, sourceDir string, destBase string) error { +func (c *tarBzipCompressor) Compress(t *testing.T, sourceDir, destBase string) error { t.Helper() tarBZ2Filename := destBase + ".tar.bz2" tarBZ2File, err := os.Create(tarBZ2Filename) require.NoError(t, err) + defer safeCloser(t, tarBZ2File) bzip2Writer, err := bzip2.NewWriter(tarBZ2File, &bzip2.WriterConfig{Level: bzip2.BestSpeed}) - defer safeCloser(t, bzip2Writer) require.NoError(t, err) + defer safeCloser(t, bzip2Writer) err = writeTar(sourceDir, bzip2Writer) require.NoError(t, err) @@ -154,16 +158,17 @@ func (c *tarBzipCompressor) Compress(t *testing.T, sourceDir string, destBase st return nil } -func (c *tarXZCompressor) Compress(t *testing.T, sourceDir string, destBase string) error { +func (c *tarXZCompressor) Compress(t *testing.T, sourceDir, destBase string) error { t.Helper() tarXZFilename := destBase + ".tar.xz" tarXZFile, err := os.Create(tarXZFilename) require.NoError(t, err) + defer safeCloser(t, tarXZFile) xzWriter, err := xz.NewWriter(tarXZFile) - defer safeCloser(t, xzWriter) require.NoError(t, err) + defer safeCloser(t, xzWriter) err = writeTar(sourceDir, xzWriter) require.NoError(t, err) @@ -171,16 +176,17 @@ func (c *tarXZCompressor) Compress(t *testing.T, sourceDir string, destBase stri return nil } -func (c *tarGzipCompressor) Compress(t *testing.T, sourceDir string, destBase string) error { +func (c *tarGzipCompressor) Compress(t *testing.T, sourceDir, destBase string) error { t.Helper() tarGZFilename := destBase + ".tar.gz" tarGZFile, err := os.Create(tarGZFilename) require.NoError(t, err) + defer safeCloser(t, tarGZFile) gzipWriter := gzip.NewWriter(tarGZFile) - defer gzipWriter.Close() require.NoError(t, err) + defer safeCloser(t, gzipWriter) err = writeTar(sourceDir, gzipWriter) require.NoError(t, err) diff --git a/util_test.go b/util_test.go index b8ff0a1..89a001b 100644 --- a/util_test.go +++ b/util_test.go @@ -14,7 +14,7 @@ import ( type testFilesInfo struct { srcFilesDir string dstFilesDir string - dataSize int + dataSize int64 fileCount int archiveCount int } @@ -30,7 +30,7 @@ Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliqu ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.` - testDataSize = 1544 + testDataSize = 1544 // equals the above * 3 and the randomDigits * 2. testFileCount = 5 testArchiveCount = 1 ) @@ -55,15 +55,9 @@ proident, sunt in culpa qui officia deserunt mollit anim id est laborum.` "level1/level2/level2.bin", } - testDataDir, err := os.MkdirTemp(".", "xtractr_test_*_data") - require.NoError(t, err, "creating temp directory failed") - t.Cleanup(func() { - os.RemoveAll(testDataDir) - }) - + testDataDir := t.TempDir() srcFilesDir := filepath.Join(testDataDir, "sources") - err = os.MkdirAll(srcFilesDir, 0o700) - require.NoError(t, err) + require.NoError(t, os.MkdirAll(srcFilesDir, 0o700)) var destFilesDir string for _, file := range testFiles { @@ -80,8 +74,7 @@ proident, sunt in culpa qui officia deserunt mollit anim id est laborum.` require.NoError(t, err) destFilesDir = filepath.Join(testDataDir, "out") - err = os.MkdirAll(destFilesDir, 0o700) - require.NoError(t, err) + require.NoError(t, os.MkdirAll(destFilesDir, 0o700)) } return &testFilesInfo{ diff --git a/zip.go b/zip.go index 3bee51f..9e35a9a 100644 --- a/zip.go +++ b/zip.go @@ -11,7 +11,7 @@ import ( /* How to extract a ZIP file. */ // ExtractZIP extracts a zip file.. to a destination. Simple enough. -func ExtractZIP(xFile *XFile) (int64, []string, error) { +func ExtractZIP(xFile *XFile) (size int64, filesList []string, err error) { zipReader, err := zip.OpenReader(xFile.FilePath) if err != nil { return 0, nil, fmt.Errorf("zip.OpenReader: %w", err) @@ -19,46 +19,53 @@ func ExtractZIP(xFile *XFile) (int64, []string, error) { defer zipReader.Close() files := []string{} - size := int64(0) + size = int64(0) - for _, zipFile := range zipReader.Reader.File { - fSize, err := xFile.unzip(zipFile) + for _, zipFile := range zipReader.File { + fSize, wfile, err := xFile.unzip(zipFile) if err != nil { return size, files, fmt.Errorf("%s: %w", xFile.FilePath, err) } - files = append(files, filepath.Join(xFile.OutputDir, zipFile.Name)) //nolint: gosec + //nolint:gosec // this is safe because we clean the paths. + files = append(files, filepath.Join(xFile.OutputDir, zipFile.Name)) size += fSize + xFile.Debugf("Wrote archived file: %s (%d bytes), total: %d files and %d bytes", wfile, fSize, len(files), size) } return size, files, nil } -func (x *XFile) unzip(zipFile *zip.File) (int64, error) { //nolint:dupl +func (x *XFile) unzip(zipFile *zip.File) (int64, string, error) { wfile := x.clean(zipFile.Name) if !strings.HasPrefix(wfile, x.OutputDir) { // The file being written is trying to write outside of our base path. Malicious archive? - return 0, fmt.Errorf("%s: %w: %s (from: %s)", zipFile.FileInfo().Name(), ErrInvalidPath, wfile, zipFile.Name) + return 0, wfile, fmt.Errorf("%s: %w: %s (from: %s)", zipFile.FileInfo().Name(), ErrInvalidPath, wfile, zipFile.Name) } - if strings.HasSuffix(wfile, "/") || zipFile.FileInfo().IsDir() { + if zipFile.FileInfo().IsDir() { + x.Debugf("Writing archived directory: %s", wfile) + if err := os.MkdirAll(wfile, x.DirMode); err != nil { - return 0, fmt.Errorf("making zipFile dir: %w", err) + return 0, wfile, fmt.Errorf("making zipFile dir: %w", err) } - return 0, nil + return 0, wfile, nil } + x.Debugf("Writing archived file: %s (packed: %d, unpacked: %d)", wfile, + zipFile.CompressedSize64, zipFile.UncompressedSize64) + zFile, err := zipFile.Open() if err != nil { - return 0, fmt.Errorf("zipFile.Open: %w", err) + return 0, wfile, fmt.Errorf("zipFile.Open: %w", err) } defer zFile.Close() s, err := writeFile(wfile, zFile, x.FileMode, x.DirMode) if err != nil { - return s, fmt.Errorf("%s: %w: %s (from: %s)", zipFile.FileInfo().Name(), err, wfile, zipFile.Name) + return s, wfile, fmt.Errorf("%s: %w: %s (from: %s)", zipFile.FileInfo().Name(), err, wfile, zipFile.Name) } - return s, nil + return s, wfile, nil } diff --git a/zip_test.go b/zip_test.go index 838bc2d..c593956 100644 --- a/zip_test.go +++ b/zip_test.go @@ -14,11 +14,30 @@ import ( func TestZip(t *testing.T) { t.Parallel() + + zip := makeZipFile(t) + + size, files, archives, err := xtractr.ExtractFile(&xtractr.XFile{ + FilePath: zip.srcFilesDir, + OutputDir: filepath.Clean(zip.dstFilesDir), + FileMode: 0o600, + DirMode: 0o700, + }) + require.NoError(t, err) + assert.Equal(t, zip.dataSize, size) + assert.Len(t, files, zip.fileCount) + assert.Len(t, archives, zip.archiveCount) +} + +func makeZipFile(t *testing.T) testFilesInfo { + t.Helper() + const ( - testDataSize = 21 - testFileCount = 5 - testArchiveCount = 1 + dataSize = int64(21) + fileCount = 5 + archiveCount = 1 ) + testFiles := []string{ "README.txt", "subdir/", @@ -27,13 +46,14 @@ func TestZip(t *testing.T) { "subdir/level2/level2file.txt", } - name, err := os.MkdirTemp(".", "xtractr_test_*_data") - require.NoError(t, err, "creating temp directory failed") - defer os.RemoveAll(name) + name := t.TempDir() zipFile, err := os.Create(filepath.Join(name, "archive.zip")) require.NoError(t, err) + defer safeCloser(t, zipFile) + zipWriter := zip.NewWriter(zipFile) + defer safeCloser(t, zipWriter) for _, file := range testFiles { if file[len(file)-1] == '/' { @@ -46,21 +66,12 @@ func TestZip(t *testing.T) { require.NoError(t, err) } } - err = zipWriter.Close() - require.NoError(t, err) - err = zipFile.Close() - require.NoError(t, err) - zipTestFile := filepath.Join(name, "archive.zip") - - size, files, archives, err := xtractr.ExtractFile(&xtractr.XFile{ - FilePath: zipTestFile, - OutputDir: filepath.Clean(name), - FileMode: 0o600, - DirMode: 0o700, - }) - require.NoError(t, err) - assert.Equal(t, int64(testDataSize), size) - assert.Len(t, files, testFileCount) - assert.Len(t, archives, testArchiveCount) + return testFilesInfo{ + srcFilesDir: filepath.Join(name, "archive.zip"), + dstFilesDir: name, + dataSize: dataSize, + fileCount: fileCount, + archiveCount: archiveCount, + } }