From 4296380b40f30f71c2875ea761bca235a77b5c1d Mon Sep 17 00:00:00 2001 From: Xynnn007 Date: Sun, 9 Mar 2025 21:28:55 +0800 Subject: [PATCH 1/5] squashfs: fix blocks start field in FileInode Due to SquashFS, the first field of both File inode and extended file inode is called 'blocks start', that means ``` The offset from the start of the archive to the first data block. ``` Link https://dr-emann.github.io/squashfs/squashfs.html#_file_inodes Signed-off-by: Xynnn007 --- filesystem/squashfs/const_internal_test.go | 8 ++++---- filesystem/squashfs/file.go | 2 +- filesystem/squashfs/finalize.go | 9 +++++---- filesystem/squashfs/inode.go | 18 +++++++++--------- filesystem/squashfs/inode_internal_test.go | 4 ++-- 5 files changed, 21 insertions(+), 20 deletions(-) diff --git a/filesystem/squashfs/const_internal_test.go b/filesystem/squashfs/const_internal_test.go index 008c28d4..6f82d297 100644 --- a/filesystem/squashfs/const_internal_test.go +++ b/filesystem/squashfs/const_internal_test.go @@ -36,7 +36,7 @@ func testGetFirstInodeHeader() *inodeHeader { } func testGetFirstInodeBody() inodeBody { return extendedFile{ - startBlock: 0, + blocksStart: 0, fragmentBlockIndex: 0, fileSize: 7, fragmentOffset: 0, @@ -95,7 +95,7 @@ var ( // this is for /foo/filename_0 testBasicFile = &basicFile{ - startBlock: 0, + blocksStart: 0, fragmentBlockIndex: 0, fileSize: 0xb, fragmentOffset: 0xc, @@ -240,7 +240,7 @@ func GetTestFileSmall(f fs.File, c Compressor) (*File, error) { } testFs.compressor = c ef := &extendedFile{ - startBlock: superblockSize, + blocksStart: superblockSize, fileSize: 7, sparse: 0, links: 0, @@ -269,7 +269,7 @@ func GetTestFileBig(f fs.File, c Compressor) (*File, error) { fragSize := uint64(5) size := uint64(testFs.blocksize) + fragSize ef := &extendedFile{ - startBlock: superblockSize, + blocksStart: superblockSize, fileSize: size, sparse: 0, links: 0, diff --git a/filesystem/squashfs/file.go b/filesystem/squashfs/file.go index d3c08f8f..d632175d 100644 --- a/filesystem/squashfs/file.go +++ b/filesystem/squashfs/file.go @@ -47,7 +47,7 @@ func (fl *File) Read(b []byte) (int, error) { // 5- read in and uncompress the necessary blocks fs := fl.filesystem size := fl.size() - fl.offset - location := int64(fl.startBlock) + location := int64(fl.blocksStart) maxRead := len(b) // if there is nothing left to read, just return EOF diff --git a/filesystem/squashfs/finalize.go b/filesystem/squashfs/finalize.go index 41e32351..a627fdd1 100644 --- a/filesystem/squashfs/finalize.go +++ b/filesystem/squashfs/finalize.go @@ -588,6 +588,7 @@ func writeDataBlocks(fileList []*finalizeFileInfo, f backend.WritableFile, ws st } allBlocks += blocks allWritten += written + location += int64(written) } return allWritten, nil } @@ -1104,7 +1105,7 @@ func createInodes(fileList []*finalizeFileInfo, idtable map[uint32]uint16, optio if e.startBlock|uint32max != uint32max || e.Size()|int64(uint32max) != int64(uint32max) || len(e.xattrs) > 0 || e.links > 0 { // use extendedFile inode ef := &extendedFile{ - startBlock: e.startBlock, + blocksStart: uint64(e.dataLocation), fileSize: uint64(e.Size()), blockSizes: e.blocks, links: e.links, @@ -1119,9 +1120,9 @@ func createInodes(fileList []*finalizeFileInfo, idtable map[uint32]uint16, optio } else { // use basicFile bf := &basicFile{ - startBlock: uint32(e.startBlock), - fileSize: uint32(e.Size()), - blockSizes: e.blocks, + blocksStart: uint32(e.dataLocation), + fileSize: uint32(e.Size()), + blockSizes: e.blocks, } if e.fragment != nil { bf.fragmentBlockIndex = e.fragment.block diff --git a/filesystem/squashfs/inode.go b/filesystem/squashfs/inode.go index 96b3c135..d278ba77 100644 --- a/filesystem/squashfs/inode.go +++ b/filesystem/squashfs/inode.go @@ -368,7 +368,7 @@ func parseDirectoryIndexes(b []byte, count int) ([]*directoryIndex, error) { // basicFile type basicFile struct { - startBlock uint32 // block count from the start of the data section where data for this file is stored + blocksStart uint32 // The offset from the start of the archive to the first data block. fragmentBlockIndex uint32 fragmentOffset uint32 fileSize uint32 @@ -391,12 +391,12 @@ func (i basicFile) equal(o inodeBody) bool { return false } } - return i.startBlock == oi.startBlock && i.fragmentOffset == oi.fragmentOffset && i.fragmentBlockIndex == oi.fragmentBlockIndex && i.fileSize == oi.fileSize + return i.blocksStart == oi.blocksStart && i.fragmentOffset == oi.fragmentOffset && i.fragmentBlockIndex == oi.fragmentBlockIndex && i.fileSize == oi.fileSize } func (i basicFile) toBytes() []byte { b := make([]byte, 16+4*len(i.blockSizes)) - binary.LittleEndian.PutUint32(b[0:4], i.startBlock) + binary.LittleEndian.PutUint32(b[0:4], i.blocksStart) binary.LittleEndian.PutUint32(b[4:8], i.fragmentBlockIndex) binary.LittleEndian.PutUint32(b[8:12], i.fragmentOffset) binary.LittleEndian.PutUint32(b[12:16], i.fileSize) @@ -413,7 +413,7 @@ func (i basicFile) xattrIndex() (uint32, bool) { } func (i basicFile) toExtended() extendedFile { return extendedFile{ - startBlock: uint64(i.startBlock), + blocksStart: uint64(i.blocksStart), fileSize: uint64(i.fileSize), sparse: 0, links: 0, @@ -433,7 +433,7 @@ func parseBasicFile(b []byte, blocksize int) (*basicFile, int, error) { } fileSize := binary.LittleEndian.Uint32(b[12:16]) d := &basicFile{ - startBlock: binary.LittleEndian.Uint32(b[0:4]), + blocksStart: binary.LittleEndian.Uint32(b[0:4]), fragmentBlockIndex: binary.LittleEndian.Uint32(b[4:8]), fragmentOffset: binary.LittleEndian.Uint32(b[8:12]), fileSize: fileSize, @@ -455,7 +455,7 @@ func parseBasicFile(b []byte, blocksize int) (*basicFile, int, error) { // extendedFile type extendedFile struct { - startBlock uint64 + blocksStart uint64 fileSize uint64 sparse uint64 links uint32 @@ -481,7 +481,7 @@ func (i extendedFile) equal(o inodeBody) bool { return false } } - return i.startBlock == oi.startBlock && + return i.blocksStart == oi.blocksStart && i.fragmentOffset == oi.fragmentOffset && i.fragmentBlockIndex == oi.fragmentBlockIndex && i.fileSize == oi.fileSize && @@ -492,7 +492,7 @@ func (i extendedFile) equal(o inodeBody) bool { func (i extendedFile) toBytes() []byte { b := make([]byte, 40+4*len(i.blockSizes)) - binary.LittleEndian.PutUint64(b[0:8], i.startBlock) + binary.LittleEndian.PutUint64(b[0:8], i.blocksStart) binary.LittleEndian.PutUint64(b[8:16], i.fileSize) binary.LittleEndian.PutUint64(b[16:24], i.sparse) binary.LittleEndian.PutUint32(b[24:28], i.links) @@ -521,7 +521,7 @@ func parseExtendedFile(b []byte, blocksize int) (*extendedFile, int, error) { } fileSize := binary.LittleEndian.Uint64(b[8:16]) d := &extendedFile{ - startBlock: binary.LittleEndian.Uint64(b[0:8]), + blocksStart: binary.LittleEndian.Uint64(b[0:8]), fileSize: fileSize, sparse: binary.LittleEndian.Uint64(b[16:24]), links: binary.LittleEndian.Uint32(b[24:28]), diff --git a/filesystem/squashfs/inode_internal_test.go b/filesystem/squashfs/inode_internal_test.go index 48910032..e8e8c459 100644 --- a/filesystem/squashfs/inode_internal_test.go +++ b/filesystem/squashfs/inode_internal_test.go @@ -260,8 +260,8 @@ func TestBasicFile(t *testing.T) { if ext.size() != f.size() { t.Errorf("Mismatched sizes actual %d expected %d", ext.size(), f.size()) } - if ext.startBlock != uint64(f.startBlock) { - t.Errorf("Mismatched startBlock actual %d expected %d", ext.startBlock, f.startBlock) + if ext.blocksStart != uint64(f.blocksStart) { + t.Errorf("Mismatched startBlock actual %d expected %d", ext.blocksStart, f.blocksStart) } if ext.fragmentOffset != f.fragmentOffset { t.Errorf("Mismatched fragmentOffset actual %d expected %d", ext.fragmentOffset, f.fragmentOffset) From 6a2389744b8b77313ae9824c9c849eb71f6ca38a Mon Sep 17 00:00:00 2001 From: Xynnn007 Date: Sun, 9 Mar 2025 21:46:15 +0800 Subject: [PATCH 2/5] squashfs: fix fragment block location The granularity of fragment data writing is block, so we don't need to return an unaligned fragment amount but 'sized chunks' everytime we write a fragmentdata block, the variable location should be added with a size of 'blocksize', which means that we have written a new fragment block. The variable `location` will be reused to tell new accumulated fragmentblocks position. Signed-off-by: Xynnn007 --- filesystem/squashfs/finalize.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/filesystem/squashfs/finalize.go b/filesystem/squashfs/finalize.go index a627fdd1..44f814ef 100644 --- a/filesystem/squashfs/finalize.go +++ b/filesystem/squashfs/finalize.go @@ -137,11 +137,11 @@ func (fs *FileSystem) Finalize(options FinalizeOptions) error { // write file fragments // fragmentBlockStart := location - fragmentBlocks, fragsWritten, err := writeFragmentBlocks(fileList, f, fs.workspace, blocksize, options, fragmentBlockStart) + fragmentBlocks, _, err := writeFragmentBlocks(fileList, f, fs.workspace, blocksize, options, fragmentBlockStart) if err != nil { return fmt.Errorf("error writing file fragment blocks: %v", err) } - location += fragsWritten + location += int64(len(fragmentBlocks) * blocksize) // extract extended attributes, and save them for later; these are written at the very end // this must be done *before* creating inodes, as inodes reference these @@ -638,6 +638,7 @@ func writeFragmentBlocks(fileList []*finalizeFileInfo, f backend.WritableFile, w compressed: compressed, location: location, }) + location += int64(blocksize) // increment as all writes will be to next block block fragmentBlockIndex++ fragmentData = fragmentData[:blocksize] From 142e1c2fe965554ac6043a1bbf031066d9e3cbcc Mon Sep 17 00:00:00 2001 From: Xynnn007 Date: Sun, 9 Mar 2025 21:52:10 +0800 Subject: [PATCH 3/5] squashfs: clear cumulative variable for fragment data The variable fragmentData is used to record if the current fragment data is enough to occupy a block. Once a block is written, it should be cleared for a new round of loop. Signed-off-by: Xynnn007 --- filesystem/squashfs/finalize.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filesystem/squashfs/finalize.go b/filesystem/squashfs/finalize.go index 44f814ef..36779b5c 100644 --- a/filesystem/squashfs/finalize.go +++ b/filesystem/squashfs/finalize.go @@ -641,7 +641,7 @@ func writeFragmentBlocks(fileList []*finalizeFileInfo, f backend.WritableFile, w location += int64(blocksize) // increment as all writes will be to next block block fragmentBlockIndex++ - fragmentData = fragmentData[:blocksize] + fragmentData = make([]byte, 0) } e.fragment = &fragmentRef{ From 1247745d1d86d17b55c90d8cc7d2275b7f76c362 Mon Sep 17 00:00:00 2001 From: Xynnn007 Date: Sun, 9 Mar 2025 21:54:13 +0800 Subject: [PATCH 4/5] squashfs: set file inode's frag index when no fragment data Due to spec, frag index of file inode https://dr-emann.github.io/squashfs/squashfs.html#_file_inodes ``` An index into the Fragment Table which describes the fragment block that the tail end of this file is stored in. If not used, this is set to 0xFFFFFFFF. ``` Signed-off-by: Xynnn007 --- filesystem/squashfs/finalize.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/filesystem/squashfs/finalize.go b/filesystem/squashfs/finalize.go index 36779b5c..81db51db 100644 --- a/filesystem/squashfs/finalize.go +++ b/filesystem/squashfs/finalize.go @@ -1128,6 +1128,8 @@ func createInodes(fileList []*finalizeFileInfo, idtable map[uint32]uint16, optio if e.fragment != nil { bf.fragmentBlockIndex = e.fragment.block bf.fragmentOffset = e.fragment.offset + } else { + bf.fragmentBlockIndex = 0xffffffff } in = bf inodeT = inodeBasicFile From e8c6ddaa19f5b5261ee4220cdaf53fdc4f961b6c Mon Sep 17 00:00:00 2001 From: Xynnn007 Date: Sun, 9 Mar 2025 22:41:43 +0800 Subject: [PATCH 5/5] lint: fix go fmt lint error Signed-off-by: Xynnn007 --- filesystem/squashfs/finalize.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/filesystem/squashfs/finalize.go b/filesystem/squashfs/finalize.go index 81db51db..b23cd994 100644 --- a/filesystem/squashfs/finalize.go +++ b/filesystem/squashfs/finalize.go @@ -1107,10 +1107,10 @@ func createInodes(fileList []*finalizeFileInfo, idtable map[uint32]uint16, optio // use extendedFile inode ef := &extendedFile{ blocksStart: uint64(e.dataLocation), - fileSize: uint64(e.Size()), - blockSizes: e.blocks, - links: e.links, - xAttrIndex: e.xAttrIndex, + fileSize: uint64(e.Size()), + blockSizes: e.blocks, + links: e.links, + xAttrIndex: e.xAttrIndex, } if e.fragment != nil { ef.fragmentBlockIndex = e.fragment.block