builder: handle estargz's chunk info for rafs v6

Signed-off-by: Yan Song <[email protected]>

Author: imeoer

rafs/src/metadata/layout/v6.rs

@@ -17,7 +17,6 @@ use lazy_static::lazy_static;
 use nydus_utils::{compress, digest, round_up, ByteSize};
 use storage::device::{BlobFeatures, BlobInfo};
 use storage::meta::{BlobMetaHeaderOndisk, BLOB_FEATURE_4K_ALIGNED};
-use storage::RAFS_MAX_CHUNK_SIZE;
 
 use crate::metadata::{layout::RafsXAttrs, RafsStore, RafsSuperFlags};
 use crate::{impl_bootstrap_converter, impl_pub_getter_setter, RafsIoReader, RafsIoWrite};
@@ -353,10 +352,7 @@ impl RafsV6SuperBlockExt {
         }
 
         let chunk_size = u32::from_le(self.s_chunk_size) as u64;
-        if !chunk_size.is_power_of_two()
-            || chunk_size < EROFS_BLOCK_SIZE
-            || chunk_size > RAFS_MAX_CHUNK_SIZE
-        {
+        if !chunk_size.is_power_of_two() || chunk_size < EROFS_BLOCK_SIZE {
             return Err(einval!("invalid chunk size in Rafs v6 extended superblock"));
         }
 
@@ -1296,11 +1292,7 @@ impl RafsV6Blob {
         }
 
         let c_size = u32::from_le(self.chunk_size) as u64;
-        if c_size.count_ones() != 1
-            || c_size < EROFS_BLOCK_SIZE
-            || c_size > RAFS_MAX_CHUNK_SIZE
-            || c_size != chunk_size as u64
-        {
+        if c_size.count_ones() != 1 || c_size < EROFS_BLOCK_SIZE || c_size != chunk_size as u64 {
             error!(
                 "RafsV6Blob: idx {} invalid c_size {}, count_ones() {}",
                 blob_index,

src/bin/nydus-image/builder/stargz.rs

@@ -18,7 +18,7 @@ use nix::sys::stat::makedev;
 use serde::{Deserialize, Serialize};
 
 use nydus_utils::digest::{self, Algorithm, DigestHasher, RafsDigest};
-use nydus_utils::ByteSize;
+use nydus_utils::{try_round_up_4k, ByteSize};
 use rafs::metadata::layout::v5::{RafsV5ChunkInfo, RafsV5Inode, RafsV5InodeFlags};
 use rafs::metadata::layout::RafsXAttrs;
 use rafs::metadata::Inode;
@@ -352,6 +352,7 @@ impl StargzIndexTreeBuilder {
         let mut tree: Option<Tree> = None;
         let mut last_reg_entry: Option<&TocEntry> = None;
 
+        let mut uncompress_offset = 0;
         for entry in toc_index.entries.iter() {
             if !entry.is_supported() {
                 continue;
@@ -370,6 +371,15 @@ impl StargzIndexTreeBuilder {
             }
 
             if (entry.is_reg() || entry.is_chunk()) && decompress_size != 0 {
+                let aligned_chunk_size = if ctx.aligned_chunk {
+                    // Safe to unwrap because `chunk_size` is much less than u32::MAX.
+                    try_round_up_4k(decompress_size).unwrap()
+                } else {
+                    decompress_size
+                };
+                let pre_uncompress_offset = uncompress_offset;
+                uncompress_offset += aligned_chunk_size;
+
                 let block_id = entry.block_id(&ctx.blob_id)?;
                 let v5_chunk_info = ChunkWrapper::V5(RafsV5ChunkInfo {
                     block_id,
@@ -381,7 +391,7 @@ impl StargzIndexTreeBuilder {
                     uncompress_size: decompress_size as u32,
                     compress_offset: entry.offset as u64,
                     // No available data on entry
-                    uncompress_offset: 0,
+                    uncompress_offset: pre_uncompress_offset,
                     file_offset: entry.chunk_offset as u64,
                     index: 0,
                     reserved: 0,
@@ -594,6 +604,8 @@ impl StargzBuilder {
         blob_ctx.set_meta_info_enabled(ctx.fs_version == RafsVersion::V6);
         blob_ctx.blob_meta_header = header;
 
+        let mut chunk_map = HashMap::new();
+
         // Set blob index and inode digest for upper nodes
         for node in &mut bootstrap_ctx.nodes {
             if node.overlay.is_lower_layer() {
@@ -603,8 +615,13 @@ impl StargzBuilder {
             let mut inode_hasher = RafsDigest::hasher(digest::Algorithm::Sha256);
 
             for chunk in node.chunks.iter_mut() {
-                let chunk_index = blob_ctx.alloc_index()?;
-                chunk.inner.set_index(chunk_index);
+                if let Some(chunk_index) = chunk_map.get(chunk.inner.id()) {
+                    chunk.inner.set_index(*chunk_index);
+                } else {
+                    let chunk_index = blob_ctx.alloc_index()?;
+                    chunk.inner.set_index(chunk_index);
+                    chunk_map.insert(*chunk.inner.id(), chunk.inner.index());
+                }
                 chunk.inner.set_blob_index(blob_index);
                 decompressed_blob_size += chunk.inner.uncompressed_size() as u64;
                 compressed_blob_size += chunk.inner.compressed_size() as u64;

src/bin/nydus-image/main.rs

@@ -952,10 +952,7 @@ impl Command {
                 let param = v.trim_start_matches("0x").trim_end_matches("0X");
                 let chunk_size =
                     u32::from_str_radix(param, 16).context(format!("invalid chunk size {}", v))?;
-                if chunk_size as u64 > RAFS_DEFAULT_CHUNK_SIZE
-                    || chunk_size < 0x1000
-                    || !chunk_size.is_power_of_two()
-                {
+                if chunk_size < 0x1000 || !chunk_size.is_power_of_two() {
                     bail!("invalid chunk size: {}", chunk_size);
                 }
                 Ok(chunk_size)