UT for buffer chunks

Signed-off-by: Athish Pranav D <[email protected]>

Author: Athishpranav2003

test/plugin/test_buffer.rb

@@ -1443,7 +1443,7 @@ def create_chunk_es(metadata, es)
 
     test 'create decompressable chunk' do
       chunk = @p.generate_chunk(create_metadata)
-      assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::Decompressable)
+      assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::GzipDecompressable)
     end
 
     test '#write compressed data which exceeds chunk_limit_size, it raises BufferChunkOverflowError' do

test/plugin/test_buffer_chunk.rb

@@ -77,6 +77,16 @@ class BufferChunkTest < Test::Unit::TestCase
       assert_raise(ArgumentError){ chunk.write_to(nil, compressed: :gzip) }
       assert_raise(ArgumentError){ chunk.append(nil, compress: :gzip) }
     end
+
+    test 'some methods raise ArgumentError with an option of `compressed: :zstd` and without extending Compressble`' do
+      meta = Object.new
+      chunk = Fluent::Plugin::Buffer::Chunk.new(meta)
+
+      assert_raise(ArgumentError){ chunk.read(compressed: :zstd) }
+      assert_raise(ArgumentError){ chunk.open(compressed: :zstd){} }
+      assert_raise(ArgumentError){ chunk.write_to(nil, compressed: :zstd) }
+      assert_raise(ArgumentError){ chunk.append(nil, compress: :zstd) }
+    end
   end
 
   class TestChunk < Fluent::Plugin::Buffer::Chunk
@@ -203,7 +213,15 @@ def open(**kwargs)
     test 'create decompressable chunk' do
       meta = Object.new
       chunk = Fluent::Plugin::Buffer::Chunk.new(meta, compress: :gzip)
-      assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::Decompressable)
+      assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::GzipDecompressable)
+    end
+  end
+
+  sub_test_case 'when compress is zstd' do
+    test 'create decompressable chunk' do
+      meta = Object.new
+      chunk = Fluent::Plugin::Buffer::Chunk.new(meta, compress: :zstd)
+      assert chunk.singleton_class.ancestors.include?(Fluent::Plugin::Buffer::Chunk::ZstdDecompressable)
     end
   end
 end

test/plugin/test_buffer_file_chunk.rb

@@ -802,6 +802,7 @@ def gen_chunk_path(prefix, unique_id)
     setup do
       @src = 'text data for compressing' * 5
       @gzipped_src = compress(@src)
+      @zstded_src = compress(@src, type: :zstd)
     end
 
     test '#append with compress option writes  compressed data to chunk when compress is gzip' do
@@ -867,5 +868,69 @@ def gen_chunk_path(prefix, unique_id)
       c.write_to(io, compressed: :gzip)
       assert_equal @gzipped_src, io.string
     end
+
+    test '#append with compress option writes  compressed data to chunk when compress is zstd' do
+      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :zstd)
+      c.append([@src, @src], compress: :zstd)
+      c.commit
+
+      # check chunk is compressed
+      assert c.read(compressed: :zstd).size < [@src, @src].join("").size
+
+      assert_equal @src + @src, c.read
+    end
+
+    test '#open passes io object having decompressed data to a block when compress is zstd' do
+      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :zstd)
+      c.concat(@zstded_src, @src.size)
+      c.commit
+
+      decomressed_data = c.open do |io|
+        v = io.read
+        assert_equal @src, v
+        v
+      end
+      assert_equal @src, decomressed_data
+    end
+
+    test '#open with compressed option passes io object having decompressed data to a block when compress is zstd' do
+      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :zstd)
+      c.concat(@zstded_src, @src.size)
+      c.commit
+
+      comressed_data = c.open(compressed: :zstd) do |io|
+        v = io.read
+        assert_equal @zstded_src, v
+        v
+      end
+      assert_equal @zstded_src, comressed_data
+    end
+
+    test '#write_to writes decompressed data when compress is zstd' do
+      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :zstd)
+      c.concat(@zstded_src, @src.size)
+      c.commit
+
+      assert_equal @src, c.read
+      assert_equal @zstded_src, c.read(compressed: :zstd)
+
+      io = StringIO.new
+      c.write_to(io)
+      assert_equal @src, io.string
+    end
+
+    test '#write_to with compressed option writes compressed data when compress is zstd' do
+      c = @klass.new(gen_metadata, File.join(@chunkdir,'test.*.log'), :create, compress: :zstd)
+      c.concat(@zstded_src, @src.size)
+      c.commit
+
+      assert_equal @src, c.read
+      assert_equal @zstded_src, c.read(compressed: :zstd)
+
+      io = StringIO.new
+      io.set_encoding(Encoding::ASCII_8BIT)
+      c.write_to(io, compressed: :zstd)
+      assert_equal @zstded_src, io.string
+    end
   end
 end

test/plugin/test_buffer_file_single_chunk.rb

@@ -542,6 +542,7 @@ def gen_chunk_path(prefix, unique_id)
     setup do
       @src = 'text data for compressing' * 5
       @gzipped_src = compress(@src)
+      @zstded_src = compress(@src, type: :zstd)
     end
 
     test '#append with compress option writes  compressed data to chunk when compress is gzip' do
@@ -607,5 +608,69 @@ def gen_chunk_path(prefix, unique_id)
       c.write_to(io, compressed: :gzip)
       assert_equal @gzipped_src, io.string
     end
+
+    test '#append with compress option writes  compressed data to chunk when compress is zstd' do
+      c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :zstd)
+      c.append([@src, @src], compress: :zstd)
+      c.commit
+
+      # check chunk is compressed
+      assert c.read(compressed: :zstd).size < [@src, @src].join("").size
+
+      assert_equal @src + @src, c.read
+    end
+
+    test '#open passes io object having decompressed data to a block when compress is zstd' do
+      c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :zstd)
+      c.concat(@zstded_src, @src.size)
+      c.commit
+
+      decomressed_data = c.open do |io|
+        v = io.read
+        assert_equal @src, v
+        v
+      end
+      assert_equal @src, decomressed_data
+    end
+
+    test '#open with compressed option passes io object having decompressed data to a block when compress is zstd' do
+      c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :zstd)
+      c.concat(@zstded_src, @src.size)
+      c.commit
+
+      comressed_data = c.open(compressed: :zstd) do |io|
+        v = io.read
+        assert_equal @zstded_src, v
+        v
+      end
+      assert_equal @zstded_src, comressed_data
+    end
+
+    test '#write_to writes decompressed data when compress is zstd' do
+      c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :zstd)
+      c.concat(@zstded_src, @src.size)
+      c.commit
+
+      assert_equal @src, c.read
+      assert_equal @zstded_src, c.read(compressed: :zstd)
+
+      io = StringIO.new
+      c.write_to(io)
+      assert_equal @src, io.string
+    end
+
+    test '#write_to with compressed option writes compressed data when compress is zstd' do
+      c = @klass.new(gen_metadata, File.join(@chunkdir,'fsb.*.buf'), :create, nil, compress: :zstd)
+      c.concat(@zstded_src, @src.size)
+      c.commit
+
+      assert_equal @src, c.read
+      assert_equal @zstded_src, c.read(compressed: :zstd)
+
+      io = StringIO.new
+      io.set_encoding(Encoding::ASCII_8BIT)
+      c.write_to(io, compressed: :zstd)
+      assert_equal @zstded_src, io.string
+    end
   end
 end

test/plugin/test_buffer_memory_chunk.rb

@@ -270,6 +270,7 @@ class BufferMemoryChunkTest < Test::Unit::TestCase
     setup do
       @src = 'text data for compressing' * 5
       @gzipped_src = compress(@src)
+      @zstded_src = compress(@src, type: :zstd)
     end
 
     test '#append with compress option writes  compressed data to chunk when compress is gzip' do
@@ -335,5 +336,69 @@ class BufferMemoryChunkTest < Test::Unit::TestCase
       c.write_to(io, compressed: :gzip)
       assert_equal @gzipped_src, io.string
     end
+
+    test '#append with compress option writes  compressed data to chunk when compress is zstd' do
+      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :zstd)
+      c.append([@src, @src], compress: :zstd)
+      c.commit
+
+      # check chunk is compressed
+      assert c.read(compressed: :zstd).size < [@src, @src].join("").size
+
+      assert_equal @src + @src, c.read
+    end
+
+    test '#open passes io object having decompressed data to a block when compress is zstd' do
+      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :zstd)
+      c.concat(@zstded_src, @src.size)
+      c.commit
+
+      decomressed_data = c.open do |io|
+        v = io.read
+        assert_equal @src, v
+        v
+      end
+      assert_equal @src, decomressed_data
+    end
+
+    test '#open with compressed option passes io object having decompressed data to a block when compress is zstd' do
+      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :zstd)
+      c.concat(@zstded_src, @src.size)
+      c.commit
+
+      comressed_data = c.open(compressed: :zstd) do |io|
+        v = io.read
+        assert_equal @zstded_src, v
+        v
+      end
+      assert_equal @zstded_src, comressed_data
+    end
+
+    test '#write_to writes decompressed data when compress is zstd' do
+      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :zstd)
+      c.concat(@zstded_src, @src.size)
+      c.commit
+
+      assert_equal @src, c.read
+      assert_equal @zstded_src, c.read(compressed: :zstd)
+
+      io = StringIO.new
+      c.write_to(io)
+      assert_equal @src, io.string
+    end
+
+    test '#write_to with compressed option writes compressed data when compress is zstd' do
+      c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new, compress: :zstd)
+      c.concat(@zstded_src, @src.size)
+      c.commit
+
+      assert_equal @src, c.read
+      assert_equal @zstded_src, c.read(compressed: :zstd)
+
+      io = StringIO.new
+      io.set_encoding(Encoding::ASCII_8BIT)
+      c.write_to(io, compressed: :zstd)
+      assert_equal @zstded_src, io.string
+    end
   end
 end