IO: implemented several compression streams to compare the speed

.dependencies.json

@@ -23,5 +23,6 @@
     "natsort": "cdd8df9602e727482ae5e051cff74b7ec7ffa07a",
     "miniz": "c883286f1a6443720e7705450f59e579a4bbb8e2",
     "imgui_test_engine": "e4ef5fda54cad3b4bd73582c2e59931c3c908614",
-    "opengametools": "0dce3f2277a32b9f646367139506b00da6f080d7"
+    "opengametools": "0dce3f2277a32b9f646367139506b00da6f080d7",
+    "lzav": "60c21d8b7617e48ab13f3dc73e534e410503f10a"
 }

src/engine-config.h.in

@@ -26,3 +26,4 @@
 #cmakedefine USE_LIBPNG 1
 #cmakedefine USE_VK_RENDERER 1
 #cmakedefine USE_GL_RENDERER 1
+#cmakedefine USE_LZ4 1

src/modules/io/CMakeLists.txt

@@ -9,6 +9,8 @@ set(SRCS
 	BufferedReadWriteStream.cpp BufferedReadWriteStream.h
 	BufferedSeekableWriteStream.h
 	BufferedWriteStream.h
+	CompressionReadStream.h
+	CompressionWriteStream.h
 	EndianStreamReadWrapper.h
 	File.cpp File.h
 	FileStream.cpp FileStream.h
@@ -17,6 +19,10 @@ set(SRCS
 	FilesystemEntry.cpp FilesystemEntry.h
 	FormatDescription.cpp FormatDescription.h
 	IOResource.h
+	LZ4ReadStream.cpp LZ4ReadStream.h
+	LZ4WriteStream.cpp LZ4WriteStream.h
+	LZAVReadStream.cpp LZAVReadStream.h
+	LZAVWriteStream.cpp LZAVWriteStream.h
 	LZFSEReadStream.cpp LZFSEReadStream.h
 	MemoryArchive.cpp MemoryArchive.h
 	MemoryReadStream.cpp MemoryReadStream.h
@@ -49,6 +55,17 @@ if (ZLIB_FOUND)
 endif()
 set(USE_ZLIB ${ZLIB_FOUND} CACHE BOOL "Use zlib for zip compression" FORCE)
 
+find_package(PkgConfig REQUIRED)
+if (PKG_CONFIG_FOUND)
+  pkg_check_modules(LIBLZ4 liblz4)
+  if (LIBLZ4_FOUND)
+	list(APPEND LIBS ${LIBLZ4_LIBRARIES})
+	set(USE_LZ4 TRUE CACHE BOOL "Use lz4 for compression" FORCE)
+  else()
+	set(USE_LZ4 FALSE CACHE BOOL "Use lz4 for compression" FORCE)
+  endif()
+endif()
+
 engine_add_module(TARGET ${LIB} SRCS ${SRCS} DEPENDENCIES ${LIBS})
 target_compile_definitions(${LIB} PRIVATE MINIZ_NO_STDIO)
 
@@ -66,6 +83,8 @@ set(TEST_SRCS
 	tests/FileStreamTest.cpp
 	tests/FormatDescriptionTest.cpp
 	tests/FileTest.cpp
+	tests/LZ4StreamTest.cpp
+	tests/LZAVStreamTest.cpp
 	tests/MemoryArchiveTest.cpp
 	tests/MemoryReadStreamTest.cpp
 	tests/StdStreamBufTest.cpp

src/modules/io/CompressionReadStream.h

@@ -0,0 +1,28 @@
+/**
+ * @file
+ */
+
+#include "engine-config.h" // USE_LZ4
+
+#if USE_LZ4
+
+#include "io/LZ4ReadStream.h"
+namespace io {
+using CompressionReadStream = LZ4ReadStream;
+}
+
+#else
+
+#if 0
+#include "io/LZAVReadStream.h"
+namespace io {
+using CompressionReadStream = LZAVReadStream;
+}
+#else
+#include "io/ZipReadStream.h"
+namespace io {
+using CompressionReadStream = ZipReadStream;
+}
+#endif
+
+#endif

src/modules/io/CompressionWriteStream.h

@@ -0,0 +1,28 @@
+/**
+ * @file
+ */
+
+#include "engine-config.h" // USE_LZ4
+
+#if USE_LZ4
+
+#include "io/LZ4WriteStream.h"
+namespace io {
+using CompressionWriteStream = LZ4WriteStream;
+}
+
+#else
+
+#if 0
+#include "io/LZAVWriteStream.h"
+namespace io {
+using CompressionWriteStream = LZAVWriteStream;
+}
+#else
+#include "io/ZipWriteStream.h"
+namespace io {
+using CompressionWriteStream = ZipWriteStream;
+}
+#endif
+
+#endif

src/modules/io/LZ4ReadStream.cpp

@@ -0,0 +1,180 @@
+/**
+ * @file
+ */
+
+#include "LZ4ReadStream.h"
+
+#ifdef USE_LZ4
+
+#include "core/Assert.h"
+#include "core/Log.h"
+#include <lz4.h>
+#include <lz4frame.h>
+
+namespace io {
+
+bool LZ4ReadStream::isLZ4Stream(io::SeekableReadStream &readStream) {
+	int64_t pos = readStream.pos();
+
+	uint32_t magic = 0;
+	if (readStream.readUInt32(magic) == -1) {
+		readStream.seek(pos);
+		return false;
+	}
+
+	readStream.seek(pos);
+	return magic == 0x184D2204;
+}
+
+LZ4ReadStream::LZ4ReadStream(io::SeekableReadStream &readStream, int size)
+	: _readStream(readStream), _size(size), _remaining(size), _srcSize(0), _srcOffset(0), _headerRead(false) {
+	// Initialize LZ4 decompression context
+	LZ4F_dctx *dctx = nullptr;
+	LZ4F_errorCode_t result = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
+	if (LZ4F_isError(result)) {
+		Log::error("Failed to create LZ4 decompression context: %s", LZ4F_getErrorName(result));
+		_err = true;
+		_stream = nullptr;
+	} else {
+		Log::debug("LZ4ReadStream created successfully");
+		_stream = dctx;
+	}
+}
+
+LZ4ReadStream::~LZ4ReadStream() {
+	if (_stream != nullptr) {
+		LZ4F_freeDecompressionContext((LZ4F_dctx *)_stream);
+		_stream = nullptr;
+	}
+}
+
+bool LZ4ReadStream::eos() const {
+	return _eos;
+}
+
+int64_t LZ4ReadStream::remaining() const {
+	if (_size >= 0) {
+		core_assert_msg(_remaining >= 0, "if size is given (%i), remaining should be >= 0 - but is %i", _size,
+						_remaining);
+		return core_min(_remaining, _readStream.remaining());
+	}
+	return _readStream.remaining();
+}
+
+int64_t LZ4ReadStream::skip(int64_t delta) {
+	int64_t bytesSkipped = 0;
+	uint8_t tempBuffer[1024];
+	while (bytesSkipped < delta) {
+		int64_t chunk = core_min(delta - bytesSkipped, (int64_t)sizeof(tempBuffer));
+		if (read(tempBuffer, chunk) < chunk) {
+			_err = true;
+			return -1;
+		}
+		bytesSkipped += chunk;
+	}
+	return bytesSkipped;
+}
+
+int LZ4ReadStream::read(void *buf, size_t size) {
+	if (_err) {
+		Log::debug("LZ4ReadStream::read() - _err is true");
+		return -1;
+	}
+	if (_stream == nullptr) {
+		Log::debug("LZ4ReadStream::read() - _stream is nullptr");
+		return -1;
+	}
+	if (_eos) {
+		return 0;
+	}
+
+	LZ4F_dctx *dctx = (LZ4F_dctx *)_stream;
+	uint8_t *targetPtr = (uint8_t *)buf;
+	size_t totalDecompressed = 0;
+
+	while (totalDecompressed < size) {
+		if (_srcOffset >= _srcSize) {
+			int64_t remainingSize = remaining();
+			if (remainingSize <= 0) {
+				if (!_headerRead) {
+					Log::debug("No data available to read LZ4 header");
+					_err = true;
+					return -1;
+				}
+				if (totalDecompressed > 0) {
+					return (int)totalDecompressed;
+				}
+				_eos = true;
+				return 0;
+			}
+
+			const size_t readSize = core_min((int64_t)sizeof(_buf), remainingSize);
+			const int bytesRead = _readStream.read(_buf, readSize);
+			if (bytesRead <= 0) {
+				if (bytesRead == -1) {
+					Log::debug("Failed to read from parent stream");
+					_err = true;
+					return -1;
+				}
+				if (!_headerRead) {
+					Log::debug("Parent stream at EOF before LZ4 header could be read");
+					_err = true;
+					return -1;
+				}
+				if (totalDecompressed > 0) {
+					return (int)totalDecompressed;
+				}
+				_eos = true;
+				return 0;
+			}
+
+			if (_size >= 0) {
+				_remaining -= bytesRead;
+			}
+
+			_srcSize = bytesRead;
+			_srcOffset = 0;
+		}
+
+		size_t dstSize = size - totalDecompressed;
+		size_t srcSizeToConsume = _srcSize - _srcOffset;
+
+		LZ4F_errorCode_t result = LZ4F_decompress(dctx, targetPtr + totalDecompressed, &dstSize, _buf + _srcOffset,
+												  &srcSizeToConsume, nullptr);
+
+		if (LZ4F_isError(result)) {
+			Log::error("LZ4 decompression error: %s", LZ4F_getErrorName(result));
+			_err = true;
+			return -1;
+		}
+
+		_srcOffset += srcSizeToConsume;
+		totalDecompressed += dstSize;
+
+		if (srcSizeToConsume > 0 || dstSize > 0) {
+			_headerRead = true;
+		}
+
+		if (result == 0) {
+			_eos = true;
+			break;
+		}
+
+		// If we didn't make any progress AND we've consumed all available source data,
+		// we need to read more source data before trying again
+		if (dstSize == 0 && srcSizeToConsume == 0) {
+			if (_srcOffset >= _srcSize) {
+				continue;
+			}
+			Log::error("LZ4 decompression stalled");
+			_err = true;
+			return -1;
+		}
+	}
+
+	return (int)totalDecompressed;
+}
+
+} // namespace io
+
+#endif // USE_LZ4

src/modules/io/LZ4ReadStream.h

@@ -0,0 +1,78 @@
+/**
+ * @file
+ */
+
+#pragma once
+
+#include "engine-config.h" // USE_LZ4
+
+#ifdef USE_LZ4
+
+#include "Stream.h"
+
+namespace io {
+
+/**
+ * @see LZ4WriteStream
+ * @ingroup IO
+ */
+class LZ4ReadStream : public io::ReadStream {
+private:
+	void *_stream;
+	io::SeekableReadStream &_readStream;
+	uint8_t _buf[256 * 1024]{};
+	const int _size;
+	int _remaining;
+	size_t _srcSize = 0;	  // Amount of valid data in _buf
+	size_t _srcOffset = 0;	  // Current read position in _buf
+	bool _headerRead = false; // Whether we've successfully read the LZ4 frame header
+	bool _eos = false;
+	bool _err = false;
+
+public:
+	/**
+	 * @param size The compressed size
+	 */
+	LZ4ReadStream(io::SeekableReadStream &readStream, int size = -1);
+	virtual ~LZ4ReadStream();
+
+	static bool isLZ4Stream(io::SeekableReadStream &stream);
+
+	/**
+	 * @brief Read an arbitrary sized amount of bytes from the input stream
+	 *
+	 * @param dataPtr The target data buffer
+	 * @param dataSize The size of the target data buffer
+	 * @return The amount of read bytes or @c -1 on error
+	 */
+	int read(void *dataPtr, size_t dataSize) override;
+	/**
+	 * @return @c true if the end of the compressed stream was found
+	 */
+	bool eos() const override;
+
+	bool err() const {
+		return _err;
+	}
+
+	/**
+	 * @brief Advances the position in the stream without reading the bytes.
+	 * @param delta the bytes to skip
+	 * @return -1 on error
+	 */
+	int64_t skip(int64_t delta);
+
+	/**
+	 * @brief The remaining amount of bytes to read from the input stream. This is
+	 * either the amount of remaining bytes in the input stream, or if the @c size
+	 * parameter was specified in the constructor, the amounts of bytes that are left
+	 * relative to the size that was specified.
+	 *
+	 * @return int64_t
+	 */
+	int64_t remaining() const;
+};
+
+} // namespace io
+
+#endif // USE_LZ4