From 9df1b0a0ee60b8c8de2dcdd37d9e7b34d8a8ea1d Mon Sep 17 00:00:00 2001 From: LukasMicroscopy Date: Tue, 25 Mar 2025 17:07:12 +0100 Subject: [PATCH 1/2] prevent duplicated reader object when writing in ndtiff dataset --- python/ndstorage/ndtiff_dataset.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/ndstorage/ndtiff_dataset.py b/python/ndstorage/ndtiff_dataset.py index fe5a9dc..b20ec16 100644 --- a/python/ndstorage/ndtiff_dataset.py +++ b/python/ndstorage/ndtiff_dataset.py @@ -258,7 +258,11 @@ def add_index_entry(self, data, new_image_updates=True): self.index[frozenset(image_coordinates.items())] = index_entry if index_entry.filename not in self._readers_by_filename: - new_reader = SingleNDTiffReader(os.path.join(self.path, index_entry.filename), file_io=self.file_io) + # prevent new reader object when writing: + if self._writable and self.current_writer.filename.split(os.sep)[-1] == index_entry.filename: + new_reader = self.current_writer.reader + else: + new_reader = SingleNDTiffReader(os.path.join(self.path, index_entry.filename), file_io=self.file_io) self._readers_by_filename[index_entry.filename] = new_reader # Should be the same on every file so resetting them is fine self.major_version, self.minor_version = new_reader.major_version, new_reader.minor_version From 887520d61b98fd7f4dfe2372acaa9331265dc02d Mon Sep 17 00:00:00 2001 From: LukasMicroscopy Date: Tue, 25 Mar 2025 17:27:38 +0100 Subject: [PATCH 2/2] speed up random access reads with mmap in SingleNDTiffWriter --- python/ndstorage/ndtiff_file.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/ndstorage/ndtiff_file.py b/python/ndstorage/ndtiff_file.py index 5fcac31..963107b 100644 --- a/python/ndstorage/ndtiff_file.py +++ b/python/ndstorage/ndtiff_file.py @@ -5,6 +5,7 @@ import time import struct import warnings +import mmap from collections import OrderedDict from io import BytesIO from .file_io import NDTiffFileIO, BUILTIN_FILE_IO @@ -314,6 +315,8 @@ def __init__(self, tiff_path, file_io: NDTiffFileIO = BUILTIN_FILE_IO, summary_m self.file_io = file_io self.tiff_path = tiff_path self.file = self.file_io.open(tiff_path, "rb") + # mmap speeds up random access + self.mmap_file = mmap.mmap(self.file.fileno(), 0, prot=mmap.PROT_READ) if summary_md is None: self.summary_md, self.first_ifd_offset = self._read_header() else: @@ -323,6 +326,7 @@ def __init__(self, tiff_path, file_io: NDTiffFileIO = BUILTIN_FILE_IO, summary_m def close(self): """ """ + self.mmap_file.close() self.file.close() def _read_header(self): @@ -364,8 +368,8 @@ def _read(self, start, end): """ convert to python ints """ - self.file.seek(int(start), 0) - return self.file.read(end - start) + self.mmap_file.seek(int(start), 0) + return self.mmap_file.read(end - start) def read_metadata(self, index): return json.loads(