diff --git a/neo/io/__init__.py b/neo/io/__init__.py index 0c8697cb9..22b6df662 100644 --- a/neo/io/__init__.py +++ b/neo/io/__init__.py @@ -10,6 +10,7 @@ Functions: .. autofunction:: neo.io.get_io +.. autofunction:: neo.io.list_candidate_ios Classes: @@ -262,7 +263,8 @@ """ -import os.path +import pathlib +from collections import Counter # try to import the neuroshare library. # if it is present, use the neuroshareapiio to load neuroshare files @@ -350,7 +352,7 @@ CedIO, EDFIO, ElanIO, - # ElphyIO, + ElphyIO, ExampleIO, IgorIO, IntanIO, @@ -359,7 +361,8 @@ MEArecIO, MaxwellIO, MicromedIO, - NixIO, # place NixIO before other IOs that use HDF5 to make it the default for .h5 files + NixIO, + NixIOFr, NeoMatlabIO, NestIO, NeuralynxIO, @@ -384,14 +387,86 @@ WinWcpIO ] +# for each supported extension list the ios supporting it +io_by_extension = {} +for current_io in iolist: # do not use `io` as variable name here as this overwrites the module io + for extension in current_io.extensions: + extension = extension.lower() + # extension handling should not be case sensitive + io_by_extension.setdefault(extension, []).append(current_io) -def get_io(filename, *args, **kwargs): + +def get_io(file_or_folder, *args, **kwargs): """ Return a Neo IO instance, guessing the type based on the filename suffix. """ - extension = os.path.splitext(filename)[1][1:] - for io in iolist: - if extension in io.extensions: - return io(filename, *args, **kwargs) + ios = list_candidate_ios(file_or_folder) + for io in ios: + try: + return io(file_or_folder, *args, **kwargs) + except: + continue + + raise IOError(f"Could not identify IO for {file_or_folder}") + - raise IOError("File extension %s not registered" % extension) +def list_candidate_ios(file_or_folder, ignore_patterns=['*.ini', 'README.txt', 'README.md']): + """ + Identify neo IO that can potentially load data in the file or folder + + Parameters + ---------- + file_or_folder (str, pathlib.Path) + Path to the file or folder to load + ignore_patterns (list) + List of patterns to ignore when scanning for known formats. See pathlib.PurePath.match(). + Default: ['ini'] + + Returns + ------- + list + List of neo io classes that are associated with the file extensions detected + """ + file_or_folder = pathlib.Path(file_or_folder) + + if file_or_folder.is_file(): + suffix = file_or_folder.suffix[1:].lower() + if suffix not in io_by_extension: + raise ValueError(f'{suffix} is not a supported format of any IO.') + return io_by_extension[suffix] + + elif file_or_folder.is_dir(): + # scan files in folder to determine io type + filenames = [f for f in file_or_folder.glob('*') if f.is_file()] + # keep only relevant filenames + filenames = [f for f in filenames if f.suffix and not any([f.match(p) for p in ignore_patterns])] + + # if no files are found in the folder, check subfolders + # this is necessary for nested-folder based formats like spikeglx + if not filenames: + filenames = [f for f in file_or_folder.glob('**/*') if f.is_file()] + # keep only relevant filenames + filenames = [f for f in filenames if f.suffix and not any([f.match(p) for p in ignore_patterns])] + + # if only file prefix was provided, e.g /mydatafolder/session1- + # to select all files sharing the `session1-` prefix + elif file_or_folder.parent.exists(): + filenames = file_or_folder.parent.glob(file_or_folder.name + '*') + + else: + raise ValueError(f'{file_or_folder} does not contain data files of a supported format') + + # find the io that fits the best with the files contained in the folder + potential_ios = [] + for filename in filenames: + for suffix in filename.suffixes: + suffix = suffix[1:].lower() + if suffix in io_by_extension: + potential_ios.extend(io_by_extension[suffix]) + + if not potential_ios: + raise ValueError(f'Could not determine IO to load {file_or_folder}') + + # return ios ordered by number of files supported + counter = Counter(potential_ios).most_common() + return [io for io, count in counter] diff --git a/neo/io/neuroshareapiio.py b/neo/io/neuroshareapiio.py index 6bc6c3cfc..145bea391 100644 --- a/neo/io/neuroshareapiio.py +++ b/neo/io/neuroshareapiio.py @@ -72,7 +72,7 @@ class NeuroshareapiIO(BaseIO): name = "Neuroshare" - extensions = [] + extensions = ['mcd'] # This object operates on neuroshare files mode = "file" diff --git a/neo/io/neurosharectypesio.py b/neo/io/neurosharectypesio.py index 10b08719c..1dad0bce0 100644 --- a/neo/io/neurosharectypesio.py +++ b/neo/io/neurosharectypesio.py @@ -108,7 +108,7 @@ class NeurosharectypesIO(BaseIO): write_params = None name = 'neuroshare' - extensions = [] + extensions = ['mcd'] mode = 'file' def __init__(self, filename='', dllname=''): diff --git a/neo/rawio/__init__.py b/neo/rawio/__init__.py index 9dbc0ff95..85b57b3ff 100644 --- a/neo/rawio/__init__.py +++ b/neo/rawio/__init__.py @@ -235,7 +235,7 @@ def get_rawio_class(filename_or_dirname): """ - Return a neo.rawio class guess from file extention. + Return a neo.rawio class guess from file extension. """ _, ext = os.path.splitext(filename_or_dirname) ext = ext[1:] diff --git a/neo/rawio/alphaomegarawio.py b/neo/rawio/alphaomegarawio.py index 9ad0b8b16..23de7cac9 100644 --- a/neo/rawio/alphaomegarawio.py +++ b/neo/rawio/alphaomegarawio.py @@ -90,16 +90,18 @@ class AlphaOmegaRawIO(BaseRawIO): def __init__(self, dirname="", lsx_files=None, prune_channels=True): super().__init__(dirname=dirname) self.dirname = Path(dirname) + self._lsx_files = lsx_files self._mpx_files = None - if self.dirname.is_dir(): - self._explore_folder() - else: - self.logger.error(f"{self.dirname} is not a folder") self._prune_channels = prune_channels self._opened_files = {} self._ignore_unknown_datablocks = True # internal debug property + if self.dirname.is_dir(): + self._explore_folder() + else: + raise ValueError(f"{self.dirname} is not a folder") + def _explore_folder(self): """ If class was instantiated with lsx_files (list of .lsx files), load only diff --git a/neo/rawio/axographrawio.py b/neo/rawio/axographrawio.py index 975637bcf..cb58e314c 100644 --- a/neo/rawio/axographrawio.py +++ b/neo/rawio/axographrawio.py @@ -218,7 +218,7 @@ class AxographRawIO(BaseRawIO): """ name = 'AxographRawIO' description = 'This IO reads .axgd/.axgx files created with AxoGraph' - extensions = ['axgd', 'axgx'] + extensions = ['axgd', 'axgx', ''] rawmode = 'one-file' def __init__(self, filename, force_single_segment=False): diff --git a/neo/rawio/biocamrawio.py b/neo/rawio/biocamrawio.py index 9b9284e63..d61d92dae 100644 --- a/neo/rawio/biocamrawio.py +++ b/neo/rawio/biocamrawio.py @@ -29,7 +29,7 @@ class BiocamRawIO(BaseRawIO): >>> float_chunk = r.rescale_signal_raw_to_float(raw_chunk, dtype='float64', channel_indexes=[0, 3, 6]) """ - extensions = ['h5'] + extensions = ['h5', 'brw'] rawmode = 'one-file' def __init__(self, filename=''): diff --git a/neo/rawio/blackrockrawio.py b/neo/rawio/blackrockrawio.py index 7b2d4e7e0..9356dfb73 100644 --- a/neo/rawio/blackrockrawio.py +++ b/neo/rawio/blackrockrawio.py @@ -122,7 +122,7 @@ class BlackrockRawIO(BaseRawIO): """ extensions = ['ns' + str(_) for _ in range(1, 7)] - extensions.extend(['nev', ]) # 'sif', 'ccf' not yet supported + extensions.extend(['nev', 'sif', 'ccf']) # 'sif', 'ccf' not yet supported rawmode = 'multi-file' def __init__(self, filename=None, nsx_override=None, nev_override=None, @@ -154,6 +154,9 @@ def __init__(self, filename=None, nsx_override=None, nev_override=None, else: self._filenames['nev'] = self.filename + self._filenames['sif'] = self.filename + self._filenames['ccf'] = self.filename + # check which files are available self._avail_files = dict.fromkeys(self.extensions, False) self._avail_nsx = [] diff --git a/neo/rawio/neuralynxrawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio/neuralynxrawio.py index cc8c812ac..8f46bb9d0 100644 --- a/neo/rawio/neuralynxrawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio/neuralynxrawio.py @@ -75,7 +75,7 @@ class NeuralynxRawIO(BaseRawIO): Display all information about signal channels, units, segment size.... """ - extensions = ['nse', 'ncs', 'nev', 'ntt'] + extensions = ['nse', 'ncs', 'nev', 'ntt', 'nvt', 'nrd'] # nvt and nrd are not yet supported rawmode = 'one-dir' _ncs_dtype = [('timestamp', 'uint64'), ('channel_id', 'uint32'), ('sample_rate', 'uint32'), diff --git a/neo/rawio/nixrawio.py b/neo/rawio/nixrawio.py index f5f097dd9..21f47cb24 100644 --- a/neo/rawio/nixrawio.py +++ b/neo/rawio/nixrawio.py @@ -31,7 +31,7 @@ class NIXRawIO(BaseRawIO): - extensions = ['nix'] + extensions = ['nix', 'h5'] rawmode = 'one-file' def __init__(self, filename=''): diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index 72c47bcff..0fd33561b 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -54,7 +54,7 @@ class OpenEphysBinaryRawIO(BaseRawIO): The current implementation does not handle spiking data, this will be added upon user request """ - extensions = [] + extensions = ['xml', 'oebin', 'txt', 'dat', 'npy'] rawmode = 'one-dir' def __init__(self, dirname='', load_sync_channel=False, experiment_names=None): diff --git a/neo/rawio/openephysrawio.py b/neo/rawio/openephysrawio.py index 9026fd128..f2a3aa0ff 100644 --- a/neo/rawio/openephysrawio.py +++ b/neo/rawio/openephysrawio.py @@ -64,7 +64,8 @@ class OpenEphysRawIO(BaseRawIO): aligned, and a warning is emitted. """ - extensions = [] + # file formats used by openephys + extensions = ['continuous', 'openephys', 'spikes', 'events', 'xml'] rawmode = 'one-dir' def __init__(self, dirname=''): diff --git a/neo/rawio/phyrawio.py b/neo/rawio/phyrawio.py index a2c23503e..027275075 100644 --- a/neo/rawio/phyrawio.py +++ b/neo/rawio/phyrawio.py @@ -33,7 +33,8 @@ class PhyRawIO(BaseRawIO): >>> spike_times = r.rescale_spike_timestamp(spike_timestamp, 'float64') """ - extensions = [] + # file formats used by phy + extensions = ['npy', 'mat', 'tsv', 'dat'] rawmode = 'one-dir' def __init__(self, dirname='', load_amplitudes=False, load_pcs=False): diff --git a/neo/rawio/rawbinarysignalrawio.py b/neo/rawio/rawbinarysignalrawio.py index ef6b1c7a5..e71210df8 100644 --- a/neo/rawio/rawbinarysignalrawio.py +++ b/neo/rawio/rawbinarysignalrawio.py @@ -27,7 +27,7 @@ class RawBinarySignalIO class RawBinarySignalRawIO(BaseRawIO): - extensions = ['raw', '*'] + extensions = ['raw', 'bin'] rawmode = 'one-file' def __init__(self, filename='', dtype='int16', sampling_rate=10000., diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py index c66c1ca23..3bececd38 100644 --- a/neo/rawio/spikeglxrawio.py +++ b/neo/rawio/spikeglxrawio.py @@ -64,7 +64,8 @@ class SpikeGLXRawIO(BaseRawIO): load_sync_channel=False/True The last channel (SY0) of each stream is a fake channel used for synchronisation. """ - extensions = [] + # file formats used by spikeglxio + extensions = ['meta', 'bin'] rawmode = 'one-dir' def __init__(self, dirname='', load_sync_channel=False, load_channel_location=False): diff --git a/neo/rawio/tdtrawio.py b/neo/rawio/tdtrawio.py index 5d3f17fcf..7fc0e3b58 100644 --- a/neo/rawio/tdtrawio.py +++ b/neo/rawio/tdtrawio.py @@ -34,6 +34,7 @@ class TdtRawIO(BaseRawIO): + extensions = ['tbk', 'tdx', 'tev', 'tin', 'tnt', 'tsq', 'sev', 'txt'] rawmode = 'one-dir' def __init__(self, dirname='', sortname=''): diff --git a/neo/test/iotest/common_io_test.py b/neo/test/iotest/common_io_test.py index d13883335..bbe0986da 100644 --- a/neo/test/iotest/common_io_test.py +++ b/neo/test/iotest/common_io_test.py @@ -9,7 +9,7 @@ The public URL is in url_for_tests. -To deposite new testing files, please create a account at +To deposit new testing files, please create a account at gin.g-node.org and upload files at NeuralEnsemble/ephy_testing_data data repo. @@ -19,7 +19,6 @@ import os import inspect -from copy import copy import unittest import pathlib @@ -27,13 +26,12 @@ from neo.io.basefromrawio import BaseFromRaw from neo.test.tools import (assert_same_sub_schema, assert_neo_object_is_compliant, - assert_sub_schema_is_lazy_loaded, - assert_children_empty) + assert_sub_schema_is_lazy_loaded) from neo.test.rawiotest.tools import can_use_network from neo.test.rawiotest.common_rawio_test import repo_for_test -from neo.utils import (download_dataset, - get_local_testing_data_folder) +from neo.utils import (download_dataset, get_local_testing_data_folder) +from neo import list_candidate_ios try: import datalad @@ -41,10 +39,8 @@ except: HAVE_DATALAD = False -from neo.test.iotest.tools import (cleanup_test_file, - close_object_safe, create_generic_io_object, +from neo.test.iotest.tools import (close_object_safe, create_generic_io_object, create_generic_reader, - create_generic_writer, get_test_file_full_path, iter_generic_io_objects, iter_generic_readers, iter_read_objects, @@ -56,18 +52,18 @@ class BaseTestIO: """ - This class make common tests for all IOs. + This class defines common tests for all IOs. - Several startegies: + Several strategies: * for IO able to read write : test_write_then_read * for IO able to read write with hash conservation (optional): test_read_then_write * for all IOs : test_assert_read_neo_object_is_compliant 2 cases: - * files are at G-node and downloaded: - download_test_files_if_not_present + * files are at G-node and downloaded * files are generated by MyIO.write() + Note: When inheriting this class use it as primary superclass in combination with the unittest.TestCase as a 2nd superclass, e.g. `NewIOTestClass(BaseTestIO, unittest.TestCase):` @@ -519,3 +515,10 @@ def test__handle_pathlib_filename(self): self.ioclass(dirname=pathlib_filename, *self.default_arguments, **self.default_keyword_arguments) + + def test_list_candidate_ios(self): + for entity in self.entities_to_test: + entity = get_test_file_full_path(self.ioclass, filename=entity, + directory=self.local_test_dir) + ios = list_candidate_ios(entity) + assert self.ioclass in ios diff --git a/neo/test/iotest/test_exampleio.py b/neo/test/iotest/test_exampleio.py index 8b87cf585..7dd3b2c76 100644 --- a/neo/test/iotest/test_exampleio.py +++ b/neo/test/iotest/test_exampleio.py @@ -2,10 +2,12 @@ Tests of neo.io.exampleio """ +import pathlib import unittest from neo.io.exampleio import ExampleIO # , HAVE_SCIPY from neo.test.iotest.common_io_test import BaseTestIO +from neo.test.iotest.tools import get_test_file_full_path from neo.io.proxyobjects import (AnalogSignalProxy, SpikeTrainProxy, EventProxy, EpochProxy) from neo import (AnalogSignal, SpikeTrain) @@ -19,10 +21,25 @@ class TestExampleIO(BaseTestIO, unittest.TestCase, ): ioclass = ExampleIO entities_to_download = [] entities_to_test = [ - 'fake1', - 'fake2', + 'fake1.fake', + 'fake2.fake', ] + def setUp(self): + super().setUp() + # ensure fake test files exist before running common tests + for entity in self.entities_to_test: + full_path = get_test_file_full_path(self.ioclass, filename=entity, + directory=self.local_test_dir) + pathlib.Path(full_path).touch() + + def tearDown(self) -> None: + super().tearDown() + for entity in self.entities_to_test: + full_path = get_test_file_full_path(self.ioclass, filename=entity, + directory=self.local_test_dir) + pathlib.Path(full_path).unlink(missing_ok=True) + # This is the minimal variables that are required # to run the common IO tests. IO specific tests # can be added here and will be run automatically diff --git a/neo/test/rawiotest/test_alphaomegarawio.py b/neo/test/rawiotest/test_alphaomegarawio.py index 353f5435b..337418568 100644 --- a/neo/test/rawiotest/test_alphaomegarawio.py +++ b/neo/test/rawiotest/test_alphaomegarawio.py @@ -82,9 +82,8 @@ def test_explore_no_folder(self): with tempfile.TemporaryDirectory() as tmpdir: # just create a temporary folder that is removed pass - with self.assertLogs(logger=self.logger, level="ERROR") as cm: + with self.assertRaisesRegex(ValueError, "is not a folder"): reader = AlphaOmegaRawIO(dirname=tmpdir) - self.assertIn("is not a folder", cm.output[0]) def test_empty_folder(self): with tempfile.TemporaryDirectory() as tmpdir: diff --git a/neo/test/rawiotest/test_spikeglxrawio.py b/neo/test/rawiotest/test_spikeglxrawio.py index 62a2f13b2..e4fd3a810 100644 --- a/neo/test/rawiotest/test_spikeglxrawio.py +++ b/neo/test/rawiotest/test_spikeglxrawio.py @@ -18,17 +18,17 @@ class TestSpikeGLXRawIO(BaseTestRawIO, unittest.TestCase): 'spikeglx/TEST_20210920_0_g0', # this is only g0 multi index - 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0/5-19-2022-CI0_g0' + 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0/5-19-2022-CI0_g0', # this is only g1 multi index - 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0/5-19-2022-CI0_g1' + 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0/5-19-2022-CI0_g1', # this mix both multi gate and multi trigger (and also multi probe) - 'spikeglx/sample_data_v2/SpikeGLX/5-19-2022-CI0', + 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0', - 'spikeglx/sample_data_v2/SpikeGLX/5-19-2022-CI1', - 'spikeglx/sample_data_v2/SpikeGLX/5-19-2022-CI2', - 'spikeglx/sample_data_v2/SpikeGLX/5-19-2022-CI3', - 'spikeglx/sample_data_v2/SpikeGLX/5-19-2022-CI4', - 'spikeglx/sample_data_v2/SpikeGLX/5-19-2022-CI5', + 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI1', + 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI2', + 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI3', + 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI4', + 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI5', ]