[IO] Extend utilities to find suitable IO for a dataset

neo/io/__init__.py

@@ -10,6 +10,7 @@
 Functions:
 
 .. autofunction:: neo.io.get_io
+.. autofunction:: neo.io.list_candidate_ios
 
 
 Classes:
@@ -262,7 +263,8 @@
 
 """
 
-import os.path
+import pathlib
+from collections import Counter
 
 # try to import the neuroshare library.
 # if it is present, use the neuroshareapiio to load neuroshare files
@@ -350,7 +352,7 @@
     CedIO,
     EDFIO,
     ElanIO,
-    # ElphyIO,
+    ElphyIO,
     ExampleIO,
     IgorIO,
     IntanIO,
@@ -359,7 +361,8 @@
     MEArecIO,
     MaxwellIO,
     MicromedIO,
-    NixIO,  # place NixIO before other IOs that use HDF5 to make it the default for .h5 files
+    NixIO,
+    NixIOFr,
     NeoMatlabIO,
     NestIO,
     NeuralynxIO,
@@ -384,14 +387,86 @@
     WinWcpIO
 ]
 
+# for each supported extension list the ios supporting it
+io_by_extension = {}
+for current_io in iolist:  # do not use `io` as variable name here as this overwrites the module io
+    for extension in current_io.extensions:
+        extension = extension.lower()
+        # extension handling should not be case sensitive
+        io_by_extension.setdefault(extension, []).append(current_io)
 
-def get_io(filename, *args, **kwargs):
+
+def get_io(file_or_folder, *args, **kwargs):
     """
     Return a Neo IO instance, guessing the type based on the filename suffix.
     """
-    extension = os.path.splitext(filename)[1][1:]
-    for io in iolist:
-        if extension in io.extensions:
-            return io(filename, *args, **kwargs)
+    ios = list_candidate_ios(file_or_folder)
+    for io in ios:
+        try:
+            return io(file_or_folder, *args, **kwargs)
+        except:
+            continue
+
+    raise IOError(f"Could not identify IO for {file_or_folder}")
+
 
-    raise IOError("File extension %s not registered" % extension)
+def list_candidate_ios(file_or_folder, ignore_patterns=['*.ini', 'README.txt', 'README.md']):
+    """
+    Identify neo IO that can potentially load data in the file or folder
+
+    Parameters
+    ----------
+    file_or_folder (str, pathlib.Path)
+        Path to the file or folder to load
+    ignore_patterns (list)
+        List of patterns to ignore when scanning for known formats. See pathlib.PurePath.match().
+        Default: ['ini']
+
+    Returns
+    -------
+    list
+        List of neo io classes that are associated with the file extensions detected
+    """
+    file_or_folder = pathlib.Path(file_or_folder)
+
+    if file_or_folder.is_file():
+        suffix = file_or_folder.suffix[1:].lower()
+        if suffix not in io_by_extension:
+            raise ValueError(f'{suffix} is not a supported format of any IO.')
+        return io_by_extension[suffix]
+
+    elif file_or_folder.is_dir():
+        # scan files in folder to determine io type
+        filenames = [f for f in file_or_folder.glob('*') if f.is_file()]
+        # keep only relevant filenames
+        filenames = [f for f in filenames if f.suffix and not any([f.match(p) for p in ignore_patterns])]
+
+        # if no files are found in the folder, check subfolders
+        # this is necessary for nested-folder based formats like spikeglx
+        if not filenames:
+            filenames = [f for f in file_or_folder.glob('**/*') if f.is_file()]
+            # keep only relevant filenames
+            filenames = [f for f in filenames if f.suffix and not any([f.match(p) for p in ignore_patterns])]
+
+    # if only file prefix was provided, e.g /mydatafolder/session1-
+    # to select all files sharing the `session1-` prefix
+    elif file_or_folder.parent.exists():
+        filenames = file_or_folder.parent.glob(file_or_folder.name + '*')
+
+    else:
+        raise ValueError(f'{file_or_folder} does not contain data files of a supported format')
+
+    # find the io that fits the best with the files contained in the folder
+    potential_ios = []
+    for filename in filenames:
+        for suffix in filename.suffixes:
+            suffix = suffix[1:].lower()
+            if suffix in io_by_extension:
+                potential_ios.extend(io_by_extension[suffix])
+
+    if not potential_ios:
+        raise ValueError(f'Could not determine IO to load {file_or_folder}')
+
+    # return ios ordered by number of files supported
+    counter = Counter(potential_ios).most_common()
+    return [io for io, count in counter]

neo/io/neuroshareapiio.py

@@ -72,7 +72,7 @@ class NeuroshareapiIO(BaseIO):
 
     name = "Neuroshare"
 
-    extensions = []
+    extensions = ['mcd']
 
     # This object operates on neuroshare files
     mode = "file"

neo/io/neurosharectypesio.py

@@ -108,7 +108,7 @@ class NeurosharectypesIO(BaseIO):
     write_params = None
 
     name = 'neuroshare'
-    extensions = []
+    extensions = ['mcd']
     mode = 'file'
 
     def __init__(self, filename='', dllname=''):

neo/rawio/__init__.py

@@ -235,7 +235,7 @@
 
 def get_rawio_class(filename_or_dirname):
     """
-    Return a neo.rawio class guess from file extention.
+    Return a neo.rawio class guess from file extension.
     """
     _, ext = os.path.splitext(filename_or_dirname)
     ext = ext[1:]

neo/rawio/alphaomegarawio.py

@@ -90,16 +90,18 @@ class AlphaOmegaRawIO(BaseRawIO):
     def __init__(self, dirname="", lsx_files=None, prune_channels=True):
         super().__init__(dirname=dirname)
         self.dirname = Path(dirname)
+
         self._lsx_files = lsx_files
         self._mpx_files = None
-        if self.dirname.is_dir():
-            self._explore_folder()
-        else:
-            self.logger.error(f"{self.dirname} is not a folder")
         self._prune_channels = prune_channels
         self._opened_files = {}
         self._ignore_unknown_datablocks = True  # internal debug property
 
+        if self.dirname.is_dir():
+            self._explore_folder()
+        else:
+            raise ValueError(f"{self.dirname} is not a folder")
+
     def _explore_folder(self):
         """
         If class was instantiated with lsx_files (list of .lsx files), load only

neo/rawio/axographrawio.py

@@ -218,7 +218,7 @@ class AxographRawIO(BaseRawIO):
     """
     name = 'AxographRawIO'
     description = 'This IO reads .axgd/.axgx files created with AxoGraph'
-    extensions = ['axgd', 'axgx']
+    extensions = ['axgd', 'axgx', '']
     rawmode = 'one-file'
 
     def __init__(self, filename, force_single_segment=False):

neo/rawio/biocamrawio.py

@@ -29,7 +29,7 @@ class BiocamRawIO(BaseRawIO):
         >>> float_chunk = r.rescale_signal_raw_to_float(raw_chunk, dtype='float64',
                                                         channel_indexes=[0, 3, 6])
     """
-    extensions = ['h5']
+    extensions = ['h5', 'brw']
     rawmode = 'one-file'
 
     def __init__(self, filename=''):

neo/rawio/blackrockrawio.py

@@ -122,7 +122,7 @@ class BlackrockRawIO(BaseRawIO):
     """
 
     extensions = ['ns' + str(_) for _ in range(1, 7)]
-    extensions.extend(['nev', ])  # 'sif', 'ccf' not yet supported
+    extensions.extend(['nev', 'sif', 'ccf'])  # 'sif', 'ccf' not yet supported
     rawmode = 'multi-file'
 
     def __init__(self, filename=None, nsx_override=None, nev_override=None,
@@ -154,6 +154,9 @@ def __init__(self, filename=None, nsx_override=None, nev_override=None,
         else:
             self._filenames['nev'] = self.filename
 
+        self._filenames['sif'] = self.filename
+        self._filenames['ccf'] = self.filename
+
         # check which files are available
         self._avail_files = dict.fromkeys(self.extensions, False)
         self._avail_nsx = []

neo/rawio/neuralynxrawio/neuralynxrawio.py

@@ -75,7 +75,7 @@ class NeuralynxRawIO(BaseRawIO):
 
             Display all information about signal channels, units, segment size....
     """
-    extensions = ['nse', 'ncs', 'nev', 'ntt']
+    extensions = ['nse', 'ncs', 'nev', 'ntt', 'nvt', 'nrd']  # nvt and nrd are not yet supported
     rawmode = 'one-dir'
 
     _ncs_dtype = [('timestamp', 'uint64'), ('channel_id', 'uint32'), ('sample_rate', 'uint32'),

neo/rawio/nixrawio.py

@@ -31,7 +31,7 @@
 
 class NIXRawIO(BaseRawIO):
 
-    extensions = ['nix']
+    extensions = ['nix', 'h5']
     rawmode = 'one-file'
 
     def __init__(self, filename=''):

neo/rawio/openephysbinaryrawio.py

@@ -54,7 +54,7 @@ class OpenEphysBinaryRawIO(BaseRawIO):
     The current implementation does not handle spiking data, this will be added upon user request
 
     """
-    extensions = []
+    extensions = ['xml', 'oebin', 'txt', 'dat', 'npy']
     rawmode = 'one-dir'
 
     def __init__(self, dirname='', load_sync_channel=False, experiment_names=None):

neo/rawio/openephysrawio.py

@@ -64,7 +64,8 @@ class OpenEphysRawIO(BaseRawIO):
         aligned,
         and a warning is emitted.
     """
-    extensions = []
+    # file formats used by openephys
+    extensions = ['continuous', 'openephys', 'spikes', 'events', 'xml']
     rawmode = 'one-dir'
 
     def __init__(self, dirname=''):

neo/rawio/phyrawio.py

@@ -33,7 +33,8 @@ class PhyRawIO(BaseRawIO):
         >>> spike_times = r.rescale_spike_timestamp(spike_timestamp, 'float64')
 
     """
-    extensions = []
+    # file formats used by phy
+    extensions = ['npy', 'mat', 'tsv', 'dat']
     rawmode = 'one-dir'
 
     def __init__(self, dirname='', load_amplitudes=False, load_pcs=False):

neo/rawio/rawbinarysignalrawio.py

@@ -27,7 +27,7 @@ class RawBinarySignalIO
 
 
 class RawBinarySignalRawIO(BaseRawIO):
-    extensions = ['raw', '*']
+    extensions = ['raw', 'bin']
     rawmode = 'one-file'
 
     def __init__(self, filename='', dtype='int16', sampling_rate=10000.,

neo/rawio/spikeglxrawio.py

@@ -64,7 +64,8 @@ class SpikeGLXRawIO(BaseRawIO):
     load_sync_channel=False/True
         The last channel (SY0) of each stream is a fake channel used for synchronisation.
     """
-    extensions = []
+    # file formats used by spikeglxio
+    extensions = ['meta', 'bin']
     rawmode = 'one-dir'
 
     def __init__(self, dirname='', load_sync_channel=False, load_channel_location=False):

neo/rawio/tdtrawio.py

@@ -34,6 +34,7 @@
 
 
 class TdtRawIO(BaseRawIO):
+    extensions = ['tbk', 'tdx', 'tev', 'tin', 'tnt', 'tsq', 'sev', 'txt']
     rawmode = 'one-dir'
 
     def __init__(self, dirname='', sortname=''):

neo/test/iotest/common_io_test.py

@@ -9,7 +9,7 @@
 
 The public URL is in url_for_tests.
 
-To deposite new testing files,  please create a account at
+To deposit new testing files, please create a account at
 gin.g-node.org and upload files at NeuralEnsemble/ephy_testing_data
 data repo.
 
@@ -19,32 +19,28 @@
 
 import os
 import inspect
-from copy import copy
 import unittest
 import pathlib
 
 from neo.core import Block, Segment
 from neo.io.basefromrawio import BaseFromRaw
 from neo.test.tools import (assert_same_sub_schema,
                             assert_neo_object_is_compliant,
-                            assert_sub_schema_is_lazy_loaded,
-                            assert_children_empty)
+                            assert_sub_schema_is_lazy_loaded)
 
 from neo.test.rawiotest.tools import can_use_network
 from neo.test.rawiotest.common_rawio_test import repo_for_test
-from neo.utils import (download_dataset,
-    get_local_testing_data_folder)
+from neo.utils import (download_dataset, get_local_testing_data_folder)
+from neo import list_candidate_ios
 
 try:
     import datalad
     HAVE_DATALAD = True
 except:
     HAVE_DATALAD = False
 
-from neo.test.iotest.tools import (cleanup_test_file,
-                                   close_object_safe, create_generic_io_object,
+from neo.test.iotest.tools import (close_object_safe, create_generic_io_object,
                                    create_generic_reader,
-                                   create_generic_writer,
                                    get_test_file_full_path,
                                    iter_generic_io_objects,
                                    iter_generic_readers, iter_read_objects,
@@ -56,18 +52,18 @@
 
 class BaseTestIO:
     """
-    This class make common tests for all IOs.
+    This class defines common tests for all IOs.
 
-    Several startegies:
+    Several strategies:
       * for IO able to read write : test_write_then_read
       * for IO able to read write with hash conservation (optional):
         test_read_then_write
       * for all IOs : test_assert_read_neo_object_is_compliant
         2 cases:
-          * files are at G-node and downloaded:
-            download_test_files_if_not_present
+          * files are at G-node and downloaded
           * files are generated by MyIO.write()
 
+
     Note: When inheriting this class use it as primary superclass in
     combination with the unittest.TestCase as a 2nd superclass, e.g.
     `NewIOTestClass(BaseTestIO, unittest.TestCase):`
@@ -518,4 +514,11 @@ def test__handle_pathlib_filename(self):
             elif self.ioclass.mode == 'dir':
                 self.ioclass(dirname=pathlib_filename,
                              *self.default_arguments,
-                             **self.default_keyword_arguments)
+                             **self.default_keyword_arguments))
+
+    def test_list_candidate_ios(self):
+        for entity in self.entities_to_test:
+            entity = get_test_file_full_path(self.ioclass, filename=entity,
+                                             directory=self.local_test_dir)
+            ios = list_candidate_ios(entity)
+            assert self.ioclass in ios