Fix a bug when reading directory twice

Author: hombit

src/nested_pandas/nestedframe/io.py

@@ -20,7 +20,8 @@
 FSSPEC_FILESYSTEMS = ("http", "https")
 FSSPEC_BLOCK_SIZE = 32 * 1024
 
-# Filesystems for which calling .is_dir() may be very slow
+# Filesystems for which calling .is_dir() may be very slow and/or .iterdir()
+# may yield non-parquet paths.
 NO_ITERDIR_FILESYSTEMS = (
     "http",
     "https",
@@ -194,13 +195,17 @@ def _read_parquet_into_table(
     # NOTE: the test for _is_local_dir is sufficient, because we're
     # preserving a path to pq.read_table, which can read local
     # directories, but not remote directories. Remote directories
-    # are supported separately via _read_parquet_directory
+    # are supported separately via _read_parquet_directory.
+    # We don't support HTTP "directories", because 1) calling .is_dir()
+    # may be very expensive, because it downloads content first,
+    # 2) because .iter_dir() is likely to return a lot of "junk"
+    # besides of the actual parquet files.
     if isinstance(data, str | Path | UPath) and not _is_local_dir(path_to_data := UPath(data)):
         storage_options = _get_storage_options(path_to_data)
         filesystem = kwargs.get("filesystem")
         if not filesystem:
             _, filesystem = _transform_read_parquet_data_arg(path_to_data)
-        # Check original string, because UPath may chomp trailing "/"
+        # Will not detect HTTP(S) directories.
         if _is_remote_dir(data, path_to_data):
             return _read_remote_parquet_directory(
                 path_to_data, filesystem, storage_options, columns, **kwargs
@@ -255,14 +260,15 @@ def _read_remote_parquet_directory(
         # handlers. This would work for e.g. S3, but not for HTTP(S).
         if _is_remote_dir(upath, upath):
             table = _read_remote_parquet_directory(upath, filesystem, storage_options, columns, **kwargs)
-        with fsspec.parquet.open_parquet_file(
-            upath.path,
-            columns=columns,
-            storage_options=storage_options,
-            fs=filesystem,
-            engine="pyarrow",
-        ) as parquet_file:
-            table = pq.read_table(parquet_file, columns=columns, **kwargs)
+        else:
+            with fsspec.parquet.open_parquet_file(
+                upath.path,
+                columns=columns,
+                storage_options=storage_options,
+                fs=filesystem,
+                engine="pyarrow",
+            ) as parquet_file:
+                table = pq.read_table(parquet_file, columns=columns, **kwargs)
         tables.append(table)
     return pa.concat_tables(tables)