Skip to content

Commit 3eb1051

Browse files
committed
use select one-liner
1 parent 7370b56 commit 3eb1051

File tree

1 file changed

+1
-17
lines changed
  • src/nested_pandas/nestedframe

1 file changed

+1
-17
lines changed

src/nested_pandas/nestedframe/io.py

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -133,23 +133,7 @@ def read_parquet(
133133
indices_to_remove = []
134134
for col, indices in nested_structures.items():
135135
# Build a struct column from the columns
136-
field_names = [table.column_names[i] for i in indices]
137-
138-
# Use iterchunks to process chunks of each column
139-
chunked_arrays = []
140-
for i in indices:
141-
column = table.column(i)
142-
if len(column.chunks) == 1:
143-
# If there is only one chunk, use it directly
144-
# avoid copy in concat_arrays
145-
chunked_arrays.append(column.chunk(0))
146-
else:
147-
# Otherwise, concatenate all chunks
148-
chunked_arrays.append(pa.concat_arrays(list(column.iterchunks())))
149-
structs[col] = pa.StructArray.from_arrays(
150-
chunked_arrays, # Child arrays
151-
field_names, # Field names
152-
)
136+
structs[col] = table.select(indices).to_struct_array()
153137
indices_to_remove.extend(indices)
154138

155139
# Remove the original columns in reverse order to avoid index shifting

0 commit comments

Comments
 (0)