From cb1d1190f1f00786aa497fc1188aed464d627066 Mon Sep 17 00:00:00 2001 From: Faeez Shaikh Date: Sat, 1 Apr 2023 16:14:59 -0500 Subject: [PATCH 1/3] Removed unused parameter from the get_readers() and removed unused import LocalFileSystem --- reader/dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/reader/dataset.py b/reader/dataset.py index 6e811cc..cb09e99 100644 --- a/reader/dataset.py +++ b/reader/dataset.py @@ -9,7 +9,7 @@ import random from typing import Optional -from fsspec.implementations.local import LocalFileSystem +from fsspec.implementations.local import pyarrow.dataset as pads import pyarrow as pa import pyarrow.parquet @@ -105,7 +105,7 @@ def pa_to_batch(self, batch: pa.RecordBatch) -> DataclassBatch: def dataloader(self, remote: bool = False): if not remote: return map(self.pa_to_batch, self.to_batches()) - readers = get_readers(2) + readers = get_readers() return map(self.pa_to_batch, reader_utils.roundrobin(*readers)) @@ -116,7 +116,7 @@ def dataloader(self, remote: bool = False): ] -def get_readers(num_readers_per_worker: int): +def get_readers(): addresses = env.get_flight_server_addresses() readers = [] From c0e30a84f0d802b2900a61c2d8191db72fa7cd52 Mon Sep 17 00:00:00 2001 From: Faeez Shaikh Date: Sat, 1 Apr 2023 16:37:41 -0500 Subject: [PATCH 2/3] Make code more readable. Replace idx --- core/test_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/test_metrics.py b/core/test_metrics.py index ac29819..f75bf93 100644 --- a/core/test_metrics.py +++ b/core/test_metrics.py @@ -58,7 +58,7 @@ def test_task_dependent_ctr(): "stuff": 0, "labels": torch.arange(0, 6).repeat(batch_size, 1), } - for idx in range(num_examples) + for _ in range(num_examples) ] for task_idx in range(5): From bf41cb51824b533ae2b557d9cca377b9a3ae16ba Mon Sep 17 00:00:00 2001 From: Faeez Shaikh Date: Sat, 1 Apr 2023 18:08:35 -0500 Subject: [PATCH 3/3] Proper removal of unused import --- reader/dataset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/reader/dataset.py b/reader/dataset.py index cb09e99..7bda972 100644 --- a/reader/dataset.py +++ b/reader/dataset.py @@ -9,7 +9,6 @@ import random from typing import Optional -from fsspec.implementations.local import pyarrow.dataset as pads import pyarrow as pa import pyarrow.parquet