diff --git a/core/test_metrics.py b/core/test_metrics.py index ac29819..f75bf93 100644 --- a/core/test_metrics.py +++ b/core/test_metrics.py @@ -58,7 +58,7 @@ def test_task_dependent_ctr(): "stuff": 0, "labels": torch.arange(0, 6).repeat(batch_size, 1), } - for idx in range(num_examples) + for _ in range(num_examples) ] for task_idx in range(5): diff --git a/reader/dataset.py b/reader/dataset.py index 6e811cc..7bda972 100644 --- a/reader/dataset.py +++ b/reader/dataset.py @@ -9,7 +9,6 @@ import random from typing import Optional -from fsspec.implementations.local import LocalFileSystem import pyarrow.dataset as pads import pyarrow as pa import pyarrow.parquet @@ -105,7 +104,7 @@ def pa_to_batch(self, batch: pa.RecordBatch) -> DataclassBatch: def dataloader(self, remote: bool = False): if not remote: return map(self.pa_to_batch, self.to_batches()) - readers = get_readers(2) + readers = get_readers() return map(self.pa_to_batch, reader_utils.roundrobin(*readers)) @@ -116,7 +115,7 @@ def dataloader(self, remote: bool = False): ] -def get_readers(num_readers_per_worker: int): +def get_readers(): addresses = env.get_flight_server_addresses() readers = []