From 88d445f7b5ff4ed1e3a8d0086050645264ea0f64 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Fri, 8 Mar 2024 14:43:07 -0500 Subject: [PATCH] Ensure dircache on ls (#612) --- gcsfs/core.py | 2 +- gcsfs/retry.py | 16 +++++++++------- gcsfs/tests/test_core.py | 19 +++++++++++++++++-- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/gcsfs/core.py b/gcsfs/core.py index f425afc0..7308e9c2 100644 --- a/gcsfs/core.py +++ b/gcsfs/core.py @@ -596,7 +596,7 @@ async def _list_objects(self, path, prefix="", versions=False, **kwargs): # Don't cache prefixed/partial listings, in addition to # not using the inventory report service to do listing directly. - if not prefix and use_snapshot_listing is False: + if not prefix and not use_snapshot_listing: self.dircache[path] = out return out diff --git a/gcsfs/retry.py b/gcsfs/retry.py index 4bb860ba..42c73a51 100644 --- a/gcsfs/retry.py +++ b/gcsfs/retry.py @@ -58,15 +58,17 @@ class ChecksumError(Exception): ) +errs = list(range(500, 505)) + [ + # Request Timeout + 408, + # Too Many Requests + 429, +] +errs = set(errs + [str(e) for e in errs]) + + def is_retriable(exception): """Returns True if this exception is retriable.""" - errs = list(range(500, 505)) + [ - # Request Timeout - 408, - # Too Many Requests - 429, - ] - errs += [str(e) for e in errs] if isinstance(exception, HttpError): return exception.code in errs diff --git a/gcsfs/tests/test_core.py b/gcsfs/tests/test_core.py index 1415d66f..37912a54 100644 --- a/gcsfs/tests/test_core.py +++ b/gcsfs/tests/test_core.py @@ -26,12 +26,27 @@ TEST_REQUESTER_PAYS_BUCKET = gcsfs.tests.settings.TEST_REQUESTER_PAYS_BUCKET -def test_simple(gcs): - assert not GoogleCredentials.tokens +def test_simple(gcs, monkeypatch): + monkeypatch.setattr(GoogleCredentials, "tokens", None) gcs.ls(TEST_BUCKET) # no error gcs.ls("/" + TEST_BUCKET) # OK to lead with '/' +def test_dircache_filled(gcs): + assert not dict(gcs.dircache) + gcs.ls(TEST_BUCKET) + assert len(gcs.dircache) == 1 + gcs.dircache[TEST_BUCKET][0]["CHECK"] = True + out = gcs.ls(TEST_BUCKET, detail=True) + assert [o for o in out if o.get("CHECK", None)] + + gcs.invalidate_cache() + assert not dict(gcs.dircache) + + gcs.find(TEST_BUCKET) + assert len(gcs.dircache) + + def test_many_connect(docker_gcs): from multiprocessing.pool import ThreadPool