From c720dcc68b451d379b1b4bcd0b1a1ec72ed62471 Mon Sep 17 00:00:00 2001 From: Sharvin-M Date: Tue, 14 Jan 2025 12:58:26 -0800 Subject: [PATCH 1/7] added client id method --- cmr/queries.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cmr/queries.py b/cmr/queries.py index cdf9f2e..2dcc928 100644 --- a/cmr/queries.py +++ b/cmr/queries.py @@ -365,6 +365,26 @@ def bearer_token(self, bearer_token: str) -> Self: return self + def client_id(self, client_id: str) -> Self: + """ + Set the value of this query's 'Client ID' header according to User's input. + + If an empty parameter is given, default is set to be + python_cmr-vX.Y.Z, where X.Y.Z is the version of python_cmr. + Otherwise, set the specified paramter option to the value along with + the suffix (python_cmr-vX.Y.Z) and a space character between the specified paramter and the suffix. + + :param client_id + :returns self + """ + + if not client_id: + self.headers.update({"Client-Id: python_cmr-v0.13.0"}) + + self.headers.update({"Client-Id": f"{client_id} python_cmr-v0.13.0"}) + + return self + def option( self, parameter: str, key: str, value: Union[str, bool, int, float, None] ) -> Self: From 9f9d247445725be3327f34b59936e1b6b789b192 Mon Sep 17 00:00:00 2001 From: Sharvin-M Date: Tue, 14 Jan 2025 18:47:31 -0800 Subject: [PATCH 2/7] updated client_id method --- cmr/queries.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/cmr/queries.py b/cmr/queries.py index 2dcc928..f0712cd 100644 --- a/cmr/queries.py +++ b/cmr/queries.py @@ -8,6 +8,7 @@ from inspect import getmembers, ismethod from re import search from typing import Iterator +from importlib.metadata import version from typing_extensions import ( Any, @@ -59,6 +60,7 @@ def __init__(self, route: str, mode: str = CMR_OPS): self.mode(mode) self.concept_id_chars: Set[str] = set() self.headers: MutableMapping[str, str] = {} + self.headers.update({"Client-Id": f"python_cmr-v{version('python_cmr')}"}) @deprecated("Use the 'results' method instead, but note that it produces an iterator.") def get(self, limit: int = 2000) -> Sequence[Any]: @@ -365,23 +367,23 @@ def bearer_token(self, bearer_token: str) -> Self: return self - def client_id(self, client_id: str) -> Self: + def client_id(self, id_: str) -> Self: """ - Set the value of this query's 'Client ID' header according to User's input. + Set the value of this query's `Client-Id` header. - If an empty parameter is given, default is set to be - python_cmr-vX.Y.Z, where X.Y.Z is the version of python_cmr. - Otherwise, set the specified paramter option to the value along with - the suffix (python_cmr-vX.Y.Z) and a space character between the specified paramter and the suffix. + Otherwise, set the header value to the specified value along with + the suffix `(python_cmr-vX.Y.Z)`, separated by a space character. :param client_id :returns self """ - - if not client_id: - self.headers.update({"Client-Id: python_cmr-v0.13.0"}) - - self.headers.update({"Client-Id": f"{client_id} python_cmr-v0.13.0"}) + + if not id_: + return self + + self.headers.update( + {"Client-Id": f"{id_} python_cmr-v{version('python_cmr')}"} + ) return self From d92aa640210a58c036663c1b0450e61e3ac9dc91 Mon Sep 17 00:00:00 2001 From: Sharvin-M Date: Tue, 14 Jan 2025 18:47:57 -0800 Subject: [PATCH 3/7] updated tests for client_id method --- tests/test_queries.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/test_queries.py b/tests/test_queries.py index 142a026..171db1d 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1,5 +1,5 @@ from cmr import Query - +from importlib.metadata import version class MockQuery(Query): def _valid_state(self) -> bool: @@ -8,7 +8,8 @@ def _valid_state(self) -> bool: def test_query_headers_initially_empty(): query = MockQuery("/foo") - assert query.headers == {} + expected_version = version("python_cmr") + assert query.headers == {"Client-Id": f"python_cmr-v{expected_version}"} def test_bearer_token_adds_header(): @@ -55,3 +56,11 @@ def test_token_replaces_existing_auth_header(): query.token("token") assert query.headers["Authorization"] == "token" + +def test_client_id_sets_header(): + query = MockQuery("/foo") + query.client_id("test_client") + query.token("token") + + expected_version = version("python_cmr") + assert query.headers["Client-Id"] == f"test_client python_cmr-v{expected_version}" From 10ec6738f79a383267bc41142d2cff5594dac5a6 Mon Sep 17 00:00:00 2001 From: Frank Greguska <89428916+frankinspace@users.noreply.github.com> Date: Wed, 15 Jan 2025 08:15:51 -0600 Subject: [PATCH 4/7] Update cmr/queries.py --- cmr/queries.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmr/queries.py b/cmr/queries.py index f0712cd..7f1fc89 100644 --- a/cmr/queries.py +++ b/cmr/queries.py @@ -369,9 +369,7 @@ def bearer_token(self, bearer_token: str) -> Self: def client_id(self, id_: str) -> Self: """ - Set the value of this query's `Client-Id` header. - - Otherwise, set the header value to the specified value along with + Set the header value to the specified value along with the suffix `(python_cmr-vX.Y.Z)`, separated by a space character. :param client_id From 12f11b13d4960e939091e0cc661e5d7abd9d0d8f Mon Sep 17 00:00:00 2001 From: Sharvin Manjrekar Date: Wed, 15 Jan 2025 09:21:52 -0800 Subject: [PATCH 5/7] Update cmr/queries.py Co-authored-by: Chuck Daniels --- cmr/queries.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cmr/queries.py b/cmr/queries.py index 7f1fc89..60956ef 100644 --- a/cmr/queries.py +++ b/cmr/queries.py @@ -369,11 +369,15 @@ def bearer_token(self, bearer_token: str) -> Self: def client_id(self, id_: str) -> Self: """ - Set the header value to the specified value along with - the suffix `(python_cmr-vX.Y.Z)`, separated by a space character. - - :param client_id - :returns self + Set the `Client-Id` header value to the specified value along with + the suffix `(python_cmr-vX.Y.Z)`, separated by a space character, + where `X.Y.Z` is the current version of the `python_cmr` library. + Set the header value to `python_cmr-vX.Y.Z` (without parentheses) + when the specified value is an empty string (or `None`), which is the + default header value even when this method is not invoked. + + :param client_id: prefix value to set on the `Client-Id` header + :returns self: """ if not id_: From 132ab4e0fab8d846f626018f2b3b10f9423e4b28 Mon Sep 17 00:00:00 2001 From: Sharvin-M Date: Wed, 15 Jan 2025 09:28:37 -0800 Subject: [PATCH 6/7] added requested changes --- cmr/queries.py | 189 ++++++++++++++++++++++-------------------- tests/test_queries.py | 4 +- 2 files changed, 103 insertions(+), 90 deletions(-) diff --git a/cmr/queries.py b/cmr/queries.py index 60956ef..53889c3 100644 --- a/cmr/queries.py +++ b/cmr/queries.py @@ -23,7 +23,8 @@ Tuple, TypeAlias, Union, - override, deprecated, + override, + deprecated, ) from urllib.parse import quote @@ -49,8 +50,16 @@ class Query: _route = "" _format = "json" _valid_formats_regex = [ - "json", "xml", "echo10", "iso", "iso19115", - "csv", "atom", "kml", "native", "stac", + "json", + "xml", + "echo10", + "iso", + "iso19115", + "csv", + "atom", + "kml", + "native", + "stac", ] def __init__(self, route: str, mode: str = CMR_OPS): @@ -60,9 +69,11 @@ def __init__(self, route: str, mode: str = CMR_OPS): self.mode(mode) self.concept_id_chars: Set[str] = set() self.headers: MutableMapping[str, str] = {} - self.headers.update({"Client-Id": f"python_cmr-v{version('python_cmr')}"}) + self.client_id() - @deprecated("Use the 'results' method instead, but note that it produces an iterator.") + @deprecated( + "Use the 'results' method instead, but note that it produces an iterator." + ) def get(self, limit: int = 2000) -> Sequence[Any]: """ Get all results up to some limit, even if spanning multiple pages. @@ -120,7 +131,9 @@ def hits(self) -> int: return int(response.headers["CMR-Hits"]) - @deprecated("Use the 'results' method instead, but note that it produces an iterator.") + @deprecated( + "Use the 'results' method instead, but note that it produces an iterator." + ) def get_all(self) -> Sequence[Any]: """ Returns all of the results for the query. This will call hits() first to determine how many @@ -129,7 +142,7 @@ def get_all(self) -> Sequence[Any]: :returns: query results as a list """ - + return list(self.get(self.hits())) def results(self, page_size: int = 2000) -> Iterator[Any]: @@ -239,13 +252,16 @@ def _build_url(self) -> str: # last chance validation for parameters if not self._valid_state(): - raise RuntimeError(("Spatial parameters must be accompanied by a collection " - "filter (ex: short_name or entry_title).")) + raise RuntimeError( + ( + "Spatial parameters must be accompanied by a collection " + "filter (ex: short_name or entry_title)." + ) + ) # encode params formatted_params = [] for key, val in self.params.items(): - # list params require slightly different formatting if isinstance(val, list): for list_val in val: @@ -263,11 +279,13 @@ def _build_url(self) -> str: formatted_options: List[str] = [] for param_key in self.options: for option_key, val in self.options[param_key].items(): - formatted_options.append(f"options[{param_key}][{option_key}]={str(val).lower()}") + formatted_options.append( + f"options[{param_key}][{option_key}]={str(val).lower()}" + ) options_as_string = "&".join(formatted_options) res = f"{self._base_url}.{self._format}?{params_as_string}&{options_as_string}" - return res.rstrip('&') + return res.rstrip("&") def concept_id(self, IDs: Union[str, Sequence[str]]) -> Self: """ @@ -308,7 +326,7 @@ def provider(self, provider: str) -> Self: if not provider: return self - self.params['provider'] = provider + self.params["provider"] = provider return self @abstractmethod @@ -380,14 +398,8 @@ def client_id(self, id_: str) -> Self: :returns self: """ - if not id_: - return self - - self.headers.update( - {"Client-Id": f"{id_} python_cmr-v{version('python_cmr')}"} - ) - - return self + python_cmr_id = f"python_cmr-v{version('python_cmr')}" + self.headers["Client-Id"] = f"{id_} ({python_cmr_id})" if id_ else python_cmr_id def option( self, parameter: str, key: str, value: Union[str, bool, int, float, None] @@ -467,14 +479,12 @@ def online_only(self, online_only: bool = True) -> Self: if "downloadable" in self.params: del self.params["downloadable"] - self.params['online_only'] = online_only + self.params["online_only"] = online_only return self def _format_date( - self, - date_from: Optional[DateLike], - date_to: Optional[DateLike] + self, date_from: Optional[DateLike], date_to: Optional[DateLike] ) -> Tuple[str, str]: """ Format dates into expected format for date queries. @@ -517,8 +527,12 @@ def convert_to_string(date: Optional[DateLike], default: datetime) -> str: return date.strftime(iso_8601) - date_from = convert_to_string(date_from, datetime(1, 1, 1, 0, 0, 0, tzinfo=timezone.utc)) - date_to = convert_to_string(date_to, datetime(1, 12, 31, 23, 59, 59, tzinfo=timezone.utc)) + date_from = convert_to_string( + date_from, datetime(1, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + ) + date_to = convert_to_string( + date_to, datetime(1, 12, 31, 23, 59, 59, tzinfo=timezone.utc) + ) # if we have both dates, make sure from isn't later than to if date_from and date_to and date_from > date_to: @@ -553,9 +567,7 @@ def revision_date( self.params["revision_date"].append(f"{date_from},{date_to}") if exclude_boundary: - self.options["revision_date"] = { - "exclude_boundary": True - } + self.options["revision_date"] = {"exclude_boundary": True} return self @@ -586,9 +598,7 @@ def temporal( self.params["temporal"].append(f"{date_from},{date_to}") if exclude_boundary: - self.options["temporal"] = { - "exclude_boundary": True - } + self.options["temporal"] = {"exclude_boundary": True} return self @@ -603,7 +613,7 @@ def short_name(self, short_name: str) -> Self: if not short_name: return self - self.params['short_name'] = short_name + self.params["short_name"] = short_name return self def version(self, version: str) -> Self: @@ -618,7 +628,7 @@ def version(self, version: str) -> Self: if not version: return self - self.params['version'] = version + self.params["version"] = version return self def point(self, lon: FloatLike, lat: FloatLike) -> Self: @@ -654,7 +664,7 @@ def circle(self, lon: FloatLike, lat: FloatLike, dist: FloatLike) -> Self: :param dist: distance in meters around waypoint (lat,lon) :returns: self """ - self.params['circle'] = f"{lon},{lat},{dist}" + self.params["circle"] = f"{lon},{lat},{dist}" return self @@ -780,7 +790,7 @@ def downloadable(self, downloadable: bool = True) -> Self: if "online_only" in self.params: del self.params["online_only"] - self.params['downloadable'] = downloadable + self.params["downloadable"] = downloadable return self @@ -794,7 +804,7 @@ def entry_title(self, entry_title: str) -> Self: entry_title = quote(entry_title) - self.params['entry_title'] = entry_title + self.params["entry_title"] = entry_title return self @@ -809,7 +819,7 @@ def platform(self, platform: str) -> Self: if not platform: raise ValueError("Please provide a value for platform") - self.params['platform'] = platform + self.params["platform"] = platform return self @@ -835,9 +845,9 @@ def orbit_number( """ if orbit2: - self.params['orbit_number'] = quote(f'{str(orbit1)},{str(orbit2)}') + self.params["orbit_number"] = quote(f"{str(orbit1)},{str(orbit2)}") else: - self.params['orbit_number'] = orbit1 + self.params["orbit_number"] = orbit1 return self @@ -891,7 +901,7 @@ def cloud_cover(self, min_cover: FloatLike = 0, max_cover: FloatLike = 100) -> S "Please ensure min_cover and max_cover are both convertible to floats" ) from None - self.params['cloud_cover'] = f"{min_cover},{max_cover}" + self.params["cloud_cover"] = f"{min_cover},{max_cover}" return self def instrument(self, instrument: str) -> Self: @@ -905,7 +915,7 @@ def instrument(self, instrument: str) -> Self: if not instrument: raise ValueError("Please provide a value for instrument") - self.params['instrument'] = instrument + self.params["instrument"] = instrument return self def sort_key(self, sort_key: str) -> Self: @@ -922,39 +932,39 @@ def sort_key(self, sort_key: str) -> Self: """ valid_sort_keys = { - 'campaign', - 'entry_title', - 'dataset_id', - 'data_size', - 'end_date', - 'granule_ur', - 'producer_granule_id', - 'project', - 'provider', - 'readable_granule_name', - 'short_name', - 'start_date', - 'version', - 'platform', - 'instrument', - 'sensor', - 'day_night_flag', - 'online_only', - 'browsable', - 'browse_only', - 'cloud_cover', - 'revision_date', + "campaign", + "entry_title", + "dataset_id", + "data_size", + "end_date", + "granule_ur", + "producer_granule_id", + "project", + "provider", + "readable_granule_name", + "short_name", + "start_date", + "version", + "platform", + "instrument", + "sensor", + "day_night_flag", + "online_only", + "browsable", + "browse_only", + "cloud_cover", + "revision_date", } # also covers if empty string and allows for '-' prefix (for descending order) - if not isinstance(sort_key, str) or sort_key.lstrip('-') not in valid_sort_keys: + if not isinstance(sort_key, str) or sort_key.lstrip("-") not in valid_sort_keys: raise ValueError( "Please provide a valid sort key for granules query. See" " https://cmr.earthdata.nasa.gov/search/site/docs/search/api.html#sorting-granule-results" " for valid sort keys." ) - self.params['sort_key'] = sort_key + self.params["sort_key"] = sort_key return self def granule_ur(self, granule_ur: str) -> Self: @@ -969,7 +979,7 @@ def granule_ur(self, granule_ur: str) -> Self: if not granule_ur: raise ValueError("Please provide a value for platform") - self.params['granule_ur'] = granule_ur + self.params["granule_ur"] = granule_ur return self def readable_granule_name( @@ -1019,7 +1029,6 @@ def collection_concept_id(self, IDs: Union[str, Sequence[str]]) -> Self: @override def _valid_state(self) -> bool: - # spatial params must be paired with a collection limiting parameter spatial_keys = ["point", "polygon", "bounding_box", "line"] collection_keys = ["short_name", "entry_title", "collection_concept_id"] @@ -1040,9 +1049,9 @@ class CollectionQuery(GranuleCollectionBaseQuery): def __init__(self, mode: str = CMR_OPS): Query.__init__(self, "collections", mode) self.concept_id_chars = {"C"} - self._valid_formats_regex.extend([ - "dif", "dif10", "opendata", "umm_json", "umm_json_v[0-9]_[0-9]" - ]) + self._valid_formats_regex.extend( + ["dif", "dif10", "opendata", "umm_json", "umm_json_v[0-9]_[0-9]"] + ) def archive_center(self, center: str) -> Self: """ @@ -1053,7 +1062,7 @@ def archive_center(self, center: str) -> Self: """ if center: - self.params['archive_center'] = center + self.params["archive_center"] = center return self @@ -1068,7 +1077,7 @@ def keyword(self, text: str) -> Self: """ if text: - self.params['keyword'] = text + self.params["keyword"] = text return self @@ -1103,7 +1112,9 @@ def tool_concept_id(self, IDs: Union[str, Sequence[str]]) -> Self: # verify we provided with tool concept IDs for ID in IDs: if ID.strip()[0] != "T": - raise ValueError(f"Only tool concept ID's can be provided (begin with 'T'): {ID}") + raise ValueError( + f"Only tool concept ID's can be provided (begin with 'T'): {ID}" + ) self.params["tool_concept_id"] = IDs @@ -1198,14 +1209,13 @@ def get(self, limit: int = 2000) -> Sequence[Any]: results: List[Any] = [] page = 1 while len(results) < limit: - response = requests.get( url, params={"page_size": page_size, "page_num": page} ) response.raise_for_status() if self._format == "json": - latest = response.json()['items'] + latest = response.json()["items"] else: latest = [response.text] @@ -1243,7 +1253,7 @@ def name(self, name: str) -> Self: if not name: return self - self.params['name'] = name + self.params["name"] = name return self @@ -1255,9 +1265,9 @@ class ToolQuery(ToolServiceVariableBaseQuery): def __init__(self, mode: str = CMR_OPS): Query.__init__(self, "tools", mode) self.concept_id_chars = {"T"} - self._valid_formats_regex.extend([ - "dif", "dif10", "opendata", "umm_json", "umm_json_v[0-9]_[0-9]" - ]) + self._valid_formats_regex.extend( + ["dif", "dif10", "opendata", "umm_json", "umm_json_v[0-9]_[0-9]"] + ) @override def _valid_state(self) -> bool: @@ -1272,9 +1282,9 @@ class ServiceQuery(ToolServiceVariableBaseQuery): def __init__(self, mode: str = CMR_OPS): Query.__init__(self, "services", mode) self.concept_id_chars = {"S"} - self._valid_formats_regex.extend([ - "dif", "dif10", "opendata", "umm_json", "umm_json_v[0-9]_[0-9]" - ]) + self._valid_formats_regex.extend( + ["dif", "dif10", "opendata", "umm_json", "umm_json_v[0-9]_[0-9]"] + ) @override def _valid_state(self) -> bool: @@ -1282,13 +1292,12 @@ def _valid_state(self) -> bool: class VariableQuery(ToolServiceVariableBaseQuery): - def __init__(self, mode: str = CMR_OPS): Query.__init__(self, "variables", mode) self.concept_id_chars = {"V"} - self._valid_formats_regex.extend([ - "dif", "dif10", "opendata", "umm_json", "umm_json_v[0-9]_[0-9]" - ]) + self._valid_formats_regex.extend( + ["dif", "dif10", "opendata", "umm_json", "umm_json_v[0-9]_[0-9]"] + ) def instance_format(self, format: Union[str, Sequence[str]]) -> Self: """ @@ -1302,7 +1311,9 @@ def instance_format(self, format: Union[str, Sequence[str]]) -> Self: if format: # Assume we have non-empty string or sequence of strings (list, tuple, etc.) - self.params['instance_format'] = [format] if isinstance(format, str) else format + self.params["instance_format"] = ( + [format] if isinstance(format, str) else format + ) return self diff --git a/tests/test_queries.py b/tests/test_queries.py index 171db1d..707c056 100644 --- a/tests/test_queries.py +++ b/tests/test_queries.py @@ -1,6 +1,7 @@ from cmr import Query from importlib.metadata import version + class MockQuery(Query): def _valid_state(self) -> bool: return True @@ -57,10 +58,11 @@ def test_token_replaces_existing_auth_header(): assert query.headers["Authorization"] == "token" + def test_client_id_sets_header(): query = MockQuery("/foo") query.client_id("test_client") query.token("token") expected_version = version("python_cmr") - assert query.headers["Client-Id"] == f"test_client python_cmr-v{expected_version}" + assert query.headers["Client-Id"] == f"test_client (python_cmr-v{expected_version})" From 38e723bf41e1e9805857a6f7aa494144411fc754 Mon Sep 17 00:00:00 2001 From: Sharvin Manjrekar Date: Fri, 17 Jan 2025 12:06:57 -0800 Subject: [PATCH 7/7] Update cmr/queries.py Co-authored-by: Chuck Daniels --- cmr/queries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmr/queries.py b/cmr/queries.py index 53889c3..623feea 100644 --- a/cmr/queries.py +++ b/cmr/queries.py @@ -385,7 +385,7 @@ def bearer_token(self, bearer_token: str) -> Self: return self - def client_id(self, id_: str) -> Self: + def client_id(self, id_: Optional[str] = None) -> Self: """ Set the `Client-Id` header value to the specified value along with the suffix `(python_cmr-vX.Y.Z)`, separated by a space character,