From 5decae15784204349494e34801799897b47e402e Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Tue, 10 May 2022 11:21:10 +0300 Subject: [PATCH 1/4] DC-2935 Api v2.0 skeleton (#60) --- requirements-test.txt | 3 +- setup.py | 2 + src/corva/api_adapter.py | 209 ++++++++++++++++++++++++++++++++++ tests/integration/test_api.py | 94 +++++++++++++++ 4 files changed, 307 insertions(+), 1 deletion(-) create mode 100644 src/corva/api_adapter.py create mode 100644 tests/integration/test_api.py diff --git a/requirements-test.txt b/requirements-test.txt index 5b6e9aa7..8f13da73 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,6 @@ coverage==5.3 freezegun==1.0.0 -pytest==6.1.2 +pytest==7.1.2 +pytest-httpx==0.20.0 pytest-mock==3.3.1 requests-mock==1.8.0 diff --git a/setup.py b/setup.py index a92b2880..1fc93821 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,8 @@ "pydantic >=1.8.2, <2.0.0", "redis >=3.5.3, <4.0.0", "requests >=2.25.0, <3.0.0", + "httpx >=0.22.0, <0.23.0", + "PyYAML >=6.0, <6.1", ], python_requires='>=3.8, <4.0', license='The Unlicense', diff --git a/src/corva/api_adapter.py b/src/corva/api_adapter.py new file mode 100644 index 00000000..94292f04 --- /dev/null +++ b/src/corva/api_adapter.py @@ -0,0 +1,209 @@ +import dataclasses +import functools +import json +import logging +import posixpath +from typing import Callable, List, Optional + +import httpx +import yaml + + +def _httpx_headers_to_dict(headers: httpx.Headers) -> dict: + return json.loads( + repr(headers) # use built-in `repr` as it obfuscates sensitive headers + .strip("Headers()") # strip obsolete data + .replace( + "'", '"' + ) # replace single quotes with double ones to get proper json string + ) + + +def _failed_request_msg( + msg: str, + request: httpx.Request, + response: Optional[httpx.Response], +) -> str: + data = {"message": f"Request failed - {msg}"} + + if response: + # log response first, so there is less chance it gets truncated + # and users are able to see server error message + data["response"] = { + "code": response.status_code, + "reason": response.reason_phrase, + "headers": _httpx_headers_to_dict(response.headers), + "content": str(response.content), + } + + data["request"] = { + "method": request.method, + "url": str(request.url), + "headers": _httpx_headers_to_dict(request.headers), + "content": str(request.content), + } + + # use yaml because it is much more readable in logs + return yaml.dump(data, sort_keys=False) + + +def logging_send(func: Callable, *, logger: logging.Logger) -> Callable: + @functools.wraps(func) + def wrapper(request: httpx.Request, *args, **kwargs): + try: + response = func(request, *args, **kwargs) + except httpx.HTTPError as exc: + # Response was not received at all + logger.error( + _failed_request_msg(msg=str(exc), request=request, response=None) + ) + raise + + try: + response.raise_for_status() + except httpx.HTTPStatusError as exc: + # Response has unsuccessful status + logger.error( + _failed_request_msg(msg=str(exc), request=request, response=response) + ) + + return response + + return wrapper + + +class DataApiV1Sdk: + def __init__(self, client: httpx.Client): + self.http = client + + def get( + self, + provider: str, + dataset: str, + *, + query: dict, + sort: dict, + limit: int, + skip: int = 0, + fields: Optional[str] = None, + ) -> List[dict]: + """Fetches data from the endpoint GET 'data/{provider}/{dataset}/'. + + Args: + provider: company name owning the dataset. + dataset: dataset name. + query: search conditions. Example: {"asset_id": 123} - will fetch data + for asset with id 123. + sort: sort conditions. Example: {"timestamp": 1} - will sort data + in ascending order by timestamp. + limit: number of data points to fecth. + Recommendation for setting the limit: + 1. The bigger ↑ each data point is - the smaller ↓ the limit; + 2. The smaller ↓ each data point is - the bigger ↑ the limit. + skip: exclude from the response the first N items of the dataset. + Note: skip should only be used for small amounts of data, having a + large skip will lead to very slow queries. + fields: comma separated list of fields to return. Example: "_id,data". + + Raises: + requests.HTTPError: if request was unsuccessful. + + Returns: + Data from dataset. + """ + + response = self.http.get( + url=f"data/{provider}/{dataset}/", + params={ + "query": json.dumps(query), + "sort": json.dumps(sort), + "fields": fields, + "limit": limit, + "skip": skip, + }, + ) + + response.raise_for_status() + + data = list(response.json()) + + return data + + +class PlatformApiV1Sdk: + def __init__(self, client: httpx.Client): + self.http = client + + +class PlatformApiV2Sdk: + def __init__(self, client: httpx.Client): + self.http = client + + +@dataclasses.dataclass(frozen=True) +class PlatformApiVersions: + v1: PlatformApiV1Sdk + v2: PlatformApiV2Sdk + + +@dataclasses.dataclass(frozen=True) +class DataApiVersions: + v1: DataApiV1Sdk + + +class UserApiSdk: + def __init__( + self, + platform_api_url: str, + data_api_url: str, + api_key: str, + app_key: str, + logger: logging.Logger, + timeout: int = 30, + ): + self._platform_api_url = platform_api_url + self._data_api_url = data_api_url + self._headers = { + "Authorization": f"API {api_key}", + "X-Corva-App": app_key, + } + self._logger = logger + self._timeout = timeout + + def __enter__(self): + data_cli = httpx.Client( + base_url=posixpath.join(self._data_api_url, "api/v1"), + headers=self._headers, + timeout=self._timeout, + ) + platform_v1_cli = httpx.Client( + base_url=posixpath.join(self._platform_api_url, "v1"), + headers=self._headers, + timeout=self._timeout, + ) + platform_v2_cli = httpx.Client( + base_url=posixpath.join(self._platform_api_url, "v2"), + headers=self._headers, + timeout=self._timeout, + ) + + data_cli.send = logging_send(func=data_cli.send, logger=self._logger) + platform_v1_cli.send = logging_send( + func=platform_v1_cli.send, logger=self._logger + ) + platform_v2_cli.send = logging_send( + func=platform_v2_cli.send, logger=self._logger + ) + + self.data = DataApiVersions(v1=DataApiV1Sdk(client=data_cli)) + self.platform = PlatformApiVersions( + v1=PlatformApiV1Sdk(client=platform_v1_cli), + v2=PlatformApiV2Sdk(client=platform_v2_cli), + ) + + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.data.v1.http.close() + self.platform.v1.http.close() + self.platform.v2.http.close() diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py new file mode 100644 index 00000000..00aa7461 --- /dev/null +++ b/tests/integration/test_api.py @@ -0,0 +1,94 @@ +import logging + +import httpx +import pytest +import pytest_httpx +import yaml + +import corva.api_adapter +import corva.configuration + + +class TestLogsFailedRequests: + def test_no_response(self, caplog: pytest.LogCaptureFixture): + caplog.handler.setFormatter(logging.Formatter('%(message)s')) + + sdk = corva.api_adapter.UserApiSdk( + platform_api_url='', + data_api_url='', + api_key='', + app_key='', + logger=logging.getLogger(), + ) + + with sdk as s, pytest.raises(httpx.HTTPError): + s.data.v1.http.get(url='whatever') + + actual = yaml.safe_load(caplog.text) + + expected = { + 'message': "Request failed - Request URL is missing an 'http://' or " + "'https://' protocol.", + 'request': { + 'method': 'GET', + 'url': 'api/v1/whatever', + 'headers': { + 'accept': '*/*', + 'accept-encoding': 'gzip, deflate', + 'connection': 'keep-alive', + 'user-agent': 'python-httpx/0.22.0', + 'authorization': '[secure]', + 'x-corva-app': '', + }, + 'content': "b''", + }, + } + + assert expected == actual + + def test_unsuccessful_response( + self, caplog: pytest.LogCaptureFixture, httpx_mock: pytest_httpx.HTTPXMock + ): + caplog.handler.setFormatter(logging.Formatter('%(message)s')) + httpx_mock.add_response(status_code=400) + + sdk = corva.api_adapter.UserApiSdk( + platform_api_url='', + data_api_url="https://test_url", + api_key='', + app_key='', + logger=logging.getLogger(), + ) + + with sdk as s: + s.data.v1.http.get(url='') + + actual = yaml.safe_load(caplog.text) + + expected = { + 'message': "Request failed - Client error '400 Bad Request' for url " + "'https://test_url/api/v1/'\nFor more information check: " + "https://httpstatuses.com/400", + 'response': { + 'code': 400, + 'reason': 'Bad Request', + 'headers': {}, + 'content': "b''", + }, + 'request': { + 'method': 'GET', + 'url': 'https://test_url/api/v1/', + 'headers': { + 'host': 'test_url', + 'accept': '*/*', + 'accept-encoding': 'gzip, deflate', + 'connection': 'keep-alive', + 'user-agent': 'python-httpx/0.22.0', + 'authorization': '[secure]', + 'x-corva-app': '', + }, + 'content': "b''", + }, + } + + assert actual == expected From 13719a3458aa325c2eb16d2097be7de44d30dbe6 Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Mon, 23 May 2022 14:15:54 +0300 Subject: [PATCH 2/4] DC-3562 Integration tests for `DataApiV1Sdk.get` (#62) --- Makefile | 10 +- requirements-test.txt | 1 + setup.cfg | 2 +- src/corva/api_adapter.py | 15 +- src/corva/configuration.py | 19 ++ .../cassettes/TestUserApiSdk.test_get.yaml | 289 ++++++++++++++++++ tests/integration/test_api.py | 188 +++++++++++- 7 files changed, 509 insertions(+), 15 deletions(-) create mode 100644 tests/integration/cassettes/TestUserApiSdk.test_get.yaml diff --git a/Makefile b/Makefile index cc0875a9..659af782 100644 --- a/Makefile +++ b/Makefile @@ -45,10 +45,16 @@ unit-tests: ## integration-tests: Run integration tests. .PHONY: integration-tests -integration-tests: export CACHE_URL = redis://localhost:6379 integration-tests: test_path = tests/integration integration-tests: - @coverage run -m pytest $(test_path) + @CACHE_URL=redis://localhost:6379 \ + PROVIDER='' \ + TEST_DATASET='' \ + API_ROOT_URL=https://platform.localhost.ai \ + DATA_API_ROOT_URL=https://data.localhost.ai \ + TEST_API_KEY='' \ + TEST_BEARER_TOKEN='' \ + coverage run -m pytest --vcr-record=none $(test_path) ## coverage: Display code coverage in the console. .PHONY: coverage diff --git a/requirements-test.txt b/requirements-test.txt index 8f13da73..eb01cd6c 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -3,4 +3,5 @@ freezegun==1.0.0 pytest==7.1.2 pytest-httpx==0.20.0 pytest-mock==3.3.1 +pytest-vcr~=1.0.2 requests-mock==1.8.0 diff --git a/setup.cfg b/setup.cfg index bc0a44fe..b70a1bbf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,7 @@ parallel = True [coverage:report] precision = 2 -fail_under = 98.09 +fail_under = 98.44 skip_covered = True show_missing = True exclude_lines = diff --git a/src/corva/api_adapter.py b/src/corva/api_adapter.py index 94292f04..7a3619e7 100644 --- a/src/corva/api_adapter.py +++ b/src/corva/api_adapter.py @@ -2,7 +2,6 @@ import functools import json import logging -import posixpath from typing import Callable, List, Optional import httpx @@ -154,14 +153,16 @@ class DataApiVersions: class UserApiSdk: def __init__( self, - platform_api_url: str, + platform_v1_url: str, + platform_v2_url: str, data_api_url: str, api_key: str, app_key: str, - logger: logging.Logger, + logger: logging.Logger = logging.getLogger(), timeout: int = 30, ): - self._platform_api_url = platform_api_url + self._platform_v1_url = platform_v1_url + self._platform_v2_url = platform_v2_url self._data_api_url = data_api_url self._headers = { "Authorization": f"API {api_key}", @@ -172,17 +173,17 @@ def __init__( def __enter__(self): data_cli = httpx.Client( - base_url=posixpath.join(self._data_api_url, "api/v1"), + base_url=self._data_api_url, headers=self._headers, timeout=self._timeout, ) platform_v1_cli = httpx.Client( - base_url=posixpath.join(self._platform_api_url, "v1"), + base_url=self._platform_v1_url, headers=self._headers, timeout=self._timeout, ) platform_v2_cli = httpx.Client( - base_url=posixpath.join(self._platform_api_url, "v2"), + base_url=self._platform_v2_url, headers=self._headers, timeout=self._timeout, ) diff --git a/src/corva/configuration.py b/src/corva/configuration.py index ebbbd7a4..42552747 100644 --- a/src/corva/configuration.py +++ b/src/corva/configuration.py @@ -1,4 +1,5 @@ import datetime +import os import pydantic @@ -25,3 +26,21 @@ class Settings(pydantic.BaseSettings): SETTINGS = Settings() + + +def get_test_api_key() -> str: + """Api key for testing""" + + return os.environ['TEST_API_KEY'] + + +def get_test_bearer() -> str: + """Bearer token for testing""" + + return os.environ['TEST_BEARER_TOKEN'] + + +def get_test_dataset() -> str: + """Dataset for testing""" + + return os.environ['TEST_DATASET'] diff --git a/tests/integration/cassettes/TestUserApiSdk.test_get.yaml b/tests/integration/cassettes/TestUserApiSdk.test_get.yaml new file mode 100644 index 00000000..7a341e9f --- /dev/null +++ b/tests/integration/cassettes/TestUserApiSdk.test_get.yaml @@ -0,0 +1,289 @@ +interactions: +- request: + body: '{"well": {"name": "deleteme-python-sdk-autotest-b307dd45"}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '59' + content-type: + - application/json + user-agent: + - python-httpx/0.22.0 + method: POST + uri: null + response: + content: '{"data":{"id":"325393","type":"well","attributes":{"name":"deleteme-python-sdk-autotest-b307dd45","status":"unknown","state":"planned"},"relationships":{}}}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 17 May 2022 12:13:38 GMT + ETag: + - W/"091ea5d8aea25ce573aabb2cc6e565ef" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - 33879c04-be0f-4457-ba80-74891ef992ba + X-Runtime: + - '0.136328' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: GET + uri: null + response: + content: '{"data":{"id":"325393","type":"well","attributes":{"asset_id":89513687},"relationships":{}}}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 17 May 2022 12:13:38 GMT + ETag: + - W/"cdb0cbdaaae6aca210587ce8dfdfbb2b" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - 3cba62ba-04d5-4d93-ad86-b38aa900701c + X-Runtime: + - '0.064769' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"deleted_count":0}' + headers: + Connection: + - keep-alive + Content-Length: + - '19' + Content-Type: + - application/json + Date: + - Tue, 17 May 2022 12:13:39 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '[{"asset_id": 89513687, "version": 1, "data": {"k": "v"}, "timestamp": + 10}, {"asset_id": 89513687, "version": 1, "data": {"k": "v"}, "timestamp": 12}, + {"asset_id": 89513687, "version": 1, "data": {"k": "v"}, "timestamp": 11}, {"asset_id": + 89513687, "version": 1, "data": {"k": "v"}, "timestamp": 13}]' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '300' + content-type: + - application/json + user-agent: + - python-httpx/0.22.0 + method: POST + uri: null + response: + content: '{"inserted_ids":["62839173c5745c7f7606c6fa","62839173c5745c7f7606c6fb","62839173c5745c7f7606c6fc","62839173c5745c7f7606c6fd"],"failed_count":0,"messages":[]}' + headers: + Connection: + - keep-alive + Content-Length: + - '157' + Content-Type: + - application/json + Date: + - Tue, 17 May 2022 12:13:39 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + x-corva-app: + - python-sdk-autotest-2022-05-17 12:13:37+00:00 + method: GET + uri: null + response: + content: '[{"_id":"62839173c5745c7f7606c6fc","timestamp":11},{"_id":"62839173c5745c7f7606c6fb","timestamp":12}]' + headers: + Connection: + - keep-alive + Content-Length: + - '101' + Content-Type: + - application/json + Date: + - Tue, 17 May 2022 12:13:40 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"status":"deleted"}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 17 May 2022 12:13:40 GMT + ETag: + - W/"4df20f95e824b2af44a61642d88daaf0" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - bf08403a-c9de-43e3-a365-bbda6b009563 + X-Runtime: + - '0.118524' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"deleted_count":4}' + headers: + Connection: + - keep-alive + Content-Length: + - '19' + Content-Type: + - application/json + Date: + - Tue, 17 May 2022 12:13:40 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +version: 1 diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py index 00aa7461..89d16e4c 100644 --- a/tests/integration/test_api.py +++ b/tests/integration/test_api.py @@ -1,4 +1,10 @@ +import contextlib +import datetime +import json import logging +import posixpath +import uuid +from typing import Iterable import httpx import pytest @@ -14,7 +20,8 @@ def test_no_response(self, caplog: pytest.LogCaptureFixture): caplog.handler.setFormatter(logging.Formatter('%(message)s')) sdk = corva.api_adapter.UserApiSdk( - platform_api_url='', + platform_v1_url='', + platform_v2_url='', data_api_url='', api_key='', app_key='', @@ -31,7 +38,7 @@ def test_no_response(self, caplog: pytest.LogCaptureFixture): "'https://' protocol.", 'request': { 'method': 'GET', - 'url': 'api/v1/whatever', + 'url': '/whatever', 'headers': { 'accept': '*/*', 'accept-encoding': 'gzip, deflate', @@ -53,7 +60,8 @@ def test_unsuccessful_response( httpx_mock.add_response(status_code=400) sdk = corva.api_adapter.UserApiSdk( - platform_api_url='', + platform_v1_url='', + platform_v2_url='', data_api_url="https://test_url", api_key='', app_key='', @@ -67,7 +75,7 @@ def test_unsuccessful_response( expected = { 'message': "Request failed - Client error '400 Bad Request' for url " - "'https://test_url/api/v1/'\nFor more information check: " + "'https://test_url/'\nFor more information check: " "https://httpstatuses.com/400", 'response': { 'code': 400, @@ -77,7 +85,7 @@ def test_unsuccessful_response( }, 'request': { 'method': 'GET', - 'url': 'https://test_url/api/v1/', + 'url': 'https://test_url/', 'headers': { 'host': 'test_url', 'accept': '*/*', @@ -92,3 +100,173 @@ def test_unsuccessful_response( } assert actual == expected + + +def vcr_before_record_request(request): + request.uri = None + + return request + + +@pytest.fixture(scope="module") +def vcr_config(): + return { + # Replace the Authorization request header + "filter_headers": ["Authorization", 'host'], + "before_record_request": vcr_before_record_request, + } + + +@pytest.fixture(scope='module') +def platform_v1_url() -> str: + return posixpath.join(corva.configuration.SETTINGS.API_ROOT_URL, 'v1') + + +@pytest.fixture(scope='module') +def platform_v2_url() -> str: + return posixpath.join(corva.configuration.SETTINGS.API_ROOT_URL, 'v2') + + +@pytest.fixture(scope='module') +def data_url() -> str: + return posixpath.join(corva.configuration.SETTINGS.DATA_API_ROOT_URL, 'api/v1') + + +@pytest.fixture(scope='module') +def headers() -> dict: + return {'Authorization': f'Bearer {corva.configuration.get_test_bearer()}'} + + +@pytest.fixture(scope='module') +def data(data_url: str, headers: dict) -> Iterable[httpx.Client]: + with httpx.Client(base_url=data_url, headers=headers) as data: + yield data + + +@pytest.fixture(scope='module') +def provider() -> str: + return corva.configuration.SETTINGS.PROVIDER + + +@pytest.fixture(scope='module') +def dataset() -> str: + return corva.configuration.get_test_dataset() + + +@pytest.fixture(scope='module') +def app_key() -> str: + now = datetime.datetime.now(tz=datetime.timezone.utc).replace(microsecond=0) + return f'python-sdk-autotest-{now}' + + +@contextlib.contextmanager +def _setup( + platform: httpx.Client, data: httpx.Client, provider: str, dataset: str +) -> Iterable[int]: + response = platform.post( + url='wells', + json={ + 'well': {'name': f'deleteme-python-sdk-autotest-{str(uuid.uuid4())[:8]}'} + }, + ) + response.raise_for_status() + well_id = int(response.json()['data']['id']) + + response = platform.get(url=f'wells/{well_id}?fields[]=well.asset_id') + response.raise_for_status() + asset_id = response.json()['data']['attributes']['asset_id'] + + data.delete( + f'data/{provider}/{dataset}/', + params={'query': json.dumps({'asset_id': asset_id})}, + ).raise_for_status() + + try: + yield asset_id + finally: + platform.delete(f'wells/{well_id}').raise_for_status() + + data.delete( + f'data/{provider}/{dataset}/', + params={'query': json.dumps({'asset_id': asset_id})}, + ).raise_for_status() + + +@pytest.fixture(scope='module') +def setup_( + platform_v2_url: str, data: httpx.Client, headers: dict, provider: str, dataset: str +) -> Iterable[int]: + with httpx.Client(base_url=platform_v2_url, headers=headers) as platform: + yield _setup(platform=platform, data=data, provider=provider, dataset=dataset) + + +@pytest.fixture(scope='function') +def sdk( + platform_v1_url: str, platform_v2_url: str, data_url: str, app_key: str +) -> Iterable[corva.api_adapter.UserApiSdk]: + sdk = corva.api_adapter.UserApiSdk( + platform_v1_url=platform_v1_url, + platform_v2_url=platform_v2_url, + data_api_url=data_url, + api_key=corva.configuration.get_test_api_key(), + app_key=app_key, + logger=logging.getLogger(), + ) + + with sdk: + yield sdk + + +class TestUserApiSdk: + @pytest.mark.vcr + def test_get( + self, + setup_: Iterable[int], + sdk: corva.api_adapter.UserApiSdk, + dataset: str, + provider: str, + data: httpx.Client, + ): + with setup_ as asset_id: + data.post( + f'data/{provider}/{dataset}/', + json=[ + { + "asset_id": asset_id, + "version": 1, + "data": {"k": "v"}, + "timestamp": 10, + }, + { + "asset_id": asset_id, + "version": 1, + "data": {"k": "v"}, + "timestamp": 12, + }, + { + "asset_id": asset_id, + "version": 1, + "data": {"k": "v"}, + "timestamp": 11, + }, + { + "asset_id": asset_id, + "version": 1, + "data": {"k": "v"}, + "timestamp": 13, + }, + ], + ).raise_for_status() + + data = sdk.data.v1.get( + provider=provider, + dataset=dataset, + query={'asset_id': asset_id}, + sort={'timestamp': 1}, + limit=2, + skip=1, + fields='timestamp', + ) + + assert len(data) == 2 + assert set(datum['timestamp'] for datum in data) == {11, 12} From 87d3d820927f1cfc6013d7333408ef1cbeeabc49 Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Mon, 23 May 2022 15:33:14 +0300 Subject: [PATCH 3/4] DC-3564 Add POST `data/{provider}/{dataset}` to `DataApiV1Sdk` (#63) --- setup.cfg | 2 +- src/corva/__init__.py | 1 + src/corva/api_adapter.py | 34 +- .../cassettes/TestUserApiSdk.test_insert.yaml | 315 ++++++++++++++++++ tests/integration/test_api.py | 86 ++++- 5 files changed, 420 insertions(+), 18 deletions(-) create mode 100644 tests/integration/cassettes/TestUserApiSdk.test_insert.yaml diff --git a/setup.cfg b/setup.cfg index b70a1bbf..addc8de2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,7 @@ parallel = True [coverage:report] precision = 2 -fail_under = 98.44 +fail_under = 98.45 skip_covered = True show_missing = True exclude_lines = diff --git a/src/corva/__init__.py b/src/corva/__init__.py index 49dd5732..1bc1f0b5 100644 --- a/src/corva/__init__.py +++ b/src/corva/__init__.py @@ -1,4 +1,5 @@ from .api import Api +from .api_adapter import InsertResult from .handlers import scheduled, stream, task from .logger import CORVA_LOGGER as Logger from .models.scheduled.scheduled import ( diff --git a/src/corva/api_adapter.py b/src/corva/api_adapter.py index 7a3619e7..3a79aca5 100644 --- a/src/corva/api_adapter.py +++ b/src/corva/api_adapter.py @@ -2,9 +2,10 @@ import functools import json import logging -from typing import Callable, List, Optional +from typing import Callable, List, Optional, Sequence import httpx +import pydantic import yaml @@ -71,6 +72,12 @@ def wrapper(request: httpx.Request, *args, **kwargs): return wrapper +class InsertResult(pydantic.BaseModel): + inserted_ids: List[str] + failed_count: int + messages: List[str] + + class DataApiV1Sdk: def __init__(self, client: httpx.Client): self.http = client @@ -105,7 +112,7 @@ def get( fields: comma separated list of fields to return. Example: "_id,data". Raises: - requests.HTTPError: if request was unsuccessful. + httpx.HTTPStatusError: if request was unsuccessful. Returns: Data from dataset. @@ -128,6 +135,29 @@ def get( return data + def insert( + self, provider: str, dataset: str, *, documents: Sequence[dict] + ) -> InsertResult: + """Inserts data using the endpoint POST 'data/{provider}/{dataset}/'. + + Args: + provider: company name owning the dataset. + dataset: dataset name. + documents: data to insert. + + Raises: + httpx.HTTPStatusError: if request was unsuccessful. + + Returns: + Insert result. + """ + + response = self.http.post(url=f"data/{provider}/{dataset}/", json=documents) + + response.raise_for_status() + + return InsertResult.parse_obj(response.json()) + class PlatformApiV1Sdk: def __init__(self, client: httpx.Client): diff --git a/tests/integration/cassettes/TestUserApiSdk.test_insert.yaml b/tests/integration/cassettes/TestUserApiSdk.test_insert.yaml new file mode 100644 index 00000000..7725a8c6 --- /dev/null +++ b/tests/integration/cassettes/TestUserApiSdk.test_insert.yaml @@ -0,0 +1,315 @@ +interactions: +- request: + body: '{"well": {"name": "deleteme-python-sdk-autotest-eb81d18c"}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '59' + content-type: + - application/json + user-agent: + - python-httpx/0.22.0 + method: POST + uri: null + response: + content: '{"data":{"id":"328567","type":"well","attributes":{"name":"deleteme-python-sdk-autotest-eb81d18c","status":"unknown","state":"planned"},"relationships":{}}}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Mon, 23 May 2022 11:39:27 GMT + ETag: + - W/"ec3494f3333667f241a2e1d1b41205ae" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - 3462393c-fe38-4cd3-aca1-a4d4a7b722ee + X-Runtime: + - '0.105799' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: GET + uri: null + response: + content: '{"data":{"id":"328567","type":"well","attributes":{"asset_id":53781610},"relationships":{}}}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Mon, 23 May 2022 11:39:27 GMT + ETag: + - W/"c412a0007563bca3e870f857bfe4cf1e" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - 099e239c-b7af-452d-9340-9456471dbcf9 + X-Runtime: + - '0.059369' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"deleted_count":0}' + headers: + Connection: + - keep-alive + Content-Length: + - '19' + Content-Type: + - application/json + Date: + - Mon, 23 May 2022 11:39:27 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: GET + uri: null + response: + content: '[]' + headers: + Connection: + - keep-alive + Content-Length: + - '2' + Content-Type: + - application/json + Date: + - Mon, 23 May 2022 11:39:28 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '[{"asset_id": 53781610, "version": 1, "data": {"k": "v"}, "timestamp": + 10}, {"asset_id": 53781610, "version": 1, "data": {"k": "v"}, "timestamp": 11}]' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '150' + content-type: + - application/json + user-agent: + - python-httpx/0.22.0 + x-corva-app: + - python-sdk-autotest-2022-05-23 11:39:26+00:00 + method: POST + uri: null + response: + content: '{"inserted_ids":["628b7271d6aff355ccea8b61","628b7271d6aff355ccea8b62"],"failed_count":0,"messages":[]}' + headers: + Connection: + - keep-alive + Content-Length: + - '103' + Content-Type: + - application/json + Date: + - Mon, 23 May 2022 11:39:29 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: GET + uri: null + response: + content: '[{"_id":"628b7271d6aff355ccea8b61","company_id":80,"asset_id":53781610,"version":1,"provider":"big-data-energy","collection":"python-sdk-autotests","data":{"k":"v"},"timestamp":10,"app_key":"big-data-energy.python_sdk_app_for_autotests"},{"_id":"628b7271d6aff355ccea8b62","company_id":80,"asset_id":53781610,"version":1,"provider":"big-data-energy","collection":"python-sdk-autotests","data":{"k":"v"},"timestamp":11,"app_key":"big-data-energy.python_sdk_app_for_autotests"}]' + headers: + Connection: + - keep-alive + Content-Length: + - '475' + Content-Type: + - application/json + Date: + - Mon, 23 May 2022 11:39:29 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"status":"deleted"}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Mon, 23 May 2022 11:39:29 GMT + ETag: + - W/"4df20f95e824b2af44a61642d88daaf0" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - 8424ca9f-13a6-4ac7-9f70-836a3011124f + X-Runtime: + - '0.099032' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"deleted_count":2}' + headers: + Connection: + - keep-alive + Content-Length: + - '19' + Content-Type: + - application/json + Date: + - Mon, 23 May 2022 11:39:29 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +version: 1 diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py index 89d16e4c..411b0421 100644 --- a/tests/integration/test_api.py +++ b/tests/integration/test_api.py @@ -112,7 +112,7 @@ def vcr_before_record_request(request): def vcr_config(): return { # Replace the Authorization request header - "filter_headers": ["Authorization", 'host'], + "filter_headers": ["Authorization", "host"], "before_record_request": vcr_before_record_request, } @@ -174,7 +174,7 @@ def _setup( response = platform.get(url=f'wells/{well_id}?fields[]=well.asset_id') response.raise_for_status() - asset_id = response.json()['data']['attributes']['asset_id'] + asset_id: int = response.json()['data']['attributes']['asset_id'] data.delete( f'data/{provider}/{dataset}/', @@ -192,7 +192,7 @@ def _setup( ).raise_for_status() -@pytest.fixture(scope='module') +@pytest.fixture(scope='function') def setup_( platform_v2_url: str, data: httpx.Client, headers: dict, provider: str, dataset: str ) -> Iterable[int]: @@ -213,8 +213,7 @@ def sdk( logger=logging.getLogger(), ) - with sdk: - yield sdk + yield sdk class TestUserApiSdk: @@ -258,15 +257,72 @@ def test_get( ], ).raise_for_status() - data = sdk.data.v1.get( - provider=provider, - dataset=dataset, - query={'asset_id': asset_id}, - sort={'timestamp': 1}, - limit=2, - skip=1, - fields='timestamp', + with sdk as s: + collection = s.data.v1.get( + provider=provider, + dataset=dataset, + query={'asset_id': asset_id}, + sort={'timestamp': 1}, + limit=2, + skip=1, + fields='timestamp', + ) + + assert len(collection) == 2 + assert set(doc['timestamp'] for doc in collection) == {11, 12} + + @pytest.mark.vcr + def test_insert( + self, + setup_: Iterable[int], + sdk: corva.api_adapter.UserApiSdk, + dataset: str, + provider: str, + data: httpx.Client, + ): + with setup_ as asset_id: + response = data.get( + url=f"data/{provider}/{dataset}/", + params={ + "query": json.dumps({'asset_id': asset_id}), + "sort": json.dumps({'timestamp': 1}), + "limit": 1, + }, + ) + response.raise_for_status() + collection = response.json() + + assert not collection + + with sdk as s: + s.data.v1.insert( + provider=provider, + dataset=dataset, + documents=[ + { + "asset_id": asset_id, + "version": 1, + "data": {"k": "v"}, + "timestamp": 10, + }, + { + "asset_id": asset_id, + "version": 1, + "data": {"k": "v"}, + "timestamp": 11, + }, + ], + ) + + response = data.get( + url=f"data/{provider}/{dataset}/", + params={ + "query": json.dumps({'asset_id': asset_id}), + "sort": json.dumps({'timestamp': 1}), + "limit": 3, + }, ) + response.raise_for_status() + collection = response.json() - assert len(data) == 2 - assert set(datum['timestamp'] for datum in data) == {11, 12} + assert len(collection) == 2 From 614b36b2fb0fe72ce7ba5463b2312a764e02018a Mon Sep 17 00:00:00 2001 From: Oleksii Symon Date: Tue, 7 Jun 2022 13:25:01 +0300 Subject: [PATCH 4/4] DC-3567 Add PUT `data/{provider}/{dataset}/{id}/` to `DataApiV1Sdk` (#64) --- Makefile | 1 + setup.cfg | 2 +- src/corva/api_adapter.py | 25 ++ src/corva/configuration.py | 6 + .../TestUserApiSdk.test_replace.yaml | 320 ++++++++++++++++++ tests/integration/test_api.py | 54 +++ 6 files changed, 407 insertions(+), 1 deletion(-) create mode 100644 tests/integration/cassettes/TestUserApiSdk.test_replace.yaml diff --git a/Makefile b/Makefile index 659af782..fe5a560e 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,7 @@ integration-tests: DATA_API_ROOT_URL=https://data.localhost.ai \ TEST_API_KEY='' \ TEST_BEARER_TOKEN='' \ + TEST_COMPANY_ID='-1' \ coverage run -m pytest --vcr-record=none $(test_path) ## coverage: Display code coverage in the console. diff --git a/setup.cfg b/setup.cfg index addc8de2..bfe79078 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,7 @@ parallel = True [coverage:report] precision = 2 -fail_under = 98.45 +fail_under = 98.47 skip_covered = True show_missing = True exclude_lines = diff --git a/src/corva/api_adapter.py b/src/corva/api_adapter.py index 3a79aca5..fcefc3be 100644 --- a/src/corva/api_adapter.py +++ b/src/corva/api_adapter.py @@ -158,6 +158,31 @@ def insert( return InsertResult.parse_obj(response.json()) + def replace(self, provider: str, dataset: str, id_: str, *, document: dict) -> dict: + """Replaces all document data. + + Replace all document data using the endpoint PUT + 'data/{provider}/{dataset}/{id}/'. + + Args: + provider: company name owning the dataset. + dataset: dataset name. + id_: document id to replace. + document: new document data. + + Raises: + httpx.HTTPStatusError: if request was unsuccessful. + + Returns: + Updated document. + """ + + response = self.http.put(url=f"data/{provider}/{dataset}/{id_}/", json=document) + + response.raise_for_status() + + return response.json() + class PlatformApiV1Sdk: def __init__(self, client: httpx.Client): diff --git a/src/corva/configuration.py b/src/corva/configuration.py index 42552747..128180cc 100644 --- a/src/corva/configuration.py +++ b/src/corva/configuration.py @@ -44,3 +44,9 @@ def get_test_dataset() -> str: """Dataset for testing""" return os.environ['TEST_DATASET'] + + +def get_test_company_id() -> int: + """Company id for testing""" + + return int(os.environ['TEST_COMPANY_ID']) diff --git a/tests/integration/cassettes/TestUserApiSdk.test_replace.yaml b/tests/integration/cassettes/TestUserApiSdk.test_replace.yaml new file mode 100644 index 00000000..149b542d --- /dev/null +++ b/tests/integration/cassettes/TestUserApiSdk.test_replace.yaml @@ -0,0 +1,320 @@ +interactions: +- request: + body: '{"well": {"name": "deleteme-python-sdk-autotest-d4a1214c"}}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '59' + content-type: + - application/json + user-agent: + - python-httpx/0.22.0 + method: POST + uri: null + response: + content: '{"data":{"id":"328568","type":"well","attributes":{"name":"deleteme-python-sdk-autotest-d4a1214c","status":"unknown","state":"planned"},"relationships":{}}}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Mon, 23 May 2022 11:41:15 GMT + ETag: + - W/"11b6d1d9eb9720313d1fd72196df9595" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - ce585d4e-d4dc-47bb-9f2a-f109e6a3a59b + X-Runtime: + - '0.101889' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: GET + uri: null + response: + content: '{"data":{"id":"328568","type":"well","attributes":{"asset_id":30719716},"relationships":{}}}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Mon, 23 May 2022 11:41:15 GMT + ETag: + - W/"b161ea51687ef8fd4b47fd4dc1ca754d" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - 05e9b6f6-ab30-4478-8f0d-dfe7e7679ac7 + X-Runtime: + - '0.044065' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"deleted_count":0}' + headers: + Connection: + - keep-alive + Content-Length: + - '19' + Content-Type: + - application/json + Date: + - Mon, 23 May 2022 11:41:16 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '[{"asset_id": 30719716, "version": 1, "data": {"k1": "v1"}, "timestamp": + 10}]' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '77' + content-type: + - application/json + user-agent: + - python-httpx/0.22.0 + method: POST + uri: null + response: + content: '{"inserted_ids":["628b72dcd6aff355ccea8b78"],"failed_count":0,"messages":[]}' + headers: + Connection: + - keep-alive + Content-Length: + - '76' + Content-Type: + - application/json + Date: + - Mon, 23 May 2022 11:41:16 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '{"data": {"k2": "v2"}, "version": 2, "company_id": 80, "timestamp": 11, + "asset_id": 30719716}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '93' + content-type: + - application/json + user-agent: + - python-httpx/0.22.0 + x-corva-app: + - python-sdk-autotest-2022-05-23 11:41:14+00:00 + method: PUT + uri: null + response: + content: '{"_id":"628b72dcd6aff355ccea8b78","company_id":80,"asset_id":30719716,"version":2,"provider":"big-data-energy","collection":"python-sdk-autotests","data":{"k2":"v2"},"timestamp":11,"app_key":"big-data-energy.python_sdk_app_for_autotests"}' + headers: + Connection: + - keep-alive + Content-Length: + - '238' + Content-Type: + - application/json + Date: + - Mon, 23 May 2022 11:41:17 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: GET + uri: null + response: + content: '{"_id":"628b72dcd6aff355ccea8b78","company_id":80,"asset_id":30719716,"version":2,"provider":"big-data-energy","collection":"python-sdk-autotests","data":{"k2":"v2"},"timestamp":11,"app_key":"big-data-energy.python_sdk_app_for_autotests"}' + headers: + Connection: + - keep-alive + Content-Length: + - '238' + Content-Type: + - application/json + Date: + - Mon, 23 May 2022 11:41:17 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"status":"deleted"}' + headers: + Cache-Control: + - max-age=0, private, must-revalidate + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json; charset=utf-8 + Date: + - Mon, 23 May 2022 11:41:18 GMT + ETag: + - W/"4df20f95e824b2af44a61642d88daaf0" + Referrer-Policy: + - strict-origin-when-cross-origin + Server: + - nginx/1.18.0 + Transfer-Encoding: + - chunked + Vary: + - Origin + X-Content-Type-Options: + - nosniff + X-Download-Options: + - noopen + X-Frame-Options: + - SAMEORIGIN + X-Permitted-Cross-Domain-Policies: + - none + X-Rack-CORS: + - miss; no-origin + X-Request-Id: + - db6d7a71-cce6-4279-8050-eb7c52cecd26 + X-Runtime: + - '0.073215' + X-XSS-Protection: + - 1; mode=block + http_version: HTTP/1.1 + status_code: 200 +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + user-agent: + - python-httpx/0.22.0 + method: DELETE + uri: null + response: + content: '{"deleted_count":1}' + headers: + Connection: + - keep-alive + Content-Length: + - '19' + Content-Type: + - application/json + Date: + - Mon, 23 May 2022 11:41:18 GMT + server: + - uvicorn + http_version: HTTP/1.1 + status_code: 200 +version: 1 diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py index 411b0421..73fcd46f 100644 --- a/tests/integration/test_api.py +++ b/tests/integration/test_api.py @@ -153,6 +153,11 @@ def dataset() -> str: return corva.configuration.get_test_dataset() +@pytest.fixture(scope='module') +def company_id() -> int: + return corva.configuration.get_test_company_id() + + @pytest.fixture(scope='module') def app_key() -> str: now = datetime.datetime.now(tz=datetime.timezone.utc).replace(microsecond=0) @@ -326,3 +331,52 @@ def test_insert( collection = response.json() assert len(collection) == 2 + + @pytest.mark.vcr + def test_replace( + self, + setup_: Iterable[int], + sdk: corva.api_adapter.UserApiSdk, + dataset: str, + provider: str, + company_id: int, + data: httpx.Client, + ): + with setup_ as asset_id: + response = data.post( + f'data/{provider}/{dataset}/', + json=[ + { + "asset_id": asset_id, + "version": 1, + "data": {"k1": "v1"}, + "timestamp": 10, + }, + ], + ) + response.raise_for_status() + id_ = response.json()['inserted_ids'][0] + + with sdk as s: + s.data.v1.replace( + provider=provider, + dataset=dataset, + id_=id_, + document={ + 'data': {'k2': 'v2'}, + 'version': 2, + 'company_id': company_id, + 'timestamp': 11, + 'asset_id': asset_id, + }, + ) + + response = data.get( + url=f"data/{provider}/{dataset}/{id_}/", + ) + response.raise_for_status() + document_after_replace = response.json() + + assert document_after_replace['data'] == {'k2': 'v2'} + assert document_after_replace['version'] == 2 + assert document_after_replace['timestamp'] == 11