diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 6d6db58..b0b923a 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -3,7 +3,7 @@ on: [push, pull_request] jobs: test-client: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: matrix: python: [3.8, 3.9, "3.10", 3.12] diff --git a/Dockerfile b/Dockerfile index bc15457..ff7c840 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ ENV LC_ALL C.UTF-8 # TODO: keep requirements in one place RUN pip install \ blinker>=1.4 \ + boto>=2.49 \ boto3>=1.1.3 \ click>=5.1 \ Flask>=0.10.1 \ @@ -29,7 +30,7 @@ RUN pip install \ pyinotify>=0.9.4, \ raven>=5.0.0 \ 'tox>4,<5' \ - 'datalake<2' + 'datalake[queuable]>2' RUN mkdir -p /opt/ COPY . /opt/ diff --git a/Makefile b/Makefile index 9a7009f..8c3b543 100644 --- a/Makefile +++ b/Makefile @@ -15,10 +15,10 @@ test-client: docker docker run --rm --entrypoint tox $(IMAGE) -c /opt/client/tox.ini test-ingester: docker - docker run --rm --entrypoint py.test $(IMAGE) ingester + docker run --rm --entrypoint py.test $(IMAGE) -vv ingester test-api: docker - docker run --rm --entrypoint py.test $(IMAGE) api + docker run --rm --entrypoint py.test $(IMAGE) -vv api .PHONY: test # Run the tests test: diff --git a/api/tests/conftest.py b/api/tests/conftest.py index bf166fa..285eadf 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -14,6 +14,7 @@ import pytest import boto3 +import os from botocore.exceptions import ( ClientError as BotoClientError, NoCredentialsError @@ -24,7 +25,8 @@ from datalake.tests import * # noqa from datalake.common import DatalakeRecord from datalake.tests import generate_random_metadata - +os.environ["AWS_REQUEST_CHECKSUM_CALCULATION"] = "when_required" +os.environ["AWS_RESPONSE_CHECKSUM_VALIDATION"] = "when_required" YEAR_2010 = 1262304000000 diff --git a/client/test/conftest.py b/client/test/conftest.py index 6e2f1b1..1587f27 100644 --- a/client/test/conftest.py +++ b/client/test/conftest.py @@ -32,6 +32,9 @@ "datalake.tests", ] +os.environ["AWS_REQUEST_CHECKSUM_CALCULATION"] = "when_required" +os.environ["AWS_RESPONSE_CHECKSUM_VALIDATION"] = "when_required" + # If we run with proper AWS credentials they will be used # This will cause moto to fail # But more critically, may impact production systems diff --git a/ingester/tests/conftest.py b/ingester/tests/conftest.py index 7c9fa9b..04c1c04 100644 --- a/ingester/tests/conftest.py +++ b/ingester/tests/conftest.py @@ -18,6 +18,8 @@ from datalake_ingester import SQSQueue +os.environ["AWS_REQUEST_CHECKSUM_CALCULATION"] = "when_required" +os.environ["AWS_RESPONSE_CHECKSUM_VALIDATION"] = "when_required" @pytest.fixture def dynamodb_connection(aws_connector):