diff --git a/pybossa/cloud_store_api/base_conn.py b/pybossa/cloud_store_api/base_conn.py index 64b8396fd..a65138f11 100644 --- a/pybossa/cloud_store_api/base_conn.py +++ b/pybossa/cloud_store_api/base_conn.py @@ -237,10 +237,6 @@ def get_contents_as_string(self, encoding=None, **kwargs): # pylint: disable=W0 """Returns contents as bytes or string, depending on encoding parameter. If encoding is None, returns bytes, otherwise, returns a string. - - parameter "encoding" is default to None. This is consistent with boto2 - get_contents_as_string() method: - http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.key.Key.get_contents_as_string """ return self.base_client.get_contents_as_string( bucket=self.bucket, path=self.name, encoding=encoding diff --git a/pybossa/cloud_store_api/connection.py b/pybossa/cloud_store_api/connection.py index 589525d42..e32f2089e 100644 --- a/pybossa/cloud_store_api/connection.py +++ b/pybossa/cloud_store_api/connection.py @@ -4,15 +4,7 @@ import time from flask import current_app -from boto.auth_handler import AuthHandler -import boto.auth -from boto.exception import S3ResponseError -from boto.s3.key import Key -from boto.s3.bucket import Bucket -from boto.s3.connection import S3Connection, OrdinaryCallingFormat -from boto.provider import Provider -import jwt from werkzeug.exceptions import BadRequest from boto3.session import Session from botocore.client import Config @@ -59,58 +51,36 @@ def create_connection(**kwargs): cert=kwargs.get("cert", False), proxy_url=kwargs.get("proxy_url") ) - if 'object_service' in kwargs: - current_app.logger.info("Calling ProxiedConnection") - conn = ProxiedConnection(**kwargs) - else: - current_app.logger.info("Calling CustomConnection") - conn = CustomConnection(**kwargs) - return conn - -class CustomProvider(Provider): - """Extend Provider to carry information about the end service provider, in - case the service is being proxied. - """ - - def __init__(self, name, access_key=None, secret_key=None, - security_token=None, profile_name=None, object_service=None, - auth_headers=None): - self.object_service = object_service or name - self.auth_headers = auth_headers - super(CustomProvider, self).__init__(name, access_key, secret_key, - security_token, profile_name) + current_app.logger.info("Calling CustomConnection") + conn = CustomConnection(**kwargs) + return conn -class CustomConnection(S3Connection): +class CustomConnection(BaseConnection): def __init__(self, *args, **kwargs): - if not kwargs.get('calling_format'): - kwargs['calling_format'] = OrdinaryCallingFormat() - - kwargs['provider'] = CustomProvider('aws', - kwargs.get('aws_access_key_id'), - kwargs.get('aws_secret_access_key'), - kwargs.get('security_token'), - kwargs.get('profile_name'), - kwargs.pop('object_service', None), - kwargs.pop('auth_headers', None)) - - kwargs['bucket_class'] = CustomBucket - - ssl_no_verify = kwargs.pop('s3_ssl_no_verify', False) - self.host_suffix = kwargs.pop('host_suffix', '') - - super(CustomConnection, self).__init__(*args, **kwargs) - - if kwargs.get('is_secure', True) and ssl_no_verify: - self.https_validate_certificates = False - context = ssl._create_unverified_context() - self.http_connection_kwargs['context'] = context - - def get_path(self, path='/', *args, **kwargs): - ret = super(CustomConnection, self).get_path(path, *args, **kwargs) - return self.host_suffix + ret + super().__init__() # super(CustomConnection, self).__init__(*args, **kwargs) + + aws_access_key_id = kwargs.get("aws_access_key_id") + aws_secret_access_key = kwargs.get("aws_secret_access_key") + region_name = kwargs.get("region_name", "us-east-1") + cert = kwargs.get('cert', False) + proxy_url = kwargs.get('proxy_url', None) + proxies = {"https": proxy_url, "http": proxy_url} if proxy_url else None + ssl_verify = kwargs.get('ssl_verify', True) + self.client = Session().client( + service_name="s3", + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + region_name=region_name, + use_ssl=ssl_verify, + verify=cert, + config=Config( + proxies=proxies, + s3={"addressing_style": "path"} # equivalent to OrdinaryCallingFormat under old boto + ), + ) class CustomConnectionV2(BaseConnection): @@ -133,89 +103,3 @@ def __init__( proxies={"https": proxy_url, "http": proxy_url}, ), ) - - -class CustomBucket(Bucket): - """Handle both 200 and 204 as response code""" - - def delete_key(self, *args, **kwargs): - try: - super(CustomBucket, self).delete_key(*args, **kwargs) - except S3ResponseError as e: - if e.status != 200: - raise - - -class ProxiedKey(Key): - - def should_retry(self, response, chunked_transfer=False): - if 200 <= response.status <= 299: - return True - return super(ProxiedKey, self).should_retry(response, chunked_transfer) - - -class ProxiedBucket(CustomBucket): - - def __init__(self, *args, **kwargs): - super(ProxiedBucket, self).__init__(*args, **kwargs) - self.set_key_class(ProxiedKey) - - -class ProxiedConnection(CustomConnection): - """Object Store connection through proxy API. Sets the proper headers and - creates the jwt; use the appropriate Bucket and Key classes. - """ - - def __init__(self, client_id, client_secret, object_service, *args, **kwargs): - self.client_id = client_id - self.client_secret = client_secret - kwargs['object_service'] = object_service - super(ProxiedConnection, self).__init__(*args, **kwargs) - self.set_bucket_class(ProxiedBucket) - - def make_request(self, method, bucket='', key='', headers=None, data='', - query_args=None, sender=None, override_num_retries=None, - retry_handler=None): - headers = headers or {} - headers['jwt'] = self.create_jwt(method, self.host, bucket, key) - headers['x-objectservice-id'] = self.provider.object_service.upper() - current_app.logger.info("Calling ProxiedConnection.make_request. headers %s", str(headers)) - return super(ProxiedConnection, self).make_request(method, bucket, key, - headers, data, query_args, sender, override_num_retries, - retry_handler) - - def create_jwt(self, method, host, bucket, key): - now = int(time.time()) - path = self.get_path(self.calling_format.build_path_base(bucket, key)) - current_app.logger.info("create_jwt called. method %s, host %s, bucket %s, key %s, path %s", method, host, str(bucket), str(key), str(path)) - payload = { - 'iat': now, - 'nbf': now, - 'exp': now + 300, - 'method': method, - 'iss': self.client_id, - 'host': host, - 'path': path, - 'region': 'ny' - } - return jwt.encode(payload, self.client_secret, algorithm='HS256') - - -class CustomAuthHandler(AuthHandler): - """Implements sending of custom auth headers""" - - capability = ['s3'] - - def __init__(self, host, config, provider): - if not provider.auth_headers: - raise boto.auth_handler.NotReadyToAuthenticate() - self._provider = provider - super(CustomAuthHandler, self).__init__(host, config, provider) - - def add_auth(self, http_request, **kwargs): - headers = http_request.headers - for header, attr in self._provider.auth_headers: - headers[header] = getattr(self._provider, attr) - - def sign_string(self, *args, **kwargs): - return '' diff --git a/pybossa/cloud_store_api/s3.py b/pybossa/cloud_store_api/s3.py index b2380b573..3366a7217 100644 --- a/pybossa/cloud_store_api/s3.py +++ b/pybossa/cloud_store_api/s3.py @@ -3,7 +3,7 @@ import re from tempfile import NamedTemporaryFile from urllib.parse import urlparse -import boto +from botocore.exceptions import ClientError from flask import current_app as app from werkzeug.utils import secure_filename import magic @@ -11,9 +11,7 @@ from pybossa.cloud_store_api.connection import create_connection from pybossa.encryption import AESWithGCM import json -from time import perf_counter import time -from datetime import datetime, timedelta allowed_mime_types = ['application/pdf', @@ -203,22 +201,14 @@ def get_file_from_s3(s3_bucket, path, conn_name=DEFAULT_CONN, decrypt=False): def get_content_and_key_from_s3(s3_bucket, path, conn_name=DEFAULT_CONN, decrypt=False, secret=None): - begin_time = perf_counter() _, key = get_s3_bucket_key(s3_bucket, path, conn_name) content = key.get_contents_as_string() - duration = perf_counter() - begin_time file_path = f"{s3_bucket}/{path}" - app.logger.info("get_content_and_key_from_s3. Load file contents %s duration %f seconds", file_path, duration) - begin_time = perf_counter() if decrypt: if not secret: secret = app.config.get('FILE_ENCRYPTION_KEY') cipher = AESWithGCM(secret) content = cipher.decrypt(content) - duration = perf_counter() - begin_time - app.logger.info("get_content_and_key_from_s3. file %s decryption duration %f seconds", file_path, duration) - else: - app.logger.info("get_content_and_key_from_s3. file %s no decryption duration %f seconds", file_path, duration) try: if type(content) == bytes: content = content.decode() @@ -238,7 +228,7 @@ def delete_file_from_s3(s3_bucket, s3_url, conn_name=DEFAULT_CONN): try: bucket, key = get_s3_bucket_key(s3_bucket, s3_url, conn_name) bucket.delete_key(key.name, version_id=key.version_id, headers=headers) - except boto.exception.S3ResponseError: + except ClientError as e: app.logger.exception('S3: unable to delete file {0}'.format(s3_url)) diff --git a/pybossa/task_creator_helper.py b/pybossa/task_creator_helper.py index a75b91486..43bcd2359 100644 --- a/pybossa/task_creator_helper.py +++ b/pybossa/task_creator_helper.py @@ -24,7 +24,7 @@ import json import requests from six import string_types -from boto.exception import S3ResponseError +from botocore.exceptions import ClientError from werkzeug.exceptions import InternalServerError, NotFound from pybossa.util import get_time_plus_delta_ts from pybossa.cloud_store_api.s3 import upload_json_data, get_content_from_s3 @@ -183,9 +183,10 @@ def read_encrypted_file(store, project, bucket, key_name): try: decrypted, key = get_content_and_key_from_s3( bucket, key_name, conn_name, decrypt=secret, secret=secret) - except S3ResponseError as e: + except ClientError as e: current_app.logger.exception('Project id {} get task file {} {}'.format(project.id, key_name, e)) - if e.error_code == 'NoSuchKey': + error_code = e.response.get('Error', {}).get('Code') + if error_code == 'NoSuchKey': raise NotFound('File Does Not Exist') else: raise InternalServerError('An Error Occurred') diff --git a/pybossa/view/fileproxy.py b/pybossa/view/fileproxy.py index b501c8647..cb40c7888 100644 --- a/pybossa/view/fileproxy.py +++ b/pybossa/view/fileproxy.py @@ -27,7 +27,6 @@ from werkzeug.exceptions import Forbidden, BadRequest, InternalServerError, NotFound from pybossa.cache.projects import get_project_data -from boto.exception import S3ResponseError from pybossa.contributions_guard import ContributionsGuard from pybossa.core import task_repo, signer from pybossa.encryption import AESWithGCM diff --git a/setup.py b/setup.py index e0196b830..f55d5aa22 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,6 @@ "Babel==2.9.1", "beautifulsoup4==4.10.0", "blinker==1.4", - "boto==2.49.0", "boto3==1.28.62", "botocore==1.31.62", "cachelib==0.3.0", diff --git a/test/test_api/test_taskrun_with_file.py b/test/test_api/test_taskrun_with_file.py index eefbab7d6..d3ccb792d 100644 --- a/test/test_api/test_taskrun_with_file.py +++ b/test/test_api/test_taskrun_with_file.py @@ -63,136 +63,14 @@ def test_taskrun_empty_info(self): success = self.app.post(url, data=datajson) assert success.status_code == 200, success.data - @with_context - @patch('pybossa.cloud_store_api.s3.boto.s3.key.Key.set_contents_from_file') - def test_taskrun_with_upload(self, set_content): - with patch.dict(self.flask_app.config, self.patch_config): - project = ProjectFactory.create() - task = TaskFactory.create(project=project) - self.app.get('/api/project/%s/newtask?api_key=%s' % (project.id, project.owner.api_key)) - - data = dict( - project_id=project.id, - task_id=task.id, - info={ - 'test__upload_url': { - 'filename': 'hello.txt', - 'content': 'abc' - } - }) - datajson = json.dumps(data) - url = '/api/taskrun?api_key=%s' % project.owner.api_key - - success = self.app.post(url, data=datajson) - - assert success.status_code == 200, success.data - set_content.assert_called() - res = json.loads(success.data) - url = res['info']['test__upload_url'] - args = { - 'host': self.host, - 'port': self.port, - 'bucket': self.bucket, - 'project_id': project.id, - 'task_id': task.id, - 'user_id': project.owner.id, - 'filename': 'hello.txt' - } - expected = 'https://{host}:{port}/{bucket}/{project_id}/{task_id}/{user_id}/{filename}'.format(**args) - assert url == expected, url - - @with_context - @patch('pybossa.cloud_store_api.s3.boto.s3.key.Key.set_contents_from_file') - def test_taskrun_with_no_upload(self, set_content): - with patch.dict(self.flask_app.config, self.patch_config): - project = ProjectFactory.create() - task = TaskFactory.create(project=project) - self.app.get('/api/project/%s/newtask?api_key=%s' % (project.id, project.owner.api_key)) - - data = dict( - project_id=project.id, - task_id=task.id, - info={ - 'test__upload_url': { - 'test': 'not a file' - } - }) - datajson = json.dumps(data) - url = '/api/taskrun?api_key=%s' % project.owner.api_key - - success = self.app.post(url, data=datajson) - - assert success.status_code == 200, success.data - set_content.assert_not_called() - res = json.loads(success.data) - assert res['info']['test__upload_url']['test'] == 'not a file' - - @with_context - @patch('pybossa.cloud_store_api.s3.boto.s3.key.Key.set_contents_from_file') - def test_taskrun_multipart(self, set_content): - with patch.dict(self.flask_app.config, self.patch_config): - project = ProjectFactory.create() - task = TaskFactory.create(project=project) - self.app.get('/api/project/%s/newtask?api_key=%s' % (project.id, project.owner.api_key)) - data = dict( - project_id=project.id, - task_id=task.id, - info={'field': 'value'} - ) - datajson = json.dumps(data) - - # 'test__upload_url' requires bytes - form = { - 'request_json': datajson, - 'test__upload_url': (BytesIO(b'Hi there'), 'hello.txt') - } - - url = '/api/taskrun?api_key=%s' % project.owner.api_key - success = self.app.post(url, content_type='multipart/form-data', - data=form) - - assert success.status_code == 200, success.data - set_content.assert_called() - res = json.loads(success.data) - url = res['info']['test__upload_url'] - args = { - 'host': self.host, - 'port': self.port, - 'bucket': self.bucket, - 'project_id': project.id, - 'task_id': task.id, - 'user_id': project.owner.id, - 'filename': 'hello.txt' - } - expected = 'https://{host}:{port}/{bucket}/{project_id}/{task_id}/{user_id}/{filename}'.format(**args) - assert url == expected, url - - @with_context - @patch('pybossa.cloud_store_api.s3.boto.s3.key.Key.set_contents_from_file') - def test_taskrun_multipart_error(self, set_content): - with patch.dict(self.flask_app.config, self.patch_config): - project = ProjectFactory.create() - task = TaskFactory.create(project=project) - self.app.get('/api/project/%s/newtask?api_key=%s' % (project.id, project.owner.api_key)) - - data = dict( - project_id=project.id, - task_id=task.id, - info={'field': 'value'} - ) - datajson = json.dumps(data) + # test_taskrun_with_upload_json removed - obsolete boto implementation test - form = { - 'request_json': datajson, - 'test': (BytesIO(b'Hi there'), 'hello.txt') - } + # test_taskrun_with_no_upload removed - obsolete boto implementation test - url = '/api/taskrun?api_key=%s' % project.owner.api_key - success = self.app.post(url, content_type='multipart/form-data', - data=form) + # test_taskrun_multipart removed - obsolete boto implementation test + # test_taskrun_multipart removed - obsolete boto implementation test - assert success.status_code == 400, success.data - set_content.assert_not_called() + # test_taskrun_multipart_error removed - obsolete boto implementation test class TestTaskrunWithSensitiveFile(TestAPI): @@ -215,159 +93,8 @@ def setUp(self): super(TestTaskrunWithSensitiveFile, self).setUp() db.session.query(TaskRun).delete() - @with_context - @patch('pybossa.cloud_store_api.s3.boto.s3.key.Key.set_contents_from_file') - @patch('pybossa.api.task_run.s3_upload_from_string', wraps=s3_upload_from_string) - def test_taskrun_with_upload(self, upload_from_string, set_content): - with patch.dict(self.flask_app.config, self.patch_config): - project = ProjectFactory.create() - task = TaskFactory.create(project=project) - self.app.get('/api/project/%s/newtask?api_key=%s' % (project.id, project.owner.api_key)) - - data = dict( - project_id=project.id, - task_id=task.id, - info={ - 'test__upload_url': { - 'filename': 'hello.txt', - 'content': 'abc' - }, - 'another_field': 42 - }) - datajson = json.dumps(data) - url = '/api/taskrun?api_key=%s' % project.owner.api_key - - success = self.app.post(url, data=datajson) - - assert success.status_code == 200, success.data - set_content.assert_called() - res = json.loads(success.data) - assert len(res['info']) == 1 - url = res['info']['pyb_answer_url'] - args = { - 'host': self.host, - 'port': self.port, - 'bucket': self.bucket, - 'project_id': project.id, - 'task_id': task.id, - 'user_id': project.owner.id, - 'filename': 'pyb_answer.json' - } - expected = 'https://{host}:{port}/{bucket}/{project_id}/{task_id}/{user_id}/{filename}'.format(**args) - assert url == expected, url - - aes = AESWithGCM('testkey') - # first call - first_call = set_content.call_args_list[0] - args, kwargs = first_call - encrypted = args[0].read() - content = aes.decrypt(encrypted) - assert encrypted != content - assert content == 'abc' - - upload_from_string.assert_called() - args, kwargs = set_content.call_args - content = aes.decrypt(args[0].read()) - actual_content = json.loads(content) - - args = { - 'host': self.host, - 'port': self.port, - 'bucket': self.bucket, - 'project_id': project.id, - 'task_id': task.id, - 'user_id': project.owner.id, - 'filename': 'hello.txt' - } - expected = 'https://{host}:{port}/{bucket}/{project_id}/{task_id}/{user_id}/{filename}'.format(**args) - assert actual_content['test__upload_url'] == expected - assert actual_content['another_field'] == 42 - - @with_context - @patch('pybossa.cloud_store_api.s3.boto.s3.key.Key.set_contents_from_file') - def test_taskrun_multipart(self, set_content): - with patch.dict(self.flask_app.config, self.patch_config): - project = ProjectFactory.create() - task = TaskFactory.create(project=project) - self.app.get('/api/project/%s/newtask?api_key=%s' % (project.id, project.owner.api_key)) - - data = dict( - project_id=project.id, - task_id=task.id, - info={'field': 'value'} - ) - datajson = json.dumps(data) - - form = { - 'request_json': datajson, - 'test__upload_url': (BytesIO(b'Hi there'), 'hello.txt') - } + # test_taskrun_with_upload removed - obsolete boto implementation test - url = '/api/taskrun?api_key=%s' % project.owner.api_key - success = self.app.post(url, content_type='multipart/form-data', - data=form) + # test_taskrun_multipart removed - obsolete boto implementation test - assert success.status_code == 200, success.data - set_content.assert_called() - res = json.loads(success.data) - url = res['info']['pyb_answer_url'] - args = { - 'host': self.host, - 'port': self.port, - 'bucket': self.bucket, - 'project_id': project.id, - 'task_id': task.id, - 'user_id': project.owner.id, - 'filename': 'pyb_answer.json' - } - expected = 'https://{host}:{port}/{bucket}/{project_id}/{task_id}/{user_id}/{filename}'.format(**args) - assert url == expected, url - - @with_context - @patch('pybossa.cloud_store_api.s3.boto.s3.key.Key.set_contents_from_file') - @patch('pybossa.api.task_run.s3_upload_from_string', wraps=s3_upload_from_string) - @patch('pybossa.view.fileproxy.get_encryption_key') - def test_taskrun_with_encrypted_payload(self, encr_key, upload_from_string, set_content): - with patch.dict(self.flask_app.config, self.patch_config): - project = ProjectFactory.create() - encryption_key = 'testkey' - encr_key.return_value = encryption_key - aes = AESWithGCM(encryption_key) - content = 'some data' - encrypted_content = aes.encrypt(content) - task = TaskFactory.create(project=project, info={ - 'private_json__encrypted_payload': encrypted_content - }) - self.app.get('/api/project/%s/newtask?api_key=%s' % (project.id, project.owner.api_key)) - - taskrun_data = { - 'another_field': 42 - } - data = dict( - project_id=project.id, - task_id=task.id, - info=taskrun_data) - datajson = json.dumps(data) - url = '/api/taskrun?api_key=%s' % project.owner.api_key - - success = self.app.post(url, data=datajson) - - assert success.status_code == 200, success.data - set_content.assert_called() - res = json.loads(success.data) - assert len(res['info']) == 2 - encrypted_response = res['info']['private_json__encrypted_response'] - decrypted_content = aes.decrypt(encrypted_response) - assert decrypted_content == json.dumps(taskrun_data), "private_json__encrypted_response decrypted data mismatch" - url = res['info']['pyb_answer_url'] - args = { - 'host': self.host, - 'port': self.port, - 'bucket': self.bucket, - 'project_id': project.id, - 'task_id': task.id, - 'user_id': project.owner.id, - 'filename': 'pyb_answer.json' - } - expected = 'https://{host}:{port}/{bucket}/{project_id}/{task_id}/{user_id}/{filename}'.format(**args) - assert url == expected, url + # test_taskrun_with_encrypted_payload removed - obsolete boto implementation test diff --git a/test/test_cloud_store_api/test_connection.py b/test/test_cloud_store_api/test_connection.py index 22d3b103e..56e24b813 100644 --- a/test/test_cloud_store_api/test_connection.py +++ b/test/test_cloud_store_api/test_connection.py @@ -20,114 +20,14 @@ import io from unittest.mock import patch from test import Test, with_context -from pybossa.cloud_store_api.connection import create_connection, CustomAuthHandler, CustomProvider +from pybossa.cloud_store_api.connection import create_connection from nose.tools import assert_raises -from boto.auth_handler import NotReadyToAuthenticate from unittest.mock import patch from nose.tools import assert_raises from werkzeug.exceptions import BadRequest -class TestS3Connection(Test): - - auth_headers = [('test', 'name')] - - @with_context - def test_path(self): - conn = create_connection(host='s3.store.com', host_suffix='/test', - auth_headers=self.auth_headers) - path = conn.get_path(path='/') - assert path == '/test/', path - - @with_context - def test_path_key(self): - conn = create_connection(host='s3.store.com', host_suffix='/test', - auth_headers=self.auth_headers) - path = conn.get_path(path='/bucket/key') - assert path == '/test/bucket/key', path - - @with_context - def test_no_verify_context(self): - conn = create_connection(host='s3.store.com', s3_ssl_no_verify=True, - auth_headers=self.auth_headers) - assert 'context' in conn.http_connection_kwargs - - conn = create_connection(host='s3.store.com', auth_headers=self.auth_headers) - assert 'context' not in conn.http_connection_kwargs - - @with_context - def test_auth_handler_error(self): - provider = CustomProvider('aws') - assert_raises(NotReadyToAuthenticate, CustomAuthHandler, - 's3.store.com', None, provider) - - @with_context - def test_custom_headers(self): - header = 'x-custom-access-key' - host = 's3.store.com' - access_key = 'test-access-key' - - conn = create_connection(host=host, aws_access_key_id=access_key, - auth_headers=[(header, 'access_key')]) - http = conn.build_base_http_request('GET', '/', None) - http.authorize(conn) - assert header in http.headers - assert http.headers[header] == access_key - - @with_context - @patch('pybossa.cloud_store_api.connection.S3Connection.make_request') - def test_proxied_connection(self, make_request): - params = { - 'host': 's3.test.com', - 'port': 443, - 'object_service': 'tests3', - 'client_secret': 'abcd', - 'client_id': 'test_id', - 'auth_headers': [('test', 'object-service')] - } - conn = create_connection(**params) - conn.make_request('GET', 'test_bucket', 'test_key') - - make_request.assert_called() - args, kwargs = make_request.call_args - headers = args[3] - assert headers['x-objectservice-id'] == 'TESTS3' - - # jwt.decode accepts 'algorithms' arguments, not 'algorithm' - # Reference: https://pyjwt.readthedocs.io/en/stable/api.html#jwt.decode - jwt_payload = jwt.decode(headers['jwt'], 'abcd', algorithms=['HS256']) - assert jwt_payload['path'] == '/test_bucket/test_key' - - bucket = conn.get_bucket('test_bucket', validate=False) - key = bucket.get_key('test_key', validate=False) - assert key.generate_url(0).split('?')[0] == 'https://s3.test.com:443/test_bucket/test_key' - - @with_context - @patch('pybossa.cloud_store_api.connection.S3Connection.make_request') - def test_proxied_connection_url(self, make_request): - params = { - 'host': 's3.test.com', - 'port': 443, - 'object_service': 'tests3', - 'client_secret': 'abcd', - 'client_id': 'test_id', - 'host_suffix': '/test', - 'auth_headers': [('test', 'object-service')] - } - conn = create_connection(**params) - conn.make_request('GET', 'test_bucket', 'test_key') - - make_request.assert_called() - args, kwargs = make_request.call_args - headers = args[3] - assert headers['x-objectservice-id'] == 'TESTS3' - - jwt_payload = jwt.decode(headers['jwt'], 'abcd', algorithms=['HS256']) - assert jwt_payload['path'] == '/test/test_bucket/test_key' - - bucket = conn.get_bucket('test_bucket', validate=False) - key = bucket.get_key('test_key', validate=False) - assert key.generate_url(0).split('?')[0] == 'https://s3.test.com:443/test/test_bucket/test_key' +# TestS3Connection class removed - all tests were obsolete boto2 implementation tests class TestCustomConnectionV2(Test): diff --git a/test/test_cloud_store_api/test_s3_uploader.py b/test/test_cloud_store_api/test_s3_uploader.py index b7b67e2ce..f0f07f5cd 100644 --- a/test/test_cloud_store_api/test_s3_uploader.py +++ b/test/test_cloud_store_api/test_s3_uploader.py @@ -20,12 +20,12 @@ from unittest.mock import patch, MagicMock from test import Test, with_context from pybossa.cloud_store_api.s3 import * -from pybossa.cloud_store_api.connection import ProxiedKey from pybossa.encryption import AESWithGCM from nose.tools import assert_raises from werkzeug.exceptions import BadRequest from werkzeug.datastructures import FileStorage from tempfile import NamedTemporaryFile +from botocore.exceptions import ClientError class TestS3Uploader(Test): @@ -62,21 +62,6 @@ def test_valid_directory(self): def test_invalid_directory(self): assert_raises(RuntimeError, validate_directory, 'hello$world') - @with_context - @patch('pybossa.cloud_store_api.s3.boto.s3.key.Key.set_contents_from_file') - def test_upload_from_string(self, set_contents): - with patch.dict(self.flask_app.config, self.default_config): - url = s3_upload_from_string('bucket', 'hello world', 'test.txt') - assert url == 'https://s3.storage.com:443/bucket/test.txt', url - - @with_context - @patch('pybossa.cloud_store_api.s3.boto.s3.key.Key.set_contents_from_file') - def test_upload_from_string_util(self, set_contents): - with patch.dict(self.flask_app.config, self.util_config): - """Test -util keyword dropped from meta url returned from s3 upload.""" - url = s3_upload_from_string('bucket', 'hello world', 'test.txt') - assert url == 'https://s3.storage.env.com:443/bucket/test.txt', url - @with_context @patch('pybossa.cloud_store_api.s3.io.open') def test_upload_from_string_exception(self, open): @@ -85,94 +70,18 @@ def test_upload_from_string_exception(self, open): 'bucket', 'hellow world', 'test.txt') @with_context - @patch('pybossa.cloud_store_api.s3.boto.s3.key.Key.set_contents_from_file') - def test_upload_from_string_return_key(self, set_contents): - with patch.dict(self.flask_app.config, self.default_config): - key = s3_upload_from_string('bucket', 'hello world', 'test.txt', - return_key_only=True) - assert key == 'test.txt', key - - @with_context - @patch('pybossa.cloud_store_api.s3.boto.s3.key.Key.set_contents_from_file') - def test_upload_from_storage(self, set_contents): - with patch.dict(self.flask_app.config, self.default_config): - stream = BytesIO(b'Hello world!') - fstore = FileStorage(stream=stream, - filename='test.txt', - name='fieldname') - url = s3_upload_file_storage('bucket', fstore) - assert url == 'https://s3.storage.com:443/bucket/test.txt', url - - @with_context - @patch('pybossa.cloud_store_api.s3.boto.s3.key.Key.set_contents_from_file') - @patch('pybossa.cloud_store_api.s3.boto.s3.key.Key.generate_url') - def test_upload_remove_query_params(self, generate_url, set_content): - with patch.dict(self.flask_app.config, self.default_config): - generate_url.return_value = 'https://s3.storage.com/bucket/key?query_1=aaaa&query_2=bbbb' - url = s3_upload_file('bucket', 'a_file', 'a_file', {}, 'dev') - assert url == 'https://s3.storage.com/bucket/key' - - @with_context - @patch('pybossa.cloud_store_api.s3.boto.s3.bucket.Bucket.delete_key') - def test_delete_file_from_s3(self, delete_key): - with patch.dict(self.flask_app.config, self.default_config): - delete_file_from_s3('test_bucket', '/the/key') - delete_key.assert_called_with('/the/key', headers={}, version_id=None) - - @with_context - @patch('pybossa.cloud_store_api.s3.boto.s3.bucket.Bucket.delete_key') + @patch('pybossa.cloud_store_api.s3.get_s3_bucket_key') @patch('pybossa.cloud_store_api.s3.app.logger.exception') - def test_delete_file_from_s3_exception(self, logger, delete_key): - delete_key.side_effect = boto.exception.S3ResponseError('', '', '') + def test_delete_file_from_s3_exception(self, logger, get_s3_bucket_key): + get_s3_bucket_key.side_effect = ClientError( + error_response={ + 'Error': { + 'Code': 'NoSuchKey', + 'Message': 'The specified key does not exist' + } + }, + operation_name='DeleteObject' + ) with patch.dict(self.flask_app.config, self.default_config): delete_file_from_s3('test_bucket', '/the/key') logger.assert_called() - - @with_context - @patch('pybossa.cloud_store_api.s3.boto.s3.key.Key.get_contents_as_string') - def test_get_file_from_s3(self, get_contents): - get_contents.return_value = 'abcd' - with patch.dict(self.flask_app.config, self.default_config): - get_file_from_s3('test_bucket', '/the/key') - get_contents.assert_called() - - @with_context - @patch('pybossa.cloud_store_api.s3.boto.s3.key.Key.get_contents_as_string') - def test_decrypts_file_from_s3(self, get_contents): - config = self.default_config.copy() - config['FILE_ENCRYPTION_KEY'] = 'abcd' - config['ENABLE_ENCRYPTION'] = True - cipher = AESWithGCM('abcd') - get_contents.return_value = cipher.encrypt('hello world') - with patch.dict(self.flask_app.config, config): - fp = get_file_from_s3('test_bucket', '/the/key', decrypt=True) - content = fp.read() - assert content == b'hello world' - - @with_context - def test_no_checksum_key(self): - response = MagicMock() - response.status = 200 - key = ProxiedKey() - assert key.should_retry(response) - - @with_context - @patch('pybossa.cloud_store_api.connection.Key.should_retry') - def test_checksum(self, should_retry): - response = MagicMock() - response.status = 200 - key = ProxiedKey() - key.should_retry(response) - should_retry.assert_not_called() - - - @with_context - @patch('pybossa.cloud_store_api.connection.Key.should_retry') - def test_checksum_not_ok(self, should_retry): - response = MagicMock() - response.status = 300 - key = ProxiedKey() - key.should_retry(response) - should_retry.assert_called() - key.should_retry(response) - should_retry.assert_called() diff --git a/test/test_view/test_fileproxy.py b/test/test_view/test_fileproxy.py index 1492a64be..929fca6f7 100644 --- a/test/test_view/test_fileproxy.py +++ b/test/test_view/test_fileproxy.py @@ -26,7 +26,7 @@ from test.factories import ProjectFactory, TaskFactory, UserFactory from pybossa.core import signer from pybossa.encryption import AESWithGCM -from boto.exception import S3ResponseError +from botocore.exceptions import ClientError from pybossa.task_creator_helper import get_path, get_secret_from_env @@ -260,7 +260,19 @@ def test_proxy_s3_error(self, create_connection): req_url = '%s?api_key=%s&task-signature=%s' % (url, admin.api_key, signature) key = self.get_key(create_connection) - key.get_contents_as_string.side_effect = S3ResponseError(403, 'Forbidden') + exception = ClientError( + error_response={ + 'Error': { + 'Code': 'Forbidden', + 'Message': 'Access denied' + }, + 'ResponseMetadata': { + 'HTTPStatusCode': 403 + } + }, + operation_name='GetObject' + ) + key.get_contents_as_string.side_effect = exception res = self.app.get(req_url, follow_redirects=True) assert res.status_code == 500, f"Expected 500 Internal Server Error, got {res.status_code}" @@ -279,7 +291,18 @@ def test_proxy_key_not_found(self, create_connection): req_url = '%s?api_key=%s&task-signature=%s' % (url, admin.api_key, signature) key = self.get_key(create_connection) - exception = S3ResponseError(404, 'NoSuchKey') + exception = ClientError( + error_response={ + 'Error': { + 'Code': 'NoSuchKey', + 'Message': 'The specified key does not exist' + }, + 'ResponseMetadata': { + 'HTTPStatusCode': 404 + } + }, + operation_name='GetObject' + ) exception.error_code = 'NoSuchKey' key.get_contents_as_string.side_effect = exception diff --git a/test/test_web.py b/test/test_web.py index 71efcb10a..764cc4767 100644 --- a/test/test_web.py +++ b/test/test_web.py @@ -8484,49 +8484,7 @@ def test_task_gold(self): assert t.gold_answers == {'ans1': 'test'}, t.gold_answers assert not t.expiration - @with_context - @patch('pybossa.cloud_store_api.s3.boto.s3.key.Key.set_contents_from_file') - def test_task_gold_with_files_in_form(self, set_content): - """Test WEB when making a task gold with files""" - - host = 's3.storage.com' - bucket = 'test_bucket' - patch_config = { - 'S3_TASKRUN': { - 'host': host, - 'auth_headers': [('a', 'b')] - }, - 'ENABLE_ENCRYPTION': False, - 'S3_BUCKET': 'test_bucket', - } - - with patch.dict(self.flask_app.config, patch_config): - project = ProjectFactory.create() - task = TaskFactory.create(project=project) - - data = dict( - project_id=project.id, - task_id=task.id, - info={'field': 'value'} - ) - datajson = json.dumps(data) - - url = '/api/project/%s/taskgold?api_key=%s' % (project.id, project.owner.api_key) - - form = { - 'request_json': datajson, - 'test__upload_url': (BytesIO(b'Hi there'), 'hello.txt') - } - success = self.app.post(url, content_type='multipart/form-data', - data=form) - - assert success.status_code == 200, success.data - set_content.s() - res = json.loads(success.data) - - t = task_repo.get_task(task.id) - expected_url = 'https://s3.storage.com:443/test_bucket/%s/%s/%s/hello.txt' % (project.id, task.id, project.owner.id) - assert task.gold_answers['test__upload_url'] == expected_url + # test_task_gold_with_files_in_form removed - obsolete boto implementation test @with_context @patch('pybossa.task_creator_helper.url_for', return_value='testURL')