Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix storages notebooks #53

Open
wants to merge 5 commits into
base: dev
Choose a base branch
from
Open

Fix storages notebooks #53

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1526e08
fix: mypy errors and renamed primary errors
l-loic Jul 11, 2024
00bd451
fix: env issue and cli list output
l-loic Jul 12, 2024
bfd1758
bump naas-models to 1.11.6
l-loic Jul 15, 2024
3eb60af
fix: mypy error
l-loic Jul 18, 2024
8b410ad
fix: NoneType' object is not subscriptable
l-loic Jul 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 30 additions & 28 deletions naas_python/domains/storage/StorageDomain.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from .models.Storage import Storage
from naas_python.domains.storage.models.Storage import Storage

from typing import Mapping
from typing import List, Mapping

from naas_python.domains.storage.StorageSchema import (
IStorageDomain,
Expand All @@ -11,18 +11,19 @@
StorageProviderNotFound
)
class StorageDomain(IStorageDomain):
def __init__(self, adaptor: IStorageAdaptor, storage_provider_adaptors : Mapping[str, IStorageProviderAdaptor]):
# List[IStorageProviderAdaptor])
#Map[str : IStorageProviderAdaptor])
self.adaptor : IStorageAdaptor = adaptor
def __init__(self, adaptor: IStorageAdaptor,
storage_provider_adaptors : Mapping[str, IStorageProviderAdaptor]
) :

self.adaptor = adaptor
self.storage_provider_adaptors : Mapping[str, IStorageProviderAdaptor] = storage_provider_adaptors

############### API ###############
def create(self,
workspace_id: str,
workspace_id: str,
storage_name: Storage.__fields__['name'],
) -> dict:
response = self.adaptor.create_workspace_storage(
response = self.adaptor.create(
workspace_id=workspace_id,
storage_name=storage_name,
)
Expand All @@ -32,7 +33,7 @@ def delete(self,
workspace_id: str,
storage_name: Storage.__fields__['name']
) -> dict:
response = self.adaptor.delete_workspace_storage(
response = self.adaptor.delete(
workspace_id=workspace_id,
storage_name=storage_name,
)
Expand All @@ -41,7 +42,7 @@ def delete(self,
def list(self,
workspace_id: str,
) -> dict:
response = self.adaptor.list_workspace_storage(
response = self.adaptor.list(
workspace_id=workspace_id,
)
return response
Expand All @@ -51,7 +52,7 @@ def list_objects(self,
storage_name: Storage.__fields__['name'],
storage_prefix: Object.__fields__['prefix'],
) -> dict:
response = self.adaptor.list_workspace_storage_object(
response = self.adaptor.list_objects(
workspace_id=workspace_id,
storage_name=storage_name,
storage_prefix=storage_prefix,
Expand All @@ -61,9 +62,9 @@ def list_objects(self,
def delete_object(self,
workspace_id: str,
storage_name: Storage.__fields__['name'],
object_name: Object.__fields__['name'],
object_name: Storage.__fields__['name'],
) -> dict:
response = self.adaptor.delete_workspace_storage_object(
response = self.adaptor.delete_object(
workspace_id=workspace_id,
storage_name=storage_name,
object_name=object_name,
Expand All @@ -74,11 +75,12 @@ def create_credentials(self,
workspace_id: str,
storage_name: Storage.__fields__['name'],
) -> dict:
credentials = self.adaptor.generate_credentials(workspace_id, storage_name)
self.__get_storage_provider_adaptor(workspace_id=workspace_id, storage_name=storage_name).save_naas_credentials(workspace_id, storage_name, credentials)
return credentials
credentials = self.adaptor.create_credentials(workspace_id, storage_name)
storage_provider_adaptor = self.__get_storage_provider_adaptor(workspace_id=workspace_id, storage_name=storage_name)
storage_provider_adaptor.save_naas_credentials(workspace_id, storage_name, credentials)
return credentials

############### BOTO ###############
############### BOTO ###############
def __get_storage_provider(self,
workspace_id: str,
storage_name: Storage.__fields__['name']
Expand All @@ -98,7 +100,7 @@ def __get_storage_provider_adaptor(self,

def post_object(self,
workspace_id: str,
storage_name: Storage.__fields__['name'],
storage_name: str,
src_file: str,
dst_file: str,
) -> dict:
Expand All @@ -108,13 +110,13 @@ def post_object(self,
if storage_provider_id not in self.storage_provider_adaptors:
raise StorageProviderNotFound(f'Provider "{storage_provider_id}" is not implemented or not loaded.')

storage_provider : IStorageProviderAdaptor = self.storage_provider_adaptors[storage_provider_id]
storage_provider_adaptor : IStorageProviderAdaptor = self.storage_provider_adaptors[storage_provider_id]

if not storage_provider.valid_naas_credentials(workspace_id, storage_name):
credentials = self.adaptor.generate_credentials(workspace_id, storage_name)
storage_provider.save_naas_credentials(workspace_id, storage_name, credentials)
if not storage_provider_adaptor.valid_naas_credentials(workspace_id, storage_name):
credentials = self.adaptor.create_credentials(workspace_id, storage_name)
storage_provider_adaptor.save_naas_credentials(workspace_id, storage_name, credentials)

response = storage_provider.post_workspace_storage_object(workspace_id=workspace_id, storage_name=storage_name, src_file=src_file, dst_file=dst_file)
response = storage_provider_adaptor.post_object(workspace_id=workspace_id, storage_name=storage_name, src_file=src_file, dst_file=dst_file)
return response

def get_object(self,
Expand All @@ -129,11 +131,11 @@ def get_object(self,
if storage_provider_id not in self.storage_provider_adaptors:
raise StorageProviderNotFound(f'Provider "{storage_provider_id}" is not implemented or not loaded.')

storage_provider : IStorageProviderAdaptor = self.storage_provider_adaptors[storage_provider_id]
storage_provider_adaptor : IStorageProviderAdaptor = self.storage_provider_adaptors[storage_provider_id]

if not storage_provider.valid_naas_credentials(workspace_id, storage_name):
credentials = self.adaptor.generate_credentials(workspace_id, storage_name)
storage_provider.save_naas_credentials(workspace_id, storage_name, credentials)
if not storage_provider_adaptor.valid_naas_credentials(workspace_id, storage_name):
credentials = self.adaptor.create_credentials(workspace_id, storage_name)
storage_provider_adaptor.save_naas_credentials(workspace_id, storage_name, credentials)

response = storage_provider.get_workspace_storage_object(workspace_id=workspace_id, storage_name=storage_name, src_file=src_file, dst_file=dst_file)
response = storage_provider_adaptor.get_object(workspace_id=workspace_id, storage_name=storage_name, src_file=src_file, dst_file=dst_file)
return response
76 changes: 48 additions & 28 deletions naas_python/domains/storage/StorageSchema.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
from abc import ABCMeta, abstractmethod
from logging import getLogger
from typing import Mapping
from typing import List, Mapping, Any
from uuid import UUID

from naas_models.pydantic.storage_p2p import *
from naas_models.pydantic.storage_p2p import Storage
from .models.Storage import Storage, Object


Expand All @@ -12,52 +13,52 @@
logger = getLogger(__name__)

class IStorageAdaptor(metaclass=ABCMeta):
@abstractmethod
def create_workspace_storage(self,
@abstractmethod
def create(self,
workspace_id: str,
storage_name: Storage.__fields__['name'],
) -> dict[str, str]:
) -> dict:
raise NotImplementedError

@abstractmethod
def delete_workspace_storage(self,
def delete(self,
workspace_id: str,
storage_name: Storage.__fields__['name'],
) -> dict:
raise NotImplementedError

@abstractmethod
def list_workspace_storage(self,
def list(self,
workspace_id: str,
) -> dict:
raise NotImplementedError

@abstractmethod
def list_workspace_storage_object(self,
def list_objects(self,
workspace_id: str,
storage_name: Storage.__fields__['name'],
storage_prefix: Object.__fields__['prefix'],
) -> dict:
raise NotImplementedError

@abstractmethod
def delete_workspace_storage_object(self,
def delete_object(self,
workspace_id: str,
storage_name: Storage.__fields__['name'],
object_name: Object.__fields__['name'],
) -> dict:
raise NotImplementedError

@abstractmethod
def generate_credentials(self, workspace_id :str, storage_name: str) -> dict:
def create_credentials(self, workspace_id : str, storage_name: Storage.__fields__['name']) -> dict:
raise NotImplementedError

class IStorageProviderAdaptor(metaclass=ABCMeta):

provider_id : str

@abstractmethod
def post_workspace_storage_object(self,
def post_object(self,
workspace_id: str,
storage_name: Storage.__fields__['name'],
src_file: str,
Expand All @@ -66,7 +67,7 @@ def post_workspace_storage_object(self,
raise NotImplementedError

@abstractmethod
def get_workspace_storage_object(self,
def get_object(self,
workspace_id: str,
storage_name: Storage.__fields__['name'],
src_file: str,
Expand All @@ -82,8 +83,6 @@ def save_naas_credentials(self, workspace_id:str, storage_name:str, credentials:
class IStorageDomain(metaclass=ABCMeta):
adaptor: IStorageAdaptor
storage_provider_adaptors : Mapping[str, IStorageProviderAdaptor]
# storage_provider_adaptors : Map[str, IStorageProviderAdaptor]
#TODO to be validated

@abstractmethod
def create(self,
Expand Down Expand Up @@ -139,7 +138,6 @@ def get_object(self,
) -> bytes:
raise NotImplementedError


@abstractmethod
def create_credentials(self,
workspace_id: str,
Expand All @@ -150,35 +148,57 @@ def create_credentials(self,
# Primary Adaptor
class IStorageInvoker(metaclass=ABCMeta):
@abstractmethod
def create_workspace_storage(self, **kwargs):
def create(self, workspace_id: str, storage_name: Storage.__fields__['name']) -> dict:
raise NotImplementedError

@abstractmethod
def delete_workspace_storage(self, **kwargs):
raise NotImplementedError
def delete(self,
workspace_id: str,
storage_name: str = Storage.__fields__['name'],
) -> dict:
raise NotImplementedError

@abstractmethod
def list_workspace_storage(self, **kwargs):
def list(self, workspace_id: str) -> dict:
raise NotImplementedError

@abstractmethod
def list_workspace_storage_object(self, **kwargs):
def create_credentials(self,
workspace_id : str,
storage_name = str,
) -> dict:
raise NotImplementedError

@abstractmethod
def delete_workspace_storage_object(self, **kwargs):
raise NotImplementedError

def list_objects(self,
workspace_id: str,
storage_name: str,
storage_prefix: str) -> dict:
raise NotImplementedError

@abstractmethod
def post_workspace_storage_object(self, **kwargs):
def delete_object(self,
workspace_id: str,
storage_name: str,
object_name: str) -> dict:
raise NotImplementedError

@abstractmethod
def get_workspace_storage_object(self, **kwargs):
raise NotImplementedError
def post_object(self,
workspace_id: str,
storage_name: str,
src_file: str,
dst_file: str,
) -> dict:
raise NotImplementedError

@abstractmethod
def create_workspace_storage_credentials(self, **kwargs):
def get_object(self,
workspace_id: str,
storage_name: str,
src_file: str,
dst_file: str,
) -> bytes:
raise NotImplementedError

# Exceptions
Expand Down
Loading
Loading