Skip to content

Commit 8dc32e1

Browse files
committed
bugfix(upload): small cleanup
1 parent 4d051d3 commit 8dc32e1

4 files changed

Lines changed: 20 additions & 35 deletions

File tree

‎QA/py/tests/userfiles_helpers.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ def make_user_dir(root: Path) -> UserFilesDirectory:
2525
ret._root_path = root
2626
ret.trash_directory = UserFilesDirectory.TRASH_DIRECTORY % 1
2727
unpacker = object.__new__(Unpacker)
28-
unpacker.accepted_mime_types = list(DEFAULT_MIMETYPES)
29-
unpacker.archive_extensions = [a.lower() for a in DEFAULT_ARCHIVES]
28+
unpacker.accepted_mime_types = set(DEFAULT_MIMETYPES)
29+
unpacker.archive_extensions = {a.lower() for a in DEFAULT_ARCHIVES}
3030
unpacker.user_id = ret.user_id
3131
unpacker.list_errors = {}
3232
unpacker._root_path = root

‎py/API_operations/BigFiles.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ async def tus_auth(
4343

4444

4545
async def _background_unpack(
46-
user_dir: UserFilesDirectory, orig_path: Path, file_path: Path, dest_dir: Path
46+
user_dir: UserFilesDirectory, file_path: Path, dest_dir: Path
4747
) -> None:
4848
try:
4949
user_dir.unpacker.switch_to_background()
@@ -98,7 +98,7 @@ async def _start_unpack(
9898
status.update({"state": "unpacking", "percent": 0})
9999
user_dir.unpacker.status_reporter = status.update
100100
task = asyncio.create_task(
101-
_background_unpack(user_dir, tus_path, dest_path, dest_dir),
101+
_background_unpack(user_dir, dest_path, dest_dir),
102102
name=f"unpack_{name}",
103103
)
104104
background_tasks.add(task)

‎py/FS/Unpack.py‎

Lines changed: 14 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,6 @@ class Unpacker(object):
285285
"""
286286

287287
ERROR_FILE_SUFFIX = ".unpack_error.log"
288-
PROGRESS_PREFIX = PROGRESS_PREFIX
289288

290289
def __init__(
291290
self,
@@ -399,13 +398,10 @@ def extract_archive(
399398
if top_level:
400399
self._status_entries_done(len(filenames), len(filenames))
401400

402-
def _is_embedded_archive(self, filename: str, filepath: Path) -> bool:
403-
return self._is_nested_archive(filename, filepath)
404-
405401
def _is_accepted_extracted_file(self, filename: str, filepath: Path) -> bool:
406402
if filepath.is_dir():
407403
return True
408-
if self._is_embedded_archive(filename, filepath):
404+
if self._is_nested_archive(filename, filepath):
409405
return True
410406
file_ext, _, mime_type = self.get_file_info(filepath.name, filepath.parent)
411407
if file_ext in TEXT_EXTENSIONS:
@@ -450,25 +446,11 @@ def get_file_info(filename: str, path: Path) -> Tuple[str, Path, Optional[str]]:
450446

451447
return file_ext, filepath, mime_type
452448

453-
def _has_accepted_format(self, path: Path, filepath: str) -> bool:
454-
# Same extension fallback as for archive members
455-
if filepath.split(".")[-1].lower() in TEXT_EXTENSIONS:
449+
def _has_accepted_format(self, filepath: Union[Path, str]) -> bool:
450+
p = Path(filepath)
451+
file_ext, _, mime_type = self.get_file_info(p.name, p.parent)
452+
if file_ext in TEXT_EXTENSIONS or mime_type in self.accepted_mime_types:
456453
return True
457-
try:
458-
py_magic = from_path(filepath)
459-
mime_type = py_magic.mime_type()
460-
except (CantMatchTypeError, Exception):
461-
mime_type = None
462-
if mime_type in self.accepted_mime_types:
463-
return True
464-
logger.info(
465-
"File format not accepted '%s' '%s' , user_id '%s'",
466-
str(path),
467-
str(filepath),
468-
str(self.user_id),
469-
)
470-
path_error = str(path.joinpath(filepath.lstrip(os.path.sep)))
471-
self.list_errors.update({"Not accepted": path_error})
472454
return False
473455

474456
def unpack_zip(self, input_path: Path, path: Path) -> None:
@@ -478,7 +460,7 @@ def unpack_zip(self, input_path: Path, path: Path) -> None:
478460
"r",
479461
allowZip64=True,
480462
filter_func=self._is_accepted_extracted_file,
481-
is_embedded_archive=self._is_embedded_archive,
463+
is_embedded_archive=self._is_nested_archive,
482464
) as archive:
483465
self.extract_archive(archive, str(input_path), path)
484466
except Exception as e:
@@ -492,7 +474,7 @@ def unpack_tar(self, input_path: Path, path: Path) -> None:
492474
input_path,
493475
"r",
494476
filter_func=self._is_accepted_extracted_file,
495-
is_embedded_archive=self._is_embedded_archive,
477+
is_embedded_archive=self._is_nested_archive,
496478
) as archive:
497479
self.extract_archive(archive, str(input_path), path)
498480
except Exception as e:
@@ -804,8 +786,14 @@ def _dispatch_nested(self, compressed_path: Path, to_path: Path) -> None:
804786
elif self._is_gz(compressed_path.as_posix()):
805787
with self._one_level_deeper():
806788
self.unpack_gz(compressed_path, to_path)
807-
elif not self._has_accepted_format(to_path, str(compressed_path)):
789+
elif not self._has_accepted_format(compressed_path):
808790
os.remove(compressed_path)
791+
logger.info(
792+
"File format not accepted '%s' '%s' , user_id '%s'",
793+
str(to_path),
794+
str(compressed_path),
795+
str(self.user_id),
796+
)
809797
self.list_errors.update({"Not accepted": str(compressed_path)})
810798
logger.info(f"Unpack done for {compressed_path}->{to_path}")
811799

‎py/FS/UserFilesDir.py‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,8 @@
1010
from starlette.datastructures import UploadFile
1111

1212
from DB.User import UserIDT
13-
from FS import Unpack
1413
from FS.CommonDir import CommonFolder, DirEntryT
15-
from FS.Unpack import Unpacker, _log_exception_throw, get_tus_files_dir
14+
from FS.Unpack import Unpacker, _log_exception_throw, get_tus_files_dir, sweep_stale
1615
from helpers.AppConfig import Config
1716
from helpers.CustomException import UnprocessableEntityException
1817
from helpers.httpexception import DETAIL_FILE_PROTECTED, DETAIL_NOTHING_DONE
@@ -27,8 +26,6 @@ class UserFilesDirectory(object):
2726

2827
USER_DIR_PATTERN = "ecotaxa_user.%d"
2928
TRASH_DIRECTORY = "trash.%d"
30-
COMPRESSED_PATTERN = "*"
31-
TSV = ".tsv"
3229

3330
def __init__(self, user_id: UserIDT):
3431
config = Config()
@@ -177,6 +174,6 @@ def get_unpack_registry_dir() -> Path:
177174

178175
def sweep_stale_unpacks() -> int:
179176
"""Clean up after the unpackings of dead processes, see Unpack."""
180-
return Unpack.sweep_stale(
177+
return sweep_stale(
181178
get_unpack_registry_dir(), Path(Config().get_users_files_dir())
182179
)

0 commit comments

Comments
 (0)