from onedrive to modelscope

Author: binary-husky

Please_Run_This_First_To_Fetch_Big_Files.py

@@ -1,47 +1,27 @@
 import os, commentjson, shutil, subprocess, tqdm, shutil
-from onedrivedownloader import download
-from distutils import dir_util
-
+import zipfile
+from modelscope import snapshot_download
 try: os.makedirs('./TEMP')
 except: pass
 
-def download_from_shared_server(key = 'cat'):
-    # download uhmap file manifest | 下载manifest目录文件
-    print('Download uhmap file manifest | 下载manifest目录文件')
-    manifest_url = "https://ageasga-my.sharepoint.com/:u:/g/personal/fuqingxu_yiteam_tech/EVmCQMSUWV5MgREWaxiz_GoBalBRV3DWBU3ToSJ5OTQaLQ?e=I8yjl9"
-    try:
-        download(manifest_url, filename="./TEMP/", force_download=True)
-    except:
-        print('Failed to connect to onedrive | 连接onedrive失败, 您可能需要翻墙才能下载资源')
-        return False
-
-    with open("./TEMP/uhmap_manifest.jsonc", "r") as f:
-        manifest = commentjson.load(f)
-
-    uhmap_url = manifest[key]
-    print('Download main files | 下载预定文件')
-    try:
-        download(uhmap_url, filename="./TEMP/DOWNLOAD", unzip=True, unzip_path='./TEMP/UNZIP')
-    except:
-        print(f'Download timeout | 下载失败, 您可能需要翻墙才能下载资源。另外如果您想手动下载的话: {uhmap_url}')
-        return False
+version = 'unreal-map-v3.4'
+model_dir = snapshot_download(f'BinaryHusky/{version}')
+zip_file_path = f'./TEMP/{version}.zip'
 
-    return True
+def combine_file(model_dir, output_file_path, num_parts):
+    with open(output_file_path, 'wb') as output_file:
+        for i in range(0, num_parts):
+            part_file_path = os.path.join(model_dir, "tensor", f"safetensor_{i+1}.pt")
+            with open(part_file_path, 'rb') as part_file:
+                output_file.write(part_file.read())
 
-def get_current_version():
-    with open('current_version', 'r', encoding='utf8') as f:
-        version = f.read()
-        return version
+extract_to_path = './'
+combine_file(model_dir, output_file_path=zip_file_path, num_parts=5)
 
-version = get_current_version()
-success = download_from_shared_server(f'uhmp-big-file-v{version}') 
+# 打开 ZIP 文件
+with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
+    # 解压所有文件到指定目录
+    zip_ref.extractall(extract_to_path)
+    print(f"files unzipped {extract_to_path}")
 
-if success:
-    print('Copying downloaded files to project root')
-    print('This will take a while if you are not using SSD...')
-    dir_util.copy_tree('./TEMP/UNZIP', './')
-    print('Download complete | 下载完成')
-else:
-    try: shutil.rmtree('./TEMP')
-    except: pass
-    print('Download fail, clear TEMP folder | 下载失败, 临时文件夹已清除')
+print("everything is ready!")