diff --git a/docs/_api/skyplane.api.rst b/docs/_api/skyplane.api.rst index 5c1a49f2b..11071c92a 100644 --- a/docs/_api/skyplane.api.rst +++ b/docs/_api/skyplane.api.rst @@ -9,9 +9,9 @@ :toctree: skyplane.api.client - skyplane.api.config skyplane.api.dataplane skyplane.api.provisioner skyplane.api.tracker + skyplane.api.transfer_config skyplane.api.transfer_job diff --git a/docs/_api/skyplane.api.config.rst b/docs/_api/skyplane.api.transfer_config.rst similarity index 76% rename from docs/_api/skyplane.api.config.rst rename to docs/_api/skyplane.api.transfer_config.rst index 70bcc6708..9a6cb92bd 100644 --- a/docs/_api/skyplane.api.config.rst +++ b/docs/_api/skyplane.api.transfer_config.rst @@ -1,7 +1,7 @@ -skyplane.api.config +skyplane.api.transfer_config =================== -.. automodule:: skyplane.api.config +.. automodule:: skyplane.api.transfer_config diff --git a/skyplane/__init__.py b/skyplane/__init__.py index d2f4ed239..bae55ac63 100644 --- a/skyplane/__init__.py +++ b/skyplane/__init__.py @@ -2,7 +2,7 @@ from skyplane import compute, exceptions from skyplane.api.client import SkyplaneClient -from skyplane.api.config import TransferConfig, AWSConfig, AzureConfig, GCPConfig +from skyplane.api.transfer_config import TransferConfig, AWSConfig, AzureConfig, GCPConfig from skyplane.api.dataplane import Dataplane from skyplane.api.pipeline import Pipeline from skyplane.api.tracker import TransferHook diff --git a/skyplane/api/client.py b/skyplane/api/client.py index d931e6625..da24fcc5d 100644 --- a/skyplane/api/client.py +++ b/skyplane/api/client.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Optional -from skyplane.api.config import TransferConfig +from skyplane.api.transfer_config import TransferConfig from skyplane.api.dataplane import Dataplane from skyplane.api.provisioner import Provisioner from skyplane.api.obj_store import ObjectStore @@ -18,7 +18,7 @@ from skyplane.api.pipeline import Pipeline if TYPE_CHECKING: - from skyplane.api.config import AWSConfig, AzureConfig, GCPConfig, TransferConfig, IBMCloudConfig + from skyplane.api.transfer_config import AWSConfig, AzureConfig, GCPConfig, TransferConfig, IBMCloudConfig class SkyplaneClient: diff --git a/skyplane/api/dataplane.py b/skyplane/api/dataplane.py index df6fab67b..773325755 100644 --- a/skyplane/api/dataplane.py +++ b/skyplane/api/dataplane.py @@ -16,7 +16,7 @@ from skyplane.exceptions import GatewayContainerStartException from skyplane.api.tracker import TransferProgressTracker, TransferHook from skyplane.api.transfer_job import CopyJob, SyncJob, TransferJob -from skyplane.api.config import TransferConfig +from skyplane.api.transfer_config import TransferConfig from skyplane.planner.topology import TopologyPlan, TopologyPlanGateway from skyplane.utils import logger from skyplane.utils.definitions import gateway_docker_image, tmp_log_dir diff --git a/skyplane/api/pipeline.py b/skyplane/api/pipeline.py index 19662f139..91080da83 100644 --- a/skyplane/api/pipeline.py +++ b/skyplane/api/pipeline.py @@ -15,7 +15,7 @@ from skyplane import compute from skyplane.api.tracker import TransferProgressTracker, TransferHook from skyplane.api.transfer_job import CopyJob, SyncJob, TransferJob -from skyplane.api.config import TransferConfig +from skyplane.api.transfer_config import TransferConfig from skyplane.planner.planner import MulticastDirectPlanner, DirectPlannerSourceOneSided, DirectPlannerDestOneSided from skyplane.planner.topology import TopologyPlanGateway diff --git a/skyplane/api/tracker.py b/skyplane/api/tracker.py index 1354b90b1..2838148b7 100644 --- a/skyplane/api/tracker.py +++ b/skyplane/api/tracker.py @@ -12,7 +12,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from skyplane import exceptions -from skyplane.api.config import TransferConfig +from skyplane.api.transfer_config import TransferConfig from skyplane.chunk import ChunkState, Chunk from skyplane.utils import logger, imports from skyplane.utils.fn import do_parallel diff --git a/skyplane/api/config.py b/skyplane/api/transfer_config.py similarity index 97% rename from skyplane/api/config.py rename to skyplane/api/transfer_config.py index 4e4979815..f9614fd59 100644 --- a/skyplane/api/config.py +++ b/skyplane/api/transfer_config.py @@ -85,7 +85,7 @@ class TransferConfig: aws_instance_class: str = "m5.8xlarge" azure_instance_class: str = "Standard_D2_v5" - gcp_instance_class: str = "n2-standard-16" + gcp_instance_class: str = "n2-standard-32" ibmcloud_instance_class: str = "bx2-2x8" gcp_use_premium_network: bool = True @@ -98,4 +98,4 @@ class TransferConfig: multipart_enabled: bool = True multipart_threshold_mb: int = 128 multipart_chunk_size_mb: int = 64 - multipart_max_chunks: int = 10000 + multipart_max_chunks: int = 9990 diff --git a/skyplane/api/transfer_job.py b/skyplane/api/transfer_job.py index ecfbaa086..f04d0256a 100644 --- a/skyplane/api/transfer_job.py +++ b/skyplane/api/transfer_job.py @@ -23,7 +23,7 @@ from functools import partial from skyplane import exceptions -from skyplane.api.config import TransferConfig +from skyplane.api.transfer_config import TransferConfig from skyplane.chunk import Chunk, ChunkRequest from skyplane.obj_store.storage_interface import StorageInterface from skyplane.obj_store.object_store_interface import ObjectStoreObject, ObjectStoreInterface diff --git a/skyplane/cli/cli_transfer.py b/skyplane/cli/cli_transfer.py index 0f4b29f6e..1356a6b04 100644 --- a/skyplane/cli/cli_transfer.py +++ b/skyplane/cli/cli_transfer.py @@ -9,7 +9,7 @@ from rich.progress import Progress, TextColumn, SpinnerColumn import skyplane -from skyplane.api.config import TransferConfig, AWSConfig, GCPConfig, AzureConfig, IBMCloudConfig +from skyplane.api.transfer_config import TransferConfig, AWSConfig, GCPConfig, AzureConfig, IBMCloudConfig from skyplane.api.transfer_job import CopyJob, SyncJob, TransferJob from skyplane.cli.impl.cp_replicate_fallback import ( replicate_onprem_cp_cmd, @@ -63,6 +63,7 @@ def __init__(self, src_region_tag: str, dst_region_tag: str, args: Dict[str, Any cloud_config.set_flag("multipart_enabled", str(self.args["multipart"])) self.transfer_config = self.make_transfer_config(skyplane_config or cloud_config) + self.client = skyplane.SkyplaneClient( aws_config=self.aws_config, azure_config=self.azure_config, @@ -103,26 +104,10 @@ def to_api_config(self, config: SkyplaneConfig): return aws_config, azure_config, gcp_config, ibmcloud_config def make_transfer_config(self, config: SkyplaneConfig) -> TransferConfig: - intraregion = self.src_region_tag == self.dst_region_tag return TransferConfig( autoterminate_minutes=config.get_flag("autoshutdown_minutes"), requester_pays=config.get_flag("requester_pays"), - use_bbr=config.get_flag("bbr"), - use_compression=config.get_flag("compress") if not intraregion else False, - use_e2ee=config.get_flag("encrypt_e2e") if not intraregion else False, - use_socket_tls=config.get_flag("encrypt_socket_tls") if not intraregion else False, - aws_use_spot_instances=config.get_flag("aws_use_spot_instances"), - azure_use_spot_instances=config.get_flag("azure_use_spot_instances"), - gcp_use_spot_instances=config.get_flag("gcp_use_spot_instances"), - aws_instance_class=config.get_flag("aws_instance_class"), - azure_instance_class=config.get_flag("azure_instance_class"), - gcp_instance_class=config.get_flag("gcp_instance_class"), - ibmcloud_instance_class=config.get_flag("ibmcloud_instance_class"), - gcp_use_premium_network=config.get_flag("gcp_use_premium_network"), multipart_enabled=config.get_flag("multipart_enabled"), - multipart_threshold_mb=config.get_flag("multipart_min_threshold_mb"), - multipart_chunk_size_mb=config.get_flag("multipart_chunk_size_mb"), - multipart_max_chunks=config.get_flag("multipart_max_chunks"), ) def check_config(self) -> bool: diff --git a/skyplane/config.py b/skyplane/config.py index f1210d8a9..5c143412f 100644 --- a/skyplane/config.py +++ b/skyplane/config.py @@ -10,31 +10,15 @@ _FLAG_TYPES = { "autoconfirm": bool, - "bbr": bool, - "compress": bool, - "encrypt_e2e": bool, - "encrypt_socket_tls": bool, + "autoshutdown_minutes": int, "verify_checksums": bool, "multipart_enabled": bool, - "multipart_min_threshold_mb": int, - "multipart_min_size_mb": int, - "multipart_chunk_size_mb": int, - "multipart_max_chunks": int, "num_connections": int, "max_instances": int, - "autoshutdown_minutes": int, - "aws_use_spot_instances": bool, - "azure_use_spot_instances": bool, - "gcp_use_spot_instances": bool, - "aws_instance_class": str, - "azure_instance_class": str, - "gcp_instance_class": str, - "ibmcloud_instance_class": str, "aws_default_region": str, "azure_default_region": str, "gcp_default_region": str, "ibmcloud_default_region": str, - "gcp_use_premium_network": bool, "usage_stats": bool, "gcp_service_account_name": str, "requester_pays": bool, @@ -44,32 +28,15 @@ _DEFAULT_FLAGS = { "autoconfirm": False, - "bbr": True, - "compress": True, - "encrypt_e2e": True, - "encrypt_socket_tls": False, + "autoshutdown_minutes": 15, "verify_checksums": True, "multipart_enabled": True, - "multipart_min_threshold_mb": 128, - "multipart_min_size_mb": 5, # 5 MiB minimum size for multipart uploads - "multipart_chunk_size_mb": 64, - "multipart_max_chunks": 9990, # AWS limit is 10k chunks "num_connections": 32, "max_instances": 1, - "autoshutdown_minutes": 15, - "aws_use_spot_instances": False, - "azure_use_spot_instances": False, - "gcp_use_spot_instances": False, - "ibmcloud_use_spot_instances": False, - "aws_instance_class": "m5.8xlarge", - "azure_instance_class": "Standard_D32_v5", - "gcp_instance_class": "n2-standard-32", - "ibmcloud_instance_class": "bx2-2x8", "aws_default_region": "us-east-1", "azure_default_region": "eastus", "gcp_default_region": "us-central1-a", "ibmcloud_default_region": "us-east", - "gcp_use_premium_network": True, "usage_stats": True, "gcp_service_account_name": "skyplane-manual", "requester_pays": False, diff --git a/skyplane/planner/planner.py b/skyplane/planner/planner.py index 6bfb5233a..b7905ecdc 100644 --- a/skyplane/planner/planner.py +++ b/skyplane/planner/planner.py @@ -5,7 +5,7 @@ import csv from skyplane import compute -from skyplane.api.config import TransferConfig +from skyplane.api.transfer_config import TransferConfig from skyplane.utils import logger from skyplane.planner.topology import TopologyPlan diff --git a/tests/integration/test_cp.py b/tests/integration/test_cp.py index 2f9cfc60e..194372128 100644 --- a/tests/integration/test_cp.py +++ b/tests/integration/test_cp.py @@ -1,5 +1,5 @@ import pytest -from skyplane.api.config import TransferConfig +from skyplane.api.transfer_config import TransferConfig from skyplane.utils import logger import time from skyplane.api.client import SkyplaneClient diff --git a/tests/unit_nocloud/test_fall_back.py b/tests/unit_nocloud/test_fall_back.py index a5dc17294..3c049a629 100644 --- a/tests/unit_nocloud/test_fall_back.py +++ b/tests/unit_nocloud/test_fall_back.py @@ -2,7 +2,7 @@ import os import pytest -from skyplane.api.config import TransferConfig +from skyplane.api.transfer_config import TransferConfig from skyplane.planner.planner import MulticastDirectPlanner