Skip to content

Commit 6efda81

Browse files
authored
Merge pull request #41 from yuchen0cc/main
update examples for OssDCPFileSystem and OssSafetensor
2 parents d738dd6 + 0f1d752 commit 6efda81

1 file changed

Lines changed: 6 additions & 4 deletions

File tree

docs/torchconnector/examples.md

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -282,18 +282,19 @@ OSS connector for AI/ML supports [PyTorch distributed checkpoints(DCP)](https://
282282
```py
283283
import torchvision
284284
import torch.distributed.checkpoint as DCP
285-
from osstorchconnector import OssFileSystem
285+
from osstorchconnector import OssDCPFileSystem
286286
import torch
287287

288288
ENDPOINT = "http://oss-cn-beijing-internal.aliyuncs.com"
289+
REGION = "cn-beijing"
289290
CONFIG_PATH = "/etc/oss-connector/config.json"
290291
CRED_PATH = "/root/.alibabacloud/credentials"
291292
OSS_URI = "oss://ossconnectorbucket/dcp-checkpoint-resnet18"
292293

293294
model = torchvision.models.resnet18()
294295

295296
# write to OSS
296-
fs = OssFileSystem(endpoint=ENDPOINT, cred_path=CRED_PATH, config_path=CONFIG_PATH)
297+
fs = OssDCPFileSystem(endpoint=ENDPOINT, cred_path=CRED_PATH, config_path=CONFIG_PATH, region=REGION)
297298
oss_storage_writer = fs.writer(OSS_URI)
298299
# DCP.save or DCP.async_save
299300
checkpoint_future = DCP.async_save(
@@ -324,11 +325,12 @@ import torch
324325
from osstorchconnector import OssSafetensor
325326

326327
ENDPOINT = "http://oss-cn-beijing-internal.aliyuncs.com"
328+
REGION = "cn-beijing"
327329
CONFIG_PATH = "/etc/oss-connector/config.json"
328330
CRED_PATH = "/root/.alibabacloud/credentials"
329331
OSS_URI = "oss://ossconnectorbucket/safetensors/model.safetensors"
330332

331-
sfts = OssSafetensor(endpoint=HTTPS_ENDPOINT, cred_path=CRED_PATH, config_path=CONFIG_PATH)
333+
sfts = OssSafetensor(endpoint=ENDPOINT, cred_path=CRED_PATH, config_path=CONFIG_PATH, region=REGION)
332334

333335
# save tensors to safetensor file on OSS
334336
tensors = {"embedding": torch.rand((512, 1024)), "attention": torch.rand((256, 256))}
@@ -344,4 +346,4 @@ with sfts.safe_open(OSS_URI, device ="cpu") as f:
344346
for key in f.keys(): # read tensors by keys
345347
tensor = f.get_tensor(key)
346348

347-
```
349+
```

0 commit comments

Comments
 (0)