Skip to content
Open
Show file tree
Hide file tree
Changes from 10 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion vllm/distributed/ec_transfer/ec_connector/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,22 @@ def save_caches(
"""
pass

@abstractmethod
def maybe_update_remote_cache_state(
self, encoder_cache: dict[str, torch.Tensor]
) -> None:
"""
Maybe update the remote cache state based on the local encoder cache.

This method can be used to synchronize or update the state of the
remote cache based on changes in the local encoder cache.

Args:
encoder_cache (dict[str, torch.Tensor]): A dictionary mapping multimodal
data hashes (`mm_hash`) to encoder cache tensors.
"""
pass

def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
Expand Down Expand Up @@ -199,7 +215,9 @@ def has_caches(
pass

@abstractmethod
def update_state_after_alloc(self, request: "Request", index: int):
def update_state_after_alloc(
self, request: "Request", index: int, local_hit: bool, remote_hit: bool
):
"""
Update ECConnector state to decide allocate cache for requests

Expand Down
Loading
Loading