Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ref(dependency-graph): Adding more context to the dependency graph #248

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion devservices/commands/down.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
from devservices.utils.console import Console
from devservices.utils.console import Status
from devservices.utils.dependencies import construct_dependency_graph
from devservices.utils.dependencies import DependencyNode
from devservices.utils.dependencies import DependencyType
from devservices.utils.dependencies import get_non_shared_remote_dependencies
from devservices.utils.dependencies import install_and_verify_dependencies
from devservices.utils.dependencies import InstalledRemoteDependency
Expand Down Expand Up @@ -234,7 +236,10 @@ def _get_dependent_service(
)
# If the service we are trying to bring down is in the dependency graph of another service,
# we should not bring it down
if service.name in dependency_graph.graph:
if (
DependencyNode(name=service.name, dependency_type=DependencyType.SERVICE)
in dependency_graph.graph
):
return other_started_service

return None
9 changes: 8 additions & 1 deletion devservices/commands/up.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
from devservices.utils.console import Console
from devservices.utils.console import Status
from devservices.utils.dependencies import construct_dependency_graph
from devservices.utils.dependencies import DependencyNode
from devservices.utils.dependencies import DependencyType
from devservices.utils.dependencies import install_and_verify_dependencies
from devservices.utils.dependencies import InstalledRemoteDependency
from devservices.utils.docker import check_all_containers_healthy
Expand Down Expand Up @@ -169,7 +171,12 @@ def _up(
dependency_graph = construct_dependency_graph(service, modes=modes)
starting_order = dependency_graph.get_starting_order()
sorted_remote_dependencies = sorted(
remote_dependencies, key=lambda dep: starting_order.index(dep.service_name)
remote_dependencies,
key=lambda dep: starting_order.index(
DependencyNode(
name=dep.service_name, dependency_type=DependencyType.SERVICE
)
),
)
# Pull all images in parallel
status.info("Pulling images")
Expand Down
82 changes: 54 additions & 28 deletions devservices/utils/dependencies.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from concurrent.futures import as_completed
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from enum import Enum
from typing import TextIO
from typing import TypeGuard

Expand Down Expand Up @@ -53,59 +54,73 @@
]


class DependencyType(str, Enum):
SERVICE = "service"
COMPOSE = "compose"


@dataclass(frozen=True, eq=True)
class DependencyNode:
name: str
dependency_type: DependencyType

def __str__(self) -> str:
return self.name

Check warning on line 68 in devservices/utils/dependencies.py

View check run for this annotation

Codecov / codecov/patch

devservices/utils/dependencies.py#L68

Added line #L68 was not covered by tests


class DependencyGraph:
def __init__(self) -> None:
self.graph: dict[str, set[str]] = dict()
self.graph: dict[DependencyNode, set[DependencyNode]] = dict()

def add_dependency(self, service_name: str) -> None:
if service_name not in self.graph:
self.graph[service_name] = set()
def add_node(self, node: DependencyNode) -> None:
if node not in self.graph:
self.graph[node] = set()

def add_edge(self, service_name: str, dependency_name: str) -> None:
# TODO: We should rename services that depend on themselves
if service_name == dependency_name:
return
if service_name not in self.graph:
self.add_dependency(service_name)
if dependency_name not in self.graph:
self.add_dependency(dependency_name)
def add_edge(self, from_node: DependencyNode, to_node: DependencyNode) -> None:
if from_node == to_node:
# TODO: Add a better exception
raise ValueError("Cannot add an edge from a node to itself")

Check warning on line 82 in devservices/utils/dependencies.py

View check run for this annotation

Codecov / codecov/patch

devservices/utils/dependencies.py#L82

Added line #L82 was not covered by tests
if from_node not in self.graph:
self.add_node(from_node)
if to_node not in self.graph:
self.add_node(to_node)

# TODO: Should we check for cycles here?

self.graph[service_name].add(dependency_name)
self.graph[from_node].add(to_node)

def topological_sort(self) -> list[str]:
def topological_sort(self) -> list[DependencyNode]:
in_degree = {service_name: 0 for service_name in self.graph}

for service_name in self.graph.keys():
for dependency in self.graph[service_name]:
in_degree[dependency] += 1
for service_node in self.graph.keys():
for dependency_node in self.graph[service_node]:
in_degree[dependency_node] += 1

queue = deque(
[
service_name
for service_name in self.graph
if in_degree[service_name] == 0
dependency_node
for dependency_node in self.graph
if in_degree[dependency_node] == 0
]
)
topological_order = list()

while queue:
service_name = queue.popleft()
topological_order.append(service_name)
service_node = queue.popleft()
topological_order.append(service_node)

for dependency in self.graph[service_name]:
in_degree[dependency] -= 1
if in_degree[dependency] == 0:
queue.append(dependency)
for dependency_node in self.graph[service_node]:
in_degree[dependency_node] -= 1
if in_degree[dependency_node] == 0:
queue.append(dependency_node)

if len(topological_order) != len(self.graph):
# TODO: Add a better exception
raise ValueError("Cycle detected in the dependency graph")

return topological_order

def get_starting_order(self) -> list[str]:
def get_starting_order(self) -> list[DependencyNode]:
return list(reversed(self.topological_sort()))


Expand Down Expand Up @@ -729,7 +744,18 @@
# Skip the dependency if it's not in the modes (since it may not be installed and we don't care about it)
if dependency_name not in service_mode_dependencies:
continue
dependency_graph.add_edge(service_config.service_name, dependency_name)
dependency_graph.add_edge(
DependencyNode(
name=service_config.service_name,
dependency_type=DependencyType.SERVICE,
),
DependencyNode(
name=dependency_name,
dependency_type=DependencyType.SERVICE
if _has_remote_config(dependency.remote)
else DependencyType.COMPOSE,
),
)
if _has_remote_config(dependency.remote):
dependency_config = get_remote_dependency_config(dependency.remote)
_construct_dependency_graph(dependency_config, [dependency.remote.mode])
Expand Down
Loading
Loading