From fe124f74e5f3400b4ceb74decbebe0a4b55c2ce2 Mon Sep 17 00:00:00 2001 From: Scott Dixon Date: Fri, 29 Mar 2024 22:49:32 -0700 Subject: [PATCH] Issue #99 proposal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a draft PR to review a proposed change to the public APIs for pydsdl. This is non-breaking change that adds a new public method and datatype ## New Method read_files - a file-oriented entry point to the front end. This takes a list of target DSDL files allowing the user to maintain an explicit list instead of depending on globular filesystem discovery. ## New Datatype DsdlFile – We define and publish a new abstract data type that encapsulates all the information collected about and from a given dsdl file. This allows a backend to associate datatypes with files for the purposes of managing dependencies (e.g. this would allow the creation of .d files by a back end). --- .gitmodules | 3 + pydsdl/__init__.py | 4 +- pydsdl/_data_type_builder.py | 42 +-- pydsdl/_dsdl.py | 194 +++++++++++++ pydsdl/_dsdl_definition.py | 67 ++--- pydsdl/_error.py | 11 +- pydsdl/_namespace.py | 465 +++++++++++++++++++++++-------- pydsdl/_namespace_reader.py | 160 +++++++++++ pydsdl/_parser.py | 12 +- pydsdl/_test.py | 11 +- setup.cfg | 2 +- test/public_regulated_data_types | 1 + test/test_public_types.py | 44 +++ 13 files changed, 833 insertions(+), 183 deletions(-) create mode 100644 .gitmodules create mode 100644 pydsdl/_dsdl.py create mode 100644 pydsdl/_namespace_reader.py create mode 160000 test/public_regulated_data_types create mode 100644 test/test_public_types.py diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..d939d93 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "test/public_regulated_data_types"] + path = test/public_regulated_data_types + url = https://github.com/OpenCyphal/public_regulated_data_types.git diff --git a/pydsdl/__init__.py b/pydsdl/__init__.py index d07f1f2..7cf20d6 100644 --- a/pydsdl/__init__.py +++ b/pydsdl/__init__.py @@ -25,8 +25,10 @@ _sys.path = [str(_Path(__file__).parent / "third_party")] + _sys.path # Never import anything that is not available here - API stability guarantees are only provided for the exposed items. +from ._dsdl import PrintOutputHandler as PrintOutputHandler +from ._dsdl import DsdlFile as DsdlFile from ._namespace import read_namespace as read_namespace -from ._namespace import PrintOutputHandler as PrintOutputHandler +from ._namespace import read_files as read_files # Error model. from ._error import FrontendError as FrontendError diff --git a/pydsdl/_data_type_builder.py b/pydsdl/_data_type_builder.py index 4572da3..e0a53b0 100644 --- a/pydsdl/_data_type_builder.py +++ b/pydsdl/_data_type_builder.py @@ -2,16 +2,12 @@ # This software is distributed under the terms of the MIT License. # Author: Pavel Kirienko -from typing import Optional, Callable, Iterable import logging from pathlib import Path -from . import _serializable -from . import _expression -from . import _error -from . import _dsdl_definition -from . import _parser -from . import _data_schema_builder -from . import _port_id_ranges +from typing import Callable, Iterable, Optional + +from . import _data_schema_builder, _error, _expression, _parser, _port_id_ranges, _serializable +from ._dsdl import DefinitionVisitor, DsdlFileBuildable class AssertionCheckFailureError(_error.InvalidDefinitionError): @@ -42,21 +38,25 @@ class MissingSerializationModeError(_error.InvalidDefinitionError): class DataTypeBuilder(_parser.StatementStreamProcessor): + + # pylint: disable=too-many-arguments def __init__( self, - definition: _dsdl_definition.DSDLDefinition, - lookup_definitions: Iterable[_dsdl_definition.DSDLDefinition], + definition: DsdlFileBuildable, + lookup_definitions: Iterable[DsdlFileBuildable], + definition_visitors: Iterable[DefinitionVisitor], print_output_handler: Callable[[int, str], None], allow_unregulated_fixed_port_id: bool, ): self._definition = definition self._lookup_definitions = list(lookup_definitions) + self._definition_visitors = definition_visitors self._print_output_handler = print_output_handler self._allow_unregulated_fixed_port_id = allow_unregulated_fixed_port_id self._element_callback = None # type: Optional[Callable[[str], None]] - assert isinstance(self._definition, _dsdl_definition.DSDLDefinition) - assert all(map(lambda x: isinstance(x, _dsdl_definition.DSDLDefinition), lookup_definitions)) + assert isinstance(self._definition, DsdlFileBuildable) + assert all(map(lambda x: isinstance(x, DsdlFileBuildable), lookup_definitions)) assert callable(self._print_output_handler) assert isinstance(self._allow_unregulated_fixed_port_id, bool) @@ -198,6 +198,7 @@ def resolve_versioned_data_type(self, name: str, version: _serializable.Version) del name found = list(filter(lambda d: d.full_name == full_name and d.version == version, self._lookup_definitions)) if not found: + # Play Sherlock to help the user with mistakes like https://forum.opencyphal.org/t/904/2 requested_ns = full_name.split(_serializable.CompositeType.NAME_COMPONENT_SEPARATOR)[0] lookup_nss = set(x.root_namespace for x in self._lookup_definitions) @@ -221,15 +222,20 @@ def resolve_versioned_data_type(self, name: str, version: _serializable.Version) raise _error.InternalError("Conflicting definitions: %r" % found) target_definition = found[0] - assert isinstance(target_definition, _dsdl_definition.DSDLDefinition) + for visitor in self._definition_visitors: + visitor.on_discover_lookup_dependent_file(self._definition, target_definition) + + assert isinstance(target_definition, DsdlFileBuildable) assert target_definition.full_name == full_name assert target_definition.version == version # Recursion is cool. - return target_definition.read( + dt = target_definition.read( lookup_definitions=self._lookup_definitions, + definition_visitors=self._definition_visitors, print_output_handler=self._print_output_handler, allow_unregulated_fixed_port_id=self._allow_unregulated_fixed_port_id, ) + return dt def _queue_attribute(self, element_callback: Callable[[str], None]) -> None: self._flush_attribute("") @@ -266,7 +272,9 @@ def _on_assert_directive(self, line_number: int, value: Optional[_expression.Any elif value is None: raise InvalidDirectiveError("Assert directive requires an expression") else: - raise InvalidDirectiveError("The assertion check expression must yield a boolean, not %s" % value.TYPE_NAME) + raise InvalidDirectiveError( + "The assertion check expression must yield a boolean, not %s" % value.TYPE_NAME + ) def _on_extent_directive(self, line_number: int, value: Optional[_expression.Any]) -> None: if self._structs[-1].serialization_mode is not None: @@ -300,7 +308,9 @@ def _on_union_directive(self, _ln: int, value: Optional[_expression.Any]) -> Non if self._structs[-1].union: raise InvalidDirectiveError("Duplicated union directive") if self._structs[-1].attributes: - raise InvalidDirectiveError("The union directive must be placed before the first " "attribute definition") + raise InvalidDirectiveError( + "The union directive must be placed before the first " "attribute definition" + ) self._structs[-1].make_union() def _on_deprecated_directive(self, _ln: int, value: Optional[_expression.Any]) -> None: diff --git a/pydsdl/_dsdl.py b/pydsdl/_dsdl.py new file mode 100644 index 0000000..d491c5a --- /dev/null +++ b/pydsdl/_dsdl.py @@ -0,0 +1,194 @@ +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT + +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Callable, Iterable, List, Optional, Set, Tuple, TypeVar, Union + +from ._serializable import CompositeType, Version + +PrintOutputHandler = Callable[[Path, int, str], None] +"""Invoked when the frontend encounters a print directive or needs to output a generic diagnostic.""" + + +class DsdlFile(ABC): + """ + Interface for DSDL files. This interface is used by the parser to abstract DSDL type details inferred from the + filesystem. + """ + + @property + @abstractmethod + def composite_type(self) -> Optional[CompositeType]: + """The composite type that was read from the DSDL file or None if the type has not been parsed yet.""" + raise NotImplementedError() + + @property + @abstractmethod + def full_name(self) -> str: + """The full name, e.g., uavcan.node.Heartbeat""" + raise NotImplementedError() + + @property + def name_components(self) -> List[str]: + """Components of the full name as a list, e.g., ['uavcan', 'node', 'Heartbeat']""" + raise NotImplementedError() + + @property + @abstractmethod + def short_name(self) -> str: + """The last component of the full name, e.g., Heartbeat of uavcan.node.Heartbeat""" + raise NotImplementedError() + + @property + @abstractmethod + def full_namespace(self) -> str: + """The full name without the short name, e.g., uavcan.node for uavcan.node.Heartbeat""" + raise NotImplementedError() + + @property + @abstractmethod + def root_namespace(self) -> str: + """The first component of the full name, e.g., uavcan of uavcan.node.Heartbeat""" + raise NotImplementedError() + + @property + @abstractmethod + def text(self) -> str: + """The source text in its raw unprocessed form (with comments, formatting intact, and everything)""" + raise NotImplementedError() + + @property + @abstractmethod + def version(self) -> Version: + """ + The version of the DSDL definition. + """ + raise NotImplementedError() + + @property + @abstractmethod + def fixed_port_id(self) -> Optional[int]: + """Either the fixed port ID as integer, or None if not defined for this type.""" + raise NotImplementedError() + + @property + @abstractmethod + def has_fixed_port_id(self) -> bool: + """ + If the type has a fixed port ID defined, this method returns True. Equivalent to ``fixed_port_id is not None``. + """ + raise NotImplementedError() + + @property + @abstractmethod + def file_path(self) -> Path: + """The path to the DSDL file on the filesystem.""" + raise NotImplementedError() + + @property + @abstractmethod + def root_namespace_path(self) -> Path: + """ + The path to the root namespace directory on the filesystem. + """ + raise NotImplementedError() + + +class DefinitionVisitor(ABC): + """ + An interface that allows visitors to index dependent types of a target DSDL file. This allows visitors to + build a closure of dependent types while parsing a set of target DSDL files. + """ + + @abstractmethod + def on_discover_lookup_dependent_file(self, target_dsdl_file: DsdlFile, dependent_type: DsdlFile) -> None: + """ + Called by the parser after if finds a dependent type but before it parses a file in a lookup namespace. + :param DsdlFile target_dsdl_file: The target DSDL file that has dependencies the parser is searching for. + :param DsdlFile dependent_type: The dependency of target_dsdl_file file the parser is about to parse. + :raises DependentFileError: If the dependent file is not allowed by the visitor. + """ + raise NotImplementedError() + + +class DsdlFileBuildable(DsdlFile): + """ + A DSDL file that can construct a composite type from its contents. + """ + + @abstractmethod + def read( + self, + lookup_definitions: Iterable["DsdlFileBuildable"], + definition_visitors: Iterable[DefinitionVisitor], + print_output_handler: Callable[[int, str], None], + allow_unregulated_fixed_port_id: bool, + ) -> CompositeType: + """ + Reads the data type definition and returns its high-level data type representation. + The output should be cached; all following invocations should read from this cache. + Caching is very important, because it is expected that the same definition may be referred to multiple + times (e.g., for composition or when accessing external constants). Re-processing a definition every time + it is accessed would be a huge waste of time. + Note, however, that this may lead to unexpected complications if one is attempting to re-read a definition + with different inputs (e.g., different lookup paths) expecting to get a different result: caching would + get in the way. That issue is easy to avoid by creating a new instance of the object. + :param lookup_definitions: List of definitions available for referring to. + :param definition_visitors: Visitors to notify about discovered dependencies. + :param print_output_handler: Used for @print and for diagnostics: (line_number, text) -> None. + :param allow_unregulated_fixed_port_id: Do not complain about fixed unregulated port IDs. + :return: The data type representation. + """ + raise NotImplementedError() + + +SortedFileT = TypeVar("SortedFileT", DsdlFile, DsdlFileBuildable) +SortedFileList = List[SortedFileT] +"""A list of DSDL files sorted by name, newest version first.""" + +FileSortKey: Callable[[SortedFileT], Tuple[str, int, int]] = lambda d: ( + d.full_name, + -d.version.major, + -d.version.minor, +) + + +def file_sort(file_list: Iterable[SortedFileT]) -> SortedFileList: + """ + Sorts a list of DSDL files lexicographically by name, newest version first. + """ + return list(sorted(file_list, key=FileSortKey)) + + +UniformCollectionT = TypeVar("UniformCollectionT", Iterable, Set, List) + + +def is_uniform_or_raise(collection: UniformCollectionT) -> UniformCollectionT: + """ + Raises an error if the collection is not uniform. + """ + first = type(next(iter(collection))) + if not all(isinstance(x, first) for x in collection): + raise TypeError(f"Not all items in collection were of type {str(first)}.") + return collection + + +PathListT = TypeVar( + "PathListT", List[Path], Set[Path], List[str], Set[str], Union[List[Path], List[str]], Union[Set[Path], Set[str]] +) + + +def normalize_paths_argument( + namespaces_or_namespace: Union[None, Path, str, Iterable[Union[Path, str]]], + type_cast: Callable[[Iterable], PathListT], +) -> PathListT: + """ + Normalizes the input argument to a list of paths. + """ + if namespaces_or_namespace is None: + return type_cast([]) + if isinstance(namespaces_or_namespace, (Path, str)): + return type_cast([Path(namespaces_or_namespace)]) + return is_uniform_or_raise(type_cast(namespaces_or_namespace)) diff --git a/pydsdl/_dsdl_definition.py b/pydsdl/_dsdl_definition.py index a8114da..025876e 100644 --- a/pydsdl/_dsdl_definition.py +++ b/pydsdl/_dsdl_definition.py @@ -2,14 +2,16 @@ # This software is distributed under the terms of the MIT License. # Author: Pavel Kirienko -import time -from typing import Iterable, Callable, Optional, List import logging +import time from pathlib import Path -from ._error import FrontendError, InvalidDefinitionError, InternalError -from ._serializable import CompositeType, Version -from . import _parser +from typing import Callable, Iterable, List, Optional +from . import _parser +from ._data_type_builder import DataTypeBuilder +from ._dsdl import DefinitionVisitor, DsdlFileBuildable +from ._error import FrontendError, InternalError, InvalidDefinitionError +from ._serializable import CompositeType, Version _logger = logging.getLogger(__name__) @@ -23,7 +25,7 @@ def __init__(self, text: str, path: Path): super().__init__(text=text, path=Path(path)) -class DSDLDefinition: +class DSDLDefinition(DsdlFileBuildable): """ A DSDL type definition source abstracts the filesystem level details away, presenting a higher-level interface that operates solely on the level of type names, namespaces, fixed identifiers, and so on. @@ -86,26 +88,16 @@ def __init__(self, file_path: Path, root_namespace_path: Path): self._cached_type: Optional[CompositeType] = None + # +-----------------------------------------------------------------------+ + # | DsdlFileBuildable :: INTERFACE | + # +-----------------------------------------------------------------------+ def read( self, - lookup_definitions: Iterable["DSDLDefinition"], + lookup_definitions: Iterable[DsdlFileBuildable], + definition_visitors: Iterable[DefinitionVisitor], print_output_handler: Callable[[int, str], None], allow_unregulated_fixed_port_id: bool, ) -> CompositeType: - """ - Reads the data type definition and returns its high-level data type representation. - The output is cached; all following invocations will read from the cache. - Caching is very important, because it is expected that the same definition may be referred to multiple - times (e.g., for composition or when accessing external constants). Re-processing a definition every time - it is accessed would be a huge waste of time. - Note, however, that this may lead to unexpected complications if one is attempting to re-read a definition - with different inputs (e.g., different lookup paths) expecting to get a different result: caching would - get in the way. That issue is easy to avoid by creating a new instance of the object. - :param lookup_definitions: List of definitions available for referring to. - :param print_output_handler: Used for @print and for diagnostics: (line_number, text) -> None. - :param allow_unregulated_fixed_port_id: Do not complain about fixed unregulated port IDs. - :return: The data type representation. - """ log_prefix = "%s.%d.%d" % (self.full_name, self.version.major, self.version.minor) if self._cached_type is not None: _logger.debug("%s: Cache hit", log_prefix) @@ -124,17 +116,17 @@ def read( ", ".join(set(sorted(map(lambda x: x.root_namespace, lookup_definitions)))), ) try: - builder = _data_type_builder.DataTypeBuilder( + builder = DataTypeBuilder( definition=self, lookup_definitions=lookup_definitions, + definition_visitors=definition_visitors, print_output_handler=print_output_handler, allow_unregulated_fixed_port_id=allow_unregulated_fixed_port_id, ) - with open(self.file_path) as f: - _parser.parse(f.read(), builder) - self._cached_type = builder.finalize() + _parser.parse(self._text, builder) + self._cached_type = builder.finalize() _logger.info( "%s: Processed in %.0f ms; category: %s, fixed port ID: %s", log_prefix, @@ -151,34 +143,35 @@ def read( except Exception as ex: # pragma: no cover raise InternalError(culprit=ex, path=self.file_path) from ex + # +-----------------------------------------------------------------------+ + # | DsdlFile :: INTERFACE | + # +-----------------------------------------------------------------------+ + @property + def composite_type(self) -> Optional[CompositeType]: + return self._cached_type + @property def full_name(self) -> str: - """The full name, e.g., uavcan.node.Heartbeat""" return self._name @property def name_components(self) -> List[str]: - """Components of the full name as a list, e.g., ['uavcan', 'node', 'Heartbeat']""" return self._name.split(CompositeType.NAME_COMPONENT_SEPARATOR) @property def short_name(self) -> str: - """The last component of the full name, e.g., Heartbeat of uavcan.node.Heartbeat""" return self.name_components[-1] @property def full_namespace(self) -> str: - """The full name without the short name, e.g., uavcan.node for uavcan.node.Heartbeat""" return str(CompositeType.NAME_COMPONENT_SEPARATOR.join(self.name_components[:-1])) @property def root_namespace(self) -> str: - """The first component of the full name, e.g., uavcan of uavcan.node.Heartbeat""" return self.name_components[0] @property def text(self) -> str: - """The source text in its raw unprocessed form (with comments, formatting intact, and everything)""" return self._text @property @@ -187,7 +180,6 @@ def version(self) -> Version: @property def fixed_port_id(self) -> Optional[int]: - """Either the fixed port ID as integer, or None if not defined for this type.""" return self._fixed_port_id @property @@ -202,6 +194,12 @@ def file_path(self) -> Path: def root_namespace_path(self) -> Path: return self._root_namespace_path + # +-----------------------------------------------------------------------+ + # | Python :: SPECIAL FUNCTIONS | + # +-----------------------------------------------------------------------+ + def __hash__(self) -> int: + return hash((self.full_name, self.version)) + def __eq__(self, other: object) -> bool: """ Two definitions will compare equal if they share the same name AND version number. @@ -220,8 +218,3 @@ def __str__(self) -> str: ) __repr__ = __str__ - - -# Moved this import here to break recursive dependency. -# Maybe I have messed up the architecture? Should think about it later. -from . import _data_type_builder # pylint: disable=wrong-import-position diff --git a/pydsdl/_error.py b/pydsdl/_error.py index d301765..bb08cf7 100644 --- a/pydsdl/_error.py +++ b/pydsdl/_error.py @@ -108,6 +108,15 @@ class InvalidDefinitionError(FrontendError): """ +class DependentFileError(RuntimeError): + """ + Raised by the DefinitionVisitor when it encounters a dependent dsdl file that is not allowed by the user. + """ + + +# +-[UNIT TESTS]------------------------------------------------------------------------------------------------------+ + + def _unittest_error() -> None: try: raise FrontendError("Hello world!") @@ -124,8 +133,8 @@ def _unittest_error() -> None: try: raise FrontendError("Hello world!", path=Path("path/to/file.dsdl")) except Exception as ex: - assert str(ex) == "path/to/file.dsdl: Hello world!" assert repr(ex) == "FrontendError: 'path/to/file.dsdl: Hello world!'" + assert str(ex) == "path/to/file.dsdl: Hello world!" def _unittest_internal_error_github_reporting() -> None: diff --git a/pydsdl/_namespace.py b/pydsdl/_namespace.py index 8e9501e..d6295d3 100644 --- a/pydsdl/_namespace.py +++ b/pydsdl/_namespace.py @@ -4,13 +4,20 @@ # pylint: disable=logging-not-lazy -from typing import Iterable, Callable, DefaultDict, List, Optional, Union, Set, Dict -import logging import collections +import logging from pathlib import Path -from . import _serializable -from . import _dsdl_definition -from . import _error +from typing import Callable, DefaultDict, Dict, Iterable, List, Optional, Set, Tuple, Union, cast + +from . import _dsdl_definition, _error, _serializable +from ._dsdl import DsdlFile, DsdlFileBuildable, PrintOutputHandler, SortedFileList +from ._dsdl import file_sort as dsdl_file_sort +from ._dsdl import normalize_paths_argument as dsdl_normalize_paths_argument +from ._dsdl import is_uniform_or_raise as dsdl_is_uniform_or_raise +from ._namespace_reader import Closure as NamespaceClosureReader + + +_logger = logging.getLogger(__name__) class RootNamespaceNameCollisionError(_error.InvalidDefinitionError): @@ -69,8 +76,13 @@ class SealingConsistencyError(_error.InvalidDefinitionError): """ -PrintOutputHandler = Callable[[Path, int, str], None] -"""Invoked when the frontend encounters a print directive or needs to output a generic diagnostic.""" +class DsdlPathInferenceError(_error.InvalidDefinitionError): + """ + Raised when the namespace, type, fixed port ID, or version cannot be inferred from a file path. + """ + + +# +--[PUBLIC API]-----------------------------------------------------------------------------------------------------+ def read_namespace( @@ -117,39 +129,19 @@ def read_namespace( :class:`OSError` if directories do not exist or inaccessible, :class:`ValueError`/:class:`TypeError` if the arguments are invalid. """ - # Add the own root namespace to the set of lookup directories, sort lexicographically, remove duplicates. - # We'd like this to be an iterable list of strings but we handle the common practice of passing in a single path. - if lookup_directories is None: - lookup_directories_path_list: List[Path] = [] - elif isinstance(lookup_directories, (str, bytes, Path)): - lookup_directories_path_list = [Path(lookup_directories)] - else: - lookup_directories_path_list = list(map(Path, lookup_directories)) - - for a in lookup_directories_path_list: - if not isinstance(a, (str, Path)): - raise TypeError("Lookup directories shall be an iterable of paths. Found in list: " + type(a).__name__) - _logger.debug(_LOG_LIST_ITEM_PREFIX + str(a)) - # Normalize paths and remove duplicates. Resolve symlinks to avoid ambiguities. root_namespace_directory = Path(root_namespace_directory).resolve() - lookup_directories_path_list.append(root_namespace_directory) - lookup_directories_path_list = list(sorted({x.resolve() for x in lookup_directories_path_list})) - _logger.debug("Lookup directories are listed below:") - for a in lookup_directories_path_list: - _logger.debug(_LOG_LIST_ITEM_PREFIX + str(a)) - - # Check for common usage errors and warn the user if anything looks suspicious. - _ensure_no_common_usage_errors(root_namespace_directory, lookup_directories_path_list, _logger.warning) - - # Check the namespaces. - _ensure_no_nested_root_namespaces(lookup_directories_path_list) - if not allow_root_namespace_name_collision: - _ensure_no_namespace_name_collisions(lookup_directories_path_list) + lookup_directories_path_list = _construct_lookup_directories_path_list( + [root_namespace_directory], + dsdl_normalize_paths_argument( + lookup_directories, cast(Callable[[Iterable], List[Path]], lambda i: [Path(it) for it in i]) + ), + allow_root_namespace_name_collision, + ) # Construct DSDL definitions from the target and the lookup dirs. - target_dsdl_definitions = _construct_dsdl_definitions_from_namespace(root_namespace_directory) + target_dsdl_definitions = _construct_dsdl_definitions_from_namespaces([root_namespace_directory]) if not target_dsdl_definitions: _logger.info("The namespace at %s is empty", root_namespace_directory) return [] @@ -157,9 +149,107 @@ def read_namespace( for x in target_dsdl_definitions: _logger.debug(_LOG_LIST_ITEM_PREFIX + str(x)) - lookup_dsdl_definitions = [] # type: List[_dsdl_definition.DSDLDefinition] - for ld in lookup_directories_path_list: - lookup_dsdl_definitions += _construct_dsdl_definitions_from_namespace(ld) + return _complete_read_function( + target_dsdl_definitions, + lookup_directories_path_list, + NamespaceClosureReader(allow_unregulated_fixed_port_id, print_output_handler), + ).direct.types + + +# pylint: disable=too-many-arguments +def read_files( + dsdl_files: Union[None, Path, str, Iterable[Union[Path, str]]], + root_namespace_directories_or_names: Union[None, Path, str, Iterable[Union[Path, str]]], + lookup_directories: Union[None, Path, str, Iterable[Union[Path, str]]] = None, + print_output_handler: Optional[PrintOutputHandler] = None, + allow_unregulated_fixed_port_id: bool = False, + allow_root_namespace_name_collision: bool = True, +) -> Tuple[List[DsdlFile], List[DsdlFile]]: + """ + This function is the main entry point of the library. + It reads all DSDL definitions from the specified root namespace directory and produces the annotated AST. + + :param root_namespace_directory: The path of the root namespace directory that will be read. + For example, ``dsdl/uavcan`` to read the ``uavcan`` namespace. + + :param lookup_directories: List of other namespace directories containing data type definitions that are + referred to from the target root namespace. For example, if you are reading a vendor-specific namespace, + the list of lookup directories should always include a path to the standard root namespace ``uavcan``, + otherwise the types defined in the vendor-specific namespace won't be able to use data types from the + standard namespace. + + :param print_output_handler: If provided, this callable will be invoked when a ``@print`` directive + is encountered or when the frontend needs to emit a diagnostic; + the arguments are: path, line number (1-based), text. + If not provided, no output will be produced except for the standard Python logging subsystem + (but ``@print`` expressions will be evaluated anyway, and a failed evaluation will be a fatal error). + + :param allow_unregulated_fixed_port_id: Do not reject unregulated fixed port identifiers. + As demanded by the specification, the frontend rejects unregulated fixed port ID by default. + This is a dangerous feature that must not be used unless you understand the risks. + Please read https://opencyphal.org/guide. + + :param allow_root_namespace_name_collision: Allow using the source root namespace name in the look up dirs or + the same root namespace name multiple times in the lookup dirs. This will enable defining a namespace + partially and let other entities define new messages or new sub-namespaces in the same root namespace. + + :return: A list of :class:`pydsdl.CompositeType` sorted lexicographically by full data type name, + then by major version (newest version first), then by minor version (newest version first). + The ordering guarantee allows the caller to always find the newest version simply by picking + the first matching occurrence. + + :raises: :class:`pydsdl.FrontendError`, :class:`MemoryError`, :class:`SystemError`, + :class:`OSError` if directories do not exist or inaccessible, + :class:`ValueError`/:class:`TypeError` if the arguments are invalid. + """ + # Normalize paths and remove duplicates. Resolve symlinks to avoid ambiguities. + target_dsdl_definitions = _construct_dsdl_definitions_from_files( + dsdl_normalize_paths_argument( + dsdl_files, cast(Callable[[Iterable], List[Path]], lambda i: [Path(it) for it in i]) + ), + dsdl_normalize_paths_argument( + root_namespace_directories_or_names, + cast(Callable[[Iterable], Union[Set[Path], Set[str]]], set), + ), + ) + if len(target_dsdl_definitions) == 0: + _logger.info("No DSDL files found in the specified directories") + return ([], []) + _logger.debug("Target DSDL definitions are listed below:") + for x in target_dsdl_definitions: + _logger.debug(_LOG_LIST_ITEM_PREFIX + str(x.file_path)) + + root_namespaces = dsdl_file_sort({f.root_namespace.resolve() for f in target_dsdl_definitions}) + lookup_directories_path_list = _construct_lookup_directories_path_list( + root_namespaces, + dsdl_normalize_paths_argument(lookup_directories, cast(Callable[[Iterable], List[Path]], list)), + allow_root_namespace_name_collision, + ) + + reader = _complete_read_function( + target_dsdl_definitions, + lookup_directories_path_list, + NamespaceClosureReader(allow_unregulated_fixed_port_id, print_output_handler), + ) + + return (reader.direct.files, reader.transitive.files) + + +# +--[INTERNAL API::PUBLIC API HELPERS]-------------------------------------------------------------------------------+ +# These are functions called by the public API before the actual processing begins. + +DSDL_FILE_SUFFIX = ".dsdl" +DSDL_FILE_GLOB = f"*{DSDL_FILE_SUFFIX}" +DSDL_FILE_SUFFIX_LEGACY = ".uavcan" +DSDL_FILE_GLOB_LEGACY = f"*{DSDL_FILE_SUFFIX_LEGACY}" +_LOG_LIST_ITEM_PREFIX = " " * 4 + + +def _complete_read_function( + target_dsdl_definitions: SortedFileList, lookup_directories_path_list: List[Path], reader: NamespaceClosureReader +) -> NamespaceClosureReader: + + lookup_dsdl_definitions = _construct_dsdl_definitions_from_namespaces(lookup_directories_path_list) # Check for collisions against the lookup definitions also. _ensure_no_collisions(target_dsdl_definitions, lookup_dsdl_definitions) @@ -177,10 +267,9 @@ def read_namespace( ", ".join(set(sorted(map(lambda t: t.root_namespace, lookup_dsdl_definitions)))), ) - # Read the constructed definitions. - types = _read_namespace_definitions( - target_dsdl_definitions, lookup_dsdl_definitions, print_output_handler, allow_unregulated_fixed_port_id - ) + # This is the biggie. All the rest of the wranging is just to get to this point. This will take the + # most time and memory. + reader.read_definitions(target_dsdl_definitions, lookup_dsdl_definitions) # Note that we check for collisions in the read namespace only. # We intentionally ignore (do not check for) possible collisions in the lookup directories, @@ -188,57 +277,108 @@ def read_namespace( # directories may contain issues and mistakes that are outside of the control of the user (e.g., # they could be managed by a third party) -- the user shouldn't be affected by mistakes committed # by the third party. - _ensure_no_fixed_port_id_collisions(types) - _ensure_minor_version_compatibility(types) + _ensure_no_fixed_port_id_collisions(reader.direct.types) + _ensure_minor_version_compatibility(reader.all.types) - return types + return reader -DSDL_FILE_GLOB = "*.dsdl" -DSDL_FILE_GLOB_LEGACY = "*.uavcan" -_LOG_LIST_ITEM_PREFIX = " " * 4 +def _construct_lookup_directories_path_list( + root_namespace_directories: List[Path], + lookup_directories_path_list: List[Path], + allow_root_namespace_name_collision: bool, +) -> List[Path]: + """ + Intermediate transformation and validation of inputs into a list of lookup directories as paths. -_logger = logging.getLogger(__name__) + :param root_namespace_directory: The path of the root namespace directory that will be read. + For example, ``dsdl/uavcan`` to read the ``uavcan`` namespace. + + :param lookup_directories: List of other namespace directories containing data type definitions that are + referred to from the target root namespace. For example, if you are reading a vendor-specific namespace, + the list of lookup directories should always include a path to the standard root namespace ``uavcan``, + otherwise the types defined in the vendor-specific namespace won't be able to use data types from the + standard namespace. + :param allow_root_namespace_name_collision: Allow using the source root namespace name in the look up dirs or + the same root namespace name multiple times in the lookup dirs. This will enable defining a namespace + partially and let other entities define new messages or new sub-namespaces in the same root namespace. -def _read_namespace_definitions( - target_definitions: List[_dsdl_definition.DSDLDefinition], - lookup_definitions: List[_dsdl_definition.DSDLDefinition], - print_output_handler: Optional[PrintOutputHandler] = None, - allow_unregulated_fixed_port_id: bool = False, -) -> List[_serializable.CompositeType]: - """ - Construct type descriptors from the specified target definitions. - Allow the target definitions to use the lookup definitions within themselves. - :param target_definitions: Which definitions to read. - :param lookup_definitions: Which definitions can be used by the processed definitions. - :return: A list of types. + :return: A list of lookup directories as paths. + + :raises: :class:`pydsdl.FrontendError`, :class:`MemoryError`, :class:`SystemError`, + :class:`OSError` if directories do not exist or inaccessible, + :class:`ValueError`/:class:`TypeError` if the arguments are invalid. """ + # Add the own root namespace to the set of lookup directories, sort lexicographically, remove duplicates. + # We'd like this to be an iterable list of strings but we handle the common practice of passing in a single path. + + # Normalize paths and remove duplicates. Resolve symlinks to avoid ambiguities. + lookup_directories_path_list.extend(root_namespace_directories) + lookup_directories_path_list = list(sorted({x.resolve() for x in lookup_directories_path_list})) + _logger.debug("Lookup directories are listed below:") + for a in lookup_directories_path_list: + _logger.debug(_LOG_LIST_ITEM_PREFIX + str(a)) - def make_print_handler(definition: _dsdl_definition.DSDLDefinition) -> Callable[[int, str], None]: - def handler(line_number: int, text: str) -> None: - if print_output_handler: # pragma: no branch - assert isinstance(line_number, int) and isinstance(text, str) - assert line_number > 0, "Line numbers must be one-based" - print_output_handler(definition.file_path, line_number, text) + # Check for common usage errors and warn the user if anything looks suspicious. + _ensure_no_common_usage_errors(root_namespace_directories, lookup_directories_path_list, _logger.warning) - return handler + # Check the namespaces. + _ensure_no_nested_root_namespaces(lookup_directories_path_list) - types = [] # type: List[_serializable.CompositeType] - for tdd in target_definitions: - try: - dt = tdd.read(lookup_definitions, make_print_handler(tdd), allow_unregulated_fixed_port_id) - except _error.FrontendError as ex: # pragma: no cover - ex.set_error_location_if_unknown(path=tdd.file_path) - raise ex - except (MemoryError, SystemError): # pragma: no cover - raise - except Exception as ex: # pragma: no cover - raise _error.InternalError(culprit=ex, path=tdd.file_path) from ex - else: - types.append(dt) + if not allow_root_namespace_name_collision: + _ensure_no_namespace_name_collisions(lookup_directories_path_list) + + return lookup_directories_path_list + + +def _construct_dsdl_definitions_from_files( + dsdl_files: List[Path], + valid_roots: Union[Set[Path], Set[str]], +) -> SortedFileList: + """ """ + output = set() # type: Set[DsdlFileBuildable] + for fp in dsdl_files: + root_namespace_path = _infer_path_to_root(fp, valid_roots) + if fp.suffix == DSDL_FILE_SUFFIX_LEGACY: + _logger.warning( + "File uses deprecated extension %r, please rename to use %r: %s", + DSDL_FILE_SUFFIX_LEGACY, + DSDL_FILE_SUFFIX, + fp, + ) + output.add(_dsdl_definition.DSDLDefinition(fp, root_namespace_path)) - return types + return dsdl_file_sort(output) + + +def _construct_dsdl_definitions_from_namespaces( + root_namespace_paths: List[Path], +) -> SortedFileList: + """ + Accepts a directory path, returns a sorted list of abstract DSDL file representations. Those can be read later. + The definitions are sorted by name lexicographically, then by major version (greatest version first), + then by minor version (same ordering as the major version). + """ + source_file_paths: Set[Path] = set() + output = [] # type: List[DsdlFileBuildable] + for root_namespace_path in root_namespace_paths: + for p in root_namespace_path.rglob(DSDL_FILE_GLOB): + source_file_paths.add(p) + for p in root_namespace_path.rglob(DSDL_FILE_GLOB_LEGACY): + source_file_paths.add(p) + _logger.warning( + "File uses deprecated extension %r, please rename to use %r: %s", + DSDL_FILE_GLOB_LEGACY, + DSDL_FILE_GLOB, + p, + ) + + for fp in sorted(source_file_paths): + dsdl_def = _dsdl_definition.DSDLDefinition(fp, root_namespace_path) + output.append(dsdl_def) + + return dsdl_file_sort(output) def _ensure_no_collisions( @@ -375,7 +515,7 @@ def _ensure_minor_version_compatibility_pairwise( def _ensure_no_common_usage_errors( - root_namespace_directory: Path, lookup_directories: Iterable[Path], reporter: Callable[[str], None] + root_namespace_directories: List[Path], lookup_directories: Iterable[Path], reporter: Callable[[str], None] ) -> None: suspicious_base_names = [ "public_regulated_data_types", @@ -391,7 +531,7 @@ def is_valid_name(s: str) -> bool: return True # resolve() will also normalize the case in case-insensitive filesystems. - all_paths = {root_namespace_directory.resolve()} | {x.resolve() for x in lookup_directories} + all_paths = {y.resolve() for y in root_namespace_directories} | {x.resolve() for x in lookup_directories} for p in all_paths: try: candidates = [x for x in p.iterdir() if x.is_dir() and is_valid_name(x.name)] @@ -435,32 +575,68 @@ def _ensure_no_namespace_name_collisions(directories: Iterable[Path]) -> None: raise RootNamespaceNameCollisionError("The name of this namespace conflicts with %s" % b, path=a) -def _construct_dsdl_definitions_from_namespace(root_namespace_path: Path) -> List[_dsdl_definition.DSDLDefinition]: +def _infer_path_to_root( + dsdl_path: Path, valid_dsdl_roots_or_path_to_root: Optional[Union[Set[Path], Set[str]]] = None +) -> Path: """ - Accepts a directory path, returns a sorted list of abstract DSDL file representations. Those can be read later. - The definitions are sorted by name lexicographically, then by major version (greatest version first), - then by minor version (same ordering as the major version). + Infer the path to the namespace root of a DSDL file path. + :param dsdl_path: The path to the alleged DSDL file. + :param valid_dsdl_roots_or_path_to_root: The set of valid root names or paths under which the type must reside. + :return The path to the root namespace directory. + :raises DsdlPathInferenceError: If the namespace root cannot be inferred from the provided information. """ - source_file_paths: Set[Path] = set() - for p in root_namespace_path.rglob(DSDL_FILE_GLOB): - source_file_paths.add(p) - for p in root_namespace_path.rglob(DSDL_FILE_GLOB_LEGACY): - source_file_paths.add(p) - _logger.warning( - "File uses deprecated extension %r, please rename to use %r: %s", DSDL_FILE_GLOB_LEGACY, DSDL_FILE_GLOB, p - ) + if dsdl_path.is_absolute(): + if valid_dsdl_roots_or_path_to_root is None: + raise DsdlPathInferenceError( + f"dsdl_path ({dsdl_path}) is absolute and no valid root names or path to root was provided. The " + "DSDL root of an absolute path cannot be inferred without this information.", + ) + if len(valid_dsdl_roots_or_path_to_root) == 0: + raise DsdlPathInferenceError( + f"dsdl_path ({dsdl_path}) is absolute and the provided valid root names are empty. The DSDL root of " + "an absolute path cannot be inferred without this information.", + ) + if isinstance(next(iter(valid_dsdl_roots_or_path_to_root)), Path): + valid_paths_to_root = cast(Set[Path], valid_dsdl_roots_or_path_to_root) + for path_to_root in valid_paths_to_root: + try: + _ = dsdl_path.relative_to(path_to_root) + except ValueError: + continue + return path_to_root + raise DsdlPathInferenceError( + f"dsdl_path ({dsdl_path}) is absolute but is not relative to " + f"any provided path to root {valid_dsdl_roots_or_path_to_root}", + ) + + if ( + valid_dsdl_roots_or_path_to_root is not None + and len(valid_dsdl_roots_or_path_to_root) > 0 + and isinstance(next(iter(valid_dsdl_roots_or_path_to_root)), str) + ): + valid_dsdl_roots = cast(Set[str], valid_dsdl_roots_or_path_to_root) + parts = list(dsdl_path.parent.parts) + namespace_parts = None + for i, part in list(enumerate(parts)): + if part in valid_dsdl_roots: + namespace_parts = parts[i:] + return Path().joinpath(*parts[: i + 1]) + # +1 to include the root folder + if namespace_parts is None: + raise DsdlPathInferenceError(f"No valid root found in path {str(dsdl_path)}") + + if not dsdl_path.is_absolute(): + return Path(dsdl_path.parts[0]) - output = [] # type: List[_dsdl_definition.DSDLDefinition] - for fp in sorted(source_file_paths): - dsdl_def = _dsdl_definition.DSDLDefinition(fp, root_namespace_path) - output.append(dsdl_def) + raise DsdlPathInferenceError(f"Could not determine a path to the namespace root of dsdl path {dsdl_path}") - # Lexicographically by name, newest version first. - return list(sorted(output, key=lambda d: (d.full_name, -d.version.major, -d.version.minor))) + +# +--[ UNIT TESTS ]---------------------------------------------------------------------------------------------------+ def _unittest_dsdl_definition_constructor() -> None: import tempfile + from ._dsdl_definition import FileNameFormatError with tempfile.TemporaryDirectory() as directory: @@ -472,7 +648,7 @@ def _unittest_dsdl_definition_constructor() -> None: (root / "nested/2.Asd.21.32.dsdl").write_text("# TEST B") (root / "nested/Foo.32.43.dsdl").write_text("# TEST C") - dsdl_defs = _construct_dsdl_definitions_from_namespace(root) + dsdl_defs = _construct_dsdl_definitions_from_namespaces([root]) print(dsdl_defs) lut = {x.full_name: x for x in dsdl_defs} # type: Dict[str, _dsdl_definition.DSDLDefinition] assert len(lut) == 3 @@ -528,7 +704,7 @@ def _unittest_dsdl_definition_constructor() -> None: (root / "nested/Malformed.MAJOR.MINOR.dsdl").touch() try: - _construct_dsdl_definitions_from_namespace(root) + _construct_dsdl_definitions_from_namespaces([root]) except FileNameFormatError as ex: print(ex) (root / "nested/Malformed.MAJOR.MINOR.dsdl").unlink() @@ -537,7 +713,7 @@ def _unittest_dsdl_definition_constructor() -> None: (root / "nested/NOT_A_NUMBER.Malformed.1.0.dsdl").touch() try: - _construct_dsdl_definitions_from_namespace(root) + _construct_dsdl_definitions_from_namespaces([root]) except FileNameFormatError as ex: print(ex) (root / "nested/NOT_A_NUMBER.Malformed.1.0.dsdl").unlink() @@ -546,26 +722,26 @@ def _unittest_dsdl_definition_constructor() -> None: (root / "nested/Malformed.dsdl").touch() try: - _construct_dsdl_definitions_from_namespace(root) + _construct_dsdl_definitions_from_namespaces([root]) except FileNameFormatError as ex: print(ex) (root / "nested/Malformed.dsdl").unlink() else: # pragma: no cover assert False - _construct_dsdl_definitions_from_namespace(root) # making sure all errors are cleared + _construct_dsdl_definitions_from_namespaces([root]) # making sure all errors are cleared (root / "nested/super.bad").mkdir() (root / "nested/super.bad/Unreachable.1.0.dsdl").touch() try: - _construct_dsdl_definitions_from_namespace(root) + _construct_dsdl_definitions_from_namespaces([root]) except FileNameFormatError as ex: print(ex) else: # pragma: no cover assert False try: - _construct_dsdl_definitions_from_namespace(root / "nested/super.bad") + _construct_dsdl_definitions_from_namespaces([root / "nested/super.bad"]) except FileNameFormatError as ex: print(ex) else: # pragma: no cover @@ -582,7 +758,7 @@ def _unittest_dsdl_definition_constructor_legacy() -> None: root = di / "foo" root.mkdir() (root / "123.Qwerty.123.234.uavcan").write_text("# TEST A") - dsdl_defs = _construct_dsdl_definitions_from_namespace(root) + dsdl_defs = _construct_dsdl_definitions_from_namespaces([root]) print(dsdl_defs) lut = {x.full_name: x for x in dsdl_defs} # type: Dict[str, _dsdl_definition.DSDLDefinition] assert len(lut) == 1 @@ -609,33 +785,34 @@ def _unittest_common_usage_errors() -> None: reports = [] # type: List[str] - _ensure_no_common_usage_errors(root_ns_dir, [], reports.append) + _ensure_no_common_usage_errors([root_ns_dir], [], reports.append) assert not reports - _ensure_no_common_usage_errors(root_ns_dir, [di / "baz"], reports.append) + _ensure_no_common_usage_errors([root_ns_dir], [di / "baz"], reports.append) assert not reports dir_dsdl = root_ns_dir / "dsdl" dir_dsdl.mkdir() - _ensure_no_common_usage_errors(dir_dsdl, [di / "baz"], reports.append) + _ensure_no_common_usage_errors([dir_dsdl], [di / "baz"], reports.append) assert not reports # Because empty. dir_dsdl_vscode = dir_dsdl / ".vscode" dir_dsdl_vscode.mkdir() - _ensure_no_common_usage_errors(dir_dsdl, [di / "baz"], reports.append) + _ensure_no_common_usage_errors([dir_dsdl], [di / "baz"], reports.append) assert not reports # Because the name is not valid. dir_dsdl_uavcan = dir_dsdl / "uavcan" dir_dsdl_uavcan.mkdir() - _ensure_no_common_usage_errors(dir_dsdl, [di / "baz"], reports.append) + _ensure_no_common_usage_errors([dir_dsdl], [di / "baz"], reports.append) (rep,) = reports reports.clear() assert str(dir_dsdl_uavcan.resolve()).lower() in rep.lower() def _unittest_nested_roots() -> None: - from pytest import raises import tempfile + from pytest import raises + with tempfile.TemporaryDirectory() as directory: di = Path(directory) (di / "a").mkdir() @@ -663,3 +840,59 @@ def _unittest_issue_71() -> None: # https://github.com/OpenCyphal/pydsdl/issues (real / "Msg.0.1.dsdl").write_text("@sealed") assert len(read_namespace(real, [real, link])) == 1 assert len(read_namespace(link, [real, link])) == 1 + + +def _unittest_type_from_path_inference() -> None: + from pytest import raises as expect_raises + + # To determine the namespace do + + dsdl_file = Path("/repo/uavcan/foo/bar/435.baz.1.0.dsdl") + path_to_root = _infer_path_to_root(dsdl_file, {"uavcan"}) + namespace_parts = dsdl_file.parent.relative_to(path_to_root.parent).parts + + assert path_to_root == Path("/repo/uavcan") + assert namespace_parts == ("uavcan", "foo", "bar") + + # The root namespace cannot be inferred in an absolute path without additional data: + + with expect_raises(DsdlPathInferenceError): + _ = _infer_path_to_root(Path("/repo/uavcan/foo/bar/435.baz.1.0.dsdl")) + + # If an absolute path is provided along with a path-to-root "hint" then the former must be relative to the + # latter: + + # dsdl file path is not contained within the root path + with expect_raises(DsdlPathInferenceError): + _ = _infer_path_to_root(Path("/repo/uavcan/foo/bar/435.baz.1.0.dsdl"), {Path("/not-a-repo")}) + + # This works + root = _infer_path_to_root(Path("/repo/uavcan/foo/bar/435.baz.1.0.dsdl"), {Path("/repo")}) + assert root == Path("/repo") + + # Either relative or absolute paths given a set of valid root names will prefer searching for the root: + + valid_roots = {"uavcan", "cyphal"} + + # absolute dsdl path using valid roots + root = _infer_path_to_root(Path("/repo/uavcan/foo/bar/435.baz.1.0.dsdl"), valid_roots) + assert root == Path("/repo/uavcan") + + # relative dsdl path using valid roots + root = _infer_path_to_root(Path("repo/uavcan/foo/bar/435.baz.1.0.dsdl"), valid_roots) + assert root == Path("repo/uavcan") + + # absolute dsdl path using valid roots but an invalid file path + with expect_raises(DsdlPathInferenceError): + _ = _infer_path_to_root(Path("/repo/crap/foo/bar/435.baz.1.0.dsdl"), valid_roots) + + # relative dsdl path using valid roots but an invalid file path + with expect_raises(DsdlPathInferenceError): + _ = _infer_path_to_root(Path("repo/crap/foo/bar/435.baz.1.0.dsdl"), valid_roots) + + # The final inference made is when relative dsdl paths are provided with no additional information. In this + # case the method assumes that the relative path is the correct and complete namespace of the type: + + # relative path + root = _infer_path_to_root(Path("uavcan/foo/bar/435.baz.1.0.dsdl")) + assert root == Path("uavcan") diff --git a/pydsdl/_namespace_reader.py b/pydsdl/_namespace_reader.py new file mode 100644 index 0000000..50df22b --- /dev/null +++ b/pydsdl/_namespace_reader.py @@ -0,0 +1,160 @@ +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT + + +import logging +from functools import partial +from pathlib import Path +from typing import List, Optional, Set + +from ._dsdl import ( + DefinitionVisitor, + DsdlFile, + DsdlFileBuildable, + PrintOutputHandler, + SortedFileList, + file_sort as dsdl_file_sort, +) +from ._error import DependentFileError, FrontendError, InternalError +from ._serializable import CompositeType + + +class FilesContainer: + + def __init__(self) -> None: + self._files: SortedFileList = [] + self._files_index: Set[DsdlFile] = set() + self._types_cache: Optional[List[CompositeType]] = None + + @property + def files(self) -> SortedFileList: + return self._files + + @property + def types(self) -> List[CompositeType]: + if self._types_cache is None: + self._types_cache = [f.composite_type for f in self._files if f.composite_type is not None] + return self._types_cache + + def __contains__(self, file: DsdlFile) -> bool: + return file in self._files_index + + +class MutableFilesContainer(FilesContainer): + + def add(self, file: DsdlFile) -> None: + if file not in self._files_index: + # TODO: Didn't realize that python 3.8 didn't have the key parameter for bisect + # I meant to keep this list sorted. I'll have to fix this later. + self._files.append(file) + self._files_index.add(file) + self._types_cache = None + + def append(self, files: Set[DsdlFile]) -> None: + for file in files: + self.add(file) + + def clear(self) -> None: + self._files = [] + self._files_index.clear() + self._types_cache = None + + def remove_if(self, file: DsdlFile) -> None: + if file in self._files_index: + self._files.remove(file) + self._files_index.remove(file) + self._types_cache = None + + def union(self, other: FilesContainer) -> FilesContainer: + union_files = MutableFilesContainer() + # pylint: disable=protected-access + union_files._files_index = self._files_index.union(other._files_index) + union_files._files = dsdl_file_sort(union_files._files_index) + return union_files + + +class Closure(DefinitionVisitor): + + @staticmethod + def print_output_to_debug_logger(logger: logging.Logger, path: Path, line_number: int, text: str) -> None: + logger.debug("%s:%d – %s", str(path), line_number, text) + + def __init__( + self, + allow_unregulated_fixed_port_id: bool, + print_output_handler: Optional[PrintOutputHandler], + ): + self._logger = logging.getLogger(__name__) + self._allow_unregulated_fixed_port_id = allow_unregulated_fixed_port_id + self._print_output_handler = print_output_handler or partial( + logging.getLogger(f"{__name__}.print_output_handler").debug, self.print_output_to_debug_logger + ) + + self._pending_definitions: Set[DsdlFileBuildable] = set() + self._direct = MutableFilesContainer() + self._transitive = MutableFilesContainer() + + # +--[DefinitionVisitor]------------------------------------------------------------------------------------------+ + def on_discover_lookup_dependent_file(self, target_dsdl_file: DsdlFile, dependent_type: DsdlFile) -> None: + if not isinstance(dependent_type, DsdlFileBuildable): + raise DependentFileError(f"Dependent file is not buildable: {dependent_type.file_path}") + self._pending_definitions.add(dependent_type) + + # --[PUBLIC]------------------------------------------------------------------------------------------------------+ + @property + def direct(self) -> FilesContainer: + return self._direct + + @property + def transitive(self) -> FilesContainer: + return self._transitive + + @property + def all(self) -> FilesContainer: + return self._direct.union(self._transitive) + + def read_definitions( + self, + target_definitions: SortedFileList, + lookup_definitions: SortedFileList, + ) -> None: + self._read_definitions(target_definitions, lookup_definitions, 0) + + # --[PRIVATE]-----------------------------------------------------------------------------------------------------+ + def _read_definitions( + self, target_definitions: SortedFileList, lookup_definitions: SortedFileList, level: int + ) -> None: + + for target_definition in target_definitions: + self._pending_definitions.clear() + if target_definition in (self._direct, self._transitive): + self._logger.debug( + "Skipping target file %s because it has already been processed", target_definition.file_path + ) + continue + if not isinstance(target_definition, DsdlFileBuildable): + raise TypeError("Expected DsdlFileBuildable, got: " + type(target_definition).__name__) + try: + target_definition.read( + lookup_definitions, + [self], + partial(self._print_output_handler, target_definition), + self._allow_unregulated_fixed_port_id, + ) + except DependentFileError: + self._logger.debug("Skipping target file %s due to dependent file error", target_definition.file_path) + except FrontendError as ex: # pragma: no cover + ex.set_error_location_if_unknown(path=target_definition.file_path) + raise ex + except (MemoryError, SystemError): # pragma: no cover + raise + except Exception as ex: # pragma: no cover + raise InternalError(culprit=ex, path=target_definition.file_path) from ex + else: + if level == 0: + self._direct.add(target_definition) + self._transitive.remove_if(target_definition) + elif target_definition not in self._direct: + self._transitive.add(target_definition) + self._read_definitions(dsdl_file_sort(self._pending_definitions), lookup_definitions, level + 1) diff --git a/pydsdl/_parser.py b/pydsdl/_parser.py index b73a533..e1de281 100644 --- a/pydsdl/_parser.py +++ b/pydsdl/_parser.py @@ -9,8 +9,8 @@ import fractions from pathlib import Path from typing import List, Tuple -import parsimonious -from parsimonious.nodes import Node as _Node +import parsimonious # type: ignore +from parsimonious.nodes import Node as _Node # type: ignore from . import _error from . import _serializable from . import _expression @@ -27,14 +27,14 @@ def parse(text: str, statement_stream_processor: "StatementStreamProcessor") -> """ pr = _ParseTreeProcessor(statement_stream_processor) try: - pr.visit(_get_grammar().parse(text)) # type: ignore + pr.visit(_get_grammar().parse(text)) except _error.FrontendError as ex: # Inject error location. If this exception is being propagated from a recursive instance, it already has # its error location populated, so nothing will happen here. ex.set_error_location_if_unknown(line=pr.current_line_number) raise ex except parsimonious.ParseError as ex: - raise DSDLSyntaxError("Syntax error", line=int(ex.line())) from None # type: ignore + raise DSDLSyntaxError("Syntax error", line=int(ex.line())) from None except parsimonious.VisitationError as ex: # pragma: no cover # noinspection PyBroadException try: @@ -89,7 +89,7 @@ def resolve_versioned_data_type(self, name: str, version: _serializable.Version) @functools.lru_cache(None) def _get_grammar() -> parsimonious.Grammar: - return parsimonious.Grammar((Path(__file__).parent / "grammar.parsimonious").read_text()) # type: ignore + return parsimonious.Grammar((Path(__file__).parent / "grammar.parsimonious").read_text()) _logger = logging.getLogger(__name__) @@ -131,7 +131,7 @@ class _ParseTreeProcessor(parsimonious.NodeVisitor): # Intentional exceptions that shall not be treated as parse errors. # Beware that those might be propagated from recursive parser instances! - unwrapped_exceptions = (_error.FrontendError, SystemError, MemoryError, SystemExit) # type: ignore + unwrapped_exceptions = (_error.FrontendError, SystemError, MemoryError, SystemExit) def __init__(self, statement_stream_processor: StatementStreamProcessor): assert isinstance(statement_stream_processor, StatementStreamProcessor) diff --git a/pydsdl/_test.py b/pydsdl/_test.py index 3e4048f..2a879c3 100644 --- a/pydsdl/_test.py +++ b/pydsdl/_test.py @@ -62,12 +62,13 @@ def parse_definition( ) -> _serializable.CompositeType: return definition.read( lookup_definitions, + [], print_output_handler=lambda line, text: print("Output from line %d:" % line, text), allow_unregulated_fixed_port_id=False, ) -@pytest.fixture() # type: ignore +@pytest.fixture() def wrkspc() -> Workspace: return Workspace() @@ -422,7 +423,7 @@ def _unittest_error(wrkspc: Workspace) -> None: def standalone(rel_path: str, definition: str, allow_unregulated: bool = False) -> _serializable.CompositeType: return wrkspc.parse_new(rel_path, definition + "\n").read( - [], lambda *_: None, allow_unregulated + [], [], lambda *_: None, allow_unregulated ) # pragma: no branch with raises(_error.InvalidDefinitionError, match="(?i).*port ID.*"): @@ -754,20 +755,20 @@ def print_handler(line_number: int, text: str) -> None: wrkspc.parse_new( "ns/A.1.0.dsdl", "# line number 1\n" "# line number 2\n" "@print 2 + 2 == 4 # line number 3\n" "# line number 4\n" "@sealed\n", - ).read([], print_handler, False) + ).read([], [], print_handler, False) assert printed_items assert printed_items[0] == 3 assert printed_items[1] == "true" - wrkspc.parse_new("ns/B.1.0.dsdl", "@print false\n@sealed").read([], print_handler, False) + wrkspc.parse_new("ns/B.1.0.dsdl", "@print false\n@sealed").read([], [], print_handler, False) assert printed_items assert printed_items[0] == 1 assert printed_items[1] == "false" wrkspc.parse_new( "ns/Offset.1.0.dsdl", "@print _offset_ # Not recorded\n" "uint8 a\n" "@print _offset_\n" "@extent 800\n" - ).read([], print_handler, False) + ).read([], [], print_handler, False) assert printed_items assert printed_items[0] == 3 assert printed_items[1] == "{8}" diff --git a/setup.cfg b/setup.cfg index 8d934ba..c58e783 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ include = pydsdl* # -------------------------------------------------- PYTEST -------------------------------------------------- [tool:pytest] -testpaths = pydsdl +testpaths = pydsdl test norecursedirs = third_party python_files = *.py python_classes = _UnitTest diff --git a/test/public_regulated_data_types b/test/public_regulated_data_types new file mode 160000 index 0000000..f9f6790 --- /dev/null +++ b/test/public_regulated_data_types @@ -0,0 +1 @@ +Subproject commit f9f67906cc0ca5d7c1b429924852f6b28f313cbf diff --git a/test/test_public_types.py b/test/test_public_types.py new file mode 100644 index 0000000..ef40aa6 --- /dev/null +++ b/test/test_public_types.py @@ -0,0 +1,44 @@ +# Copyright (C) OpenCyphal Development Team +# Copyright Amazon.com Inc. or its affiliates. +# SPDX-License-Identifier: MIT + +# pylint: disable=redefined-outer-name +# pylint: disable=logging-fstring-interpolation +import cProfile +import io +import logging +import pstats +from pathlib import Path +from pstats import SortKey + +import pytest + +import pydsdl + + +@pytest.fixture +def public_types() -> Path: + return Path("test") / "public_regulated_data_types" / "uavcan" + + +def dsdl_printer(dsdl_file: Path, line: int, message: str) -> None: + """ + Prints the DSDL file. + """ + logging.info(f"{dsdl_file}:{line}: {message}") + + +def _unittest_public_types(public_types: Path) -> None: + """ + Sanity check to ensure that the public types can be read. This also allows us to debug + against a real dataset. + """ + pr = cProfile.Profile() + pr.enable() + _ = pydsdl.read_namespace(public_types) + pr.disable() + s = io.StringIO() + sortby = SortKey.TIME + ps = pstats.Stats(pr, stream=s).sort_stats(sortby) + ps.print_stats() + print(s.getvalue())