diff --git a/colcon_cache/base_handler/__init__.py b/colcon_cache/base_handler/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/colcon_cache/base_handler/cache.py b/colcon_cache/base_handler/cache.py new file mode 100644 index 0000000..329c477 --- /dev/null +++ b/colcon_cache/base_handler/cache.py @@ -0,0 +1,31 @@ +# Copyright 2021 Ruffin White +# Licensed under the Apache License, Version 2.0 + +import os + +from colcon_clean.base_handler import BaseHandlerExtensionPoint +from colcon_core.plugin_system import satisfies_version + +BASE_PATH = 'cache' + + +class CacheBaseHandler(BaseHandlerExtensionPoint): + """Determin how cache paths for the workspace should be cleaned.""" + + def __init__(self): # noqa: D107 + super().__init__(BASE_PATH) + satisfies_version( + BaseHandlerExtensionPoint.EXTENSION_POINT_VERSION, '^1.0') + + def add_arguments(self, *, parser): # noqa: D102 + parser.add_argument( + '--cache-base', + default=self.base_path, + help='The base path for all cache directories ' + '(default: {self.base_path})'.format_map(locals())) + + def get_workspace_paths(self, *, args): # noqa: D102 + return [args.cache_base] + + def get_package_paths(self, *, args, pkg): # noqa: D102 + return [os.path.join(args.cache_base, pkg.name)] diff --git a/colcon_cache/cache/__init__.py b/colcon_cache/cache/__init__.py index 8009d3f..8af078d 100644 --- a/colcon_cache/cache/__init__.py +++ b/colcon_cache/cache/__init__.py @@ -98,16 +98,16 @@ def dump(self, path): # noqa: D102 sort_keys=True) -def get_lockfile_path(package_build_base, verb_name): +def get_lockfile_path(package_base_path, verb_name): """ Get the lockfile path of a verb from the package build directory. - :param str package_build_base: The build directory of a package + :param str package_base_path: The base path of a package :param str verb_name: The invoked verb name :returns: The path for the lockfile :rtype: Path """ return pathlib.Path( - package_build_base, + package_base_path, 'cache', LOCKFILE_FILENAME.format_map(locals())) diff --git a/colcon_cache/event_handler/__init__.py b/colcon_cache/event_handler/__init__.py index 698b14b..a8d1c29 100644 --- a/colcon_cache/event_handler/__init__.py +++ b/colcon_cache/event_handler/__init__.py @@ -6,16 +6,16 @@ from colcon_cache.cache import get_lockfile_path -def get_previous_lockfile(package_build_base, verb_name): +def get_previous_lockfile(package_base_path, verb_name): """ - Get the lockfile of a verb from the package build directory. + Get the lockfile of a verb from the package base path. - :param str package_build_base: The build directory of a package + :param str package_base_path: The base path of a package :param str verb_name: The invoked verb name :returns: The previously persisted lockfile, otherwise None :rtype: CacheLockfile """ - path = get_lockfile_path(package_build_base, verb_name) + path = get_lockfile_path(package_base_path, verb_name) if not path.exists(): return None lockfile = CacheLockfile() @@ -23,14 +23,14 @@ def get_previous_lockfile(package_build_base, verb_name): return lockfile -def set_lockfile(package_build_base, verb_name, lockfile): +def set_lockfile(package_base_path, verb_name, lockfile): """ Persist the lockfile of a verb in the package build directory. - :param str package_build_base: The build directory of a package + :param str package_base_path: The build directory of a package :param str verb_name: The invoked verb name :param CacheLockfile lockfile: The lockfile of the invocation """ - path = get_lockfile_path(package_build_base, verb_name) + path = get_lockfile_path(package_base_path, verb_name) path.parent.mkdir(parents=True, exist_ok=True) lockfile.dump(path) diff --git a/colcon_cache/package_selection/modified.py b/colcon_cache/package_selection/modified.py index e9bd149..cfcddf8 100644 --- a/colcon_cache/package_selection/modified.py +++ b/colcon_cache/package_selection/modified.py @@ -76,11 +76,8 @@ def select_packages(self, args, decorators): # noqa: D102 pkg = decorator.descriptor - package_build_base = os.path.join( - args.build_base, pkg.name) - verb_lockfile = verb_handler_extension\ - .get_current_lockfile(package_build_base) + .get_current_lockfile(args, pkg.name) package_kind = None if verb_lockfile is None: diff --git a/colcon_cache/package_selection/valid.py b/colcon_cache/package_selection/valid.py index 1f39a64..c65e7a2 100644 --- a/colcon_cache/package_selection/valid.py +++ b/colcon_cache/package_selection/valid.py @@ -2,8 +2,6 @@ # Copyright 2021 Ruffin White # Licensed under the Apache License, Version 2.0 -import os - from colcon_cache.verb_handler import get_verb_handler_extensions from colcon_core.package_selection import logger from colcon_core.package_selection import PackageSelectionExtensionPoint @@ -45,14 +43,6 @@ def select_packages(self, args, decorators): # noqa: D102 else: assert False - if not hasattr(args, 'build_base'): - logger.warning( - "Ignoring '{argument}' since the invoked verb doesn't have a " - "'--build-base' argument and therefore can't access " - 'information about the relative state of a package' - .format_map(locals())) - return - verb_name = args.packages_select_cache_key if not verb_name: verb_name = args.verb_name @@ -76,13 +66,10 @@ def select_packages(self, args, decorators): # noqa: D102 pkg = decorator.descriptor - package_build_base = os.path.join( - args.build_base, pkg.name) - verb_lockfile = verb_handler_extension\ - .get_current_lockfile(package_build_base) + .get_current_lockfile(args, pkg.name) reference_lockfile = verb_handler_extension\ - .get_reference_lockfile(package_build_base) + .get_reference_lockfile(args, pkg.name) reference_name = verb_handler_extension.reference_name package_kind = None diff --git a/colcon_cache/subverb/lock.py b/colcon_cache/subverb/lock.py index 9682b70..ead6960 100644 --- a/colcon_cache/subverb/lock.py +++ b/colcon_cache/subverb/lock.py @@ -8,6 +8,7 @@ from pathlib import Path from colcon_cache.subverb import CacheSubverbExtensionPoint +from colcon_cache.verb_handler import add_verb_handler_arguments from colcon_core.argument_parser.destination_collector \ import DestinationCollectorDecorator from colcon_core.event.job import JobUnselected @@ -71,17 +72,13 @@ def __init__(self): # noqa: D107 CacheSubverbExtensionPoint.EXTENSION_POINT_VERSION, '^1.0') def add_arguments(self, *, parser): # noqa: D102 - parser.add_argument( - '--build-base', - default='build', - help='The base path for all build directories (default: build)') - parser.add_argument( '--ignore-dependencies', action='store_true', help='Ignore dependencies when capturing caches (default: false)') add_executor_arguments(parser) + add_verb_handler_arguments(parser) add_event_handler_arguments(parser) add_packages_arguments(parser) diff --git a/colcon_cache/task/lock/__init__.py b/colcon_cache/task/lock/__init__.py index 209e2f6..ed13f81 100644 --- a/colcon_cache/task/lock/__init__.py +++ b/colcon_cache/task/lock/__init__.py @@ -17,7 +17,7 @@ def get_dependencies_lockfiles(args, dependencies): # noqa: D103 lockfile = _cached_lockfiles[dep_path] else: lockfile = get_previous_lockfile( - package_build_base=dep_path, + package_base_path=dep_path, verb_name='cache') _cached_lockfiles[dep_path] = lockfile dependencies_checksums[dep_name] = lockfile diff --git a/colcon_cache/task/lock/dirhash.py b/colcon_cache/task/lock/dirhash.py index 3699287..465c2c2 100644 --- a/colcon_cache/task/lock/dirhash.py +++ b/colcon_cache/task/lock/dirhash.py @@ -171,9 +171,9 @@ async def lock(self, *, additional_hooks=None): # noqa: D102 "Capturing dirhash cache of package in '{args.path}'" .format_map(locals())) - cache_base = Path(args.build_base, 'cache') + cache_base = Path(args.cache_base, 'cache') cache_base.mkdir(parents=True, exist_ok=True) - lockfile = get_previous_lockfile(args.build_base, 'cache') + lockfile = get_previous_lockfile(args.cache_base, 'cache') if lockfile is None: lockfile = CacheLockfile(lock_type=ENTRY_TYPE) assert lockfile.lock_type == ENTRY_TYPE @@ -193,7 +193,7 @@ async def lock(self, *, additional_hooks=None): # noqa: D102 lockfile.checksums.current pkg.metadata['lockfile'] = lockfile - set_lockfile(args.build_base, 'cache', lockfile) + set_lockfile(args.cache_base, 'cache', lockfile) return 0 diff --git a/colcon_cache/task/lock/git.py b/colcon_cache/task/lock/git.py index dc52252..4eca2f0 100644 --- a/colcon_cache/task/lock/git.py +++ b/colcon_cache/task/lock/git.py @@ -78,9 +78,9 @@ async def lock(self, *, additional_hooks=None): # noqa: D102 "Capturing git cache of package in '{args.path}'" .format_map(locals())) - cache_base = Path(args.build_base, 'cache') + cache_base = Path(args.cache_base, 'cache') cache_base.mkdir(parents=True, exist_ok=True) - lockfile = get_previous_lockfile(args.build_base, 'cache') + lockfile = get_previous_lockfile(args.cache_base, 'cache') if lockfile is None: lockfile = CacheLockfile(lock_type=ENTRY_TYPE) assert lockfile.lock_type == ENTRY_TYPE @@ -96,7 +96,7 @@ async def lock(self, *, additional_hooks=None): # noqa: D102 self.compute_current_checksums(args, lockfile) pkg.metadata['lockfile'] = lockfile - set_lockfile(args.build_base, 'cache', lockfile) + set_lockfile(args.cache_base, 'cache', lockfile) return 0 diff --git a/colcon_cache/verb_handler/__init__.py b/colcon_cache/verb_handler/__init__.py index 24b4482..513c367 100644 --- a/colcon_cache/verb_handler/__init__.py +++ b/colcon_cache/verb_handler/__init__.py @@ -2,6 +2,8 @@ # Copyright 2021 Ruffin White # Licensed under the Apache License, Version 2.0 +import os + from colcon_cache.event_handler import get_previous_lockfile from colcon_core.plugin_system import instantiate_extensions from colcon_core.plugin_system import order_extensions_by_name @@ -21,32 +23,51 @@ class VerbHandlerExtensionPoint: """The version of the package selection extension interface.""" EXTENSION_POINT_VERSION = '1.0' - def __init__(self, verb_name, reference_name): # noqa: D107 + def __init__(self, base_path, verb_name, reference_name): # noqa: D107 # TODO: find better alternative than perhaps using params + self.base_path = base_path self.verb_name = verb_name self.reference_name = reference_name - def get_current_lockfile(self, package_build_base): + def add_arguments(self, *, parser): + """ + Add command line arguments specific to the workspace base. + + This method must be overridden in a subclass. + + :param parser: The argument parser + """ + raise NotImplementedError() + + def get_current_lockfile(self, args, pkg_name): """ Get current lockfile for verb. This method can be overridden in a subclass. - :param package_build_base: Base build path for package + :param args: Args with respective base path + :param pkg_name: Name for package :returns: A lockfile, or None """ - return get_previous_lockfile(package_build_base, self.verb_name) + reference_base_path = getattr(args, self.reference_name + '_base') + package_reference_base = os.path.join(reference_base_path, pkg_name) + return get_previous_lockfile( + package_reference_base, self.verb_name) - def get_reference_lockfile(self, package_build_base): + def get_reference_lockfile(self, args, pkg_name): """ Get reference lockfile for verb. This method can be overridden in a subclass. - :param package_build_base: Base build path for package + :param args: Args with respective base path + :param pkg_name: Name for package :returns: A lockfile, or None """ - return get_previous_lockfile(package_build_base, self.reference_name) + reference_base_path = getattr(args, self.reference_name + '_base') + package_reference_base = os.path.join(reference_base_path, pkg_name) + return get_previous_lockfile( + package_reference_base, self.reference_name) def get_job_lockfile(self, job): """ @@ -57,7 +78,24 @@ def get_job_lockfile(self, job): :param jobs: The job from `event[1]` :returns: A lockfile, or None """ - return self.get_reference_lockfile(job.task_context.args.build_base) + reference_pkg_base_path = getattr( + job.task_context.args, self.reference_name + '_base') + return get_previous_lockfile( + reference_pkg_base_path, self.reference_name) + + +def add_verb_handler_arguments(parser): + """ + Add the command line arguments for the verb handler extensions. + + :param parser: The argument parser + """ + group = parser.add_argument_group(title='Verb handler arguments') + extensions = get_verb_handler_extensions() + + for key in sorted(extensions.keys()): + extension = extensions[key] + extension.add_arguments(parser=group) def get_verb_handler_extensions(): diff --git a/colcon_cache/verb_handler/build.py b/colcon_cache/verb_handler/build.py index b2a0c8d..f2aeaaf 100644 --- a/colcon_cache/verb_handler/build.py +++ b/colcon_cache/verb_handler/build.py @@ -4,6 +4,7 @@ from colcon_cache.verb_handler import VerbHandlerExtensionPoint from colcon_core.plugin_system import satisfies_version +BASE_PATH = 'build' VERB_NAME = 'build' REFERENCE_NAME = 'cache' @@ -12,6 +13,13 @@ class BuildVerbHandler(VerbHandlerExtensionPoint): """Determin how lockfiles for the build verb should be handled.""" def __init__(self): # noqa: D107 - super().__init__(VERB_NAME, REFERENCE_NAME) + super().__init__(BASE_PATH, VERB_NAME, REFERENCE_NAME) satisfies_version( VerbHandlerExtensionPoint.EXTENSION_POINT_VERSION, '^1.0') + + def add_arguments(self, *, parser): # noqa: D102 + parser.add_argument( + '--build-base', + default=self.base_path, + help='The base path for all build directories ' + '(default: {self.base_path})'.format_map(locals())) diff --git a/colcon_cache/verb_handler/cache.py b/colcon_cache/verb_handler/cache.py index f45848e..3716ebc 100644 --- a/colcon_cache/verb_handler/cache.py +++ b/colcon_cache/verb_handler/cache.py @@ -4,6 +4,7 @@ from colcon_cache.verb_handler import VerbHandlerExtensionPoint from colcon_core.plugin_system import satisfies_version +BASE_PATH = 'cache' VERB_NAME = 'cache' REFERENCE_NAME = None @@ -12,11 +13,18 @@ class CacheVerbHandler(VerbHandlerExtensionPoint): """Determin how lockfiles for the cache verb should be handled.""" def __init__(self): # noqa: D107 - super().__init__(VERB_NAME, REFERENCE_NAME) + super().__init__(BASE_PATH, VERB_NAME, REFERENCE_NAME) satisfies_version( VerbHandlerExtensionPoint.EXTENSION_POINT_VERSION, '^1.0') - def get_reference_lockfile(self, package_build_base): # noqa: D102 + def add_arguments(self, *, parser): # noqa: D102 + parser.add_argument( + '--cache-base', + default=self.base_path, + help='The base path for all cache directories ' + '(default: {self.base_path})'.format_map(locals())) + + def get_reference_lockfile(self, args, pkg_name): # noqa: D102 return None def get_job_lockfile(self, job): # noqa: D102 diff --git a/colcon_cache/verb_handler/list.py b/colcon_cache/verb_handler/list.py new file mode 100644 index 0000000..7dabb2b --- /dev/null +++ b/colcon_cache/verb_handler/list.py @@ -0,0 +1,27 @@ +# Copyright 2021 Ruffin White +# Licensed under the Apache License, Version 2.0 + +from colcon_cache.verb_handler import VerbHandlerExtensionPoint +from colcon_core.plugin_system import satisfies_version + +BASE_PATH = None +VERB_NAME = None +REFERENCE_NAME = None + + +class ListVerbHandler(VerbHandlerExtensionPoint): + """Determin how lockfiles for the list verb should be handled.""" + + def __init__(self): # noqa: D107 + super().__init__(BASE_PATH, VERB_NAME, REFERENCE_NAME) + satisfies_version( + VerbHandlerExtensionPoint.EXTENSION_POINT_VERSION, '^1.0') + + def add_arguments(self, *, parser): # noqa: D102 + pass + + def get_reference_lockfile(self, args, pkg_name): # noqa: D102 + return None + + def get_job_lockfile(self, job): # noqa: D102 + return None diff --git a/colcon_cache/verb_handler/test.py b/colcon_cache/verb_handler/test.py index f7afe62..d7bdc03 100644 --- a/colcon_cache/verb_handler/test.py +++ b/colcon_cache/verb_handler/test.py @@ -4,6 +4,7 @@ from colcon_cache.verb_handler import VerbHandlerExtensionPoint from colcon_core.plugin_system import satisfies_version +BASE_PATH = 'build' VERB_NAME = 'test' REFERENCE_NAME = 'build' @@ -12,6 +13,13 @@ class TestVerbHandler(VerbHandlerExtensionPoint): """Determin how lockfiles for the test verb should be handled.""" def __init__(self): # noqa: D107 - super().__init__(VERB_NAME, REFERENCE_NAME) + super().__init__(BASE_PATH, VERB_NAME, REFERENCE_NAME) satisfies_version( VerbHandlerExtensionPoint.EXTENSION_POINT_VERSION, '^1.0') + + def add_arguments(self, *, parser): # noqa: D102 + parser.add_argument( + '--test-result-base', + default=self.base_path, + help='The base path for all test_result directories ' + '(default: {self.base_path})'.format_map(locals())) diff --git a/setup.cfg b/setup.cfg index 584e3d6..a965cb7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -68,6 +68,8 @@ filterwarnings = junit_suite_name = colcon-cache [options.entry_points] +colcon_clean.base_handler = + cache = colcon_cache.base_handler.cache:CacheBaseHandler colcon_core.event_handler = cache_lockfile = colcon_cache.event_handler.lockfile:LockfileEventHandler colcon_core.extension_point = @@ -91,7 +93,7 @@ colcon_cache.task.lock = colcon_cache.verb_handler = build = colcon_cache.verb_handler.build:BuildVerbHandler cache = colcon_cache.verb_handler.cache:CacheVerbHandler - list = colcon_cache.verb_handler.cache:CacheVerbHandler + list = colcon_cache.verb_handler.list:ListVerbHandler test = colcon_cache.verb_handler.test:TestVerbHandler [flake8] diff --git a/test/package_selection/test_valid.py b/test/package_selection/test_valid.py index 92a91c8..6880fef 100644 --- a/test/package_selection/test_valid.py +++ b/test/package_selection/test_valid.py @@ -18,11 +18,13 @@ def test_valid(): # TODO: check event log for warning by mocking logger args = Object() + args.packages_select_cache_key = 'cache' args.packages_select_cache_invalid = True args.packages_skip_cache_valid = False valid_package_selection.select_packages(args, decorators) args = Object() + args.packages_select_cache_key = 'cache' args.packages_select_cache_invalid = False args.packages_skip_cache_valid = True valid_package_selection.select_packages(args, decorators)