diff --git a/README.md b/README.md index abd4575..e3c5903 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,8 @@ sources: - repo: "https://github.com/google/material-design-icons" name: material-design-icons rev: master + copies: + - target: Icons groups: - name: code diff --git a/docs/setup/git.md b/docs/setup/git.md index f284743..901a983 100644 --- a/docs/setup/git.md +++ b/docs/setup/git.md @@ -52,7 +52,7 @@ The token can also be written to `.netrc` during builds, see the guide for [Trav ## Symlinks on Windows -If you're using Windows, there are some additional prerequisites to ensure Gitman works seamlessly with symbolic links. +If you're using Windows, there are some additional prerequisites to ensure Gitman works seamlessly with symbolic links. You could consider using copies instead. ### Grant Permissions diff --git a/docs/use-cases/multiple-copies.md b/docs/use-cases/multiple-copies.md new file mode 100644 index 0000000..93360d6 --- /dev/null +++ b/docs/use-cases/multiple-copies.md @@ -0,0 +1,48 @@ +# Using Multiple Copies + +This feature can be used to create as many copies as you need from one repository. This can be helpful e.g. on Windows where you need administrator priviledges in order to create a symbolic link. + +## The Syntax + +Let's say we have a simple project structure: + +```text +|- include +|- src +|- docs +``` + +with the following `gitman.yml`: + +```yaml +location: .gitman + +sources: + - repo: + name: my_dependency + rev: v1.0.3 + copies: + - source: include + target: vendor/partial_repo + - target: vendor/full_repo +``` + +This will result in the following copies: + +- `/.gitman/my_dependency/include` -> `/vendor/partial_repo` +- `/.gitman/my_dependency` -> `/vendor/full_repo` + +## Alternative Syntax + +```yaml +location: vendor + +sources: + - repo: + name: my_dependency + rev: v1.0.3 + copies: + - { source: src, target: partial_repo } + - { target: full_repo } +``` + diff --git a/gitman.yml b/gitman.yml index 8e69277..fd0bfb4 100644 --- a/gitman.yml +++ b/gitman.yml @@ -10,6 +10,8 @@ sources: links: - source: '' target: demo/example + copies: + - scripts: - cat .noserc - make foobar @@ -22,6 +24,9 @@ sources: - links: - + copies: + - source: '' + target: demo/example scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -33,6 +38,8 @@ sources: - links: - + copies: + - scripts: - echo "Hello, World!" - pwd @@ -45,6 +52,8 @@ sources: - links: - + copies: + - scripts: - sources_locked: @@ -57,6 +66,8 @@ sources_locked: - links: - + copies: + - scripts: - cat .noserc - make foobar @@ -69,6 +80,8 @@ sources_locked: - links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -80,6 +93,8 @@ sources_locked: - links: - + copies: + - scripts: - echo "Hello, World!" - pwd @@ -92,6 +107,8 @@ sources_locked: - links: - + copies: + - scripts: - groups: diff --git a/gitman/models/config.py b/gitman/models/config.py index 57e23fe..993bbe7 100644 --- a/gitman/models/config.py +++ b/gitman/models/config.py @@ -116,6 +116,8 @@ def install_dependencies( assert self.root, f"Missing root: {self}" source.create_links(self.root, force=force) common.newline() + source.create_copies(self.root, force=force) + common.newline() count += 1 config = load_config(search=False) diff --git a/gitman/models/source.py b/gitman/models/source.py index 886fbf7..c62a519 100644 --- a/gitman/models/source.py +++ b/gitman/models/source.py @@ -16,6 +16,12 @@ class Link: target: str = "" +@dataclass +class Copy: + source: str = "" + target: str = "" + + @dataclass class Source: """Represents a repository to clone and options for controlling checkout. @@ -29,6 +35,7 @@ class Source: | `params` | Additional arguments for `clone` | No | `null` | | `sparse_paths` | Controls partial checkout | No | `[]` | | `links` | Creates symlinks within a project | No | `[]` | + | `copies` | Creates copies within of a file or folder | No | `[]` | | `scripts` | Shell commands to run after checkout | No | `[]` | ### Params @@ -51,6 +58,10 @@ class Source: See [using multiple links][using-multiple-links] for more information. + ### Copies + + See [using multiple copies][using-multiple-copies] for more information. + ### Scripts Scripts can be used to run post-checkout commands such us build steps. For example: @@ -72,6 +83,7 @@ class Source: params: Optional[str] = None sparse_paths: List[str] = field(default_factory=list) links: List[Link] = field(default_factory=list) + copies: List[Copy] = field(default_factory=list) scripts: List[str] = field(default_factory=list) @@ -205,6 +217,19 @@ def create_links(self, root: str, *, force: bool = False): source = os.path.join(relpath, os.path.normpath(link.source)) create_sym_link(source, target, force=force) + def create_copies(self, root: str, *, force: bool = False): + """Create copies from source to target.""" + if not self.copies: + return + + for copy in self.copies: + target = os.path.join(root, os.path.normpath(copy.target)) + relpath = os.path.relpath(os.getcwd(), os.path.dirname(target)) + source = os.path.join( + root, os.path.join(relpath, copy.source) if copy.source else relpath + ) + create_copy(source, target, force=force) + def run_scripts(self, force: bool = False, show_shell_stdout: bool = False): log.info("Running install scripts...") @@ -317,6 +342,7 @@ def lock( name=self.name, rev=rev, links=self.links, + copies=self.copies, scripts=self.scripts, sparse_paths=self.sparse_paths, ) @@ -349,3 +375,24 @@ def create_sym_link(source: str, target: str, *, force: bool): raise exceptions.UncommittedChanges(msg) shell.ln(source, target) + + +def create_copy(source: str, target: str, *, force: bool): + log.info("Creating a copy...") + + if os.path.islink(target): + os.remove(target) + elif os.path.exists(target): + if force: + shell.rm(target) + elif os.path.isfile(target) and os.path.isdir(source): + msg = "Preexisting file location to be replaced by folder at {}".format( + target + ) + raise exceptions.UncommittedChanges(msg) + elif os.path.isfile(target) and os.path.isfile(source): + shell.rm(target) + else: + msg = "Preexisting target location at {}".format(target) + log.warn(msg) + shell.cp(source, target) diff --git a/gitman/shell.py b/gitman/shell.py index 336e47e..3b390af 100644 --- a/gitman/shell.py +++ b/gitman/shell.py @@ -1,6 +1,7 @@ """Utilities to call shell programs.""" import os +import shutil import subprocess import log @@ -127,6 +128,16 @@ def ln(source, target): os.symlink(source, target) +def cp(source, target): + dirpath = os.path.dirname(target) + if not os.path.isdir(dirpath): + mkdir(dirpath) + if os.path.isdir(source): + shutil.copytree(src=source, dst=target, dirs_exist_ok=True) + else: + shutil.copy2(src=source, dst=target) + + def rm(path): if os.name == "nt": if os.path.isfile(path): diff --git a/mkdocs.yml b/mkdocs.yml index d1acf1e..911e0af 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -36,6 +36,7 @@ nav: - Sparse Checkouts: use-cases/sparse-checkouts.md - Default Groups: use-cases/default-groups.md - Multiple Links: use-cases/multiple-links.md + - Multiple Copies: use-cases/multiple-copies.md - Extras: - Git SVN Bridge: extras/git-svn-bridge.md - Bundled Application: extras/bundled-application.md diff --git a/tests/test_api.py b/tests/test_api.py index 623b647..c93a664 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -32,6 +32,8 @@ - links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -43,6 +45,8 @@ - links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -54,6 +58,8 @@ - links: - + copies: + - scripts: - sources_locked: @@ -105,6 +111,8 @@ def it_creates_a_new_config_file(tmpdir): - links: - + copies: + - scripts: - sources_locked: @@ -117,6 +125,8 @@ def it_creates_a_new_config_file(tmpdir): - links: - + copies: + - scripts: - default_group: '' @@ -156,6 +166,8 @@ def it_merges_sources(config): rev: main links: - + copies: + - scripts: - sources_locked: @@ -165,6 +177,8 @@ def it_merges_sources(config): rev: example-branch links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -173,6 +187,8 @@ def it_merges_sources(config): rev: 7bd138fe7359561a8c2ff9d195dff238794ccc04 links: - + copies: + - scripts: - """ @@ -202,6 +218,8 @@ def it_can_handle_missing_locked_sources(config): rev: example-branch links: - + copies: + - scripts: - sources_locked: @@ -211,6 +229,8 @@ def it_can_handle_missing_locked_sources(config): rev: 7bd138fe7359561a8c2ff9d195dff238794ccc04 links: - + copies: + - scripts: - """ @@ -244,6 +264,8 @@ def config_with_link(config): rev: 7bd138fe7359561a8c2ff9d195dff238794ccc04 links: - target: my_link + copies: + - scripts: - """ @@ -290,6 +312,8 @@ def config_with_links(config): target: gmd_3 - source: gitman_sources/gmd_4 target: gmd_4 + copies: + - scripts: - """ @@ -321,6 +345,109 @@ def it_overwrites_files_with_force(config_with_links): expect(gitman.install(depth=1, force=True)) == True + def describe_copies(): + @pytest.fixture + def config_with_copy(config): + config.datafile.text = strip( + """ + location: deps + sources: + - name: gitman_1 + repo: https://github.com/jacebrowning/gitman-demo + rev: 954e166c17d61935037fbd0799fbe0176c29df10 + links: + - + copies: + - target: my_copy + - source: gdm/common.py + target: libCommon.py + scripts: + - + """ + ) + config.datafile.load() + + return config + + def it_should_create_copies(config_with_copy): + expect(gitman.install(depth=1)) == True + + expect(os.listdir()).contains("my_copy") + expect(os.listdir()).contains("libCommon.py") + + def it_should_overwrite_files(config_with_copy): + os.system("touch libCommon.py") + expect(os.listdir()).contains("libCommon.py") + + def it_should_merge_non_empty_directories(config_with_copy): + os.system("mkdir my_copy") + os.system("touch my_copy/my_copy") + + expect(os.listdir("my_copy")).contains("my_copy") + + def it_overwrites_folders_with_force(config_with_copy): + os.system("touch my_copy") + + expect(gitman.install(depth=1, force=True)) == True + + def describe_multi_copies(): + @pytest.fixture + def config_with_copies(config): + config.datafile.text = strip( + """ + location: deps + sources: + - name: gitman_1 + repo: https://github.com/jacebrowning/gitman-demo + rev: e6e7595cef573589bcfbbf6c9398b7c166cdda9d + links: + - + copies: + - source: gdm/test + target: gdm_test + - source: gdm/commands.py + target: gdmCommands.py + scripts: + - + """ + ) + config.datafile.load() + + return config + + def it_should_create_copies(config_with_copies): + expect(gitman.install(depth=1)) == True + expect(os.listdir()).contains("gdm_test") + expect(os.listdir()).contains("gdmCommands.py") + + def it_should_not_overwrite_files_with_folders(config_with_copies): + os.system("touch gdm_test") + + with pytest.raises(RuntimeError): + gitman.install(depth=1) + + def it_should_merge_with_non_empty_directories(config_with_copies): + os.system("mkdir gdm_test") + os.system("touch gdm_test/my_copy") + + expect(gitman.install(depth=1)) == True + expect(os.listdir("gdm_test")).contains("my_copy") + + def it_should_overwrite_files(config_with_copies): + os.system("mkdir gdm_test") + os.system("touch gdm_test/test_all.py") + os.system("touch gdmCommands.py") + + expect(gitman.install(depth=1)) == True + expect(os.listdir("gdm_test")).contains("test_all.py") + expect(os.listdir()).contains("gdmCommands.py") + + def it_overwrites_files_with_force(config_with_copies): + os.system("mkdir gdm_test") + os.system("touch gdm_test/my_copy") + expect(gitman.install(depth=1, force=True)) == True + expect("my_copy" not in os.listdir("gdm_test")) == True + def describe_scripts(): @pytest.fixture def config_with_scripts(config): @@ -335,6 +462,8 @@ def config_with_scripts(config): rev: 7bd138fe7359561a8c2ff9d195dff238794ccc04 links: - + copies: + - scripts: - make foobar """ @@ -366,6 +495,8 @@ def config_with_scripts(config): rev: ddbe17ef173538d1fda29bd99a14bab3c5d86e78 links: - + copies: + - scripts: - """ @@ -401,6 +532,8 @@ def config_with_group(config): rev: example-branch links: - + copies: + - scripts: - - name: gitman_2 @@ -412,6 +545,8 @@ def config_with_group(config): rev: example-tag links: - + copies: + - scripts: - groups: @@ -465,6 +600,8 @@ def config_with_default_group(config): rev: example-branch links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -476,6 +613,8 @@ def config_with_default_group(config): rev: example-tag links: - + copies: + - scripts: - groups: @@ -507,6 +646,8 @@ def config_without_default_group(config): rev: example-branch links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -518,6 +659,8 @@ def config_without_default_group(config): rev: example-tag links: - + copies: + - scripts: - groups: @@ -648,6 +791,8 @@ def it_locks_previously_unlocked_dependencies(config): rev: example-branch links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -659,6 +804,8 @@ def it_locks_previously_unlocked_dependencies(config): rev: example-tag links: - + copies: + - scripts: - sources_locked: @@ -671,6 +818,8 @@ def it_locks_previously_unlocked_dependencies(config): rev: (old revision) links: - + copies: + - scripts: - groups: @@ -695,6 +844,8 @@ def it_locks_previously_unlocked_dependencies(config): - links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -706,6 +857,8 @@ def it_locks_previously_unlocked_dependencies(config): - links: - + copies: + - scripts: - sources_locked: @@ -718,6 +871,8 @@ def it_locks_previously_unlocked_dependencies(config): - links: - + copies: + - scripts: - groups: @@ -740,6 +895,8 @@ def it_should_not_lock_dependencies_when_disabled(config): - links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -751,6 +908,8 @@ def it_should_not_lock_dependencies_when_disabled(config): rev: example-tag links: - + copies: + - scripts: - sources_locked: @@ -763,6 +922,8 @@ def it_should_not_lock_dependencies_when_disabled(config): - links: - + copies: + - scripts: - groups: @@ -786,6 +947,8 @@ def it_should_not_lock_dependencies_when_disabled(config): - links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -797,6 +960,8 @@ def it_should_not_lock_dependencies_when_disabled(config): - links: - + copies: + - scripts: - sources_locked: @@ -809,6 +974,8 @@ def it_should_not_lock_dependencies_when_disabled(config): - links: - + copies: + - scripts: - groups: @@ -858,6 +1025,8 @@ def it_locks_previously_locked_dependencies_by_group_name(config): rev: example-branch links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -869,6 +1038,8 @@ def it_locks_previously_locked_dependencies_by_group_name(config): rev: example-tag links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -880,6 +1051,8 @@ def it_locks_previously_locked_dependencies_by_group_name(config): rev: example-tag links: - + copies: + - scripts: - sources_locked: @@ -892,6 +1065,8 @@ def it_locks_previously_locked_dependencies_by_group_name(config): rev: (old revision) links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -903,6 +1078,8 @@ def it_locks_previously_locked_dependencies_by_group_name(config): rev: (old revision) links: - + copies: + - scripts: - groups: @@ -930,6 +1107,8 @@ def it_locks_previously_locked_dependencies_by_group_name(config): - links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -941,6 +1120,8 @@ def it_locks_previously_locked_dependencies_by_group_name(config): - links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -952,6 +1133,8 @@ def it_locks_previously_locked_dependencies_by_group_name(config): - links: - + copies: + - scripts: - sources_locked: @@ -964,6 +1147,8 @@ def it_locks_previously_locked_dependencies_by_group_name(config): - links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -975,6 +1160,8 @@ def it_locks_previously_locked_dependencies_by_group_name(config): - links: - + copies: + - scripts: - groups: @@ -1016,6 +1203,8 @@ def git_changes( rev: example-tag links: - + copies: + - scripts: - sources_locked: @@ -1028,6 +1217,8 @@ def git_changes( rev: (old revision) links: - + copies: + - scripts: - groups: @@ -1051,6 +1242,8 @@ def git_changes( - links: - + copies: + - scripts: - sources_locked: @@ -1063,6 +1256,8 @@ def git_changes( - links: - + copies: + - scripts: - groups: @@ -1108,6 +1303,8 @@ def git_changes( rev: example-tag links: - + copies: + - scripts: - sources_locked: @@ -1120,6 +1317,8 @@ def git_changes( rev: (old revision) links: - + copies: + - scripts: - groups: @@ -1144,6 +1343,8 @@ def git_changes( - links: - + copies: + - scripts: - sources_locked: @@ -1156,6 +1357,8 @@ def git_changes( - links: - + copies: + - scripts: - groups: @@ -1176,6 +1379,8 @@ def it_merges_sources(config): rev: example-branch links: - + copies: + - scripts: - sources_locked: @@ -1185,6 +1390,8 @@ def it_merges_sources(config): rev: example-branch links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -1193,6 +1400,8 @@ def it_merges_sources(config): rev: 7bd138fe7359561a8c2ff9d195dff238794ccc04 links: - + copies: + - scripts: - """ @@ -1262,6 +1471,8 @@ def it_records_all_versions_when_no_arguments(config): - links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -1273,6 +1484,8 @@ def it_records_all_versions_when_no_arguments(config): - links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -1284,6 +1497,8 @@ def it_records_all_versions_when_no_arguments(config): - links: - + copies: + - scripts: - """ @@ -1308,6 +1523,8 @@ def it_records_specified_dependencies(config): - links: - + copies: + - scripts: - - repo: https://github.com/jacebrowning/gitman-demo @@ -1319,6 +1536,8 @@ def it_records_specified_dependencies(config): - links: - + copies: + - scripts: - """