Merge pull request borgbackup#9272 from ThomasWaldmann/windows-ci

windows CI

Author: ThomasWaldmann

.github/workflows/ci.yml

@@ -544,10 +544,10 @@ jobs:
 
   windows_tests:
 
-    if: false  # can be used to temporarily disable the build
+    if: true  # can be used to temporarily disable the build
     runs-on: windows-latest
-    timeout-minutes: 120
-    needs: native_tests
+    timeout-minutes: 90
+    needs: [lint, security]
 
     env:
       PY_COLORS: 1
@@ -583,14 +583,15 @@ jobs:
           # build borg.exe
           . env/bin/activate
           pip install -e .
-          pyinstaller -y scripts/borg.exe.spec
+          mkdir -p dist/binary
+          pyinstaller -y --clean --distpath=dist/binary scripts/borg.exe.spec
           # build sdist and wheel in dist/...
           python -m build
 
       - uses: actions/upload-artifact@v4
         with:
           name: borg-windows
-          path: dist/borg.exe
+          path: dist/binary/borg.exe
 
       - name: Run tests
         run: |

src/borg/helpers/parseformat.py

@@ -18,6 +18,7 @@
 from string import Formatter
 
 from ..logger import create_logger
+from ..platformflags import is_win32
 
 logger = create_logger()
 
@@ -453,8 +454,8 @@ class Location:
         (?P<path>.+)
     """
 
-    # abs_path must start with a slash.
-    abs_path_re = r"(?P<path>/.+)"
+    # abs_path must start with a slash (or drive letter on Windows).
+    abs_path_re = r"(?P<path>[A-Za-z]:/.+)" if is_win32 else r"(?P<path>/.+)"
 
     # path may or may not start with a slash.
     abs_or_rel_path_re = r"(?P<path>.+)"
@@ -493,7 +494,8 @@ class Location:
 
     rclone_re = re.compile(r"(?P<proto>rclone):(?P<path>(.*))", re.VERBOSE)
 
-    file_or_socket_re = re.compile(r"(?P<proto>(file|socket))://" + abs_path_re, re.VERBOSE)
+    sl = "/" if is_win32 else ""
+    file_or_socket_re = re.compile(r"(?P<proto>(file|socket))://" + sl + abs_path_re, re.VERBOSE)
 
     local_re = re.compile(local_path_re, re.VERBOSE)
 

src/borg/legacyrepository.py

@@ -27,6 +27,7 @@
 from .repoobj import RepoObj
 from .checksums import crc32, StreamingXXH64
 from .crypto.file_integrity import IntegrityCheckedFile, FileIntegrityError
+from .repository import _local_abspath_to_file_url
 
 logger = create_logger(__name__)
 
@@ -191,7 +192,7 @@ class PathPermissionDenied(Error):
 
     def __init__(self, path, create=False, exclusive=False, lock_wait=None, lock=True, send_log_cb=None):
         self.path = os.path.abspath(path)
-        self._location = Location("file://%s" % self.path)
+        self._location = Location(_local_abspath_to_file_url(self.path))
         self.version = None
         # long-running repository methods which emit log or progress output are responsible for calling
         # the ._send_log method periodically to get log and progress output transferred to the borg client

src/borg/repository.py

@@ -106,7 +106,7 @@ def __init__(
         if isinstance(path_or_location, Location):
             location = path_or_location
             if location.proto == "file":
-                url = _local_abspath_to_file_url(location.path)  # frequently users give without file:// prefix
+                url = _local_abspath_to_file_url(location.path)
             else:
                 url = location.processed  # location as given by user, processed placeholders
         else:

src/borg/testsuite/archiver/diff_cmd_test.py

@@ -428,8 +428,8 @@ def test_sort_by_all_keys_with_directions(archivers, request, sort_key):
 
 
 @pytest.mark.skipif(
-    not are_hardlinks_supported() or is_freebsd or is_netbsd,
-    reason="hardlinks not supported or test failing on freebsd and netbsd",
+    not are_hardlinks_supported() or is_freebsd or is_netbsd or is_win32,
+    reason="hardlinks not supported or test failing on freebsd, netbsd and windows",
 )
 def test_hard_link_deletion_and_replacement(archivers, request):
     archiver = request.getfixturevalue(archivers)

src/borg/testsuite/archiver/lock_cmds_test.py

@@ -8,7 +8,8 @@
 from ...constants import *  # NOQA
 from . import cmd, generate_archiver_tests, RK_ENCRYPTION
 from ...helpers import CommandError
-from ...platformflags import is_haiku
+from ...platformflags import is_haiku, is_win32
+from ...repository import _local_abspath_to_file_url
 
 pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary")  # NOQA
 
@@ -19,11 +20,11 @@ def test_break_lock(archivers, request):
     cmd(archiver, "break-lock")
 
 
-@pytest.mark.skipif(is_haiku, reason="does not find borg python module on Haiku OS")
+@pytest.mark.skipif(is_haiku or is_win32, reason="does not find borg python module on Haiku OS and Windows")
 def test_with_lock(tmp_path):
     repo_path = tmp_path / "repo"
     env = os.environ.copy()
-    env["BORG_REPO"] = "file://" + str(repo_path)
+    env["BORG_REPO"] = _local_abspath_to_file_url(str(repo_path.absolute()))
     # test debug output:
     print("sys.path: %r" % sys.path)
     print("PYTHONPATH: %s" % env.get("PYTHONPATH", ""))

src/borg/testsuite/fslocking_test.py

@@ -20,6 +20,7 @@
     NotLocked,
     NotMyLock,
 )
+from ..platformflags import is_win32
 
 ID1 = "foo", 1, 1
 ID2 = "bar", 2, 2
@@ -105,6 +106,7 @@ def test_migrate_lock(self, lockpath):
         assert lock.by_me()  # we still have the lock
         assert old_unique_name != new_unique_name  # Locking filename is different now.
 
+    @pytest.mark.skipif(is_win32, reason="broken on windows")
     def test_race_condition(self, lockpath):
         class SynchronizedCounter:
             def __init__(self, count=0):

src/borg/testsuite/helpers/fs_test.py

@@ -375,19 +375,20 @@ def open_dir(path):
     assert dir_is_tagged(path=str(normal_dir), exclude_caches=False) == []
 
     # Test 2: exclude_caches with file-descriptor-based operations
-    with open_dir(str(cache_dir)) as fd:
-        assert dir_is_tagged(dir_fd=fd, exclude_caches=True) == [CACHE_TAG_NAME]
-    with open_dir(str(invalid_cache_dir)) as fd:
-        assert dir_is_tagged(dir_fd=fd, exclude_caches=True) == []
-    with open_dir(str(normal_dir)) as fd:
-        assert dir_is_tagged(dir_fd=fd, exclude_caches=True) == []
-
-    with open_dir(str(cache_dir)) as fd:
-        assert dir_is_tagged(dir_fd=fd, exclude_caches=False) == []
-    with open_dir(str(invalid_cache_dir)) as fd:
-        assert dir_is_tagged(dir_fd=fd, exclude_caches=False) == []
-    with open_dir(str(normal_dir)) as fd:
-        assert dir_is_tagged(dir_fd=fd, exclude_caches=False) == []
+    if not is_win32:
+        with open_dir(str(cache_dir)) as fd:
+            assert dir_is_tagged(dir_fd=fd, exclude_caches=True) == [CACHE_TAG_NAME]
+        with open_dir(str(invalid_cache_dir)) as fd:
+            assert dir_is_tagged(dir_fd=fd, exclude_caches=True) == []
+        with open_dir(str(normal_dir)) as fd:
+            assert dir_is_tagged(dir_fd=fd, exclude_caches=True) == []
+
+        with open_dir(str(cache_dir)) as fd:
+            assert dir_is_tagged(dir_fd=fd, exclude_caches=False) == []
+        with open_dir(str(invalid_cache_dir)) as fd:
+            assert dir_is_tagged(dir_fd=fd, exclude_caches=False) == []
+        with open_dir(str(normal_dir)) as fd:
+            assert dir_is_tagged(dir_fd=fd, exclude_caches=False) == []
 
     # Test 3: exclude_if_present with path-based operations
     tags = [".NOBACKUP"]
@@ -401,21 +402,22 @@ def open_dir(path):
     assert dir_is_tagged(path=str(normal_dir), exclude_if_present=tags) == []
 
     # Test 4: exclude_if_present with file descriptor-based operations
-    tags = [".NOBACKUP"]
-    with open_dir(str(tagged_dir)) as fd:
-        assert dir_is_tagged(dir_fd=fd, exclude_if_present=tags) == [".NOBACKUP"]
-    with open_dir(str(other_tagged_dir)) as fd:
-        assert dir_is_tagged(dir_fd=fd, exclude_if_present=tags) == []
-    with open_dir(str(normal_dir)) as fd:
-        assert dir_is_tagged(dir_fd=fd, exclude_if_present=tags) == []
-
-    tags = [".NOBACKUP", ".DONOTBACKUP"]
-    with open_dir(str(tagged_dir)) as fd:
-        assert dir_is_tagged(dir_fd=fd, exclude_if_present=tags) == [".NOBACKUP"]
-    with open_dir(str(other_tagged_dir)) as fd:
-        assert dir_is_tagged(dir_fd=fd, exclude_if_present=tags) == [".DONOTBACKUP"]
-    with open_dir(str(normal_dir)) as fd:
-        assert dir_is_tagged(dir_fd=fd, exclude_if_present=tags) == []
+    if not is_win32:
+        tags = [".NOBACKUP"]
+        with open_dir(str(tagged_dir)) as fd:
+            assert dir_is_tagged(dir_fd=fd, exclude_if_present=tags) == [".NOBACKUP"]
+        with open_dir(str(other_tagged_dir)) as fd:
+            assert dir_is_tagged(dir_fd=fd, exclude_if_present=tags) == []
+        with open_dir(str(normal_dir)) as fd:
+            assert dir_is_tagged(dir_fd=fd, exclude_if_present=tags) == []
+
+        tags = [".NOBACKUP", ".DONOTBACKUP"]
+        with open_dir(str(tagged_dir)) as fd:
+            assert dir_is_tagged(dir_fd=fd, exclude_if_present=tags) == [".NOBACKUP"]
+        with open_dir(str(other_tagged_dir)) as fd:
+            assert dir_is_tagged(dir_fd=fd, exclude_if_present=tags) == [".DONOTBACKUP"]
+        with open_dir(str(normal_dir)) as fd:
+            assert dir_is_tagged(dir_fd=fd, exclude_if_present=tags) == []
 
     # Test 5: both exclude types with path-based operations
     assert sorted(dir_is_tagged(path=str(both_dir), exclude_caches=True, exclude_if_present=[".NOBACKUP"])) == [
@@ -427,14 +429,15 @@ def open_dir(path):
     assert dir_is_tagged(path=str(normal_dir), exclude_caches=True, exclude_if_present=[".NOBACKUP"]) == []
 
     # Test 6: both exclude types with file descriptor-based operations
-    with open_dir(str(both_dir)) as fd:
-        assert sorted(dir_is_tagged(dir_fd=fd, exclude_caches=True, exclude_if_present=[".NOBACKUP"])) == [
-            ".NOBACKUP",
-            CACHE_TAG_NAME,
-        ]
-    with open_dir(str(cache_dir)) as fd:
-        assert dir_is_tagged(dir_fd=fd, exclude_caches=True, exclude_if_present=[".NOBACKUP"]) == [CACHE_TAG_NAME]
-    with open_dir(str(tagged_dir)) as fd:
-        assert dir_is_tagged(dir_fd=fd, exclude_caches=True, exclude_if_present=[".NOBACKUP"]) == [".NOBACKUP"]
-    with open_dir(str(normal_dir)) as fd:
-        assert dir_is_tagged(dir_fd=fd, exclude_caches=True, exclude_if_present=[".NOBACKUP"]) == []
+    if not is_win32:
+        with open_dir(str(both_dir)) as fd:
+            assert sorted(dir_is_tagged(dir_fd=fd, exclude_caches=True, exclude_if_present=[".NOBACKUP"])) == [
+                ".NOBACKUP",
+                CACHE_TAG_NAME,
+            ]
+        with open_dir(str(cache_dir)) as fd:
+            assert dir_is_tagged(dir_fd=fd, exclude_caches=True, exclude_if_present=[".NOBACKUP"]) == [CACHE_TAG_NAME]
+        with open_dir(str(tagged_dir)) as fd:
+            assert dir_is_tagged(dir_fd=fd, exclude_caches=True, exclude_if_present=[".NOBACKUP"]) == [".NOBACKUP"]
+        with open_dir(str(normal_dir)) as fd:
+            assert dir_is_tagged(dir_fd=fd, exclude_caches=True, exclude_if_present=[".NOBACKUP"]) == []

src/borg/testsuite/helpers/parseformat_test.py

@@ -26,6 +26,7 @@
     ChunkerParams,
 )
 from ...helpers.time import format_timedelta, parse_timestamp
+from ...platformflags import is_win32
 
 
 def test_bin_to_hex():
@@ -194,31 +195,31 @@ def test_sftp(self, monkeypatch, keys_dir):
 
     def test_socket(self, monkeypatch, keys_dir):
         monkeypatch.delenv("BORG_REPO", raising=False)
+        url = "socket:///c:/repo/path" if is_win32 else "socket:///repo/path"
+        path = "c:/repo/path" if is_win32 else "/repo/path"
         assert (
-            repr(Location("socket:///repo/path"))
-            == "Location(proto='socket', user=None, pass=None, host=None, port=None, path='/repo/path')"
+            repr(Location(url))
+            == f"Location(proto='socket', user=None, pass=None, host=None, port=None, path='{path}')"
         )
-        assert Location("socket:///some/path").to_key_filename() == keys_dir + "_some_path"
+        assert Location(url).to_key_filename().endswith("_repo_path")
 
     def test_file(self, monkeypatch, keys_dir):
         monkeypatch.delenv("BORG_REPO", raising=False)
+        url = "file:///c:/repo/path" if is_win32 else "file:///repo/path"
+        path = "c:/repo/path" if is_win32 else "/repo/path"
         assert (
-            repr(Location("file:///some/path"))
-            == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/some/path')"
+            repr(Location(url)) == f"Location(proto='file', user=None, pass=None, host=None, port=None, path='{path}')"
         )
-        assert (
-            repr(Location("file:///some/path"))
-            == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/some/path')"
-        )
-        assert Location("file:///some/path").to_key_filename() == keys_dir + "_some_path"
+        assert Location(url).to_key_filename().endswith("_repo_path")
 
+    @pytest.mark.skipif(is_win32, reason="still broken")
     def test_smb(self, monkeypatch, keys_dir):
         monkeypatch.delenv("BORG_REPO", raising=False)
         assert (
             repr(Location("file:////server/share/path"))
             == "Location(proto='file', user=None, pass=None, host=None, port=None, path='//server/share/path')"
         )
-        assert Location("file:////server/share/path").to_key_filename() == keys_dir + "__server_share_path"
+        assert Location("file:////server/share/path").to_key_filename().endswith("__server_share_path")
 
     def test_folder(self, monkeypatch, keys_dir):
         monkeypatch.delenv("BORG_REPO", raising=False)
@@ -230,6 +231,7 @@ def test_folder(self, monkeypatch, keys_dir):
         )
         assert Location("path").to_key_filename().endswith(rel_path)
 
+    @pytest.mark.skipif(is_win32, reason="Windows has drive letters in abs paths")
     def test_abspath(self, monkeypatch, keys_dir):
         monkeypatch.delenv("BORG_REPO", raising=False)
         assert (
@@ -259,6 +261,7 @@ def test_relpath(self, monkeypatch, keys_dir):
         )
         assert Location("ssh://user@host/relative/path").to_key_filename() == keys_dir + "host__relative_path"
 
+    @pytest.mark.skipif(is_win32, reason="Windows does not support colons in paths")
     def test_with_colons(self, monkeypatch, keys_dir):
         monkeypatch.delenv("BORG_REPO", raising=False)
         assert (
@@ -281,9 +284,6 @@ def test_canonical_path(self, monkeypatch):
         monkeypatch.delenv("BORG_REPO", raising=False)
         locations = [
             "relative/path",
-            "/absolute/path",
-            "file:///absolute/path",
-            "socket:///absolute/path",
             "ssh://host/relative/path",
             "ssh://host//absolute/path",
             "ssh://user@host:1234/relative/path",
@@ -292,6 +292,9 @@ def test_canonical_path(self, monkeypatch):
             "sftp://user@host:1234/relative/path",
             "rclone:remote:path",
         ]
+        locations.insert(1, "c:/absolute/path" if is_win32 else "/absolute/path")
+        locations.insert(2, "file:///c:/absolute/path" if is_win32 else "file:///absolute/path")
+        locations.insert(3, "socket:///c:/absolute/path" if is_win32 else "socket:///absolute/path")
         for location in locations:
             assert (
                 Location(location).canonical_path() == Location(Location(location).canonical_path()).canonical_path()

src/borg/testsuite/storelocking_test.py

@@ -4,15 +4,17 @@
 
 from borgstore.store import Store
 
+from ..repository import _local_abspath_to_file_url
 from ..storelocking import Lock, NotLocked, LockTimeout
 
 ID1 = "foo", 1, 1
 ID2 = "bar", 2, 2
 
 
 @pytest.fixture()
-def lockstore(tmpdir):
-    store = Store("file://" + str(tmpdir / "lockstore"), levels={"locks/": [0]})
+def lockstore(tmp_path):
+    lockstore_path = tmp_path / "lockstore"
+    store = Store(_local_abspath_to_file_url(str(lockstore_path.absolute())), levels={"locks/": [0]})
     store.create()
     with store:
         yield store