Merge pull request borgbackup#9264 from ThomasWaldmann/path-sep-windows

path sep and windows related changes

Author: ThomasWaldmann

requirements.d/pyinstaller.txt

@@ -1 +1 @@
-pyinstaller==6.14.2
+pyinstaller==6.18.0

src/borg/archive.py

@@ -2,6 +2,7 @@
 import errno
 import json
 import os
+import posixpath
 import stat
 import sys
 import time
@@ -1243,8 +1244,8 @@ def __init__(
     @contextmanager
     def create_helper(self, path, st, status=None, hardlinkable=True, strip_prefix=None):
         if strip_prefix is not None:
-            assert not path.endswith(os.sep)
-            if strip_prefix.startswith(path + os.sep):
+            assert not path.endswith("/")
+            if strip_prefix.startswith(path + "/"):
                 # still on a directory level that shall be stripped - do not create an item for this!
                 yield None, "x", False, None
                 return
@@ -1547,7 +1548,7 @@ def s_to_ns(s):
 
             # if the tar has names starting with "./", normalize them like borg create also does.
             # ./dir/file must become dir/file in the borg archive.
-            normalized_path = os.path.normpath(tarinfo.name)
+            normalized_path = posixpath.normpath(tarinfo.name)
             item = Item(
                 path=make_path_safe(normalized_path),
                 mode=tarinfo.mode | type,
@@ -1608,7 +1609,7 @@ def process_symlink(self, *, tarinfo, status, type):
     def process_hardlink(self, *, tarinfo, status, type):
         with self.create_helper(tarinfo, status, type) as (item, status):
             # create a not hardlinked borg item, reusing the chunks, see HardLinkManager.__doc__
-            normalized_path = os.path.normpath(tarinfo.linkname)
+            normalized_path = posixpath.normpath(tarinfo.linkname)
             safe_path = make_path_safe(normalized_path)
             chunks = self.hlm.retrieve(safe_path)
             if chunks is not None:

src/borg/archiver/create_cmd.py

@@ -3,6 +3,7 @@
 import argparse
 import logging
 import os
+import posixpath
 import stat
 import subprocess
 import time
@@ -16,11 +17,11 @@
 from ..cache import Cache
 from ..constants import *  # NOQA
 from ..compress import CompressionSpec
-from ..helpers import comment_validator, ChunkerParams, PathSpec
+from ..helpers import comment_validator, ChunkerParams, FilesystemPathSpec
 from ..helpers import archivename_validator, FilesCacheMode
 from ..helpers import eval_escapes
 from ..helpers import timestamp, archive_ts_now
-from ..helpers import get_cache_dir, os_stat, get_strip_prefix
+from ..helpers import get_cache_dir, os_stat, get_strip_prefix, slashify
 from ..helpers import dir_is_tagged
 from ..helpers import log_multi
 from ..helpers import basic_json_data, json_print
@@ -106,8 +107,9 @@ def create_inner(archive, cache, fso):
                     pipe_bin = sys.stdin.buffer
                 pipe = TextIOWrapper(pipe_bin, errors="surrogateescape")
                 for path in iter_separated(pipe, paths_sep):
+                    path = slashify(path)
                     strip_prefix = get_strip_prefix(path)
-                    path = os.path.normpath(path)
+                    path = posixpath.normpath(path)
                     try:
                         with backup_io("stat"):
                             st = os_stat(path=path, parent_fd=None, name=None, follow_symlinks=False)
@@ -160,7 +162,7 @@ def create_inner(archive, cache, fso):
                         continue
 
                     strip_prefix = get_strip_prefix(path)
-                    path = os.path.normpath(path)
+                    path = posixpath.normpath(path)
                     try:
                         with backup_io("stat"):
                             st = os_stat(path=path, parent_fd=None, name=None, follow_symlinks=False)
@@ -489,7 +491,7 @@ def _rec_walk(
                                             path=path, fd=child_fd, st=st, strip_prefix=strip_prefix
                                         )
                                     for tag_name in tag_names:
-                                        tag_path = os.path.join(path, tag_name)
+                                        tag_path = posixpath.join(path, tag_name)
                                         self._rec_walk(
                                             path=tag_path,
                                             parent_fd=child_fd,
@@ -523,7 +525,7 @@ def _rec_walk(
                         with backup_io("scandir"):
                             entries = helpers.scandir_inorder(path=path, fd=child_fd)
                         for dirent in entries:
-                            normpath = os.path.normpath(os.path.join(path, dirent.name))
+                            normpath = posixpath.normpath(posixpath.join(path, dirent.name))
                             self._rec_walk(
                                 path=normpath,
                                 parent_fd=child_fd,
@@ -962,5 +964,5 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
 
         subparser.add_argument("name", metavar="NAME", type=archivename_validator, help="specify the archive name")
         subparser.add_argument(
-            "paths", metavar="PATH", nargs="*", type=PathSpec, action="extend", help="paths to archive"
+            "paths", metavar="PATH", nargs="*", type=FilesystemPathSpec, action="extend", help="paths to archive"
         )

src/borg/archiver/extract_cmd.py

@@ -1,7 +1,6 @@
 import sys
 import argparse
 import logging
-import os
 import stat
 
 from ._common import with_repository, with_archive
@@ -60,7 +59,7 @@ def do_extract(self, args, repository, manifest, archive):
         for item in archive.iter_items():
             orig_path = item.path
             if strip_components:
-                stripped_path = os.sep.join(orig_path.split(os.sep)[strip_components:])
+                stripped_path = "/".join(orig_path.split("/")[strip_components:])
                 if not stripped_path:
                     continue
                 item.path = stripped_path

src/borg/archiver/help_cmd.py

@@ -35,11 +35,14 @@ class HelpMixIn:
             start with ``src``.
           - When you back up relative paths like ``../../src``, the archived paths
             start with ``src``.
+          - On native Windows, archived absolute paths look like ``C/Windows/System32``.
 
         Borg supports different pattern styles. To define a non-default
         style for a specific pattern, prefix it with two characters followed
         by a colon ':' (i.e. ``fm:path/*``, ``sh:path/**``).
 
+        Note: Windows users must only use forward slashes in patterns, not backslashes.
+
         The default pattern style for ``--exclude`` differs from ``--pattern``, see below.
 
         `Fnmatch <https://docs.python.org/3/library/fnmatch.html>`_, selector ``fm:``
@@ -48,8 +51,8 @@ class HelpMixIn:
             any number of characters, '?' matching any single character, '[...]'
             matching any single character specified, including ranges, and '[!...]'
             matching any character not specified. For the purpose of these patterns,
-            the path separator (backslash for Windows and '/' on other systems) is not
-            treated specially. Wrap meta-characters in brackets for a literal
+            the path separator (forward slash '/') is not treated specially.
+            Wrap meta-characters in brackets for a literal
             match (i.e. ``[?]`` to match the literal character '?'). For a path
             to match a pattern, the full path must match, or it must match
             from the start of the full path to just before a path separator. Except
@@ -69,9 +72,7 @@ class HelpMixIn:
         `Regular expressions <https://docs.python.org/3/library/re.html>`_, selector ``re:``
             Unlike shell patterns, regular expressions are not required to match the full
             path and any substring match is sufficient. It is strongly recommended to
-            anchor patterns to the start ('^'), to the end ('$') or both. Path
-            separators (backslash for Windows and '/' on other systems) in paths are
-            always normalized to a forward slash '/' before applying a pattern.
+            anchor patterns to the start ('^'), to the end ('$') or both.
 
         Path prefix, selector ``pp:``
             This pattern style is useful to match whole subdirectories. The pattern
@@ -103,6 +104,20 @@ class HelpMixIn:
             cannot supply ``re:`` patterns. Further, ensure that ``sh:`` and
             ``fm:`` patterns only contain a handful of wildcards at most.
 
+        .. note::
+
+            **Windows path handling**: All paths in Borg archives use forward slashes (``/``)
+            as path separators, regardless of the platform. When creating archives on Windows,
+            backslashes from filesystem paths are automatically converted to forward slashes.
+
+        .. note::
+
+            **Windows reserved characters**: On Windows, when extracting archives created on
+            POSIX systems, paths may contain characters that are reserved from being used in
+            file or directory names (like: ``< > : " \\ | ? *``).
+            These are replaced by characters in the unicode private use area (``U+F0xx``) like
+            the CIFS mapchars feature also does it. It won't be pretty, but at least it works.
+
         Exclusions can be passed via the command line option ``--exclude``. When used
         from within a shell, the patterns should be quoted to protect them from
         expansion.

src/borg/helpers/__init__.py

@@ -20,15 +20,23 @@
 from .fs import ensure_dir, join_base_dir, get_socket_filename
 from .fs import get_security_dir, get_keys_dir, get_base_dir, get_cache_dir, get_config_dir, get_runtime_dir
 from .fs import dir_is_tagged, dir_is_cachedir, remove_dotdot_prefixes, make_path_safe, scandir_inorder
-from .fs import secure_erase, safe_unlink, dash_open, os_open, os_stat, get_strip_prefix, umount
+from .fs import secure_erase, safe_unlink, dash_open, os_open, os_stat, get_strip_prefix, umount, slashify
 from .fs import O_, flags_dir, flags_special_follow, flags_special, flags_base, flags_normal, flags_noatime
 from .fs import HardLinkManager
 from .misc import sysinfo, log_multi, consume
 from .misc import ChunkIteratorFileWrapper, open_item, chunkit, iter_separated, ErrorIgnoringTextIOWrapper
 from .parseformat import bin_to_hex, hex_to_bin, safe_encode, safe_decode
 from .parseformat import text_to_json, binary_to_json, remove_surrogates, join_cmd
 from .parseformat import eval_escapes, decode_dict, positive_int_validator, interval
-from .parseformat import PathSpec, SortBySpec, ChunkerParams, FilesCacheMode, partial_format, DatetimeWrapper
+from .parseformat import (
+    PathSpec,
+    FilesystemPathSpec,
+    SortBySpec,
+    ChunkerParams,
+    FilesCacheMode,
+    partial_format,
+    DatetimeWrapper,
+)
 from .parseformat import format_file_size, parse_file_size, FileSize
 from .parseformat import sizeof_fmt, sizeof_fmt_iec, sizeof_fmt_decimal, Location, text_validator
 from .parseformat import format_line, replace_placeholders, PlaceholderError, relative_time_marker_validator

src/borg/helpers/fs.py

@@ -249,13 +249,55 @@ def make_path_safe(path):
     For reasons of security, a ValueError is raised should
     `path` contain any '..' elements.
     """
+    if "\\.." in path or "..\\" in path:
+        raise ValueError(f"unexpected '..' element in path {path!r}")
+
+    path = map_chars(path)
+
     path = path.lstrip("/")
     if path.startswith("../") or "/../" in path or path.endswith("/..") or path == "..":
         raise ValueError(f"unexpected '..' element in path {path!r}")
     path = posixpath.normpath(path)
     return path
 
 
+def slashify(path):
+    """
+    Replace backslashes with forward slashes if running on Windows.
+
+    Use case: we always want to use forward slashes, even on Windows.
+    """
+    return path.replace("\\", "/") if is_win32 else path
+
+
+# Bijective mapping to Unicode Private Use Area (like cifs mapchars)
+WINDOWS_MAP_CHARS = str.maketrans(
+    {
+        "<": "\uF03C",
+        ">": "\uF03E",
+        ":": "\uF03A",
+        '"': "\uF022",
+        "\\": "\uF05C",
+        "|": "\uF07C",
+        "?": "\uF03F",
+        "*": "\uF02A",
+    }
+)
+
+
+def map_chars(path):
+    """
+    Map reserved characters if running on Windows.
+
+    Use case: if an archived path contains reserved characters (that are not reserved on POSIX)
+    we need to replace them with replacements to make the path usable on Windows.
+    """
+    if not is_win32:
+        return path
+
+    return path.translate(WINDOWS_MAP_CHARS)
+
+
 def get_strip_prefix(path):
     # similar to how rsync does it, we allow users to give paths like:
     # /this/gets/stripped/./this/is/kept
@@ -265,7 +307,7 @@ def get_strip_prefix(path):
     pos = path.find("/./")  # detect slashdot hack
     if pos > 0:
         # found a prefix to strip! make sure it ends with one "/"!
-        return os.path.normpath(path[:pos]) + os.sep
+        return posixpath.normpath(path[:pos]) + "/"
     else:
         # no or empty prefix, nothing to strip!
         return None
@@ -276,15 +318,14 @@ def get_strip_prefix(path):
 
 def remove_dotdot_prefixes(path):
     """
-    Remove '../'s at the beginning of `path`. Additionally,
-    the path is made relative.
+    Remove '../'s at the beginning of `path`. Additionally, the path is made relative.
 
-    `path` is expected to be normalized already (e.g. via `os.path.normpath()`).
+    `path` is expected to be normalized already (e.g. via `posixpath.normpath()`).
     """
+    assert "\\" not in path
     if is_win32:
         if len(path) > 1 and path[1] == ":":
             path = path.replace(":", "", 1)
-        path = path.replace("\\", "/")
 
     path = path.lstrip("/")
     path = _dotdot_re.sub("", path)

src/borg/helpers/parseformat.py

@@ -18,17 +18,17 @@
 from string import Formatter
 
 from ..logger import create_logger
-from ..platformflags import is_win32
 
 logger = create_logger()
 
 from .errors import Error
-from .fs import get_keys_dir, make_path_safe
+from .fs import get_keys_dir, make_path_safe, slashify
 from .msgpack import Timestamp
 from .time import OutputTimestamp, format_time, safe_timestamp
 from .. import __version__ as borg_version
 from .. import __version_tuple__ as borg_version_tuple
 from ..constants import *  # NOQA
+from ..platformflags import is_win32
 
 if TYPE_CHECKING:
     from ..item import ItemDiff
@@ -335,6 +335,12 @@ def PathSpec(text):
     return text
 
 
+def FilesystemPathSpec(text):
+    if not text:
+        raise argparse.ArgumentTypeError("Empty strings are not accepted as paths.")
+    return slashify(text)
+
+
 def SortBySpec(text):
     from ..manifest import AI_HUMAN_SORT_KEYS
 
@@ -558,7 +564,8 @@ def _parse(self, text):
         m = self.local_re.match(text)
         if m:
             self.proto = "file"
-            self.path = os.path.abspath(os.path.normpath(m.group("path")))
+            path = m.group("path")
+            self.path = slashify(os.path.abspath(path)) if is_win32 else os.path.abspath(path)
             return True
         return False
 

src/borg/item.pyx

@@ -7,7 +7,7 @@ from cpython.bytes cimport PyBytes_AsStringAndSize
 from .constants import ITEM_KEYS, ARCHIVE_KEYS
 from .helpers import StableDict
 from .helpers import format_file_size
-from .helpers.fs import assert_sanitized_path, to_sanitized_path
+from .helpers.fs import assert_sanitized_path, to_sanitized_path, map_chars, slashify
 from .helpers.msgpack import timestamp_to_int, int_to_timestamp, Timestamp
 from .helpers.time import OutputTimestamp, safe_timestamp
 
@@ -265,7 +265,7 @@ cdef class Item(PropDict):
 
     path = PropDictProperty(str, 'surrogate-escaped str', encode=assert_sanitized_path, decode=to_sanitized_path)
     source = PropDictProperty(str, 'surrogate-escaped str')  # legacy borg 1.x. borg 2: see .target
-    target = PropDictProperty(str, 'surrogate-escaped str')
+    target = PropDictProperty(str, 'surrogate-escaped str', encode=slashify, decode=map_chars)
     user = PropDictProperty(str, 'surrogate-escaped str')
     group = PropDictProperty(str, 'surrogate-escaped str')
 

src/borg/legacyrepository.py

@@ -5,6 +5,7 @@
 import stat
 import struct
 import time
+from pathlib import Path
 from collections import defaultdict
 from configparser import ConfigParser
 from functools import partial
@@ -27,7 +28,6 @@
 from .repoobj import RepoObj
 from .checksums import crc32, StreamingXXH64
 from .crypto.file_integrity import IntegrityCheckedFile, FileIntegrityError
-from .repository import _local_abspath_to_file_url
 
 logger = create_logger(__name__)
 
@@ -191,8 +191,9 @@ class PathPermissionDenied(Error):
         exit_mcode = 21
 
     def __init__(self, path, create=False, exclusive=False, lock_wait=None, lock=True, send_log_cb=None):
-        self.path = os.path.abspath(path)
-        self._location = Location(_local_abspath_to_file_url(self.path))
+        p = Path(path).absolute()
+        self.path = str(p)
+        self._location = Location(p.as_uri())
         self.version = None
         # long-running repository methods which emit log or progress output are responsible for calling
         # the ._send_log method periodically to get log and progress output transferred to the borg client