Merge pull request #9213 from ThomasWaldmann/mfusepy

integrate highlevel fuse lib mfusepy

Author: ThomasWaldmann

.github/workflows/ci.yml

@@ -136,19 +136,19 @@ jobs:
             "include": [
               {"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "mypy"},
               {"os": "ubuntu-22.04", "python-version": "3.11", "toxenv": "docs"},
-              {"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "py310-fuse2"},
-              {"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-fuse3"}
+              {"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "py310-llfuse"},
+              {"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-mfusepy"}
             ]
           }' || '{
             "include": [
               {"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "mypy"},
               {"os": "ubuntu-22.04", "python-version": "3.11", "toxenv": "docs"},
-              {"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "py310-fuse2"},
-              {"os": "ubuntu-22.04", "python-version": "3.11", "toxenv": "py311-fuse2", "binary": "borg-linux-glibc235-x86_64-gh"},
-              {"os": "ubuntu-22.04-arm", "python-version": "3.11", "toxenv": "py311-fuse2", "binary": "borg-linux-glibc235-arm64-gh"},
-              {"os": "ubuntu-24.04", "python-version": "3.12", "toxenv": "py312-fuse3"},
-              {"os": "ubuntu-24.04", "python-version": "3.13", "toxenv": "py313-fuse3"},
-              {"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-fuse3"},
+              {"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "py310-llfuse"},
+              {"os": "ubuntu-22.04", "python-version": "3.11", "toxenv": "py311-llfuse", "binary": "borg-linux-glibc235-x86_64-gh"},
+              {"os": "ubuntu-22.04-arm", "python-version": "3.11", "toxenv": "py311-llfuse", "binary": "borg-linux-glibc235-arm64-gh"},
+              {"os": "ubuntu-24.04", "python-version": "3.12", "toxenv": "py312-pyfuse3"},
+              {"os": "ubuntu-24.04", "python-version": "3.13", "toxenv": "py313-pyfuse3"},
+              {"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-mfusepy"},
               {"os": "macos-15-intel", "python-version": "3.11", "toxenv": "py311-none", "binary": "borg-macos-15-x86_64-gh"},
               {"os": "macos-15", "python-version": "3.11", "toxenv": "py311-none", "binary": "borg-macos-15-arm64-gh"}
             ]
@@ -190,9 +190,9 @@ jobs:
         sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev
         sudo apt-get install -y bash zsh fish  # for shell completion tests
         sudo apt-get install -y rclone openssh-server curl
-        if [[ "$TOXENV" == *"fuse2"* ]]; then
+        if [[ "$TOXENV" == *"llfuse"* ]]; then
           sudo apt-get install -y libfuse-dev fuse  # Required for Python llfuse module
-        elif [[ "$TOXENV" == *"fuse3"* ]]; then
+        elif [[ "$TOXENV" == *"pyfuse3"* || "$TOXENV" == *"mfusepy"* ]]; then
           sudo apt-get install -y libfuse3-dev fuse3  # Required for Python pyfuse3 module
         fi
 
@@ -266,10 +266,12 @@ jobs:
 
     - name: Install borgbackup
       run: |
-        if [[ "$TOXENV" == *"fuse2"* ]]; then
+        if [[ "$TOXENV" == *"llfuse"* ]]; then
           pip install -ve ".[llfuse]"
-        elif [[ "$TOXENV" == *"fuse3"* ]]; then
+        elif [[ "$TOXENV" == *"pyfuse3"* ]]; then
           pip install -ve ".[pyfuse3]"
+        elif [[ "$TOXENV" == *"mfusepy"* ]]; then
+          pip install -ve ".[mfusepy]"
         else
           pip install -ve .
         fi
@@ -423,8 +425,8 @@ jobs:
                 pip -V
                 python -m pip install --upgrade pip wheel
                 pip install -r requirements.d/development.txt
-                pip install -e ".[llfuse]"
-                tox -e py311-fuse2
+                pip install -e ".[mfusepy]"
+                tox -e py311-mfusepy
 
                 if [[ "${{ matrix.do_binaries }}" == "true" && "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then
                   python -m pip install 'pyinstaller==6.14.2'

Vagrantfile

@@ -373,7 +373,7 @@ Vagrant.configure(2) do |config|
     b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
     b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
     b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("freebsd13")
-    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("freebsd13", ".*(fuse3|none).*")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("freebsd13", ".*(pyfuse3|none).*")
   end
 
   config.vm.define "freebsd14" do |b|
@@ -390,7 +390,7 @@ Vagrant.configure(2) do |config|
     b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
     b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
     b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("freebsd14")
-    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("freebsd14", ".*(fuse3|none).*")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("freebsd14", ".*(pyfuse3|none).*")
   end
 
   config.vm.define "openbsd7" do |b|
@@ -413,7 +413,7 @@ Vagrant.configure(2) do |config|
     b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
     b.vm.provision "packages netbsd", :type => :shell, :inline => packages_netbsd
     b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("netbsd9")
-    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false)
+    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("nofuse")
     b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("netbsd9", ".*fuse.*")
   end
 

docs/global.rst.inc

@@ -23,6 +23,7 @@
 .. _msgpack: https://msgpack.org/
 .. _`msgpack-python`: https://pypi.python.org/pypi/msgpack-python/
 .. _llfuse: https://pypi.python.org/pypi/llfuse/
+.. _mfusepy: https://pypi.python.org/pypi/mfusepy/
 .. _pyfuse3: https://pypi.python.org/pypi/pyfuse3/
 .. _userspace filesystems: https://en.wikipedia.org/wiki/Filesystem_in_Userspace
 .. _Cython: http://cython.org/

docs/installation.rst

@@ -175,6 +175,7 @@ development header files (sometimes in a separate `-dev` or `-devel` package).
 * Optionally, if you wish to mount an archive as a FUSE filesystem, you need
   a FUSE implementation for Python:
 
+  - mfusepy_ >= 3.1.0 (for fuse 2 and fuse 3, use `pip install borgbackup[mfusepy]`), or
   - pyfuse3_ >= 3.1.1 (for fuse 3, use `pip install borgbackup[pyfuse3]`), or
   - llfuse_ >= 1.3.8 (for fuse 2, use `pip install borgbackup[llfuse]`).
   - Additionally, your OS will need to have FUSE support installed

docs/usage/general/environment.rst.inc

@@ -88,8 +88,9 @@ General:
         This is a comma-separated list of implementation names, they are tried in the
         given order, e.g.:
 
-        - ``pyfuse3,llfuse``: default, first try to load pyfuse3, then try to load llfuse.
+        - ``mfusepy,pyfuse3,llfuse``: default, first try to load mfusepy, then pyfuse3, then llfuse.
         - ``llfuse,pyfuse3``: first try to load llfuse, then try to load pyfuse3.
+        - ``mfusepy``: only try to load mfusepy
         - ``pyfuse3``: only try to load pyfuse3
         - ``llfuse``: only try to load llfuse
         - ``none``: do not try to load an implementation

pyproject.toml

@@ -41,8 +41,10 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-llfuse = ["llfuse >= 1.3.8"]
-pyfuse3 = ["pyfuse3 >= 3.1.1"]
+llfuse = ["llfuse >= 1.3.8"]  # fuse 2, low-level
+pyfuse3 = ["pyfuse3 >= 3.1.1"]  # fuse 3, low-level, async
+mfusepy = ["mfusepy >= 3.1.0, <4.0.0"]  # fuse 2+3, high-level
+mfusepym = ["mfusepy @ git+https://github.com/mxmlnkn/mfusepy.git@master"]
 nofuse = []
 s3 = ["borgstore[s3] ~= 0.3.0"]
 sftp = ["borgstore[sftp] ~= 0.3.0"]
@@ -166,7 +168,7 @@ ignore_missing_imports = true
 requires = ["tox>=4.19", "pkgconfig", "cython", "wheel", "setuptools_scm"]
 # Important: when adding/removing Python versions here,
 #            also update the section "Test environments with different FUSE implementations" accordingly.
-env_list = ["py{310,311,312,313,314}-{none,fuse2,fuse3}", "docs", "ruff", "mypy", "bandit"]
+env_list = ["py{310,311,312,313,314}-{none,llfuse,pyfuse3,mfusepy}", "docs", "ruff", "mypy", "bandit"]
 
 [tool.tox.env_run_base]
 package = "editable-legacy"  # without this it does not find setup_docs when running under fakeroot
@@ -180,54 +182,74 @@ pass_env = ["*"]  # needed by tox4, so env vars are visible for building borg
 # Test environments with different FUSE implementations
 [tool.tox.env.py310-none]
 
-[tool.tox.env.py310-fuse2]
+[tool.tox.env.py310-llfuse]
 set_env = {BORG_FUSE_IMPL = "llfuse"}
 extras = ["llfuse", "sftp", "s3"]
 
-[tool.tox.env.py310-fuse3]
+[tool.tox.env.py310-pyfuse3]
 set_env = {BORG_FUSE_IMPL = "pyfuse3"}
 extras = ["pyfuse3", "sftp", "s3"]
 
+[tool.tox.env.py310-mfusepy]
+set_env = {BORG_FUSE_IMPL = "mfusepy"}
+extras = ["mfusepy", "sftp", "s3"]
+
 [tool.tox.env.py311-none]
 
-[tool.tox.env.py311-fuse2]
+[tool.tox.env.py311-llfuse]
 set_env = {BORG_FUSE_IMPL = "llfuse"}
 extras = ["llfuse", "sftp", "s3"]
 
-[tool.tox.env.py311-fuse3]
+[tool.tox.env.py311-pyfuse3]
 set_env = {BORG_FUSE_IMPL = "pyfuse3"}
 extras = ["pyfuse3", "sftp", "s3"]
 
+[tool.tox.env.py311-mfusepy]
+set_env = {BORG_FUSE_IMPL = "mfusepy"}
+extras = ["mfusepy", "sftp", "s3"]
+
 [tool.tox.env.py312-none]
 
-[tool.tox.env.py312-fuse2]
+[tool.tox.env.py312-llfuse]
 set_env = {BORG_FUSE_IMPL = "llfuse"}
 extras = ["llfuse", "sftp", "s3"]
 
-[tool.tox.env.py312-fuse3]
+[tool.tox.env.py312-pyfuse3]
 set_env = {BORG_FUSE_IMPL = "pyfuse3"}
 extras = ["pyfuse3", "sftp", "s3"]
 
+[tool.tox.env.py312-mfusepy]
+set_env = {BORG_FUSE_IMPL = "mfusepy"}
+extras = ["mfusepy", "sftp", "s3"]
+
 [tool.tox.env.py313-none]
 
-[tool.tox.env.py313-fuse2]
+[tool.tox.env.py313-llfuse]
 set_env = {BORG_FUSE_IMPL = "llfuse"}
 extras = ["llfuse", "sftp", "s3"]
 
-[tool.tox.env.py313-fuse3]
+[tool.tox.env.py313-pyfuse3]
 set_env = {BORG_FUSE_IMPL = "pyfuse3"}
 extras = ["pyfuse3", "sftp", "s3"]
 
+[tool.tox.env.py313-mfusepy]
+set_env = {BORG_FUSE_IMPL = "mfusepy"}
+extras = ["mfusepy", "sftp", "s3"]
+
 [tool.tox.env.py314-none]
 
-[tool.tox.env.py314-fuse2]
+[tool.tox.env.py314-llfuse]
 set_env = {BORG_FUSE_IMPL = "llfuse"}
 extras = ["llfuse", "sftp", "s3"]
 
-[tool.tox.env.py314-fuse3]
+[tool.tox.env.py314-pyfuse3]
 set_env = {BORG_FUSE_IMPL = "pyfuse3"}
 extras = ["pyfuse3", "sftp", "s3"]
 
+[tool.tox.env.py314-mfusepy]
+set_env = {BORG_FUSE_IMPL = "mfusepy"}
+extras = ["mfusepy", "sftp", "s3"]
+
 [tool.tox.env.ruff]
 skip_install = true
 deps = ["ruff"]

src/borg/archiver/mount_cmds.py

@@ -19,9 +19,9 @@ def do_mount(self, args):
         """Mounts an archive or an entire repository as a FUSE filesystem."""
         # Perform these checks before opening the repository and asking for a passphrase.
 
-        from ..fuse_impl import llfuse, BORG_FUSE_IMPL
+        from ..fuse_impl import llfuse, has_mfusepy, BORG_FUSE_IMPL
 
-        if llfuse is None:
+        if llfuse is None and not has_mfusepy:
             raise RTError("borg mount not available: no FUSE support, BORG_FUSE_IMPL=%s." % BORG_FUSE_IMPL)
 
         if not os.path.isdir(args.mountpoint):
@@ -34,16 +34,31 @@ def do_mount(self, args):
 
     @with_repository(compatibility=(Manifest.Operation.READ,))
     def _do_mount(self, args, repository, manifest):
-        from ..fuse import FuseOperations
+        from ..fuse_impl import has_mfusepy
 
-        with cache_if_remote(repository, decrypted_cache=manifest.repo_objs) as cached_repo:
-            operations = FuseOperations(manifest, args, cached_repo)
+        if has_mfusepy:
+            # Use mfusepy implementation
+            from ..hlfuse import borgfs
+
+            operations = borgfs(manifest, args, repository)
             logger.info("Mounting filesystem")
             try:
                 operations.mount(args.mountpoint, args.options, args.foreground, args.show_rc)
             except RuntimeError:
                 # Relevant error message already printed to stderr by FUSE
                 raise RTError("FUSE mount failed")
+        else:
+            # Use llfuse/pyfuse3 implementation
+            from ..fuse import FuseOperations
+
+            with cache_if_remote(repository, decrypted_cache=manifest.repo_objs) as cached_repo:
+                operations = FuseOperations(manifest, args, cached_repo)
+                logger.info("Mounting filesystem")
+                try:
+                    operations.mount(args.mountpoint, args.options, args.foreground, args.show_rc)
+                except RuntimeError:
+                    # Relevant error message already printed to stderr by FUSE
+                    raise RTError("FUSE mount failed")
 
     def do_umount(self, args):
         """Unmounts the FUSE filesystem."""

src/borg/conftest.py

@@ -12,7 +12,7 @@
 setup_logging()
 
 from borg.archiver import Archiver  # noqa: E402
-from borg.testsuite import has_lchflags, has_llfuse, has_pyfuse3  # noqa: E402
+from borg.testsuite import has_lchflags, has_llfuse, has_pyfuse3, has_mfusepy  # noqa: E402
 from borg.testsuite import are_symlinks_supported, are_hardlinks_supported, is_utime_fully_supported  # noqa: E402
 from borg.testsuite.archiver import BORG_EXES
 from borg.testsuite.platform.platform_test import fakeroot_detected  # noqa: E402
@@ -37,8 +37,9 @@ def clean_env(tmpdir_factory, monkeypatch):
 def pytest_report_header(config, start_path):
     tests = {
         "BSD flags": has_lchflags,
-        "fuse2": has_llfuse,
-        "fuse3": has_pyfuse3,
+        "llfuse": has_llfuse,
+        "pyfuse3": has_pyfuse3,
+        "mfusepy": has_mfusepy,
         "root": not fakeroot_detected(),
         "symlinks": are_symlinks_supported(),
         "hardlinks": are_hardlinks_supported(),

src/borg/fuse.py

@@ -9,10 +9,17 @@
 import time
 from collections import defaultdict, Counter
 from signal import SIGINT
+from typing import TYPE_CHECKING
 
 from .constants import ROBJ_FILE_STREAM, zeros
-from .fuse_impl import llfuse, has_pyfuse3
-from .platform import ENOATTR
+
+if TYPE_CHECKING:
+    # For type checking, assume llfuse is available
+    # This allows mypy to understand llfuse.Operations
+    import llfuse
+    from .fuse_impl import has_pyfuse3, ENOATTR
+else:
+    from .fuse_impl import llfuse, has_pyfuse3, ENOATTR
 
 if has_pyfuse3:
     import trio

src/borg/fuse_impl.py

@@ -1,36 +1,63 @@
 """
-Loads the library for the low-level FUSE implementation.
+Loads the library for the FUSE implementation.
 """
 
 import os
+import types
 
-BORG_FUSE_IMPL = os.environ.get("BORG_FUSE_IMPL", "pyfuse3,llfuse")
+from .platform import ENOATTR  # noqa
+
+BORG_FUSE_IMPL = os.environ.get("BORG_FUSE_IMPL", "mfusepy,pyfuse3,llfuse")
+
+hlfuse: types.ModuleType | None = None
+llfuse: types.ModuleType | None = None
 
 for FUSE_IMPL in BORG_FUSE_IMPL.split(","):
     FUSE_IMPL = FUSE_IMPL.strip()
     if FUSE_IMPL == "pyfuse3":
         try:
-            import pyfuse3 as llfuse
+            import pyfuse3
         except ImportError:
             pass
         else:
+            llfuse = pyfuse3
             has_llfuse = False
             has_pyfuse3 = True
+            has_mfusepy = False
+            has_any_fuse = True
+            hlfuse = None  # noqa
             break
     elif FUSE_IMPL == "llfuse":
         try:
-            import llfuse
+            import llfuse as llfuse_module
         except ImportError:
             pass
         else:
+            llfuse = llfuse_module
             has_llfuse = True
             has_pyfuse3 = False
+            has_mfusepy = False
+            has_any_fuse = True
+            hlfuse = None  # noqa
+            break
+    elif FUSE_IMPL == "mfusepy":
+        try:
+            import mfusepy
+        except ImportError:
+            pass
+        else:
+            hlfuse = mfusepy
+            has_llfuse = False
+            has_pyfuse3 = False
+            has_mfusepy = True
+            has_any_fuse = True
             break
     elif FUSE_IMPL == "none":
         pass
     else:
         raise RuntimeError("Unknown FUSE implementation in BORG_FUSE_IMPL: '%s'." % BORG_FUSE_IMPL)
 else:
-    llfuse = None  # noqa
     has_llfuse = False
     has_pyfuse3 = False
+    has_mfusepy = False
+    has_any_fuse = False