Merge pull request borgbackup#9676 from ThomasWaldmann/support-rest-http-stdio

Support rest http stdio

Author: ThomasWaldmann

.github/workflows/canary.yml

@@ -136,7 +136,7 @@ jobs:
         run: |
           # build borg.exe
           . env/bin/activate
-          pip install -e ".[cockpit,s3,sftp,rest,rclone]"
+          pip install -e ".[cockpit,s3,sftp,rclone]"
           mkdir -p dist/binary
           pyinstaller -y --clean --distpath=dist/binary scripts/borg.exe.spec
           # build sdist and wheel in dist/...

.github/workflows/ci.yml

@@ -243,8 +243,13 @@ jobs:
         # Start ssh-agent and add our key so paramiko can use the agent
         eval "$(ssh-agent -s)"
         ssh-add ~/.ssh/id_ed25519
+        sudo python3 -m venv /opt/borgstore-venv
+        sudo /opt/borgstore-venv/bin/pip install -U pip setuptools wheel
+        sudo /opt/borgstore-venv/bin/pip install "borgstore[rest]"
+        sudo ln -sf /opt/borgstore-venv/bin/borgstore-server-rest /usr/local/bin/borgstore-server-rest
         # Export SFTP test URL for tox via GITHUB_ENV
         echo "BORG_TEST_SFTP_REPO=sftp://sftpuser@localhost:22/borg/sftp-repo" >> $GITHUB_ENV
+        echo "BORG_TEST_REST_REPO=rest://sftpuser@localhost:22/borg/rest-repo" >> $GITHUB_ENV
 
     - name: Install and configure MinIO S3 server (test only)
       if: ${{ runner.os == 'Linux' && !contains(matrix.toxenv, 'mypy') && !contains(matrix.toxenv, 'docs') }}
@@ -281,13 +286,13 @@ jobs:
     - name: Install borgbackup
       run: |
         if [[ "$TOXENV" == *"llfuse"* ]]; then
-          pip install -ve ".[llfuse,cockpit,s3,sftp,rest,rclone]"
+          pip install -ve ".[llfuse,cockpit,s3,sftp,rclone]"
         elif [[ "$TOXENV" == *"pyfuse3"* ]]; then
-          pip install -ve ".[pyfuse3,cockpit,s3,sftp,rest,rclone]"
+          pip install -ve ".[pyfuse3,cockpit,s3,sftp,rclone]"
         elif [[ "$TOXENV" == *"mfusepy"* ]]; then
-          pip install -ve ".[mfusepy,cockpit,s3,sftp,rest,rclone]"
+          pip install -ve ".[mfusepy,cockpit,s3,sftp,rclone]"
         else
-          pip install -ve ".[cockpit,s3,sftp,rest,rclone]"
+          pip install -ve ".[cockpit,s3,sftp,rclone]"
         fi
 
     - name: Build Borg fat binaries (${{ matrix.binary }})
@@ -461,7 +466,7 @@ jobs:
                 pip -V
                 python -m pip install --upgrade pip wheel
                 pip install -r requirements.d/development.lock.txt
-                pip install -e ".[mfusepy,cockpit,s3,sftp,rest,rclone]"
+                pip install -e ".[mfusepy,cockpit,s3,sftp,rclone]"
                 tox -e py311-mfusepy
 
                 if [[ "${{ matrix.do_binaries }}" == "true" && "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then
@@ -659,7 +664,7 @@ jobs:
         run: |
           # build borg.exe
           . env/bin/activate
-          pip install -e ".[cockpit,s3,sftp,rest,rclone]"
+          pip install -e ".[cockpit,s3,sftp,rclone]"
           mkdir -p dist/binary
           pyinstaller -y --clean --distpath=dist/binary scripts/borg.exe.spec
           # build sdist and wheel in dist/...

docs/usage/general/repository-urls.rst.inc

@@ -12,7 +12,13 @@ expanded by your shell).
 
 Note: You may also prepend ``file://`` to a filesystem path to use URL style.
 
-**Remote repositories** accessed via SSH user@host:
+**Remote repositories** accessed via SSH user@host (REST http over stdio):
+
+``rest://user@host:port//abs/path/to/repo`` — absolute path
+
+``rest://user@host:port/rel/path/to/repo`` — path relative to the current directory
+
+**Remote repositories** accessed via SSH user@host (legacy borg RPC protocol):
 
 ``ssh://user@host:port//abs/path/to/repo`` — absolute path
 

pyproject.toml

@@ -31,7 +31,7 @@ license = "BSD-3-Clause"
 license-files = ["LICENSE", "AUTHORS"]
 dependencies = [
   "borghash ~= 0.1.0",
-  "borgstore ~= 0.5.0",
+  "borgstore[rest] ~= 0.5.1",
   "msgpack >=1.0.3, <=1.1.2",
   "packaging",
   "platformdirs >=3.0.0, <5.0.0; sys_platform == 'darwin'",  # for macOS: breaking changes in 3.0.0.
@@ -51,10 +51,9 @@ mfusepy = ["mfusepy >= 3.1.0, <4.0.0"]  # fuse 2+3, high-level
 # a pypi release of borgbackup can't contain a dependency on github!
 # mfusepym = ["mfusepy @ git+https://github.com/mxmlnkn/mfusepy.git@master"]
 nofuse = []
-s3 = ["borgstore[s3] ~= 0.5.0"]
-sftp = ["borgstore[sftp] ~= 0.5.0"]
-rclone = ["borgstore[rclone] ~= 0.5.0"]
-rest = ["borgstore[rest] ~= 0.5.0"]
+s3 = ["borgstore[rest,s3] ~= 0.5.1"]
+sftp = ["borgstore[rest,sftp] ~= 0.5.1"]
+rclone = ["borgstore[rest,rclone] ~= 0.5.1"]
 cockpit = ["textual>=6.8.0"]  # might also work with older versions, untested
 
 [project.urls]
@@ -192,71 +191,71 @@ pass_env = ["*"]  # needed by tox4, so env vars are visible for building borg
 
 [tool.tox.env.py310-llfuse]
 set_env = {BORG_FUSE_IMPL = "llfuse"}
-extras = ["llfuse", "sftp", "s3", "rest", "rclone"]
+extras = ["llfuse", "sftp", "s3", "rclone"]
 
 [tool.tox.env.py310-pyfuse3]
 set_env = {BORG_FUSE_IMPL = "pyfuse3"}
-extras = ["pyfuse3", "sftp", "s3", "rest", "rclone"]
+extras = ["pyfuse3", "sftp", "s3", "rclone"]
 
 [tool.tox.env.py310-mfusepy]
 set_env = {BORG_FUSE_IMPL = "mfusepy"}
-extras = ["mfusepy", "sftp", "s3", "rest", "rclone"]
+extras = ["mfusepy", "sftp", "s3", "rclone"]
 
 [tool.tox.env.py311-none]
 
 [tool.tox.env.py311-llfuse]
 set_env = {BORG_FUSE_IMPL = "llfuse"}
-extras = ["llfuse", "sftp", "s3", "rest", "rclone"]
+extras = ["llfuse", "sftp", "s3", "rclone"]
 
 [tool.tox.env.py311-pyfuse3]
 set_env = {BORG_FUSE_IMPL = "pyfuse3"}
-extras = ["pyfuse3", "sftp", "s3", "rest", "rclone"]
+extras = ["pyfuse3", "sftp", "s3", "rclone"]
 
 [tool.tox.env.py311-mfusepy]
 set_env = {BORG_FUSE_IMPL = "mfusepy"}
-extras = ["mfusepy", "sftp", "s3", "rest", "rclone"]
+extras = ["mfusepy", "sftp", "s3", "rclone"]
 
 [tool.tox.env.py312-none]
 
 [tool.tox.env.py312-llfuse]
 set_env = {BORG_FUSE_IMPL = "llfuse"}
-extras = ["llfuse", "sftp", "s3", "rest", "rclone"]
+extras = ["llfuse", "sftp", "s3", "rclone"]
 
 [tool.tox.env.py312-pyfuse3]
 set_env = {BORG_FUSE_IMPL = "pyfuse3"}
-extras = ["pyfuse3", "sftp", "s3", "rest", "rclone"]
+extras = ["pyfuse3", "sftp", "s3", "rclone"]
 
 [tool.tox.env.py312-mfusepy]
 set_env = {BORG_FUSE_IMPL = "mfusepy"}
-extras = ["mfusepy", "sftp", "s3", "rest", "rclone"]
+extras = ["mfusepy", "sftp", "s3", "rclone"]
 
 [tool.tox.env.py313-none]
 
 [tool.tox.env.py313-llfuse]
 set_env = {BORG_FUSE_IMPL = "llfuse"}
-extras = ["llfuse", "sftp", "s3", "rest", "rclone"]
+extras = ["llfuse", "sftp", "s3", "rclone"]
 
 [tool.tox.env.py313-pyfuse3]
 set_env = {BORG_FUSE_IMPL = "pyfuse3"}
-extras = ["pyfuse3", "sftp", "s3", "rest", "rclone"]
+extras = ["pyfuse3", "sftp", "s3", "rclone"]
 
 [tool.tox.env.py313-mfusepy]
 set_env = {BORG_FUSE_IMPL = "mfusepy"}
-extras = ["mfusepy", "sftp", "s3", "rest", "rclone"]
+extras = ["mfusepy", "sftp", "s3", "rclone"]
 
 [tool.tox.env.py314-none]
 
 [tool.tox.env.py314-llfuse]
 set_env = {BORG_FUSE_IMPL = "llfuse"}
-extras = ["llfuse", "sftp", "s3", "rest", "rclone"]
+extras = ["llfuse", "sftp", "s3", "rclone"]
 
 [tool.tox.env.py314-pyfuse3]
 set_env = {BORG_FUSE_IMPL = "pyfuse3"}
-extras = ["pyfuse3", "sftp", "s3", "rest", "rclone"]
+extras = ["pyfuse3", "sftp", "s3", "rclone"]
 
 [tool.tox.env.py314-mfusepy]
 set_env = {BORG_FUSE_IMPL = "mfusepy"}
-extras = ["mfusepy", "sftp", "s3", "rest", "rclone"]
+extras = ["mfusepy", "sftp", "s3", "rclone"]
 
 [tool.tox.env.ruff]
 skip_install = true

src/borg/archiver/_common.py

@@ -42,7 +42,7 @@ def get_repository(location, *, create, exclusive, lock_wait, lock, args, v1_leg
         )
 
     elif (
-        location.proto in ("sftp", "file", "http", "https", "rclone", "s3", "b2") and not v1_legacy
+        location.proto in ("rest", "sftp", "file", "http", "https", "rclone", "s3", "b2") and not v1_legacy
     ):  # stuff directly supported by borgstore
         repository = Repository(location, create=create, exclusive=exclusive, lock_wait=lock_wait, lock=lock)
 

src/borg/conftest.py

@@ -96,7 +96,7 @@ def __init__(self):
         self.patterns_file_path: str | None = None
 
     def get_kind(self) -> str:
-        if self.repository_location.startswith("ssh://__testsuite__"):
+        if self.repository_location.startswith("ssh://__testsuite__") or self.repository_location.startswith("rest://"):
             return "remote"
         elif self.EXE == "borg.exe":
             return "binary"
@@ -152,7 +152,7 @@ def maybe_clear_flags_and_retry(func, path, _exc_info):
 
 @pytest.fixture()
 def remote_archiver(archiver):
-    archiver.repository_location = "ssh://__testsuite__/" + str(archiver.repository_path)
+    archiver.repository_location = "rest://" + "/" + str(archiver.repository_path)
     yield archiver
 
 

src/borg/helpers/parseformat.py

@@ -552,6 +552,19 @@ class Location:
         re.VERBOSE,
     )
 
+    # REST http via stdio (via ssh, if host given):
+    rest_re = re.compile(
+        r"(?P<proto>(rest))://"
+        + r"("
+        + optional_user_re
+        + host_re
+        + optional_port_re
+        + r")?"
+        + r"/"  # this is the separator, not part of the path!
+        + abs_or_rel_path_re,
+        re.VERBOSE,
+    )
+
     # BorgStore REST server
     # (http|https)://user:pass@host:port/
     http_re = re.compile(
@@ -624,6 +637,14 @@ def parse(self, text, overrides={}):
 
     def _parse(self, text):
         m = self.ssh_or_sftp_re.match(text)
+        if m:
+            self.proto = m.group("proto")
+            self.user = m.group("user")
+            self._host = m.group("host")
+            self.port = m.group("port") and int(m.group("port")) or None
+            self.path = os.path.normpath(m.group("path"))
+            return True
+        m = self.rest_re.match(text)
         if m:
             self.proto = m.group("proto")
             self.user = m.group("user")
@@ -692,7 +713,7 @@ def canonical_path(self):
             return self.path
         if self.proto == "rclone":
             return f"{self.proto}:{self.path}"
-        if self.proto in ("sftp", "ssh", "s3", "b2", "http", "https"):
+        if self.proto in ("rest", "sftp", "ssh", "s3", "b2", "http", "https"):
             return (
                 f"{self.proto}://"
                 f"{(self.user + '@') if self.user else ''}"

src/borg/testsuite/archiver/__init__.py

@@ -178,7 +178,10 @@ def open_archive(repo_path, name):
 
 def open_repository(archiver):
     if archiver.get_kind() == "remote":
-        return RemoteRepository(Location(archiver.repository_location))
+        location = Location(archiver.repository_location)
+        if location.proto == "rest":
+            return Repository(location, exclusive=True)
+        return RemoteRepository(location)
     else:
         return Repository(archiver.repository_path, exclusive=True)
 

src/borg/testsuite/archiver/checks_test.py

@@ -278,6 +278,8 @@ def test_unknown_mandatory_feature_in_cache(archivers, request):
 
 # Begin Remote Tests
 def test_remote_repo_restrict_to_path(remote_archiver):
+    if remote_archiver.repository_location.startswith("rest://"):
+        pytest.skip("Not applicable for rest:// protocol")
     original_location, repo_path = remote_archiver.repository_location, remote_archiver.repository_path
     # restricted to repo directory itself:
     with patch.object(RemoteRepository, "extra_test_args", ["--restrict-to-path", repo_path]):
@@ -306,6 +308,8 @@ def test_remote_repo_restrict_to_path(remote_archiver):
 
 
 def test_remote_repo_restrict_to_repository(remote_archiver):
+    if remote_archiver.repository_location.startswith("rest://"):
+        pytest.skip("Not applicable for rest:// protocol")
     repo_path = remote_archiver.repository_path
     # restricted to repo directory itself:
     with patch.object(RemoteRepository, "extra_test_args", ["--restrict-to-repository", repo_path]):

src/borg/testsuite/archiver/remote_repo_test.py

@@ -10,6 +10,7 @@
 
 
 SFTP_URL = os.environ.get("BORG_TEST_SFTP_REPO")
+REST_URL = os.environ.get("BORG_TEST_REST_REPO")
 S3_URL = os.environ.get("BORG_TEST_S3_REPO")
 
 
@@ -57,6 +58,30 @@ def test_rclone_repo_basics(archiver, tmp_path):
     cmd(archiver, "repo-delete")
 
 
+@pytest.mark.skipif(not REST_URL, reason="BORG_TEST_REST_REPO not set.")
+def test_rest_repo_basics(archiver):
+    create_regular_file(archiver.input_path, "file1", size=100 * 1024)
+    create_regular_file(archiver.input_path, "file2", size=10 * 1024)
+    archiver.repository_location = REST_URL
+    archive_name = "test-archive"
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    cmd(archiver, "create", archive_name, "input")
+    list_output = cmd(archiver, "repo-list")
+    assert archive_name in list_output
+    archive_list_output = cmd(archiver, "list", archive_name)
+    assert "input/file1" in archive_list_output
+    assert "input/file2" in archive_list_output
+    with changedir("output"):
+        cmd(archiver, "extract", archive_name)
+    assert_dirs_equal(
+        archiver.input_path, os.path.join(archiver.output_path, "input"), ignore_flags=True, ignore_xattrs=True
+    )
+    cmd(archiver, "delete", "-a", archive_name)
+    list_output = cmd(archiver, "repo-list")
+    assert archive_name not in list_output
+    cmd(archiver, "repo-delete")
+
+
 @pytest.mark.skipif(not SFTP_URL, reason="BORG_TEST_SFTP_REPO not set.")
 def test_sftp_repo_basics(archiver):
     create_regular_file(archiver.input_path, "file1", size=100 * 1024)