Implement OSFS.copy using sendfile or shutil.copy (#216)

* Use `shutil.copy2` as the backend implementation of `fs.osfs.OSFS.copy`

* Add support for `pyfastcopy` extra in `fs.osfs` if installed

* Include backport of `shutil.copyfile` using `sendfile` to `fs.osfs`

* Add tests to check for unsupported `sendfile` in `tests.test_osfs`

* Refactor `OSFS.copy` into a single method for both implementations

* Add test for line not covered in `OSFS.copy`

* Use `os.fstat` in `OSFS.copy` to get the size for `sendfile`

Author: Martin Larralde

fs/osfs.py

@@ -14,6 +14,7 @@
 import logging
 import os
 import platform
+import shutil
 import stat
 import sys
 import typing
@@ -28,13 +29,21 @@
     except ImportError:  # pragma: no cover
         scandir = None
 
+try:
+    from os import sendfile
+except ImportError:
+    try:
+        from sendfile import sendfile
+    except ImportError:
+        sendfile = None
+
 from . import errors
 from .errors import FileExists
 from .base import FS
 from .enums import ResourceType
 from ._fscompat import fsencode, fsdecode, fspath
 from .info import Info
-from .path import basename
+from .path import basename, dirname
 from .permissions import Permissions
 from .error_tools import convert_os_errors
 from .mode import Mode, validate_open_mode
@@ -361,90 +370,79 @@ def removedir(self, path):
     # Optional Methods
     # --------------------------------------------------------
 
+    # --- Type hint for opendir ------------------------------
+
     if False:  # typing.TYPE_CHECKING
 
         def opendir(self, path, factory=None):
             # type: (_O, Text, Optional[_OpendirFactory]) -> SubFS[_O]
             pass
 
-    def getsyspath(self, path):
-        # type: (Text) -> Text
-        sys_path = os.path.join(self._root_path, path.lstrip("/").replace("/", os.sep))
-        return sys_path
-
-    def geturl(self, path, purpose="download"):
-        # type: (Text, Text) -> Text
-        if purpose != "download":
-            raise NoURL(path, purpose)
-        return "file://" + self.getsyspath(path)
 
-    def gettype(self, path):
-        # type: (Text) -> ResourceType
-        self.check()
-        sys_path = self._to_sys_path(path)
-        with convert_os_errors("gettype", path):
-            stat = os.stat(sys_path)
-        resource_type = self._get_type_from_stat(stat)
-        return resource_type
+    # --- Backport of os.sendfile for Python < 3.8 -----------
+
+    def _check_copy(self, src_path, dst_path, overwrite=False):
+        # validate individual paths
+        _src_path = self.validatepath(src_path)
+        _dst_path = self.validatepath(dst_path)
+        # check src_path exists and is a file
+        if self.gettype(src_path) is not ResourceType.file:
+            raise errors.FileExpected(src_path)
+        # check dst_path does not exist if we are not overwriting
+        if not overwrite and self.exists(_dst_path):
+            raise errors.DestinationExists(dst_path)
+        # check parent dir of _dst_path exists and is a directory
+        if self.gettype(dirname(dst_path)) is not ResourceType.directory:
+            raise errors.DirectoryExpected(dirname(dst_path))
+        return _src_path, _dst_path
+
+
+    if sys.version_info[:2] < (3, 8) and sendfile is not None:
+
+        _sendfile_error_codes = frozenset({
+            errno.EIO,
+            errno.EINVAL,
+            errno.ENOSYS,
+            errno.ENOTSUP,
+            errno.EBADF,
+            errno.ENOTSOCK,
+            errno.EOPNOTSUPP,
+        })
+
+        def copy(self, src_path, dst_path, overwrite=False):
+            # type: (Text, Text, bool) -> None
+            with self._lock:
+                # validate and canonicalise paths
+                _src_path, _dst_path = self._check_copy(src_path, dst_path, overwrite)
+                _src_sys, _dst_sys = self.getsyspath(_src_path), self.getsyspath(_dst_path)
+                # attempt using sendfile
+                try:
+                    # initialise variables to pass to sendfile
+                    # open files to obtain a file descriptor
+                    with io.open(_src_sys, 'r') as src:
+                        with io.open(_dst_sys, 'w') as dst:
+                            fd_src, fd_dst = src.fileno(), dst.fileno()
+                            sent = maxsize = os.fstat(fd_src).st_size
+                            offset = 0
+                            while sent > 0:
+                                sent = sendfile(fd_dst, fd_src, offset, maxsize)
+                                offset += sent
+                except OSError as e:
+                    # the error is not a simple "sendfile not supported" error
+                    if e.errno not in self._sendfile_error_codes:
+                        raise
+                    # fallback using the shutil implementation
+                    shutil.copy2(_src_sys, _dst_sys)
 
-    def islink(self, path):
-        # type: (Text) -> bool
-        self.check()
-        _path = self.validatepath(path)
-        sys_path = self._to_sys_path(_path)
-        if not self.exists(path):
-            raise errors.ResourceNotFound(path)
-        with convert_os_errors("islink", path):
-            return os.path.islink(sys_path)
+    else:
 
-    def open(
-        self,
-        path,  # type: Text
-        mode="r",  # type: Text
-        buffering=-1,  # type: int
-        encoding=None,  # type: Optional[Text]
-        errors=None,  # type: Optional[Text]
-        newline="",  # type: Text
-        line_buffering=False,  # type: bool
-        **options  # type: Any
-    ):
-        # type: (...) -> IO
-        _mode = Mode(mode)
-        validate_open_mode(mode)
-        self.check()
-        _path = self.validatepath(path)
-        sys_path = self._to_sys_path(_path)
-        with convert_os_errors("open", path):
-            if six.PY2 and _mode.exclusive:
-                sys_path = os.open(sys_path, os.O_RDWR | os.O_CREAT | os.O_EXCL)
-            _encoding = encoding or "utf-8"
-            return io.open(
-                sys_path,
-                mode=_mode.to_platform(),
-                buffering=buffering,
-                encoding=None if _mode.binary else _encoding,
-                errors=errors,
-                newline=None if _mode.binary else newline,
-                **options
-            )
+        def copy(self, src_path, dst_path, overwrite=False):
+            # type: (Text, Text, bool) -> None
+            with self._lock:
+                _src_path, _dst_path = self._check_copy(src_path, dst_path, overwrite)
+                shutil.copy2(self.getsyspath(_src_path), self.getsyspath(_dst_path))
 
-    def setinfo(self, path, info):
-        # type: (Text, RawInfo) -> None
-        self.check()
-        _path = self.validatepath(path)
-        sys_path = self._to_sys_path(_path)
-        if not os.path.exists(sys_path):
-            raise errors.ResourceNotFound(path)
-        if "details" in info:
-            details = info["details"]
-            if "accessed" in details or "modified" in details:
-                _accessed = typing.cast(int, details.get("accessed"))
-                _modified = typing.cast(int, details.get("modified", _accessed))
-                accessed = int(_modified if _accessed is None else _accessed)
-                modified = int(_modified)
-                if accessed is not None or modified is not None:
-                    with convert_os_errors("setinfo", path):
-                        os.utime(sys_path, (accessed, modified))
+    # --- Backport of os.scandir for Python < 3.5 ------------
 
     if scandir:
 
@@ -545,3 +543,84 @@ def scandir(
             start, end = page
             iter_info = itertools.islice(iter_info, start, end)
         return iter_info
+
+    # --- Miscellaneous --------------------------------------
+
+    def getsyspath(self, path):
+        # type: (Text) -> Text
+        sys_path = os.path.join(self._root_path, path.lstrip("/").replace("/", os.sep))
+        return sys_path
+
+    def geturl(self, path, purpose="download"):
+        # type: (Text, Text) -> Text
+        if purpose != "download":
+            raise NoURL(path, purpose)
+        return "file://" + self.getsyspath(path)
+
+    def gettype(self, path):
+        # type: (Text) -> ResourceType
+        self.check()
+        sys_path = self._to_sys_path(path)
+        with convert_os_errors("gettype", path):
+            stat = os.stat(sys_path)
+        resource_type = self._get_type_from_stat(stat)
+        return resource_type
+
+    def islink(self, path):
+        # type: (Text) -> bool
+        self.check()
+        _path = self.validatepath(path)
+        sys_path = self._to_sys_path(_path)
+        if not self.exists(path):
+            raise errors.ResourceNotFound(path)
+        with convert_os_errors("islink", path):
+            return os.path.islink(sys_path)
+
+    def open(
+        self,
+        path,  # type: Text
+        mode="r",  # type: Text
+        buffering=-1,  # type: int
+        encoding=None,  # type: Optional[Text]
+        errors=None,  # type: Optional[Text]
+        newline="",  # type: Text
+        line_buffering=False,  # type: bool
+        **options  # type: Any
+    ):
+        # type: (...) -> IO
+        _mode = Mode(mode)
+        validate_open_mode(mode)
+        self.check()
+        _path = self.validatepath(path)
+        sys_path = self._to_sys_path(_path)
+        with convert_os_errors("open", path):
+            if six.PY2 and _mode.exclusive:
+                sys_path = os.open(sys_path, os.O_RDWR | os.O_CREAT | os.O_EXCL)
+            _encoding = encoding or "utf-8"
+            return io.open(
+                sys_path,
+                mode=_mode.to_platform(),
+                buffering=buffering,
+                encoding=None if _mode.binary else _encoding,
+                errors=errors,
+                newline=None if _mode.binary else newline,
+                **options
+            )
+
+    def setinfo(self, path, info):
+        # type: (Text, RawInfo) -> None
+        self.check()
+        _path = self.validatepath(path)
+        sys_path = self._to_sys_path(_path)
+        if not os.path.exists(sys_path):
+            raise errors.ResourceNotFound(path)
+        if "details" in info:
+            details = info["details"]
+            if "accessed" in details or "modified" in details:
+                _accessed = typing.cast(int, details.get("accessed"))
+                _modified = typing.cast(int, details.get("modified", _accessed))
+                accessed = int(_modified if _accessed is None else _accessed)
+                modified = int(_modified)
+                if accessed is not None or modified is not None:
+                    with convert_os_errors("setinfo", path):
+                        os.utime(sys_path, (accessed, modified))

tests/test_osfs.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import errno
 import io
 import os
 import mock
@@ -14,7 +15,6 @@
 
 from fs.test import FSTestCases
 
-
 from six import text_type
 
 
@@ -68,6 +68,24 @@ def test_not_exists(self):
         with self.assertRaises(errors.CreateFailed):
             fs = osfs.OSFS("/does/not/exists/")
 
+    @unittest.skipIf(osfs.sendfile is None, 'sendfile not supported')
+    def test_copy_sendfile(self):
+        # try copying using sendfile
+        with mock.patch.object(osfs, 'sendfile') as sendfile:
+            sendfile.side_effect = OSError(errno.ENOTSUP, 'sendfile not supported')
+            self.test_copy()
+        # check other errors are transmitted
+        self.fs.touch('foo')
+        with mock.patch.object(osfs, 'sendfile') as sendfile:
+            sendfile.side_effect = OSError(errno.EWOULDBLOCK)
+            with self.assertRaises(OSError):
+                self.fs.copy('foo', 'foo_copy')
+        # check parent exist and is dir
+        with self.assertRaises(errors.ResourceNotFound):
+            self.fs.copy('foo', 'spam/eggs')
+        with self.assertRaises(errors.DirectoryExpected):
+            self.fs.copy('foo', 'foo_copy/foo')
+
     def test_create(self):
         """Test create=True"""