Opener refinements (#245)

* mypy fixes and some opener refinements

* typing ignores, and formatting

* simplify

* version bump

Author: willmcgugan

CHANGELOG.md

@@ -1,36 +1,40 @@
 # Change Log
+
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
-## [Unreleased]
+## [2.1.3] - 2018-12-24
 
 ### Fixed
 
-- Incomplete FTPFile.write when using `workers`  @geoffjukes
+- Incomplete FTPFile.write when using `workers` @geoffjukes
+- Fixed AppFS not creating directory
+
+### Added
+
+- Added load_extern switch to opener, fixes #228 @althanos
 
-## [2.1.2] - 20180-11-10
+## [2.1.2] - 2018-11-10
 
 ### Added
 
 - Support for Windows NT FTP servers @sspross
 
-
 ### Fixed
 
 - Root dir of MemoryFS accesible as a file
 - Packaging issues @televi
 - Deprecation warning re collections.Mapping
 
-
 ## [2.1.1] - 2018-10-03
 
 ### Added
 
 - Added PEP 561 py.typed files
 - Use sendfile for faster copies @althonos
-- Atomic exclusive mode  in Py2.7 @sqwishy
+- Atomic exclusive mode in Py2.7 @sqwishy
 
 ### Fixed
 
@@ -63,7 +67,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 
 - workers parameter to fs.copy, fs.move, and fs.mirror for concurrent
-    copies
+  copies
 
 ## [2.0.24] - 2018-06-28
 
@@ -99,7 +103,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ### Changed
 
 - Changed path.splitext so that 'leading periods on the basename are
-    ignored', which is the behaviour of os.path.splitext
+  ignored', which is the behaviour of os.path.splitext
 
 ## [2.0.20] - 2018-03-13
 
@@ -206,16 +210,19 @@ No changes, pushed wrong branch to PyPi.
 ## [2.0.7] - 2017-08-06
 
 ### Fixes
+
 - Fixed entry point breaking pip
 
 ## [2.0.6] - 2017-08-05
 
 ### Fixes
+
 - Opener refinements
 
 ## [2.0.5] - 2017-08-02
 
 ### Fixed
+
 - Fixed potential for deadlock in MemoryFS
 
 ### Added
@@ -242,15 +249,15 @@ No changes, pushed wrong branch to PyPi.
 ### Changed
 
 - More specific error when `validatepath` throws an error about the path
-    argument being the wrong type, and changed from a ValueError to a
-    TypeError.
+  argument being the wrong type, and changed from a ValueError to a
+  TypeError.
 - Deprecated `encoding` parameter in OSFS.
 
 ## [2.0.3] - 2017-04-22
 
 ### Added
 
-- New `copy_if_newer' functionality in `copy` module.
+- New `copy_if_newer' functionality in`copy` module.
 
 ### Fixed
 
@@ -259,12 +266,14 @@ No changes, pushed wrong branch to PyPi.
 ## [2.0.2] - 2017-03-12
 
 ### Changed
+
 - Improved FTP support for non-compliant servers
 - Fix for ZipFS implied directories
 
 ## [2.0.1] - 2017-03-11
 
 ### Added
+
 - TarFS contributed by Martin Larralde
 
 ### Fixed

fs/_version.py

@@ -1,3 +1,3 @@
 """Version, used in module and setup.py.
 """
-__version__ = "2.1.2"
+__version__ = "2.1.3"

fs/opener/registry.py

@@ -18,7 +18,17 @@
 from .parse import parse_fs_url
 
 if False:  # typing.TYPE_CHECKING
-    from typing import Iterator, List, Optional, Text, Tuple, Union
+    from typing import (
+        Callable,
+        Dict,
+        Iterator,
+        List,
+        Optional,
+        Text,
+        Type,
+        Tuple,
+        Union,
+    )
     from ..base import FS
 
 
@@ -35,12 +45,12 @@ def __init__(self, default_opener="osfs", load_extern=False):
                 is not supplied. The default is to use 'osfs', so that the
                 FS URL is treated as a system path if no protocol is given.
             load_extern (bool, optional): Set to `True` to load openers from
-                Pyfilesystem2 extensions. Defaults to `False`.
+                PyFilesystem2 extensions. Defaults to `False`.
 
         """
         self.default_opener = default_opener
         self.load_extern = load_extern
-        self._protocols = {}  # type: Mapping[Text, Opener]
+        self._protocols = {}  # type: Dict[Text, Opener]
 
     def __repr__(self):
         # type: () -> Text
@@ -63,6 +73,7 @@ class ArchiveOpener(Opener):
         """
         if not isinstance(opener, Opener):
             opener = opener()
+        assert isinstance(opener, Opener), "Opener instance required"
         assert opener.protocols, "must list one or more protocols"
         for protocol in opener.protocols:
             self._protocols[protocol] = opener
@@ -72,12 +83,15 @@ def protocols(self):
         # type: () -> List[Text]
         """`list`: the list of supported protocols.
         """
-        # we use OrderedDict to build an ordered set of protocols
-        _protocols = collections.OrderedDict((k, None) for k in self._protocols)
+
+        _protocols = list(self._protocols)
         if self.load_extern:
-            for entry_point in pkg_resources.iter_entry_points("fs.opener"):
-                _protocols[entry_point.name] = None
-        return list(_protocols.keys())
+            _protocols.extend(
+                entry_point.name
+                for entry_point in pkg_resources.iter_entry_points("fs.opener")
+            )
+            _protocols = list(collections.OrderedDict.fromkeys(_protocols))
+        return _protocols
 
     def get_opener(self, protocol):
         # type: (Text) -> Opener

fs/osfs.py

@@ -33,7 +33,7 @@
     from os import sendfile
 except ImportError:
     try:
-        from sendfile import sendfile
+        from sendfile import sendfile  # type: ignore
     except ImportError:
         sendfile = None
 
@@ -116,7 +116,7 @@ def __init__(
             root_path = fsdecode(root_path)
         self.root_path = root_path
         _drive, _root_path = os.path.splitdrive(fsdecode(fspath(root_path)))
-        _root_path = _drive + (_root_path or '/') if _drive else _root_path
+        _root_path = _drive + (_root_path or "/") if _drive else _root_path
         _root_path = os.path.expanduser(os.path.expandvars(_root_path))
         _root_path = os.path.normpath(os.path.abspath(_root_path))
         self._root_path = _root_path
@@ -378,7 +378,6 @@ def opendir(self, path, factory=None):
             # type: (_O, Text, Optional[_OpendirFactory]) -> SubFS[_O]
             pass
 
-
     # --- Backport of os.sendfile for Python < 3.8 -----------
 
     def _check_copy(self, src_path, dst_path, overwrite=False):
@@ -396,31 +395,35 @@ def _check_copy(self, src_path, dst_path, overwrite=False):
             raise errors.DirectoryExpected(dirname(dst_path))
         return _src_path, _dst_path
 
-
     if sys.version_info[:2] < (3, 8) and sendfile is not None:
 
-        _sendfile_error_codes = frozenset({
-            errno.EIO,
-            errno.EINVAL,
-            errno.ENOSYS,
-            errno.ENOTSUP,
-            errno.EBADF,
-            errno.ENOTSOCK,
-            errno.EOPNOTSUPP,
-        })
+        _sendfile_error_codes = frozenset(
+            {
+                errno.EIO,
+                errno.EINVAL,
+                errno.ENOSYS,
+                errno.ENOTSUP,  # type: ignore
+                errno.EBADF,
+                errno.ENOTSOCK,
+                errno.EOPNOTSUPP,
+            }
+        )
 
         def copy(self, src_path, dst_path, overwrite=False):
             # type: (Text, Text, bool) -> None
             with self._lock:
                 # validate and canonicalise paths
                 _src_path, _dst_path = self._check_copy(src_path, dst_path, overwrite)
-                _src_sys, _dst_sys = self.getsyspath(_src_path), self.getsyspath(_dst_path)
+                _src_sys, _dst_sys = (
+                    self.getsyspath(_src_path),
+                    self.getsyspath(_dst_path),
+                )
                 # attempt using sendfile
                 try:
                     # initialise variables to pass to sendfile
                     # open files to obtain a file descriptor
-                    with io.open(_src_sys, 'r') as src:
-                        with io.open(_dst_sys, 'w') as dst:
+                    with io.open(_src_sys, "r") as src:
+                        with io.open(_dst_sys, "w") as dst:
                             fd_src, fd_dst = src.fileno(), dst.fileno()
                             sent = maxsize = os.fstat(fd_src).st_size
                             offset = 0

tests/test_opener.py

@@ -101,6 +101,9 @@ def test_parse_params_decode(self):
 
 
 class TestRegistry(unittest.TestCase):
+    def test_protocols(self):
+        self.assertIsInstance(opener.registry.protocols, list)
+
     def test_registry_protocols(self):
         # Check registry.protocols list the names of all available extension
         extensions = [