change pyfilesystem2 to fsspec / morefs; bump to 0.0.13

tutankalex · tutankalex · commit 6a5bfb24a4cd · 2024-11-14T22:58:35.000+08:00
pyfilesystem2 is no longer actively maintained, but it uses a deprecated function, leading to these warnings: ``` python3.11/site-packages/fs/__init__.py:4: DeprecationWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html ``` see e.g. PyFilesystem/pyfilesystem2#590 fortunately, fsspec and https://github.com/iterative/morefs are sufficient to cover a direct transition
diff --git a/nbs/00_core.ipynb b/nbs/00_core.ipynb
@@ -33,9 +33,11 @@
     "\n",
     "import dotenv\n",
     "import jsonschema\n",
-    "from fs.base import FS\n",
-    "from fs.osfs import OSFS\n",
+    "from fsspec.spec import AbstractFileSystem\n",
+    "from fsspec.implementations.local import LocalFileSystem\n",
+    "from fsspec.implementations.dirfs import DirFileSystem\n",
     "from jsonschema import ValidationError, validate\n",
+    "from io import StringIO\n",
     "\n",
     "logger = logging.getLogger(__name__)"
    ]
@@ -241,35 +243,36 @@
     "    # this is a risky value: it gets reused across instances;\n",
     "    # the idea is to maybe set it once and use it multiple times.\n",
     "    # but in testing this smells bad\n",
-    "    DEFAULT_STORAGE_DRIVER: FS = None  # defaults to OSFS\n",
+    "    DEFAULT_STORAGE_DRIVER: AbstractFileSystem = None  # defaults to DirFileSystem\n",
     "    \n",
     "    CONFIG_VALIDATOR_JSON_SCHEMA_ENVVAR_NAME = 'CONFIG_VALIDATOR_JSON_SCHEMA'\n",
     "    \n",
     "    @classmethod\n",
     "    def get_default_storage_driver(cls):\n",
     "        if cls.DEFAULT_STORAGE_DRIVER is None:\n",
-    "            cls.DEFAULT_STORAGE_DRIVER = OSFS(os.getcwd())\n",
+    "            cls.DEFAULT_STORAGE_DRIVER = DirFileSystem(os.getcwd())\n",
     "        return cls.DEFAULT_STORAGE_DRIVER\n",
     "\n",
     "    \n",
     "    @classmethod\n",
     "    def _get_maybe_abspath_driver(cls, maybe_abspath: str):\n",
     "        if os.path.isabs(maybe_abspath):  # special case\n",
-    "            return OSFS('/')\n",
+    "            return DirFileSystem('/')\n",
     "        else:\n",
     "            return cls.get_default_storage_driver()\n",
     "        \n",
     "    @classmethod\n",
-    "    def load_json(cls, json_source: Union[str, dict]=None, storage_driver: FS = None) -> dict:\n",
+    "    def load_json(cls, json_source: Union[str, dict]=None, storage_driver: AbstractFileSystem = None) -> dict:\n",
     "        \"\"\"\n",
     "        Convenience method to return a dictionary from either a file path or an already-loaded dictionary.\n",
     "\n",
     "        Args:\n",
     "            - `json_source` (Union[str, dict], optional): The JSON source to load.\n",
     "                                This can be a file path (str) \n",
     "                                or an already loaded dictionary (dict). \n",
-    "            - `storage_driver` (FS, optional): An instance of the storage driver used to load the JSON file. \n",
-    "                                               If not provided, OSFS from the current working dir is used.\n",
+    "            - `storage_driver` (AbstractFileSystem, optional): An instance of the storage driver used to\n",
+    "                                load the JSON file. If not provided, DirFileSystem from the current\n",
+    "                                working dir is used.\n",
     "\n",
     "        Returns:\n",
     "            dict: A dictionary that was loaded from the provided `json_source`.\n",
@@ -283,14 +286,14 @@
     "            return json.load(ifile)\n",
     "\n",
     "    @classmethod\n",
-    "    def get_default_json_schema(cls, storage_driver: FS = None) -> dict:\n",
+    "    def get_default_json_schema(cls, storage_driver: AbstractFileSystem = None) -> dict:\n",
     "        if cls.CONFIG_VALIDATOR_JSON_SCHEMA_ENVVAR_NAME in os.environ:\n",
     "            expected_json_schema_path = \\\n",
     "                os.environ[cls.CONFIG_VALIDATOR_JSON_SCHEMA_ENVVAR_NAME]\n",
     "            return cls.load_json(expected_json_schema_path, storage_driver)\n",
     "        return None\n",
     "\n",
-    "    def __init__(self, json_schema: Union[str, dict]=None, storage_driver: FS=None):\n",
+    "    def __init__(self, json_schema: Union[str, dict]=None, storage_driver: AbstractFileSystem=None):\n",
     "        \"\"\"\n",
     "        Initialize the instance with a JSON schema and a storage driver.\n",
     "\n",
@@ -299,8 +302,8 @@
     "                                                          If no value is provided, it will fall back to looking for an environment \n",
     "                                                          variable corresponding to the class variable \n",
     "                                                          `CONFIG_VALIDATOR_JSON_SCHEMA_ENVVAR_NAME` to find a JSON schema file.\n",
-    "            - `storage_driver` (FS, optional): The storage driver to use. If no value is provided, \n",
-    "                                               `self.__class__.DEFAULT_STORAGE_DRIVER` is used.\n",
+    "            - `storage_driver` (AbstractFileSystem, optional): The storage driver to use. If no value is provided, \n",
+    "                                                          `self.__class__.DEFAULT_STORAGE_DRIVER` is used.\n",
     "\n",
     "        Raises:\n",
     "            Exception: An exception is raised if no valid JSON schema is provided or found.\n",
@@ -336,7 +339,7 @@
     "    def load_dotenv(cls,\n",
     "                    json_schema: Union[str, dict]=None,\n",
     "                    dotenv_path: str=None,\n",
-    "                    storage_driver: FS=None,\n",
+    "                    storage_driver: AbstractFileSystem=None,\n",
     "                    override: bool=False,\n",
     "                   ):\n",
     "        \"\"\"\n",
@@ -349,15 +352,15 @@
     "                                            (such as an environment variable or default schema) is used.\n",
     "            - `dotenv_path` (str, optional): Path to the .env file to load the variables from.\n",
     "                                             If not provided, loads an empty dict to start.\n",
-    "            - `storage_driver` (FS, optional): The storage driver to use for loading files. If not given,\n",
-    "                                               \".env\" will be attempted from the current working directory;\n",
-    "                                               if that does not exist, an empty dict will be used.\n",
+    "            - `storage_driver` (AbstractFileSystem, optional): The storage driver to use for loading files.\n",
+    "                                             If not given, \".env\" will be attempted from the current working\n",
+    "                                             directory;  if that does not exist, an empty dict will be used.\n",
     "            - `override` (bool, optional): If True, variables from the .env file or schema default override existing\n",
     "                                           `os.environ` variables.\n",
     "        \"\"\"\n",
     "\n",
     "        # WARN this sidesteps storage_driver!\n",
-    "        # it will cause breakage if storage_driver != OSFS AND `.env` exists in PWD\n",
+    "        # it will cause breakage if storage_driver != DirFileSystem AND `.env` exists in PWD\n",
     "        if dotenv_path is None:\n",
     "            maybe_dotenv_path = dotenv.find_dotenv()  # '' if not exist; else abspath\n",
     "            if maybe_dotenv_path:\n",
@@ -369,13 +372,13 @@
     "        if dotenv_path:\n",
     "            dotenv_storage_driver = storage_driver or cls._get_maybe_abspath_driver(dotenv_path)\n",
     "            with dotenv_storage_driver.open(dotenv_path) as ifile:\n",
-    "                config = dotenv.dotenv_values(stream=ifile)\n",
+    "                config = dotenv.dotenv_values(stream=StringIO(ifile.read().decode('utf-8')))\n",
     "                \n",
     "        if config is None:\n",
     "            dotenv_storage_driver = storage_driver or cls.get_default_storage_driver()\n",
     "            if dotenv_storage_driver.exists('.env'):  # unlike dotenv.find_dotenv, stay relative!\n",
     "                with dotenv_storage_driver.open('.env') as ifile:\n",
-    "                    config = dotenv.dotenv_values(stream=ifile)\n",
+    "                    config = dotenv.dotenv_values(stream=StringIO(ifile.read().decode('utf-8')))\n",
     "        \n",
     "        if config is None:\n",
     "            config = {}\n",
@@ -580,7 +583,7 @@
    "outputs": [],
    "source": [
     "#| hide\n",
-    "from fs.memoryfs import MemoryFS"
+    "from morefs.memory import MemFS"
    ]
   },
   {
@@ -590,14 +593,15 @@
    "outputs": [],
    "source": [
     "#| hide\n",
-    "# test ability to override the storage driver (memoryfs here)\n",
+    "# test ability to override the storage driver (MemFS here)\n",
     "\n",
-    "memfs = MemoryFS()\n",
+    "memfs = MemFS()\n",
     "\n",
-    "memfs.makedirs('extra-long-directory-place', recreate=True)\n",
-    "with memfs.open('extra-long-directory-place/schema.json', 'w') as ofile:\n",
+    "memfs.makedirs('extra-long-directory-place', exist_ok=True)\n",
+    "temp_config_validator_json_schema_path = 'extra-long-directory-place/schema.json'\n",
+    "with memfs.open(temp_config_validator_json_schema_path, 'w') as ofile:\n",
     "    ofile.write(json.dumps(example_properties_schema))\n",
-    "    os.environ['CONFIG_VALIDATOR_JSON_SCHEMA'] = ofile.name\n",
+    "    os.environ['CONFIG_VALIDATOR_JSON_SCHEMA'] = temp_config_validator_json_schema_path\n",
     "\n",
     "validator = ConfigValidator(storage_driver=memfs)\n",
     "validated_config = validator.load_config({\n",
@@ -615,7 +619,7 @@
     "\n",
     "# test loading dotenv from an arbitrary file\n",
     "\n",
-    "memfs.makedirs('special-bespoke-location', recreate=True)\n",
+    "memfs.makedirs('special-bespoke-location', exist_ok=True)\n",
     "with memfs.open('special-bespoke-location/my-own.env', 'w') as ofile:\n",
     "    ofile.write('\\n'.join([\n",
     "        'string_value_with_enum=only',\n",
@@ -647,7 +651,7 @@
     "#| hide\n",
     "# test non-os FS with fallback .env path (=$PWD/.env)\n",
     "\n",
-    "memfs_fallback = MemoryFS()\n",
+    "memfs_fallback = MemFS()\n",
     "\n",
     "with memfs_fallback.open('schema.json', 'w') as ofile:\n",
     "    ofile.write(json.dumps(example_properties_schema))\n",
@@ -714,7 +718,7 @@
     "        }\n",
     "    }))\n",
     "    \n",
-    "memfs.makedirs('precedence-test', recreate=True)\n",
+    "memfs.makedirs('precedence-test', exist_ok=True)\n",
     "with memfs.open('precedence-test/.env', 'w') as ofile:\n",
     "    ofile.write('\\n'.join([\n",
     "        'A_VALUE_TO_OVERRIDE=in dotenv',\n",
diff --git a/schematized_config/__init__.py b/schematized_config/__init__.py
@@ -1 +1 @@
-__version__ = "0.0.12"
+__version__ = "0.0.13"
diff --git a/schematized_config/core.py b/schematized_config/core.py
@@ -12,9 +12,11 @@
 
 import dotenv
 import jsonschema
-from fs.base import FS
-from fs.osfs import OSFS
+from fsspec.spec import AbstractFileSystem
+from fsspec.implementations.local import LocalFileSystem
+from fsspec.implementations.dirfs import DirFileSystem
 from jsonschema import ValidationError, validate
+from io import StringIO
 
 logger = logging.getLogger(__name__)
 
@@ -103,35 +105,36 @@ class ConfigValidator(object):
     # this is a risky value: it gets reused across instances;
     # the idea is to maybe set it once and use it multiple times.
     # but in testing this smells bad
-    DEFAULT_STORAGE_DRIVER: FS = None  # defaults to OSFS
+    DEFAULT_STORAGE_DRIVER: AbstractFileSystem = None  # defaults to DirFileSystem
     
     CONFIG_VALIDATOR_JSON_SCHEMA_ENVVAR_NAME = 'CONFIG_VALIDATOR_JSON_SCHEMA'
     
     @classmethod
     def get_default_storage_driver(cls):
         if cls.DEFAULT_STORAGE_DRIVER is None:
-            cls.DEFAULT_STORAGE_DRIVER = OSFS(os.getcwd())
+            cls.DEFAULT_STORAGE_DRIVER = DirFileSystem(os.getcwd())
         return cls.DEFAULT_STORAGE_DRIVER
 
     
     @classmethod
     def _get_maybe_abspath_driver(cls, maybe_abspath: str):
         if os.path.isabs(maybe_abspath):  # special case
-            return OSFS('/')
+            return DirFileSystem('/')
         else:
             return cls.get_default_storage_driver()
         
     @classmethod
-    def load_json(cls, json_source: Union[str, dict]=None, storage_driver: FS = None) -> dict:
+    def load_json(cls, json_source: Union[str, dict]=None, storage_driver: AbstractFileSystem = None) -> dict:
         """
         Convenience method to return a dictionary from either a file path or an already-loaded dictionary.
 
         Args:
             - `json_source` (Union[str, dict], optional): The JSON source to load.
                                 This can be a file path (str) 
                                 or an already loaded dictionary (dict). 
-            - `storage_driver` (FS, optional): An instance of the storage driver used to load the JSON file. 
-                                               If not provided, OSFS from the current working dir is used.
+            - `storage_driver` (AbstractFileSystem, optional): An instance of the storage driver used to
+                                load the JSON file. If not provided, DirFileSystem from the current
+                                working dir is used.
 
         Returns:
             dict: A dictionary that was loaded from the provided `json_source`.
@@ -145,14 +148,14 @@ def load_json(cls, json_source: Union[str, dict]=None, storage_driver: FS = None
             return json.load(ifile)
 
     @classmethod
-    def get_default_json_schema(cls, storage_driver: FS = None) -> dict:
+    def get_default_json_schema(cls, storage_driver: AbstractFileSystem = None) -> dict:
         if cls.CONFIG_VALIDATOR_JSON_SCHEMA_ENVVAR_NAME in os.environ:
             expected_json_schema_path = \
                 os.environ[cls.CONFIG_VALIDATOR_JSON_SCHEMA_ENVVAR_NAME]
             return cls.load_json(expected_json_schema_path, storage_driver)
         return None
 
-    def __init__(self, json_schema: Union[str, dict]=None, storage_driver: FS=None):
+    def __init__(self, json_schema: Union[str, dict]=None, storage_driver: AbstractFileSystem=None):
         """
         Initialize the instance with a JSON schema and a storage driver.
 
@@ -161,8 +164,8 @@ def __init__(self, json_schema: Union[str, dict]=None, storage_driver: FS=None):
                                                           If no value is provided, it will fall back to looking for an environment 
                                                           variable corresponding to the class variable 
                                                           `CONFIG_VALIDATOR_JSON_SCHEMA_ENVVAR_NAME` to find a JSON schema file.
-            - `storage_driver` (FS, optional): The storage driver to use. If no value is provided, 
-                                               `self.__class__.DEFAULT_STORAGE_DRIVER` is used.
+            - `storage_driver` (AbstractFileSystem, optional): The storage driver to use. If no value is provided, 
+                                                          `self.__class__.DEFAULT_STORAGE_DRIVER` is used.
 
         Raises:
             Exception: An exception is raised if no valid JSON schema is provided or found.
@@ -198,7 +201,7 @@ def load_validated_environment(cls, json_schema: Union[str, dict]=None, **kwargs
     def load_dotenv(cls,
                     json_schema: Union[str, dict]=None,
                     dotenv_path: str=None,
-                    storage_driver: FS=None,
+                    storage_driver: AbstractFileSystem=None,
                     override: bool=False,
                    ):
         """
@@ -211,15 +214,15 @@ def load_dotenv(cls,
                                             (such as an environment variable or default schema) is used.
             - `dotenv_path` (str, optional): Path to the .env file to load the variables from.
                                              If not provided, loads an empty dict to start.
-            - `storage_driver` (FS, optional): The storage driver to use for loading files. If not given,
-                                               ".env" will be attempted from the current working directory;
-                                               if that does not exist, an empty dict will be used.
+            - `storage_driver` (AbstractFileSystem, optional): The storage driver to use for loading files.
+                                             If not given, ".env" will be attempted from the current working
+                                             directory;  if that does not exist, an empty dict will be used.
             - `override` (bool, optional): If True, variables from the .env file or schema default override existing
                                            `os.environ` variables.
         """
 
         # WARN this sidesteps storage_driver!
-        # it will cause breakage if storage_driver != OSFS AND `.env` exists in PWD
+        # it will cause breakage if storage_driver != DirFileSystem AND `.env` exists in PWD
         if dotenv_path is None:
             maybe_dotenv_path = dotenv.find_dotenv()  # '' if not exist; else abspath
             if maybe_dotenv_path:
@@ -231,13 +234,13 @@ def load_dotenv(cls,
         if dotenv_path:
             dotenv_storage_driver = storage_driver or cls._get_maybe_abspath_driver(dotenv_path)
             with dotenv_storage_driver.open(dotenv_path) as ifile:
-                config = dotenv.dotenv_values(stream=ifile)
+                config = dotenv.dotenv_values(stream=StringIO(ifile.read().decode('utf-8')))
                 
         if config is None:
             dotenv_storage_driver = storage_driver or cls.get_default_storage_driver()
             if dotenv_storage_driver.exists('.env'):  # unlike dotenv.find_dotenv, stay relative!
                 with dotenv_storage_driver.open('.env') as ifile:
-                    config = dotenv.dotenv_values(stream=ifile)
+                    config = dotenv.dotenv_values(stream=StringIO(ifile.read().decode('utf-8')))
         
         if config is None:
             config = {}
diff --git a/settings.ini b/settings.ini
@@ -1,7 +1,7 @@
 [DEFAULT]
 repo = python-schematized-config
 lib_name = python-schematized-config
-version = 0.0.12
+version = 0.0.13
 min_python = 3.7
 license = apache2
 black_formatting = False
@@ -26,7 +26,7 @@ keywords = nbdev jupyter notebook python
 language = English
 status = 3
 user = tutankalex
-requirements = jsonschema>=4.17.3 python-dotenv>=1.0.0 fs>=2.4.16 fastcore>=1.5
+requirements = fastcore>=1.5 jsonschema>=4.17.3 python-dotenv>=1.0.0 pygtrie>=2.5.0 morefs>=0.2.2
 readme_nb = index.ipynb
 allowed_metadata_keys = 
 allowed_cell_metadata_keys = 

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-__version__ = "0.0.12"`
	`1`	`+__version__ = "0.0.13"`