DOC: trim docs, fix inaccuracies and rendering issues

datalad_fuse/__init__.py

@@ -94,7 +94,9 @@ class FuseFS(Interface):
             args=("--caching",),
             choices=["none", "ondisk"],
             default="none",
-            doc="Whether to cache fsspec'ed files on disk on not at all",
+            doc="""Whether to keep the data fetched from remote URLs in an
+            on-disk cache ('ondisk'), or only in memory while a file is open
+            ('none', the default).""",
         ),
         # TODO: (might better become config vars?)
         # --cache=persist

datalad_fuse/fsspec.py

@@ -62,8 +62,8 @@ class DatasetAdapter:
     caching : bool
         If true, keep the data fetched from remote URLs in a sparse on-disk
         cache under ``<path>/.git/datalad/cache/fsspec/``, to be reused by
-        subsequent reads (for up to a week).  If false, data are only
-        buffered in memory while a file is open.
+        subsequent reads (for a week after they were first cached).  If false,
+        data are only buffered in memory while a file is open.
     mode_transparent : bool
         If true, paths of key files under ``.git/annex/objects/`` (the targets
         of annexed symlinks) are opened as annexed content, fetched from a

datalad_fuse/fsspec_head.py

@@ -57,7 +57,9 @@ class FsspecHead(Interface):
             args=("--caching",),
             choices=["none", "ondisk"],
             default="none",
-            doc="Whether to cache fsspec'ed files on disk on not at all",
+            doc="""Whether to keep the data fetched from remote URLs in an
+            on-disk cache ('ondisk'), or only in memory while a file is open
+            ('none', the default).""",
         ),
         "path": Parameter(
             args=("path",),

docs/source/_static/custom.css

@@ -0,0 +1,29 @@
+/* Let table cells wrap instead of overflowing the content column */
+.wy-table-responsive table td,
+.wy-table-responsive table th {
+    white-space: normal;
+}
+
+/* Let long inline code break, e.g. on narrow (phone) screens */
+code.literal,
+code.literal span.pre {
+    white-space: normal;
+    overflow-wrap: anywhere;
+}
+
+/* On narrow screens, stack field lists (parameters etc. in the API
+   reference), and let long titles, signatures and index tables wrap */
+@media screen and (max-width: 768px) {
+    html.writer-html5 .rst-content dl.field-list {
+        display: block;
+    }
+    html.writer-html5 .rst-content dl.field-list > dd {
+        margin-left: 1em;
+    }
+    .rst-content h1,
+    .rst-content dt.sig,
+    .rst-content table.autosummary td,
+    .rst-content table.indextable td {
+        overflow-wrap: anywhere;
+    }
+}

docs/source/api.rst

@@ -33,6 +33,7 @@ Opening files: ``datalad_fuse.fsspec``
 .. autoclass:: FileState
    :members:
    :undoc-members:
+   :show-inheritance:
 
 
 git-annex helpers: ``datalad_fuse.utils``
@@ -55,3 +56,4 @@ FUSE file system: ``datalad_fuse.fuse_``
 
 .. autoclass:: DataLadFUSE
    :no-members:
+   :show-inheritance:

docs/source/changelog.rst

@@ -1,5 +1,5 @@
 Changelog
 *********
 
-.. include:: ../../CHANGELOG.md
+.. include:: generated/CHANGELOG.md.inc
    :parser: myst_parser.sphinx_

docs/source/concepts.rst

@@ -73,43 +73,21 @@ in order until one can be opened:
    versions cannot be listed, the file's current version is used, with a
    warning that it may not be the right one.
 
-   .. note::
-      This fallback is newer than the 0.6.0 release.
-
 If none of the candidates can be opened, opening the file fails with
 ``Could not find a usable URL for <path> within <dataset>``.
 
 To see which URLs are tried, enable debug logging (see
 :ref:`troubleshooting-logging`).
 
 Requests answered with a server error (HTTP 5xx) are retried up to four
-times, waiting up to 36 seconds in between, and each retry logs a warning
-"Retrying request to ...".  A server that keeps failing can thus delay
-opening a file by a couple of minutes before the next URL is tried.
-Connection errors are not retried: the next candidate URL is tried right
-away.
-
-
-Access is anonymous
--------------------
-
-URLs are accessed without credentials, unless credentials are part of the URL
-itself (e.g. ``https://user:token@host/...``, as can be the case for git
-remotes).  DataLad's credential store, ``~/.netrc`` or AWS credentials are not
-consulted, so content that requires authentication (for example embargoed
-DANDI data or private repositories) cannot be read this way; use ``datalad
-get`` for it.
-
-HTTP proxies configured with the ``http_proxy``/``https_proxy`` environment
-variables are not used either, so where the internet can only be reached
-through a proxy, remote content cannot be read.
-
-Content is not verified
------------------------
+times, with delays of up to 36 seconds (each retry logs "Retrying request to
+..."); after a connection error, the next URL is tried right away.
 
-``datalad get`` and ``git annex get`` verify downloaded content against its
-key's checksum.  ``datalad-fuse`` reads parts of files and does not verify
-them, so it relies on the URLs serving the right content.
+URLs are accessed anonymously: credentials are only used if they are part of
+the URL itself (e.g. ``https://user:token@host/...`` for some git remotes),
+and HTTP proxies set with ``http_proxy``/``https_proxy`` are not used.  Unlike
+``datalad get``, ``datalad-fuse`` does not verify content against the
+checksum in its key.  See :ref:`limitations` for a summary.
 
 
 Reading only what is needed
@@ -123,9 +101,9 @@ to 5 MiB, while reading the metadata and a few arrays of a multi-gigabyte
 NWB/HDF5 file transfers only a small fraction of it.
 
 This works best for file formats designed for partial access, such as HDF5
-and NWB, and with tools that only read the parts they need.  Reading a whole file, e.g. to decompress a ``.nii.gz`` file or compute
-a checksum, transfers all of it, one 5 MiB request after the other; if you
-need entire files, ``datalad get`` is usually faster.
+and NWB, and with tools that only read the parts they need.  Reading whole
+files transfers all of them, one 5 MiB request after the other, and is
+usually faster with ``datalad get``.
 
 The first access to a remote file takes a little time, as git-annex has to be
 queried and a connection established; subsequent reads of the same open file
@@ -143,12 +121,13 @@ ondisk`` for ``datalad fusefs`` and ``datalad fsspec-head``), fsspec's
 `CachingFileSystem
 <https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.implementations.cached.CachingFileSystem>`_
 stores the fetched blocks on disk and reuses them when the same file is read
-again, also in later sessions, for up to a week; after that, fsspec considers
-them expired and fetches the data again.
+again, also in later sessions.  A cached file expires a week after it was
+first cached; its data are then fetched again.
 
 - The cache of a dataset is located at ``.git/datalad/cache/fsspec/`` inside
   that dataset, so each (sub)dataset has its own.
-- Files are cached *sparsely*: only the blocks that were read are stored.
+- Files are cached *sparsely*: only the blocks that were read are stored, so
+  the cache takes less disk space than the files' sizes suggest.
 - The cache is separate from the git-annex object store: cached files do not
   count as present content for ``git annex`` or ``datalad``.
 - The cache does not shrink by itself.  Remove it with

docs/source/conf.py

@@ -3,8 +3,10 @@
 # See https://www.sphinx-doc.org/en/master/usage/configuration.html
 
 import datetime
+import os
 from os.path import abspath, dirname, exists
 from os.path import join as opj
+import re
 import subprocess
 import sys
 
@@ -34,6 +36,15 @@
         check=True,
     )
 
+# CHANGELOG.md is generated by `auto` from PR titles, some of which, such as
+# "[gh-actions](deps): ...", read as Markdown links to a document "deps".
+# changelog.rst includes a copy with such links escaped.
+with open(opj(repo_root, "CHANGELOG.md"), encoding="utf-8") as f:
+    _changelog = re.sub(r"\[([^]]+)\]\(([\w-]+)\)", r"\\[\1\\](\2)", f.read())
+os.makedirs(opj(docs_source, "generated"), exist_ok=True)
+with open(opj(docs_source, "generated", "CHANGELOG.md.inc"), "w", encoding="utf-8") as f:
+    f.write(_changelog)
+
 # Methods decorated with methodtools.lru_cache are hidden behind a descriptor
 # that autodoc does not recognize as a method.  For the documentation only,
 # replace them with the functions they wrap, so that they are documented.
@@ -74,11 +85,14 @@
 
 pygments_style = "sphinx"
 
+# Do not turn "--" into dashes: option names must stay copyable
+smartquotes_action = "qe"
+
 # for the module reference
 autosummary_generate = True
 autodoc_member_order = "bysource"
 autodoc_typehints = "description"
-autodoc_default_options = {"members": True, "show-inheritance": True}
+autodoc_default_options = {"members": True}
 # mfusepy needs libfuse at import time; mock it if libfuse is not installed
 try:
     import mfusepy  # noqa: F401
@@ -98,15 +112,15 @@
 copybutton_exclude = ".linenos, .gp, .go"
 
 myst_heading_anchors = 3
-# CHANGELOG.md is generated by `auto`: its headings jump from H1 to H4, and
-# PR titles such as "[gh-actions](deps): ..." look like links to documents
-suppress_warnings = ["myst.header", "myst.xref_missing"]
+# CHANGELOG.md is generated by `auto`: its headings jump from H1 to H4
+suppress_warnings = ["myst.header"]
 
 # -- Options for HTML output ----------------------------------------------
 
 html_theme = "sphinx_rtd_theme"
 html_logo = "_static/datalad_logo.png"
 html_static_path = ["_static"]
+html_css_files = ["custom.css"]
 html_show_sourcelink = False
 html_context = {
     "display_github": True,
@@ -115,3 +129,24 @@
     "github_version": "master",
     "conf_py_path": "/docs/source/",
 }
+
+
+def _hide_edit_link(_app, pagename, _templatename, context, _doctree):
+    # These pages are generated or included, so there is no source to edit
+    if pagename.startswith("generated/") or pagename in (
+        "changelog",
+        "genindex",
+        "search",
+    ):
+        context["display_github"] = False
+
+
+def _hide_enum_signature(_app, what, name, _obj, _options, signature, ret):
+    if what == "class" and name == "datalad_fuse.fsspec.FileState":
+        return "", None
+    return signature, ret
+
+
+def setup(app):
+    app.connect("html-page-context", _hide_edit_link)
+    app.connect("autodoc-process-signature", _hide_enum_signature)

docs/source/contributing.rst

@@ -1,2 +1,4 @@
+:github_url: https://github.com/datalad/datalad-fuse/blob/master/CONTRIBUTING.md
+
 .. include:: ../../CONTRIBUTING.md
    :parser: myst_parser.sphinx_

docs/source/index.rst

@@ -33,31 +33,14 @@ get`` is usually faster, and keeps the files for later.
 
 There are two ways to use it:
 
-.. list-table::
-   :header-rows: 1
-   :stub-columns: 1
-   :widths: 16 42 42
-
-   * -
-     - FUSE mount
-     - Python
-   * - What
-     - ``datalad fusefs`` presents a dataset as a read-only directory tree in
-       which annexed files can be opened like local files.
-     - ``FsspecAdapter`` and ``DatasetAdapter`` return Python file objects
-       for files of a dataset.
-   * - Works with
-     - Any program: ``h5ls``, MATLAB, ``nwbinspector``, shell tools, ...
-     - Python libraries that accept file objects: h5py, pynwb, pandas,
-       json, ...
-   * - Needs
-     - Linux with FUSE installed
-     - Just Python (nothing to install beyond ``datalad-fuse``)
-
-Use Python if the library you read files with accepts Python file objects
-(h5py, PyNWB, pandas, ...); use the FUSE mount for tools that need a file
-name: MATLAB, FSL, command-line tools, or Python functions that only accept
-paths.
+- **A FUSE mount** (``datalad fusefs``, Linux with FUSE): the dataset appears
+  as a read-only directory tree, so any program can open its files by name:
+  ``h5ls``, MATLAB, FSL, ``nwbinspector``, shell tools, ...
+- **From Python** (``DatasetAdapter``, ``FsspecAdapter``; no FUSE needed): you
+  get Python file objects, which libraries such as h5py, PyNWB or pandas can
+  read from.
+
+Both need git-annex (see :doc:`installation`).
 
 
 A quick look
@@ -104,7 +87,7 @@ Or open files directly from Python:
 
    from datalad_fuse.fsspec import DatasetAdapter
 
-   # caching=False: keep fetched data in memory only (see "Caching")
+   # caching=False: keep fetched data in memory only
    with closing(DatasetAdapter("000582", caching=False)) as dsa:
        with dsa.open("sub-10073/sub-10073_ses-17010302_behavior+ecephys.nwb") as f:
            with h5py.File(f, "r") as h5, pynwb.NWBHDF5IO(file=h5) as io:
@@ -115,14 +98,14 @@ The :doc:`tutorial` walks through both in more detail.
 
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
    :caption: Getting started
 
    installation
    tutorial
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
    :caption: User guide
 
    concepts
@@ -131,7 +114,7 @@ The :doc:`tutorial` walks through both in more detail.
    troubleshooting
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
    :caption: Reference
 
    api
@@ -143,11 +126,3 @@ The :doc:`tutorial` walks through both in more detail.
    :caption: Development
 
    contributing
-
-
-Indices
-=======
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`

docs/source/installation.rst

@@ -38,8 +38,8 @@ along with it:
 
    $ python3 -m pip install datalad-fuse
 
-To install the development version, which may include features that are not
-released yet:
+This documentation describes the development version, which may include
+features that are not released yet.  To install it:
 
 .. code-block:: console
 
@@ -49,20 +49,12 @@ released yet:
 Checking the installation
 =========================
 
-``datalad-fuse`` is a DataLad extension: it adds commands to DataLad.  Check
-that DataLad finds them and that git-annex is available:
+``datalad-fuse`` is a DataLad extension: it adds commands to DataLad.  These
+commands should print their help, and the version of git-annex and of
+``datalad-fuse``:
 
 .. code-block:: console
 
    $ datalad fusefs --help
-   Usage: datalad fusefs [-h] [-d DATASET] [-f] [--mode-transparent]
-   ...
    $ git annex version
-   git-annex version: 10.20260901
-   ...
-
-and that the Python package can be imported:
-
-.. code-block:: console
-
    $ python3 -c "import datalad_fuse; print(datalad_fuse.__version__)"