diff --git a/.gitignore b/.gitignore index a1b4f35..e81dead 100644 --- a/.gitignore +++ b/.gitignore @@ -104,4 +104,3 @@ venv.bak/ # mypy .mypy_cache/ - diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..61dfa7e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,26 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.2.3 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: flake8 + - id: check-merge-conflict + - id: debug-statements + - id: no-commit-to-branch + +- repo: https://github.com/asottile/seed-isort-config + rev: v1.9.2 + hooks: + - id: seed-isort-config + +- repo: https://github.com/ambv/black + rev: 23.1.0 + hooks: + - id: black + language_version: python3.11 + +- repo: https://github.com/pre-commit/mirrors-isort + rev: v4.3.20 + hooks: + - id: isort diff --git a/LICENSE b/LICENSE index 14e2f77..a612ad9 100644 --- a/LICENSE +++ b/LICENSE @@ -35,7 +35,7 @@ Mozilla Public License Version 2.0 means any form of the work other than Source Code Form. 1.7. "Larger Work" - means a work that combines Covered Software with other material, in + means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. 1.8. "License" diff --git a/README.md b/README.md index 36ecc4f..42ab1fd 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ django-ipfs-storage =================== -Store [Django file-uploads](https://docs.djangoproject.com/en/1.11/topics/files/) +Store [Django file-uploads](https://docs.djangoproject.com/en/4.0/topics/files/) on the [Interplanetary File System](https://ipfs.io/). Uploads are added and pinned to the configured IPFS node, @@ -10,9 +10,9 @@ This hash is the name that is saved to your database. Duplicate content will also have the same address, saving disk space. -Because of this only file creation and reading is supported. +Because of this, only file creation and reading is supported. -Other IPFS users access and reseed a piece of content +Other IPFS users access and reseed a piece of content through its unique content ID. Differently-distributed (i.e. normal HTTP) users can access the uploads through an HTTP→IPFS gateway. @@ -24,7 +24,7 @@ Installation ```bash pip install django-ipfs-storage ``` - +It uses the only Python maintained library for IPFS (as of March 2023) [IPFS-Toolkit](https://github.com/emendir/IPFS-Toolkit-Python) Configuration ------------- @@ -34,11 +34,8 @@ and returns URLs pointing to the public HTTP Gateway To customise this, set the following variables in your `settings.py`: -- `IPFS_STORAGE_API_URL`: defaults to `'http://localhost:5001/api/v0/'`. -- `IPFS_GATEWAY_API_URL`: defaults to `'https://ipfs.io/ipfs/'`. - -Set `IPFS_GATEWAY_API_URL` to `'http://localhost:8080/ipfs/'` to serve content -through your local daemon's HTTP gateway. +- `IPFS_STORAGE_API_URL`: defaults to `'/ip4/0.0.0.0/tcp/5001'`. +- `IPFS_GATEWAY_API_URL`: defaults to `'/ip4/0.0.0.0/tcp/8080'`. Usage @@ -55,9 +52,11 @@ Use IPFS as [Django's default file storage backend](https://docs.djangoproject.c DEFAULT_FILE_STORAGE = 'ipfs_storage.InterPlanetaryFileSystemStorage' -IPFS_STORAGE_API_URL = 'http://localhost:5001/api/v0/' -IPFS_STORAGE_GATEWAY_URL = 'http://localhost:8080/ipfs/' -``` +IPFS_STORAGE_API_URL = '/ip4/localhost/tcp/5001' + +IPFS_STORAGE_GATEWAY_URL = '/ip4/localhost/tcp/8080' +IPFS_STORAGE_GATEWAY_API_URL = 'http://localhost:8080/ipfs' +``` ### For a specific FileField @@ -67,12 +66,12 @@ Alternatively, you may only want to use the IPFS storage backend for a single fi ```python from django.db import models -from ipfs_storage import InterPlanetaryFileSystemStorage +from ipfs_storage import InterPlanetaryFileSystemStorage class MyModel(models.Model): # … - file_stored_on_ipfs = models.FileField(storage=InterPlanetaryFileSystemStorage()) + file_stored_on_ipfs = models.FileField(storage=InterPlanetaryFileSystemStorage()) other_file = models.FileField() # will still use DEFAULT_FILE_STORAGE ``` @@ -84,7 +83,7 @@ FAQ ### Why IPFS? -Not my department. See . +Not my department. See . ### How do I ensure my uploads are always available? @@ -99,7 +98,7 @@ See above. ### How do I delete an upload? Because of the distributed nature of IPFS, anyone who accesses a piece -of content keeps a copy, and reseeds it for you automatically until it's +of content keeps a copy, and reseeds it for you automatically until it's evicted from their node's local cache. Yay bandwidth costs! Boo censorship! Unfortunately, if you're trying to censor yourself (often quite necessary), diff --git a/ipfs_storage/__init__.py b/ipfs_storage/__init__.py index d41fafc..4028652 100644 --- a/ipfs_storage/__init__.py +++ b/ipfs_storage/__init__.py @@ -1,82 +1 @@ -from urllib.parse import urlparse - -from django.conf import settings -from django.core.files.base import File, ContentFile -from django.core.files.storage import Storage -from django.utils.deconstruct import deconstructible -import ipfsapi - - -__version__ = '0.0.4' - - -@deconstructible -class InterPlanetaryFileSystemStorage(Storage): - """IPFS Django storage backend. - - Only file creation and reading is supported - due to the nature of the IPFS protocol. - """ - - def __init__(self, api_url=None, gateway_url=None): - """Connect to Interplanetary File System daemon API to add/pin files. - - :param api_url: IPFS control API base URL. - Also configurable via `settings.IPFS_STORAGE_API_URL`. - Defaults to 'http://localhost:5001/api/v0/'. - :param gateway_url: Base URL for IPFS Gateway (for HTTP-only clients). - Also configurable via `settings.IPFS_STORAGE_GATEWAY_URL`. - Defaults to 'https://ipfs.io/ipfs/'. - """ - parsed_api_url = urlparse(api_url or getattr(settings, 'IPFS_STORAGE_API_URL', 'http://localhost:5001/api/v0/')) - self._ipfs_client = ipfsapi.connect( - parsed_api_url.hostname, - parsed_api_url.port, - parsed_api_url.path.strip('/') - ) - self.gateway_url = gateway_url or getattr(settings, 'IPFS_STORAGE_GATEWAY_URL', 'https://ipfs.io/ipfs/') - - def _open(self, name: str, mode='rb') -> File: - """Retrieve the file content identified by multihash. - - :param name: IPFS Content ID multihash. - :param mode: Ignored. The returned File instance is read-only. - """ - return ContentFile(self._ipfs_client.cat(name), name=name) - - def _save(self, name: str, content: File) -> str: - """Add and pin content to IPFS daemon. - - :param name: Ignored. Provided to comply with `Storage` interface. - :param content: Django File instance to save. - :return: IPFS Content ID multihash. - """ - multihash = self._ipfs_client.add_bytes(content.__iter__()) - self._ipfs_client.pin_add(multihash) - return multihash - - def get_valid_name(self, name): - """Returns name. Only provided for compatibility with Storage interface.""" - return name - - def get_available_name(self, name, max_length=None): - """Returns name. Only provided for compatibility with Storage interface.""" - return name - - def size(self, name: str) -> int: - """Total size, in bytes, of IPFS content with multihash `name`.""" - return self._ipfs_client.object_stat(name)['CumulativeSize'] - - def delete(self, name: str): - """Unpin IPFS content from the daemon.""" - self._ipfs_client.pin_rm(name) - - def url(self, name: str): - """Returns an HTTP-accessible Gateway URL by default. - - Override this if you want direct `ipfs://…` URLs or something. - - :param name: IPFS Content ID multihash. - :return: HTTP URL to access the content via an IPFS HTTP Gateway. - """ - return '{gateway_url}{multihash}'.format(gateway_url=self.gateway_url, multihash=name) +from .storage import InterPlanetaryFileSystemStorage diff --git a/ipfs_storage/storage.py b/ipfs_storage/storage.py new file mode 100644 index 0000000..6f8210f --- /dev/null +++ b/ipfs_storage/storage.py @@ -0,0 +1,68 @@ +from urllib.parse import urlparse + +from ipfs_api import ipfshttpclient + +from django.conf import settings +from django.core.files.base import File, ContentFile +from django.utils.deconstruct import deconstructible +from django.core.files.storage import Storage + + +@deconstructible +class InterPlanetaryFileSystemStorage(Storage): + """IPFS Django storage backend. + + Only file creation and reading is supported due to the nature of the IPFS protocol. + """ + + def __init__(self, api_url=None, gateway_url=None): + """Connect to Interplanetary File System daemon API to add/pin files.""" + self._ipfs_client = ipfshttpclient.connect(settings.IPFS_STORAGE_API_URL) + self._ipfs_client.config.set( + "Addresses.Gateway", settings.IPFS_STORAGE_GATEWAY_URL + ) + + def _open(self, name: str, mode="rb") -> File: + """Retrieve the file content identified by multihash. + + :param name: IPFS Content ID multihash. + :param mode: Ignored. The returned File instance is read-only. + """ + return ContentFile(self._ipfs_client.cat(name), name=name) + + def _save(self, name: str, content: File) -> str: + """Add and pin content to IPFS daemon. + + :param name: Ignored. Provided to comply with `Storage` interface. + :param content: Django File instance to save. + :return: IPFS Content ID multihash. + """ + multihash = self._ipfs_client.add_bytes(content.__iter__()) + self._ipfs_client.pin.add(multihash) + return multihash + + def get_valid_name(self, name): + """Returns name. Only provided for compatibility with Storage interface.""" + return name + + def get_available_name(self, name, max_length=None): + """Returns name. Only provided for compatibility with Storage interface.""" + return name + + def size(self, name: str) -> int: + """Total size, in bytes, of IPFS content with multihash `name`.""" + return self._ipfs_client.object.stat(name)["CumulativeSize"] + + def delete(self, name: str): + """Unpin IPFS content from the daemon.""" + self._ipfs_client.pin.rm(name) + + def url(self, name: str): + """Returns an HTTP-accessible Gateway URL by default. + + Override this if you want direct `ipfs://…` URLs or something. + + :param name: IPFS Content ID multihash. + :return: HTTP URL to access the content via an IPFS HTTP Gateway. + """ + return f"{settings.IPFS_STORAGE_GATEWAY_API_URL}/ipfs/{name}" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..21d88cd --- /dev/null +++ b/setup.cfg @@ -0,0 +1,16 @@ +[flake8] +ignore = E203, E266, E501 +max-line-length = 100 +max-complexity = 18 +select = B,C,E,F,W,T4,B9 + +[isort] +use_parentheses = True +multi_line_output = 3 +length_sort = 1 +lines_between_types = 0 +known_django = django +known_third_party = ipfs_api,pytest,setuptools +sections = FUTURE, STDLIB, THIRDPARTY, DJANGO, FIRSTPARTY, LOCALFOLDER +no_lines_before = LOCALFOLDER +known_first_party = skatepedia,scraper diff --git a/setup.py b/setup.py index 9f0bf0b..74f153d 100644 --- a/setup.py +++ b/setup.py @@ -1,44 +1,32 @@ -from setuptools import setup, find_packages -from codecs import open - -from ipfs_storage import __version__ +import os +from setuptools import setup, find_packages -try: - import pypandoc - long_description = pypandoc.convert('README.md', 'rst') -except(IOError, ImportError): - with open('README.rst', encoding='utf-8') as f: - long_description = f.read() +HERE = os.path.dirname(os.path.abspath(__file__)) +README = open(os.path.join(HERE, "README.md")).read() +__version__ = "0.1.0" setup( - name='django-ipfs-storage', - description='IPFS storage backend for Django.', - long_description=long_description, - keywords='django ipfs storage', + name="django-ipfs-storage", + description="IPFS storage backend for Django.", + long_description=README, + keywords="django ipfs storage", version=__version__, - license='MPL 2.0', - - author='Ben Jeffrey', - author_email='mail@benjeffrey.net', - url='https://github.com/jeffbr13/django-ipfs-storage', - + license="MPL 2.0", + author="Ben Jeffrey", + author_email="mail@benjeffrey.net", + url="https://github.com/skatepedia/django-ipfs-storage", classifiers=( - 'Development Status :: 3 - Alpha', - 'Programming Language :: Python :: 3', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', - 'Framework :: Django', + "Programming Language :: Python :: 3", + "Intended Audience :: Developers", + "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", + "Framework :: Django", ), - packages=find_packages(), - install_requires=( - 'django', - 'ipfsapi', + "Django", + "IPFS-Toolkit", ), - setup_requires=( - 'pypandoc', - ) + test_requires=("pytest"), ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..2c02ad5 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,27 @@ +from unittest import mock + +import pytest + +from django.conf import settings + + +def pytest_configure(): + settings.configure( + IPFS_STORAGE_API_URL="/ip4/0.0.0.0/tcp/5001", + IPFS_STORAGE_GATEWAY_URL="/ip4/0.0.0.0/tcp/8080", + IPFS_STORAGE_GATEWAY_API_URL="http://0.0.0.0:8080", + ) + + +@pytest.fixture +def ipfs_client(): + """Return an ipfshttpclient.Client mock. + Used for instantation of :class:`ipfs_storage.InterPlanetaryFileSystemStorage`. + Introduce it in tests as a function argument `ipfs_client`. + """ + with ( + mock.patch("ipfs_storage.storage.ipfshttpclient.connect") as ipfs_conn_mock, + mock.patch("ipfs_storage.storage.ipfshttpclient.Client") as client_mock, + ): + ipfs_conn_mock.return_value = client_mock + yield client_mock diff --git a/tests/test_storage.py b/tests/test_storage.py new file mode 100644 index 0000000..a3a3f21 --- /dev/null +++ b/tests/test_storage.py @@ -0,0 +1,51 @@ +from unittest import mock + +from django.conf import settings +from django.core.files.base import ContentFile + +from ipfs_storage import InterPlanetaryFileSystemStorage + + +def test_storage_save(ipfs_client): + name = "test_storage_save.txt" + content = ContentFile(b"new content") + storage = InterPlanetaryFileSystemStorage() + storage.save(name, content) + + assert ipfs_client.add_bytes.called + assert ipfs_client.pin.add.called + + +def test_storage_open(ipfs_client): + storage = InterPlanetaryFileSystemStorage() + name = "test_storage_open.txt" + ipfs_client.cat = mock.MagicMock(return_value=b"new content") + storage.open(name) + + assert ipfs_client.cat.called + assert ipfs_client.cat.call_args[0][0] == name + + +def test_storage_delete(ipfs_client): + storage = InterPlanetaryFileSystemStorage() + name = "test_storage_open.txt" + storage.delete(name) + assert ipfs_client.pin.rm.called + assert ipfs_client.pin.rm.call_args[0][0] == name + + +def test_storage_size(ipfs_client): + storage = InterPlanetaryFileSystemStorage() + name = "test_storage_save.txt" + content = ContentFile(b"new content") + ipfs_client.object.stat.return_value = {"CumulativeSize": 100} + size = storage.size(name) + assert ipfs_client.object.stat.called + assert size == 100 + + +def test_storage_url(ipfs_client): + storage = InterPlanetaryFileSystemStorage() + cid = "QmQU6KTY5w4uuxQkLYKNXSJ89naNnkmkJhtne4f1yBquzv" + url = storage.url(cid) + assert f"{settings.IPFS_STORAGE_GATEWAY_API_URL}/ipfs/{cid}" == url