Add support for connecting to the IPFS daemon over Unix domain sockets (fixes #197)

Author: ntninja

CHANGELOG.md

@@ -1,9 +1,11 @@
 py-ipfs-http-client 0.6.1 (XX.XX.20XX)
 --------------------------------------
 
- * Add typings for most of the public and private API and enable type checking with `mypy`
+ * Added typings for most of the public and private API and enable type checking with `mypy`
+ * Added support for connecting to the IPFS daemon using Unix domain sockets (implemented for both the requests and HTTPx backend)
  * Deprecate `.repo.gc(…)`s `return_result` parameter in favour of the newly introduced `quiet` parameter to match the newer HTTP API
  * If you use the undocumented `return_result` parameter anywhere else consider such use deprecated, support for this parameter will be removed in 0.7.X everywhere
+    * Rationale: This parameter used to map to using the HTTP HEAD method perform the given request without any reply being returned, but this feature has been dropped with go-IPFS 0.5 from the API.
 
 Bugfixes:
 

README.md

@@ -157,13 +157,19 @@ Use an IPFS server with basic auth (replace username and password with real cred
 >>> client = ipfshttpclient.connect('/dns/ipfs-api.example.com/tcp/443/https', auth=("username", "password"))
 ```
 
-Pass custom headers to your IPFS api with each request:
+Pass custom headers to the IPFS daemon with each request:
 ```py
 >>> import ipfshttpclient
 >>> headers = {"CustomHeader": "foobar"}
 >>> client = ipfshttpclient.connect('/dns/ipfs-api.example.com/tcp/443/https', headers=headers)
 ```
 
+Connect to the IPFS daemon using a Unix domain socket (plain HTTP only):
+```py
+>>> import ipfshttpclient
+>>> client = ipfshttpclient.connect("/unix/run/ipfs/ipfs.sock")
+```
+
 
 
 ## Documentation

ipfshttpclient/http_common.py

@@ -7,13 +7,16 @@
 
 import multiaddr  # type: ignore[import]
 from multiaddr.protocols import (P_DNS, P_DNS4, P_DNS6,  # type: ignore[import]
-                                 P_HTTP, P_HTTPS, P_IP4, P_IP6, P_TCP)
+                                 P_HTTP, P_HTTPS, P_IP4, P_IP6, P_TCP, P_UNIX)
 
 from . import encoding
 from . import exceptions
 from . import utils
 
 
+AF_UNIX = getattr(socket, "AF_UNIX", NotImplemented)
+
+
 if ty.TYPE_CHECKING:
 	import http.cookiejar  # noqa: F401
 	from typing_extensions import Literal, Protocol  # noqa: F401
@@ -240,7 +243,7 @@ def close(self) -> None:
 
 
 def multiaddr_to_url_data(addr: addr_t, base: str  # type: ignore[no-any-unimported]
-) -> ty.Tuple[str, socket.AddressFamily, bool]:
+) -> ty.Tuple[str, ty.Optional[str], socket.AddressFamily, bool]:
 	try:
 		addr = multiaddr.Multiaddr(addr)
 	except multiaddr.exceptions.ParseError as error:
@@ -255,17 +258,31 @@ def multiaddr_to_url_data(addr: addr_t, base: str  # type: ignore[no-any-unimpor
 		family = socket.AF_UNSPEC
 		host_numeric = proto.code in (P_IP4, P_IP6)
 
+		uds_path = None  # type: ty.Optional[str]
 		if proto.code in (P_IP4, P_DNS4):
 			family = socket.AF_INET
 		elif proto.code in (P_IP6, P_DNS6):
 			family = socket.AF_INET6
+		elif proto.code == P_UNIX and AF_UNIX is not NotImplemented:
+			family = AF_UNIX
+			uds_path = host
 		elif proto.code != P_DNS:
 			raise exceptions.AddressError(addr)
 
-		# Read port value
-		proto, port = next(addr_iter)
-		if proto.code != P_TCP:
-			raise exceptions.AddressError(addr)
+		if family == AF_UNIX:
+			assert uds_path is not None
+			netloc = urllib.parse.quote(uds_path, safe="")
+		else:
+			# Read port value for IP-based transports
+			proto, port = next(addr_iter)
+			if proto.code != P_TCP:
+				raise exceptions.AddressError(addr)
+			
+			# Pre-format network location URL part based on host+port
+			if ":" in host and not host.startswith("["):
+				netloc = "[{0}]:{1}".format(host, port)
+			else:
+				netloc = "{0}:{1}".format(host, port)
 
 		# Read application-level protocol name
 		secure = False
@@ -291,17 +308,15 @@ def multiaddr_to_url_data(addr: addr_t, base: str  # type: ignore[no-any-unimpor
 
 	# Convert the parsed `addr` values to a URL base and parameters for the
 	# HTTP library
-	if ":" in host and not host.startswith("["):
-		host = "[{0}]".format(host)
 	base_url = urllib.parse.SplitResult(
 		scheme   = "http" if not secure else "https",
-		netloc   = "{0}:{1}".format(host, port),
+		netloc   = netloc,
 		path     = base,
 		query    = "",
 		fragment = ""
 	).geturl()
 
-	return base_url, family, host_numeric
+	return base_url, uds_path, family, host_numeric
 
 
 def map_args_to_params(

ipfshttpclient/http_httpx.py

@@ -75,10 +75,11 @@ def map_args_to_httpx(
 
 
 class ClientSync(ClientSyncBase[httpx.Client]):
-	__slots__ = ("_session_base", "_session_kwargs", "_session_laddr")
+	__slots__ = ("_session_base", "_session_kwargs", "_session_laddr", "_session_uds_path")
 	_session_base: "httpx._types.URLTypes"
 	_session_kwargs: RequestArgs
 	_session_laddr: ty.Optional[str]
+	_session_uds_path: ty.Optional[str]
 
 	def _init(self, addr: addr_t, base: str, *,  # type: ignore[no-any-unimported]
 	          auth: auth_t,
@@ -87,16 +88,20 @@ def _init(self, addr: addr_t, base: str, *,  # type: ignore[no-any-unimported]
 	          params: params_t,
 	          timeout: timeout_t) -> None:
 		base_url: str
+		uds_path: ty.Optional[str]
 		family: socket.AddressFamily
 		host_numeric: bool
-		base_url, family, host_numeric = multiaddr_to_url_data(addr, base)
+		base_url, uds_path, family, host_numeric = multiaddr_to_url_data(addr, base)
 
 		self._session_laddr = None
+		self._session_uds_path = None
 		if family != socket.AF_UNSPEC:
 			if family == socket.AF_INET:
 				self._session_laddr = "0.0.0.0"
 			elif family == socket.AF_INET6:
 				self._session_laddr = "::"
+			elif family == socket.AF_UNIX:
+				self._session_uds_path = uds_path
 			else:
 				assert False, ("multiaddr_to_url_data should only return a socket "
 				               "address family of AF_INET, AF_INET6 or AF_UNSPEC")
@@ -113,6 +118,7 @@ def _init(self, addr: addr_t, base: str, *,  # type: ignore[no-any-unimported]
 	def _make_session(self) -> httpx.Client:
 		connection_pool = httpcore.SyncConnectionPool(
 			local_address = self._session_laddr,
+			uds = self._session_uds_path,
 
 			#XXX: Argument values duplicated from httpx._client.Client._init_transport:
 			keepalive_expiry          = 5.0,  #XXX: Value duplicated from httpx._client.KEEPALIVE_EXPIRY

ipfshttpclient/http_requests.py

@@ -71,8 +71,9 @@ def map_args_to_requests(
 
 
 class ClientSync(ClientSyncBase[requests.Session]):  # type: ignore[name-defined]
-	__slots__ = ("_base_url", "_session_props", "_default_timeout")
+	__slots__ = ("_base_url", "_default_timeout", "_session_props")
 	#_base_url: str
+	#_default_timeout: timeout_t
 	#_session_props: ty.Dict[str, ty.Any]
 
 	def _init(self, addr: addr_t, base: str, *,  # type: ignore[no-any-unimported]
@@ -81,7 +82,7 @@ def _init(self, addr: addr_t, base: str, *,  # type: ignore[no-any-unimported]
 	          headers: headers_t,
 	          params: params_t,
 	          timeout: timeout_t) -> None:
-		self._base_url, family, host_numeric = multiaddr_to_url_data(addr, base)
+		self._base_url, _, family, host_numeric = multiaddr_to_url_data(addr, base)
 
 		self._session_props = map_args_to_requests(
 			auth=auth,

ipfshttpclient/requests_wrapper.py

@@ -18,6 +18,8 @@
 	int(socket.AF_INET):  "ip4",
 	int(socket.AF_INET6): "ip6",
 }
+if hasattr(socket, "AF_UNIX"):
+	AF2NAME[int(socket.AF_UNIX)] = "unix"
 NAME2AF = {name: af for af, name in AF2NAME.items()}
 
 
@@ -40,14 +42,20 @@ def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
 	if not family or family == socket.AF_UNSPEC:
 		family = urllib3.util.connection.allowed_gai_family()
 
-	for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
+	# Extension for Unix domain sockets
+	if hasattr(socket, "AF_UNIX") and family == socket.AF_UNIX:
+		gai_result = [(socket.AF_UNIX, socket.SOCK_STREAM, 0, "", host)]
+	else:
+		gai_result = socket.getaddrinfo(host, port, family, socket.SOCK_STREAM)
+
+	for res in gai_result:
 		af, socktype, proto, canonname, sa = res
 		sock = None
 		try:
 			sock = socket.socket(af, socktype, proto)
 
 			# If provided, set socket level options before connecting.
-			if socket_options is not None:
+			if socket_options is not None and family != getattr(socket, "AF_UNIX", NotImplemented):
 				for opt in socket_options:
 					sock.setsockopt(*opt)
 
@@ -94,6 +102,8 @@ def _new_conn(self):
 
 		try:
 			dns_host = getattr(self, "_dns_host", self.host)
+			if hasattr(socket, "AF_UNIX") and extra_kw["family"] == socket.AF_UNIX:
+				dns_host = urllib.parse.unquote(dns_host)
 			conn = create_connection(
 				(dns_host, self.port), self.timeout, **extra_kw)
 		except socket.timeout:

test/unit/test_http.py

@@ -8,7 +8,9 @@
 """
 
 import json
+import os
 import socket
+import tempfile
 import time
 
 import pytest
@@ -23,7 +25,7 @@
 @pytest.fixture(scope="module")
 def http_server(request):
 	"""
-	Slightly modified version of the ``pytest_localserver.plugin.http_server``
+	Slightly modified version of the :func:`pytest_localserver.plugin.httpserver`
 	fixture that will only start and stop the server application once for test
 	performance reasons.
 	"""
@@ -33,18 +35,48 @@ def http_server(request):
 	return server
 
 
+@pytest.fixture(scope="module")
+def http_server_uds(request):
+	"""Like :func:`http_server` but will listen on a Unix domain socket instead
+
+	If the current platform does not support Unix domain sockets, the
+	corresponding test will be skipped.
+	"""
+	if not hasattr(socket, "AF_UNIX"):
+		pytest.skip("Platform does not support Unix domain sockets")
+	
+	uds_path = tempfile.mktemp(".sock")
+	def remove_uds_path():
+		try:
+			os.remove(uds_path)
+		except FileNotFoundError:
+			pass
+	request.addfinalizer(remove_uds_path)
+	
+	server = pytest_localserver.http.ContentServer("unix://{0}".format(uds_path))
+	server.start()
+	request.addfinalizer(server.stop)
+	return server
+
+
 @pytest.fixture
 def http_client(http_server):
 	return ipfshttpclient.http.ClientSync(
 		"/ip4/{0}/tcp/{1}/http".format(*http_server.server_address),
-		ipfshttpclient.DEFAULT_BASE
+		ipfshttpclient.DEFAULT_BASE,
+	)
+
+
+@pytest.fixture
+def http_client_uds(http_server_uds):
+	return ipfshttpclient.http.ClientSync(
+		"/unix/{0}".format(http_server_uds.server_address.lstrip("/")),
+		ipfshttpclient.DEFAULT_BASE,
 	)
 
 
 def broken_http_server_app(environ, start_response):
-	"""
-	HTTP server application that will be slower to respond (0.5 seconds)
-	"""
+	"""HTTP server application that will return a malformed response"""
 	start_response("0 What the heck?", [])
 
 	yield b""
@@ -58,9 +90,7 @@ def broken_http_server(request):
 
 
 def slow_http_server_app(environ, start_response):
-	"""
-	HTTP server application that will be slower to respond (0.5 seconds)
-	"""
+	"""HTTP server application that will be slower to respond (0.5 seconds)"""
 	start_response("400 Bad Request", [
 		("Content-Type", "text/json")
 	])
@@ -86,13 +116,28 @@ def test_successful_request(http_client, http_server):
 	res = http_client.request("/okay")
 	assert res == b"okay"
 
+def test_successful_request_uds(http_client_uds, http_server_uds):
+	"""Tests that a successful http request returns the proper message."""
+	http_server_uds.serve_content("okay", 200)
+	
+	res = http_client_uds.request("/okay")
+	assert res == b"okay"
+	print(http_server_uds.requests[0].headers)
+
 def test_generic_failure(http_client, http_server):
 	"""Tests that a failed http request raises an HTTPError."""
 	http_server.serve_content("fail", 500)
 
 	with pytest.raises(ipfshttpclient.exceptions.StatusError):
 		http_client.request("/fail")
 
+def test_generic_failure_uds(http_client_uds, http_server_uds):
+	"""Tests that a failed http request raises an HTTPError."""
+	http_server_uds.serve_content("fail", 500)
+	
+	with pytest.raises(ipfshttpclient.exceptions.StatusError):
+		http_client_uds.request("/fail")
+
 def test_http_client_failure(http_client, http_server):
 	"""Tests that an http client failure raises an ipfsHTTPClientError."""
 	http_server.serve_content(json.dumps({
@@ -366,26 +411,33 @@ def test_readable_stream_wrapper_read_single_bytes(mocker):
 
 @pytest.mark.parametrize("args,expected", [
 	(("/dns/localhost/tcp/5001", "api/v0"),
-	 ("http://localhost:5001/api/v0/", socket.AF_UNSPEC, False)),
+	 ("http://localhost:5001/api/v0/", None, socket.AF_UNSPEC, False)),
 	
 	(("/dns/localhost/tcp/5001/http", "api/v0"),
-	 ("http://localhost:5001/api/v0/", socket.AF_UNSPEC, False)),
+	 ("http://localhost:5001/api/v0/", None, socket.AF_UNSPEC, False)),
 	
 	(("/dns4/localhost/tcp/5001/http", "api/v0"),
-	 ("http://localhost:5001/api/v0/", socket.AF_INET, False)),
+	 ("http://localhost:5001/api/v0/", None, socket.AF_INET, False)),
 	
 	(("/dns6/localhost/tcp/5001/http", "api/v0/"),
-	 ("http://localhost:5001/api/v0/", socket.AF_INET6, False)),
+	 ("http://localhost:5001/api/v0/", None, socket.AF_INET6, False)),
 	
 	(("/ip4/127.0.0.1/tcp/5001/https", "api/v1/"),
-	 ("https://127.0.0.1:5001/api/v1/", socket.AF_INET, True)),
+	 ("https://127.0.0.1:5001/api/v1/", None, socket.AF_INET, True)),
 	
 	(("/ip6/::1/tcp/5001/https", "api/v1"),
-	 ("https://[::1]:5001/api/v1/", socket.AF_INET6, True)),
+	 ("https://[::1]:5001/api/v1/", None, socket.AF_INET6, True)),
 	
 	(("/dns4/ietf.org/tcp/443/https", "/base/"),
-	 ("https://ietf.org:443/base/", socket.AF_INET, False)),
-])
+	 ("https://ietf.org:443/base/", None, socket.AF_INET, False)),
+] + ([  # Unix domain sockets aren't supported on all target platforms
+	(("/unix/run/ipfs/ipfs.sock", "api/v0"),
+	 ("http://%2Frun%2Fipfs%2Fipfs.sock/api/v0/", "/run/ipfs/ipfs.sock", socket.AF_UNIX, False)),
+	# Stupid, but standard behaviour: There is no way to append a target protocol item, after
+	# a path protocol like /unix, so terminating it with /https ends up part of the /unix path
+	(("/unix/run/ipfs/ipfs.sock/https", "api/v0"),
+	 ("http://%2Frun%2Fipfs%2Fipfs.sock%2Fhttps/api/v0/", "/run/ipfs/ipfs.sock/https", socket.AF_UNIX, False)),
+] if hasattr(socket, "AF_UNIX") else []))
 def test_multiaddr_to_url_data(args, expected):
 	assert ipfshttpclient.http_common.multiaddr_to_url_data(*args) == expected
 
@@ -395,12 +447,18 @@ def test_multiaddr_to_url_data(args, expected):
 	("/ip4/192.168.250.1/tcp/4001/p2p/QmVgNoP89mzpgEAAqK8owYoDEyB97MkcGvoWZir8otE9Uc", "api/v1/"),
 	("/ip4/::1/sctp/5001/https", "api/v1/"),
 	("/sctp/5001/http", "api/v0"),
+	("/unix", "api/v0"),
+	
+	# Should work, but needs support in py-multiaddr first (tls protocol)
+	("/ip6/::1/tcp/5001/tls/http", "api/v1"),
 	
-	# Should work, but currently doesn't (no proxy support)
-	("/dns/localhost/tcp/5001/socks5/dns/ietf.org/tcp/80/http", "/base/"),
-	("/dns/proxy-servers.example/tcp/5001/https/dns/ietf.org/tcp/80/http", "/base/"),
+	# Should work, but needs support in py-multiaddr first (proxying protocols)
+	("/dns/localhost/tcp/1080/socks5/dns/ietf.org/tcp/80/http", "/base/"),
+	("/dns/localhost/tcp/1080/socks5/ip6/2001:1234:5678:9ABC::1/tcp/80/http", "/base/"),
+	("/dns/localhost/tcp/80/http-tunnel/dns/mgnt.my-server.example/tcp/443/https", "/srv/ipfs/api/v0"),
+	("/dns/proxy-servers.example/tcp/443/tls/http-tunnel/dns/my-server.example/tcp/5001/http", "/base/"),
 	
-	# Maybe should also work, but currently doesn't (HTTP/3)
+	# Maybe should also work eventually, but currently doesn't (HTTP/3)
 	("/dns/localhost/udp/5001/quic/http", "/base"),
 ])
 def test_multiaddr_to_url_data_invalid(args):