From 6f5bb0212097f4f1ce24798012e3ad5c124ac8c0 Mon Sep 17 00:00:00 2001 From: Frank Villaro-Dixon Date: Mon, 6 May 2024 23:43:37 +0200 Subject: [PATCH 1/3] http: remove `chunk` capability There is no need to support chunks in python3 as the `read()` interface does that for us. Furthermore, this improves fetch performance in large queries (c.f. #111). Signed-off-by: Frank Villaro-Dixon --- overpy/__init__.py | 17 ++--------------- tests/test_request.py | 9 --------- 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/overpy/__init__.py b/overpy/__init__.py index 3de3c1b..2d79ce6 100644 --- a/overpy/__init__.py +++ b/overpy/__init__.py @@ -1,7 +1,6 @@ from datetime import datetime from decimal import Decimal -from functools import partial -from urllib.request import urlopen +from urllib.request import urlopen, Request from urllib.error import HTTPError from xml.sax import handler, make_parser import xml.etree.ElementTree @@ -60,7 +59,6 @@ class Overpass: """ Class to access the Overpass API - :param read_chunk_size: Max size of each chunk read from the server response :param url: Optional URL of the Overpass server. Defaults to http://overpass-api.de/api/interpreter :param xml_parser: The xml parser to use :param max_retry_count: Max number of retries (Default: default_max_retry_count) @@ -70,9 +68,6 @@ class Overpass: #: Global max number of retries (Default: 0) default_max_retry_count: ClassVar[int] = 0 - #: Max size of each chunk read from the server response - default_read_chunk_size: ClassVar[int] = 4096 - #: Global time to wait between tries (Default: 1.0s) default_retry_timeout: ClassVar[float] = 1.0 @@ -81,7 +76,6 @@ class Overpass: def __init__( self, - read_chunk_size: Optional[int] = None, url: Optional[str] = None, xml_parser: int = XML_PARSER_SAX, max_retry_count: int = None, @@ -94,11 +88,6 @@ def __init__( self._regex_extract_error_msg = re.compile(br"\(?P\") self._regex_remove_tag = re.compile(b"<[^>]*?>") - if read_chunk_size is None: - read_chunk_size = self.default_read_chunk_size - - #: The chunk size for this instance - self.read_chunk_size = read_chunk_size if max_retry_count is None: max_retry_count = self.default_max_retry_count @@ -151,9 +140,7 @@ def query(self, query: Union[bytes, str]) -> "Result": response = b"" try: with urlopen(self.url, query) as f: - f_read = partial(f.read, self.read_chunk_size) - for data in iter(f_read, b""): - response += data + response = f.read() except HTTPError as exc: f = exc diff --git a/tests/test_request.py b/tests/test_request.py index a545035..c7257bf 100644 --- a/tests/test_request.py +++ b/tests/test_request.py @@ -128,15 +128,6 @@ def do_POST(self): class TestQuery: - def test_chunk_size(self): - url, server = new_server_thread(HandleResponseJSON) - - api = overpy.Overpass(read_chunk_size=128) - api.url = url - result = api.query("[out:json];node(50.745,7.17,50.75,7.18);out;") - stop_server_thread(server) - assert len(result.nodes) > 0 - def test_overpass_syntax_error(self): url, server = new_server_thread(HandleOverpassBadRequest) From 5bb1b71ac22799f1e9f3c9b216331ed3fc5d81e0 Mon Sep 17 00:00:00 2001 From: Frank Villaro-Dixon Date: Mon, 6 May 2024 23:51:29 +0200 Subject: [PATCH 2/3] http: add support for `headers` It is useful to be able to specify additional headers to send when making overpass requests. For example a special user agent, or an API token. Signed-off-by: Frank Villaro-Dixon --- overpy/__init__.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/overpy/__init__.py b/overpy/__init__.py index 2d79ce6..6cc4a5b 100644 --- a/overpy/__init__.py +++ b/overpy/__init__.py @@ -63,6 +63,7 @@ class Overpass: :param xml_parser: The xml parser to use :param max_retry_count: Max number of retries (Default: default_max_retry_count) :param retry_timeout: Time to wait between tries (Default: default_retry_timeout) + :param headers: Dict of headers to send with the request """ #: Global max number of retries (Default: 0) @@ -74,18 +75,25 @@ class Overpass: #: Default URL of the Overpass server default_url: ClassVar[str] = "http://overpass-api.de/api/interpreter" + default_headers: ClassVar[Dict[str, str]] = {} + def __init__( self, url: Optional[str] = None, xml_parser: int = XML_PARSER_SAX, max_retry_count: int = None, - retry_timeout: float = None): + retry_timeout: float = None, + headers: dict[str, str] = None): #: URL to use for this instance self.url = self.default_url if url is not None: self.url = url + self.headers = self.default_headers + if headers is not None: + self.headers = headers + self._regex_extract_error_msg = re.compile(br"\(?P\") self._regex_remove_tag = re.compile(b"<[^>]*?>") @@ -139,7 +147,8 @@ def query(self, query: Union[bytes, str]) -> "Result": response = b"" try: - with urlopen(self.url, query) as f: + req = Request(self.url, data=query, headers=self.headers) + with urlopen(req) as f: response = f.read() except HTTPError as exc: f = exc From 8829d9ce02f7972b4f6a628732d03ff97c5ee187 Mon Sep 17 00:00:00 2001 From: Frank Villaro-Dixon Date: Mon, 6 May 2024 23:52:38 +0200 Subject: [PATCH 3/3] http: header: add default `User-Agent` header As customary with API libraries, it is useful for server operators to know which library makes requests to it. This sets the default to the library + version. Signed-off-by: Frank Villaro-Dixon --- overpy/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/overpy/__init__.py b/overpy/__init__.py index 6cc4a5b..3725142 100644 --- a/overpy/__init__.py +++ b/overpy/__init__.py @@ -75,7 +75,9 @@ class Overpass: #: Default URL of the Overpass server default_url: ClassVar[str] = "http://overpass-api.de/api/interpreter" - default_headers: ClassVar[Dict[str, str]] = {} + default_headers: ClassVar[Dict[str, str]] = { + 'User-Agent': 'python-overpy/{} (urllib)'.format(__version__), + } def __init__( self,