From eaa73534e7766de034df257a8f2faee6c58264de Mon Sep 17 00:00:00 2001 From: Joe Gordon Date: Thu, 8 Feb 2018 10:53:02 -0800 Subject: [PATCH 1/5] Add tox and update .gitignore In order to make it easy to test against python2 and python3 use tox. Also update .gitignore to cover related files --- .gitignore | 4 ++++ tox.ini | 6 ++++++ 2 files changed, 10 insertions(+) create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index f582195..1b692ae 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ pip-log.txt build dist MANIFEST +*.pyc +.pytest_cache/ +.tox/ +__pycache__/ diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..f53598a --- /dev/null +++ b/tox.ini @@ -0,0 +1,6 @@ +[tox] +envlist=py27,py3 + +[testenv] +deps = pytest +commands = pytest test_urlnorm.py From 22dfd88259c4292ed31fa21ad471291a906fb415 Mon Sep 17 00:00:00 2001 From: Joe Gordon Date: Thu, 8 Feb 2018 11:00:55 -0800 Subject: [PATCH 2/5] First pass at python3 support * use print from future * New notation for longs --- test_urlnorm.py | 3 ++- urlnorm.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test_urlnorm.py b/test_urlnorm.py index 6edc4c8..b8593fc 100644 --- a/test_urlnorm.py +++ b/test_urlnorm.py @@ -1,6 +1,7 @@ """ this is a py.test test file """ +from __future__ import print_function import urlnorm from urlnorm import _unicode @@ -106,7 +107,7 @@ def pytest_generate_tests(metafunc): def test_invalid_urls(url): try: output = urlnorm.norm(url) - print '%r' % output + print('%r' % output) except urlnorm.InvalidUrl: return assert 1 == 0, "this should have raised an InvalidUrl exception" diff --git a/urlnorm.py b/urlnorm.py index 96c1bf5..4eed789 100644 --- a/urlnorm.py +++ b/urlnorm.py @@ -203,7 +203,7 @@ def norm_path(scheme, path): return '/' return path -MAX_IP = 0xffffffffL +MAX_IP = 0xffffffff def int2ip(ipnum): From 052e77c526a17e6ce54fccc714acdf1fa5dcd853 Mon Sep 17 00:00:00 2001 From: Joe Gordon Date: Thu, 8 Feb 2018 13:13:47 -0800 Subject: [PATCH 3/5] Manually update for python3 compatability * Stop using decode('utf8') in tests * str.lower() instead of string.lower(str) --- test_urlnorm.py | 10 +++++----- urlnorm.py | 5 ++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/test_urlnorm.py b/test_urlnorm.py index b8593fc..53e72f9 100644 --- a/test_urlnorm.py +++ b/test_urlnorm.py @@ -21,7 +21,7 @@ def pytest_generate_tests(metafunc): 'http://USER:pass@www.Example.COM/foo/bar': 'http://USER:pass@www.example.com/foo/bar', 'http://www.example.com./': 'http://www.example.com/', 'http://test.example/?a=%26&b=1': 'http://test.example/?a=%26&b=1', # should not un-encode the & that is part of a parameter value - 'http://test.example/?a=%e3%82%82%26': 'http://test.example/?a=\xe3\x82\x82%26'.decode('utf8'), # should return a unicode character + 'http://test.example/?a=%e3%82%82%26': u'http://test.example/?a=\u3082%26', # should return a unicode character # note: this breaks the internet for parameters that are positional (stupid nextel) and/or don't have an = sign # 'http://test.example/?a=1&b=2&a=3': 'http://test.example/?a=1&a=3&b=2', # should be in sorted/grouped order @@ -30,12 +30,12 @@ def pytest_generate_tests(metafunc): 'http://test.example?': 'http://test.example/', # with trailing / 'http://a.COM/path/?b&a' : 'http://a.com/path/?b&a', # test utf8 and unicode - u'http://XBLA\u306eXbox.com': 'http://xbla\xe3\x81\xaexbox.com/'.decode('utf8'), - u'http://XBLA\u306eXbox.com'.encode('utf8'): 'http://xbla\xe3\x81\xaexbox.com/'.decode('utf8'), - u'http://XBLA\u306eXbox.com': 'http://xbla\xe3\x81\xaexbox.com/'.decode('utf8'), + u'http://XBLA\u306eXbox.com': u'http://xbla\u306exbox.com/', + u'http://XBLA\u306eXbox.com'.encode('utf8'): u'http://xbla\u306exbox.com/', + u'http://XBLA\u306eXbox.com': u'http://xbla\u306exbox.com/', # test idna + utf8 domain # u'http://xn--q-bga.XBLA\u306eXbox.com'.encode('utf8'): 'http://q\xc3\xa9.xbla\xe3\x81\xaexbox.com'.decode('utf8'), - 'http://ja.wikipedia.org/wiki/%E3%82%AD%E3%83%A3%E3%82%BF%E3%83%94%E3%83%A9%E3%83%BC%E3%82%B8%E3%83%A3%E3%83%91%E3%83%B3': 'http://ja.wikipedia.org/wiki/\xe3\x82\xad\xe3\x83\xa3\xe3\x82\xbf\xe3\x83\x94\xe3\x83\xa9\xe3\x83\xbc\xe3\x82\xb8\xe3\x83\xa3\xe3\x83\x91\xe3\x83\xb3'.decode('utf8'), + 'http://ja.wikipedia.org/wiki/%E3%82%AD%E3%83%A3%E3%82%BF%E3%83%94%E3%83%A9%E3%83%BC%E3%82%B8%E3%83%A3%E3%83%91%E3%83%B3': u'http://ja.wikipedia.org/wiki/\u30ad\u30e3\u30bf\u30d4\u30e9\u30fc\u30b8\u30e3\u30d1\u30f3', 'http://test.example/\xe3\x82\xad': 'http://test.example/\xe3\x82\xad', # check that %23 (#) is not escaped where it shouldn't be diff --git a/urlnorm.py b/urlnorm.py index 4eed789..5cae4b3 100644 --- a/urlnorm.py +++ b/urlnorm.py @@ -68,7 +68,6 @@ __version__ = "1.1.4" from urlparse import urlparse, urlunparse -from string import lower import re @@ -160,7 +159,7 @@ def norm(url): def norm_tuple(scheme, authority, path, parameters, query, fragment): """given individual url components, return its normalized form""" - scheme = lower(scheme) + scheme = scheme.lower() if not scheme: raise InvalidUrl('missing URL scheme') authority = norm_netloc(scheme, authority) @@ -238,7 +237,7 @@ def norm_netloc(scheme, netloc): if '.' not in host and not (host[0] == '[' and host[-1] == ']'): raise InvalidUrl('host %r is not valid' % host) - authority = lower(host) + authority = host.lower() if 'xn--' in authority: subdomains = [_idn(subdomain) for subdomain in authority.split('.')] authority = '.'.join(subdomains) From 1b4636512b5a0da0163cf11a753f35a929b035f6 Mon Sep 17 00:00:00 2001 From: Joe Gordon Date: Thu, 8 Feb 2018 15:41:27 -0800 Subject: [PATCH 4/5] Add python3 support via modernize Add python3 support using six. --- setup.py | 1 + test_urlnorm.py | 3 ++- urlnorm.py | 27 ++++++++++++++++----------- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index 697a6cf..8a156e4 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,7 @@ description="Normalize a URL to a standard unicode encoding", py_modules=['urlnorm'], license='MIT License', + install_requires=['six'], author='Jehiah Czebotar', author_email='jehiah@gmail.com', url='http://github.com/jehiah/urlnorm', diff --git a/test_urlnorm.py b/test_urlnorm.py index 53e72f9..119ef29 100644 --- a/test_urlnorm.py +++ b/test_urlnorm.py @@ -1,3 +1,4 @@ +# -*- coding: utf8 -*- """ this is a py.test test file """ @@ -42,7 +43,7 @@ def pytest_generate_tests(metafunc): 'http://test.example/?p=%23val#test-%23-val%25': 'http://test.example/?p=%23val#test-%23-val%25', # check that %25 is not unescaped to % 'http://test.example/%25/?p=val%25ue' : 'http://test.example/%25/?p=val%25ue', - "http://test.domain/I%C3%B1t%C3%ABrn%C3%A2ti%C3%B4n%EF%BF%BDliz%C3%A6ti%C3%B8n" : "http://test.domain/I\xc3\xb1t\xc3\xabrn\xc3\xa2ti\xc3\xb4n\xef\xbf\xbdliz\xc3\xa6ti\xc3\xb8n", + "http://test.domain/I%C3%B1t%C3%ABrn%C3%A2ti%C3%B4n%EF%BF%BDliz%C3%A6ti%C3%B8n" : u"http://test.domain/Iñtërnâtiôn�lizætiøn", # check that %20 in paths, params, query strings, and fragments are unescaped 'http://test.example/abcde%20def?que%20ry=str%20ing#frag%20ment' : 'http://test.example/abcde def?que ry=str ing#frag ment', # check that spaces are collated to '+' diff --git a/urlnorm.py b/urlnorm.py index 5cae4b3..82cbda1 100644 --- a/urlnorm.py +++ b/urlnorm.py @@ -41,6 +41,10 @@ - more fine-grained authority parsing and normalisation """ +from __future__ import absolute_import +from six import unichr +import six +from six.moves import range __license__ = """ Copyright (c) 1999-2002 Mark Nottingham Copyright (c) 2010 Jehiah Czebotar @@ -67,7 +71,7 @@ # also update in setup.py __version__ = "1.1.4" -from urlparse import urlparse, urlunparse +from six.moves.urllib.parse import urlparse, urlunparse, unquote import re @@ -107,8 +111,8 @@ class InvalidUrl(Exception): qs_unsafe_list = set('?&=+%#') fragment_unsafe_list = set('+%#') path_unsafe_list = set('/?;%+#') -_hextochr = dict(('%02x' % i, chr(i)) for i in range(256)) -_hextochr.update(('%02X' % i, chr(i)) for i in range(256)) +_hextochr = dict((b'%02x' % i, six.int2byte(i)) for i in range(256)) +_hextochr.update((b'%02X' % i, six.int2byte(i)) for i in range(256)) def unquote_path(s): @@ -131,22 +135,23 @@ def unquote_safe(s, unsafe_list): """unquote percent escaped string except for percent escape sequences that are in unsafe_list""" # note: this build utf8 raw strings ,then does a .decode('utf8') at the end. # as a result it's doing .encode('utf8') on each block of the string as it's processed. - res = _utf8(s).split('%') - for i in xrange(1, len(res)): + unsafe_list = [_utf8(i) for i in unsafe_list] + res = _utf8(s).split(b'%') + for i in range(1, len(res)): item = res[i] try: raw_chr = _hextochr[item[:2]] if raw_chr in unsafe_list or ord(raw_chr) < 20: # leave it unescaped (but uppercase the percent escape) - res[i] = '%' + item[:2].upper() + item[2:] + res[i] = b'%' + item[:2].upper() + item[2:] else: res[i] = raw_chr + item[2:] except KeyError: - res[i] = '%' + item + res[i] = b'%' + item except UnicodeDecodeError: # note: i'm not sure what this does res[i] = unichr(int(item[:2], 16)) + item[2:] - o = "".join(res) + o = b"".join(res) return _unicode(o) @@ -259,14 +264,14 @@ def _idn(subdomain): def _utf8(value): - if isinstance(value, unicode): + if isinstance(value, six.text_type): return value.encode("utf-8") assert isinstance(value, str) return value def _unicode(value): - if isinstance(value, str): + if isinstance(value, six.binary_type): return value.decode("utf-8") - assert isinstance(value, unicode) + assert isinstance(value, six.text_type) return value From 376f5864f5a8b3048bd9fe8939b31e2d0d4f9709 Mon Sep 17 00:00:00 2001 From: Joe Gordon Date: Thu, 8 Feb 2018 15:57:16 -0800 Subject: [PATCH 5/5] Add python 3 classifiers So https://github.com/brettcannon/caniusepython3 will detect urlnorm as supporting python3. --- setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.py b/setup.py index 8a156e4..e8edb0d 100644 --- a/setup.py +++ b/setup.py @@ -14,4 +14,10 @@ author_email='jehiah@gmail.com', url='http://github.com/jehiah/urlnorm', download_url="http://github.com/downloads/jehiah/urlnorm/urlnorm-%s.tar.gz" % version, + classifiers=[ + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + ], )