From aa37425a9bf9eb335d0172b7d7897a32cfd69158 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Wed, 1 Jan 2020 23:15:49 -0500 Subject: [PATCH 1/3] move smart_list into sub-package/multiple files Step one of refactoring - making SmartList into its own package, with each class having its own file. No code changes were made. Note that SmartList and ListProxy import each other, so had to import SmartList as a full package name rather than use from ... import ... construct. --- mwparserfromhell/smart_list.py | 456 ----------------------- mwparserfromhell/smart_list/ListProxy.py | 240 ++++++++++++ mwparserfromhell/smart_list/SmartList.py | 162 ++++++++ mwparserfromhell/smart_list/__init__.py | 1 + mwparserfromhell/smart_list/utils.py | 59 +++ mwparserfromhell/wikicode.py | 5 +- tests/test_smart_list.py | 5 +- 7 files changed, 469 insertions(+), 459 deletions(-) delete mode 100644 mwparserfromhell/smart_list.py create mode 100644 mwparserfromhell/smart_list/ListProxy.py create mode 100644 mwparserfromhell/smart_list/SmartList.py create mode 100644 mwparserfromhell/smart_list/__init__.py create mode 100644 mwparserfromhell/smart_list/utils.py diff --git a/mwparserfromhell/smart_list.py b/mwparserfromhell/smart_list.py deleted file mode 100644 index e7fa59f6..00000000 --- a/mwparserfromhell/smart_list.py +++ /dev/null @@ -1,456 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2012-2016 Ben Kurtovic -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -""" -This module contains the :class:`.SmartList` type, as well as its -:class:`._ListProxy` child, which together implement a list whose sublists -reflect changes made to the main list, and vice-versa. -""" - -from __future__ import unicode_literals -from sys import maxsize -from weakref import ref - -from .compat import py3k - -__all__ = ["SmartList"] - -def inheritdoc(method): - """Set __doc__ of *method* to __doc__ of *method* in its parent class. - - Since this is used on :class:`.SmartList`, the "parent class" used is - ``list``. This function can be used as a decorator. - """ - method.__doc__ = getattr(list, method.__name__).__doc__ - return method - - -class _SliceNormalizerMixIn(object): - """MixIn that provides a private method to normalize slices.""" - - def _normalize_slice(self, key, clamp=False): - """Return a slice equivalent to the input *key*, standardized.""" - if key.start is None: - start = 0 - else: - start = (len(self) + key.start) if key.start < 0 else key.start - if key.stop is None or key.stop == maxsize: - stop = len(self) if clamp else None - else: - stop = (len(self) + key.stop) if key.stop < 0 else key.stop - return slice(start, stop, key.step or 1) - - -class SmartList(_SliceNormalizerMixIn, list): - """Implements the ``list`` interface with special handling of sublists. - - When a sublist is created (by ``list[i:j]``), any changes made to this - list (such as the addition, removal, or replacement of elements) will be - reflected in the sublist, or vice-versa, to the greatest degree possible. - This is implemented by having sublists - instances of the - :class:`._ListProxy` type - dynamically determine their elements by storing - their slice info and retrieving that slice from the parent. Methods that - change the size of the list also change the slice info. For example:: - - >>> parent = SmartList([0, 1, 2, 3]) - >>> parent - [0, 1, 2, 3] - >>> child = parent[2:] - >>> child - [2, 3] - >>> child.append(4) - >>> child - [2, 3, 4] - >>> parent - [0, 1, 2, 3, 4] - """ - - def __init__(self, iterable=None): - if iterable: - super(SmartList, self).__init__(iterable) - else: - super(SmartList, self).__init__() - self._children = {} - - def __getitem__(self, key): - if not isinstance(key, slice): - return super(SmartList, self).__getitem__(key) - key = self._normalize_slice(key, clamp=False) - sliceinfo = [key.start, key.stop, key.step] - child = _ListProxy(self, sliceinfo) - child_ref = ref(child, self._delete_child) - self._children[id(child_ref)] = (child_ref, sliceinfo) - return child - - def __setitem__(self, key, item): - if not isinstance(key, slice): - return super(SmartList, self).__setitem__(key, item) - item = list(item) - super(SmartList, self).__setitem__(key, item) - key = self._normalize_slice(key, clamp=True) - diff = len(item) + (key.start - key.stop) // key.step - if not diff: - return - values = self._children.values if py3k else self._children.itervalues - for child, (start, stop, step) in values(): - if start > key.stop: - self._children[id(child)][1][0] += diff - if stop is not None and stop >= key.stop: - self._children[id(child)][1][1] += diff - - def __delitem__(self, key): - super(SmartList, self).__delitem__(key) - if isinstance(key, slice): - key = self._normalize_slice(key, clamp=True) - else: - key = slice(key, key + 1, 1) - diff = (key.stop - key.start) // key.step - values = self._children.values if py3k else self._children.itervalues - for child, (start, stop, step) in values(): - if start > key.start: - self._children[id(child)][1][0] -= diff - if stop is not None and stop >= key.stop: - self._children[id(child)][1][1] -= diff - - if not py3k: - def __getslice__(self, start, stop): - return self.__getitem__(slice(start, stop)) - - def __setslice__(self, start, stop, iterable): - self.__setitem__(slice(start, stop), iterable) - - def __delslice__(self, start, stop): - self.__delitem__(slice(start, stop)) - - def __add__(self, other): - return SmartList(list(self) + other) - - def __radd__(self, other): - return SmartList(other + list(self)) - - def __iadd__(self, other): - self.extend(other) - return self - - def _delete_child(self, child_ref): - """Remove a child reference that is about to be garbage-collected.""" - del self._children[id(child_ref)] - - def _detach_children(self): - """Remove all children and give them independent parent copies.""" - children = [val[0] for val in self._children.values()] - for child in children: - child()._parent = list(self) - self._children.clear() - - @inheritdoc - def append(self, item): - head = len(self) - self[head:head] = [item] - - @inheritdoc - def extend(self, item): - head = len(self) - self[head:head] = item - - @inheritdoc - def insert(self, index, item): - self[index:index] = [item] - - @inheritdoc - def pop(self, index=None): - if index is None: - index = len(self) - 1 - item = self[index] - del self[index] - return item - - @inheritdoc - def remove(self, item): - del self[self.index(item)] - - @inheritdoc - def reverse(self): - self._detach_children() - super(SmartList, self).reverse() - - if py3k: - @inheritdoc - def sort(self, key=None, reverse=None): - self._detach_children() - kwargs = {} - if key is not None: - kwargs["key"] = key - if reverse is not None: - kwargs["reverse"] = reverse - super(SmartList, self).sort(**kwargs) - else: - @inheritdoc - def sort(self, cmp=None, key=None, reverse=None): - self._detach_children() - kwargs = {} - if cmp is not None: - kwargs["cmp"] = cmp - if key is not None: - kwargs["key"] = key - if reverse is not None: - kwargs["reverse"] = reverse - super(SmartList, self).sort(**kwargs) - - -class _ListProxy(_SliceNormalizerMixIn, list): - """Implement the ``list`` interface by getting elements from a parent. - - This is created by a :class:`.SmartList` object when slicing. It does not - actually store the list at any time; instead, whenever the list is needed, - it builds it dynamically using the :meth:`_render` method. - """ - - def __init__(self, parent, sliceinfo): - super(_ListProxy, self).__init__() - self._parent = parent - self._sliceinfo = sliceinfo - - def __repr__(self): - return repr(self._render()) - - def __lt__(self, other): - if isinstance(other, _ListProxy): - return self._render() < list(other) - return self._render() < other - - def __le__(self, other): - if isinstance(other, _ListProxy): - return self._render() <= list(other) - return self._render() <= other - - def __eq__(self, other): - if isinstance(other, _ListProxy): - return self._render() == list(other) - return self._render() == other - - def __ne__(self, other): - if isinstance(other, _ListProxy): - return self._render() != list(other) - return self._render() != other - - def __gt__(self, other): - if isinstance(other, _ListProxy): - return self._render() > list(other) - return self._render() > other - - def __ge__(self, other): - if isinstance(other, _ListProxy): - return self._render() >= list(other) - return self._render() >= other - - if py3k: - def __bool__(self): - return bool(self._render()) - else: - def __nonzero__(self): - return bool(self._render()) - - def __len__(self): - return max((self._stop - self._start) // self._step, 0) - - def __getitem__(self, key): - if isinstance(key, slice): - key = self._normalize_slice(key, clamp=True) - keystart = min(self._start + key.start, self._stop) - keystop = min(self._start + key.stop, self._stop) - adjusted = slice(keystart, keystop, key.step) - return self._parent[adjusted] - else: - return self._render()[key] - - def __setitem__(self, key, item): - if isinstance(key, slice): - key = self._normalize_slice(key, clamp=True) - keystart = min(self._start + key.start, self._stop) - keystop = min(self._start + key.stop, self._stop) - adjusted = slice(keystart, keystop, key.step) - self._parent[adjusted] = item - else: - length = len(self) - if key < 0: - key = length + key - if key < 0 or key >= length: - raise IndexError("list assignment index out of range") - self._parent[self._start + key] = item - - def __delitem__(self, key): - if isinstance(key, slice): - key = self._normalize_slice(key, clamp=True) - keystart = min(self._start + key.start, self._stop) - keystop = min(self._start + key.stop, self._stop) - adjusted = slice(keystart, keystop, key.step) - del self._parent[adjusted] - else: - length = len(self) - if key < 0: - key = length + key - if key < 0 or key >= length: - raise IndexError("list assignment index out of range") - del self._parent[self._start + key] - - def __iter__(self): - i = self._start - while i < self._stop: - yield self._parent[i] - i += self._step - - def __reversed__(self): - i = self._stop - 1 - while i >= self._start: - yield self._parent[i] - i -= self._step - - def __contains__(self, item): - return item in self._render() - - if not py3k: - def __getslice__(self, start, stop): - return self.__getitem__(slice(start, stop)) - - def __setslice__(self, start, stop, iterable): - self.__setitem__(slice(start, stop), iterable) - - def __delslice__(self, start, stop): - self.__delitem__(slice(start, stop)) - - def __add__(self, other): - return SmartList(list(self) + other) - - def __radd__(self, other): - return SmartList(other + list(self)) - - def __iadd__(self, other): - self.extend(other) - return self - - def __mul__(self, other): - return SmartList(list(self) * other) - - def __rmul__(self, other): - return SmartList(other * list(self)) - - def __imul__(self, other): - self.extend(list(self) * (other - 1)) - return self - - @property - def _start(self): - """The starting index of this list, inclusive.""" - return self._sliceinfo[0] - - @property - def _stop(self): - """The ending index of this list, exclusive.""" - if self._sliceinfo[1] is None: - return len(self._parent) - return self._sliceinfo[1] - - @property - def _step(self): - """The number to increase the index by between items.""" - return self._sliceinfo[2] - - def _render(self): - """Return the actual list from the stored start/stop/step.""" - return list(self._parent)[self._start:self._stop:self._step] - - @inheritdoc - def append(self, item): - self._parent.insert(self._stop, item) - - @inheritdoc - def count(self, item): - return self._render().count(item) - - @inheritdoc - def index(self, item, start=None, stop=None): - if start is not None: - if stop is not None: - return self._render().index(item, start, stop) - return self._render().index(item, start) - return self._render().index(item) - - @inheritdoc - def extend(self, item): - self._parent[self._stop:self._stop] = item - - @inheritdoc - def insert(self, index, item): - if index < 0: - index = len(self) + index - self._parent.insert(self._start + index, item) - - @inheritdoc - def pop(self, index=None): - length = len(self) - if index is None: - index = length - 1 - elif index < 0: - index = length + index - if index < 0 or index >= length: - raise IndexError("pop index out of range") - return self._parent.pop(self._start + index) - - @inheritdoc - def remove(self, item): - index = self.index(item) - del self._parent[self._start + index] - - @inheritdoc - def reverse(self): - item = self._render() - item.reverse() - self._parent[self._start:self._stop:self._step] = item - - if py3k: - @inheritdoc - def sort(self, key=None, reverse=None): - item = self._render() - kwargs = {} - if key is not None: - kwargs["key"] = key - if reverse is not None: - kwargs["reverse"] = reverse - item.sort(**kwargs) - self._parent[self._start:self._stop:self._step] = item - else: - @inheritdoc - def sort(self, cmp=None, key=None, reverse=None): - item = self._render() - kwargs = {} - if cmp is not None: - kwargs["cmp"] = cmp - if key is not None: - kwargs["key"] = key - if reverse is not None: - kwargs["reverse"] = reverse - item.sort(**kwargs) - self._parent[self._start:self._stop:self._step] = item - - -del inheritdoc diff --git a/mwparserfromhell/smart_list/ListProxy.py b/mwparserfromhell/smart_list/ListProxy.py new file mode 100644 index 00000000..2e3b94a0 --- /dev/null +++ b/mwparserfromhell/smart_list/ListProxy.py @@ -0,0 +1,240 @@ +# SmartList has to be a full import in order to avoid cyclical import errors +import mwparserfromhell.smart_list.SmartList +from .utils import _SliceNormalizerMixIn, inheritdoc +from ..compat import py3k + + +class _ListProxy(_SliceNormalizerMixIn, list): + """Implement the ``list`` interface by getting elements from a parent. + + This is created by a :class:`.SmartList` object when slicing. It does not + actually store the list at any time; instead, whenever the list is needed, + it builds it dynamically using the :meth:`_render` method. + """ + + def __init__(self, parent, sliceinfo): + super(_ListProxy, self).__init__() + self._parent = parent + self._sliceinfo = sliceinfo + + def __repr__(self): + return repr(self._render()) + + def __lt__(self, other): + if isinstance(other, _ListProxy): + return self._render() < list(other) + return self._render() < other + + def __le__(self, other): + if isinstance(other, _ListProxy): + return self._render() <= list(other) + return self._render() <= other + + def __eq__(self, other): + if isinstance(other, _ListProxy): + return self._render() == list(other) + return self._render() == other + + def __ne__(self, other): + if isinstance(other, _ListProxy): + return self._render() != list(other) + return self._render() != other + + def __gt__(self, other): + if isinstance(other, _ListProxy): + return self._render() > list(other) + return self._render() > other + + def __ge__(self, other): + if isinstance(other, _ListProxy): + return self._render() >= list(other) + return self._render() >= other + + if py3k: + def __bool__(self): + return bool(self._render()) + else: + def __nonzero__(self): + return bool(self._render()) + + def __len__(self): + return max((self._stop - self._start) // self._step, 0) + + def __getitem__(self, key): + if isinstance(key, slice): + key = self._normalize_slice(key, clamp=True) + keystart = min(self._start + key.start, self._stop) + keystop = min(self._start + key.stop, self._stop) + adjusted = slice(keystart, keystop, key.step) + return self._parent[adjusted] + else: + return self._render()[key] + + def __setitem__(self, key, item): + if isinstance(key, slice): + key = self._normalize_slice(key, clamp=True) + keystart = min(self._start + key.start, self._stop) + keystop = min(self._start + key.stop, self._stop) + adjusted = slice(keystart, keystop, key.step) + self._parent[adjusted] = item + else: + length = len(self) + if key < 0: + key = length + key + if key < 0 or key >= length: + raise IndexError("list assignment index out of range") + self._parent[self._start + key] = item + + def __delitem__(self, key): + if isinstance(key, slice): + key = self._normalize_slice(key, clamp=True) + keystart = min(self._start + key.start, self._stop) + keystop = min(self._start + key.stop, self._stop) + adjusted = slice(keystart, keystop, key.step) + del self._parent[adjusted] + else: + length = len(self) + if key < 0: + key = length + key + if key < 0 or key >= length: + raise IndexError("list assignment index out of range") + del self._parent[self._start + key] + + def __iter__(self): + i = self._start + while i < self._stop: + yield self._parent[i] + i += self._step + + def __reversed__(self): + i = self._stop - 1 + while i >= self._start: + yield self._parent[i] + i -= self._step + + def __contains__(self, item): + return item in self._render() + + if not py3k: + def __getslice__(self, start, stop): + return self.__getitem__(slice(start, stop)) + + def __setslice__(self, start, stop, iterable): + self.__setitem__(slice(start, stop), iterable) + + def __delslice__(self, start, stop): + self.__delitem__(slice(start, stop)) + + def __add__(self, other): + return mwparserfromhell.smart_list.SmartList(list(self) + other) + + def __radd__(self, other): + return mwparserfromhell.smart_list.SmartList(other + list(self)) + + def __iadd__(self, other): + self.extend(other) + return self + + def __mul__(self, other): + return mwparserfromhell.smart_list.SmartList(list(self) * other) + + def __rmul__(self, other): + return mwparserfromhell.smart_list.SmartList(other * list(self)) + + def __imul__(self, other): + self.extend(list(self) * (other - 1)) + return self + + @property + def _start(self): + """The starting index of this list, inclusive.""" + return self._sliceinfo[0] + + @property + def _stop(self): + """The ending index of this list, exclusive.""" + if self._sliceinfo[1] is None: + return len(self._parent) + return self._sliceinfo[1] + + @property + def _step(self): + """The number to increase the index by between items.""" + return self._sliceinfo[2] + + def _render(self): + """Return the actual list from the stored start/stop/step.""" + return list(self._parent)[self._start:self._stop:self._step] + + @inheritdoc + def append(self, item): + self._parent.insert(self._stop, item) + + @inheritdoc + def count(self, item): + return self._render().count(item) + + @inheritdoc + def index(self, item, start=None, stop=None): + if start is not None: + if stop is not None: + return self._render().index(item, start, stop) + return self._render().index(item, start) + return self._render().index(item) + + @inheritdoc + def extend(self, item): + self._parent[self._stop:self._stop] = item + + @inheritdoc + def insert(self, index, item): + if index < 0: + index = len(self) + index + self._parent.insert(self._start + index, item) + + @inheritdoc + def pop(self, index=None): + length = len(self) + if index is None: + index = length - 1 + elif index < 0: + index = length + index + if index < 0 or index >= length: + raise IndexError("pop index out of range") + return self._parent.pop(self._start + index) + + @inheritdoc + def remove(self, item): + index = self.index(item) + del self._parent[self._start + index] + + @inheritdoc + def reverse(self): + item = self._render() + item.reverse() + self._parent[self._start:self._stop:self._step] = item + + if py3k: + @inheritdoc + def sort(self, key=None, reverse=None): + item = self._render() + kwargs = {} + if key is not None: + kwargs["key"] = key + if reverse is not None: + kwargs["reverse"] = reverse + item.sort(**kwargs) + self._parent[self._start:self._stop:self._step] = item + else: + @inheritdoc + def sort(self, cmp=None, key=None, reverse=None): + item = self._render() + kwargs = {} + if cmp is not None: + kwargs["cmp"] = cmp + if key is not None: + kwargs["key"] = key + if reverse is not None: + kwargs["reverse"] = reverse + item.sort(**kwargs) + self._parent[self._start:self._stop:self._step] = item diff --git a/mwparserfromhell/smart_list/SmartList.py b/mwparserfromhell/smart_list/SmartList.py new file mode 100644 index 00000000..e442252c --- /dev/null +++ b/mwparserfromhell/smart_list/SmartList.py @@ -0,0 +1,162 @@ +from _weakref import ref + +from .ListProxy import _ListProxy +from .utils import _SliceNormalizerMixIn, inheritdoc +from ..compat import py3k + + +class SmartList(_SliceNormalizerMixIn, list): + """Implements the ``list`` interface with special handling of sublists. + + When a sublist is created (by ``list[i:j]``), any changes made to this + list (such as the addition, removal, or replacement of elements) will be + reflected in the sublist, or vice-versa, to the greatest degree possible. + This is implemented by having sublists - instances of the + :class:`._ListProxy` type - dynamically determine their elements by storing + their slice info and retrieving that slice from the parent. Methods that + change the size of the list also change the slice info. For example:: + + >>> parent = SmartList([0, 1, 2, 3]) + >>> parent + [0, 1, 2, 3] + >>> child = parent[2:] + >>> child + [2, 3] + >>> child.append(4) + >>> child + [2, 3, 4] + >>> parent + [0, 1, 2, 3, 4] + """ + + def __init__(self, iterable=None): + if iterable: + super(SmartList, self).__init__(iterable) + else: + super(SmartList, self).__init__() + self._children = {} + + def __getitem__(self, key): + if not isinstance(key, slice): + return super(SmartList, self).__getitem__(key) + key = self._normalize_slice(key, clamp=False) + sliceinfo = [key.start, key.stop, key.step] + child = _ListProxy(self, sliceinfo) + child_ref = ref(child, self._delete_child) + self._children[id(child_ref)] = (child_ref, sliceinfo) + return child + + def __setitem__(self, key, item): + if not isinstance(key, slice): + return super(SmartList, self).__setitem__(key, item) + item = list(item) + super(SmartList, self).__setitem__(key, item) + key = self._normalize_slice(key, clamp=True) + diff = len(item) + (key.start - key.stop) // key.step + if not diff: + return + values = self._children.values if py3k else self._children.itervalues + for child, (start, stop, step) in values(): + if start > key.stop: + self._children[id(child)][1][0] += diff + if stop is not None and stop >= key.stop: + self._children[id(child)][1][1] += diff + + def __delitem__(self, key): + super(SmartList, self).__delitem__(key) + if isinstance(key, slice): + key = self._normalize_slice(key, clamp=True) + else: + key = slice(key, key + 1, 1) + diff = (key.stop - key.start) // key.step + values = self._children.values if py3k else self._children.itervalues + for child, (start, stop, step) in values(): + if start > key.start: + self._children[id(child)][1][0] -= diff + if stop is not None and stop >= key.stop: + self._children[id(child)][1][1] -= diff + + if not py3k: + def __getslice__(self, start, stop): + return self.__getitem__(slice(start, stop)) + + def __setslice__(self, start, stop, iterable): + self.__setitem__(slice(start, stop), iterable) + + def __delslice__(self, start, stop): + self.__delitem__(slice(start, stop)) + + def __add__(self, other): + return SmartList(list(self) + other) + + def __radd__(self, other): + return SmartList(other + list(self)) + + def __iadd__(self, other): + self.extend(other) + return self + + def _delete_child(self, child_ref): + """Remove a child reference that is about to be garbage-collected.""" + del self._children[id(child_ref)] + + def _detach_children(self): + """Remove all children and give them independent parent copies.""" + children = [val[0] for val in self._children.values()] + for child in children: + child()._parent = list(self) + self._children.clear() + + @inheritdoc + def append(self, item): + head = len(self) + self[head:head] = [item] + + @inheritdoc + def extend(self, item): + head = len(self) + self[head:head] = item + + @inheritdoc + def insert(self, index, item): + self[index:index] = [item] + + @inheritdoc + def pop(self, index=None): + if index is None: + index = len(self) - 1 + item = self[index] + del self[index] + return item + + @inheritdoc + def remove(self, item): + del self[self.index(item)] + + @inheritdoc + def reverse(self): + self._detach_children() + super(SmartList, self).reverse() + + if py3k: + @inheritdoc + def sort(self, key=None, reverse=None): + self._detach_children() + kwargs = {} + if key is not None: + kwargs["key"] = key + if reverse is not None: + kwargs["reverse"] = reverse + super(SmartList, self).sort(**kwargs) + else: + @inheritdoc + def sort(self, cmp=None, key=None, reverse=None): + self._detach_children() + kwargs = {} + if cmp is not None: + kwargs["cmp"] = cmp + if key is not None: + kwargs["key"] = key + if reverse is not None: + kwargs["reverse"] = reverse + super(SmartList, self).sort(**kwargs) diff --git a/mwparserfromhell/smart_list/__init__.py b/mwparserfromhell/smart_list/__init__.py new file mode 100644 index 00000000..302c9f81 --- /dev/null +++ b/mwparserfromhell/smart_list/__init__.py @@ -0,0 +1 @@ +from .SmartList import SmartList diff --git a/mwparserfromhell/smart_list/utils.py b/mwparserfromhell/smart_list/utils.py new file mode 100644 index 00000000..48ab0d88 --- /dev/null +++ b/mwparserfromhell/smart_list/utils.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2012-2016 Ben Kurtovic +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +This module contains the :class:`.SmartList` type, as well as its +:class:`._ListProxy` child, which together implement a list whose sublists +reflect changes made to the main list, and vice-versa. +""" + +from __future__ import unicode_literals + +from sys import maxsize + +__all__ = [] + + +def inheritdoc(method): + """Set __doc__ of *method* to __doc__ of *method* in its parent class. + + Since this is used on :class:`.SmartList`, the "parent class" used is + ``list``. This function can be used as a decorator. + """ + method.__doc__ = getattr(list, method.__name__).__doc__ + return method + + +class _SliceNormalizerMixIn(object): + """MixIn that provides a private method to normalize slices.""" + + def _normalize_slice(self, key, clamp=False): + """Return a slice equivalent to the input *key*, standardized.""" + if key.start is None: + start = 0 + else: + start = (len(self) + key.start) if key.start < 0 else key.start + if key.stop is None or key.stop == maxsize: + stop = len(self) if clamp else None + else: + stop = (len(self) + key.stop) if key.stop < 0 else key.stop + return slice(start, stop, key.step or 1) diff --git a/mwparserfromhell/wikicode.py b/mwparserfromhell/wikicode.py index 840d8ed1..1a966e2e 100644 --- a/mwparserfromhell/wikicode.py +++ b/mwparserfromhell/wikicode.py @@ -21,13 +21,14 @@ # SOFTWARE. from __future__ import unicode_literals -from itertools import chain + import re +from itertools import chain from .compat import bytes, py3k, range, str from .nodes import (Argument, Comment, ExternalLink, Heading, HTMLEntity, Node, Tag, Template, Text, Wikilink) -from .smart_list import _ListProxy +from .smart_list.ListProxy import _ListProxy from .string_mixin import StringMixIn from .utils import parse_anything diff --git a/tests/test_smart_list.py b/tests/test_smart_list.py index 3c9f7119..8deddd51 100644 --- a/tests/test_smart_list.py +++ b/tests/test_smart_list.py @@ -21,10 +21,13 @@ # SOFTWARE. from __future__ import unicode_literals + import unittest from mwparserfromhell.compat import py3k, range -from mwparserfromhell.smart_list import SmartList, _ListProxy +from mwparserfromhell.smart_list import SmartList +from mwparserfromhell.smart_list.ListProxy import _ListProxy + class TestSmartList(unittest.TestCase): """Test cases for the SmartList class and its child, _ListProxy.""" From 406562415032328f5f54f738cda8f976e67b3ebc Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Wed, 1 Jan 2020 23:23:30 -0500 Subject: [PATCH 2/3] updated comment/license --- mwparserfromhell/smart_list/ListProxy.py | 23 +++++++++++++++++++ mwparserfromhell/smart_list/SmartList.py | 23 +++++++++++++++++++ mwparserfromhell/smart_list/__init__.py | 29 ++++++++++++++++++++++++ mwparserfromhell/smart_list/utils.py | 7 +----- 4 files changed, 76 insertions(+), 6 deletions(-) diff --git a/mwparserfromhell/smart_list/ListProxy.py b/mwparserfromhell/smart_list/ListProxy.py index 2e3b94a0..6d4b85cf 100644 --- a/mwparserfromhell/smart_list/ListProxy.py +++ b/mwparserfromhell/smart_list/ListProxy.py @@ -1,3 +1,26 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2012-2016 Ben Kurtovic +# Copyright (C) 2019-2020 Yuri Astrakhan +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + # SmartList has to be a full import in order to avoid cyclical import errors import mwparserfromhell.smart_list.SmartList from .utils import _SliceNormalizerMixIn, inheritdoc diff --git a/mwparserfromhell/smart_list/SmartList.py b/mwparserfromhell/smart_list/SmartList.py index e442252c..30d2b1e2 100644 --- a/mwparserfromhell/smart_list/SmartList.py +++ b/mwparserfromhell/smart_list/SmartList.py @@ -1,3 +1,26 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2012-2016 Ben Kurtovic +# Copyright (C) 2019-2020 Yuri Astrakhan +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + from _weakref import ref from .ListProxy import _ListProxy diff --git a/mwparserfromhell/smart_list/__init__.py b/mwparserfromhell/smart_list/__init__.py index 302c9f81..81d4fb19 100644 --- a/mwparserfromhell/smart_list/__init__.py +++ b/mwparserfromhell/smart_list/__init__.py @@ -1 +1,30 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2012-2016 Ben Kurtovic +# Copyright (C) 2019-2020 Yuri Astrakhan +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +This module contains the :class:`.SmartList` type, as well as its +:class:`._ListProxy` child, which together implement a list whose sublists +reflect changes made to the main list, and vice-versa. +""" + from .SmartList import SmartList diff --git a/mwparserfromhell/smart_list/utils.py b/mwparserfromhell/smart_list/utils.py index 48ab0d88..609b095c 100644 --- a/mwparserfromhell/smart_list/utils.py +++ b/mwparserfromhell/smart_list/utils.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2012-2016 Ben Kurtovic +# Copyright (C) 2019-2020 Yuri Astrakhan # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -20,12 +21,6 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -""" -This module contains the :class:`.SmartList` type, as well as its -:class:`._ListProxy` child, which together implement a list whose sublists -reflect changes made to the main list, and vice-versa. -""" - from __future__ import unicode_literals from sys import maxsize From 44ec68ff10c556fc03ba84d5764d56adc76d4974 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Mon, 6 Jan 2020 14:32:59 -0500 Subject: [PATCH 3/3] WIP - rewrote smart list This is a first draft of the smart list. It passes all of the existing smartlist tests (except for full-list reversal, whose behavior has changed), but the rest of the code has not yet been updated. --- mwparserfromhell/parser/builder.py | 6 +- mwparserfromhell/smart_list/ListProxy.py | 263 -------------- mwparserfromhell/smart_list/SmartList.py | 185 ---------- mwparserfromhell/smart_list/__init__.py | 2 +- mwparserfromhell/smart_list/smart_list.py | 409 ++++++++++++++++++++++ mwparserfromhell/smart_list/utils.py | 18 - mwparserfromhell/utils.py | 8 +- mwparserfromhell/wikicode.py | 5 +- tests/_test_tree_equality.py | 4 +- tests/test_smart_list.py | 36 +- tests/test_wikicode.py | 4 +- 11 files changed, 439 insertions(+), 501 deletions(-) delete mode 100644 mwparserfromhell/smart_list/ListProxy.py delete mode 100644 mwparserfromhell/smart_list/SmartList.py create mode 100644 mwparserfromhell/smart_list/smart_list.py diff --git a/mwparserfromhell/parser/builder.py b/mwparserfromhell/parser/builder.py index f1b9689f..1e5a1b3f 100644 --- a/mwparserfromhell/parser/builder.py +++ b/mwparserfromhell/parser/builder.py @@ -27,7 +27,7 @@ from ..nodes import (Argument, Comment, ExternalLink, Heading, HTMLEntity, Tag, Template, Text, Wikilink) from ..nodes.extras import Attribute, Parameter -from ..smart_list import SmartList +from ..smart_list import smart_list from ..wikicode import Wikicode __all__ = ["Builder"] @@ -67,7 +67,7 @@ def _pop(self): The raw node list is wrapped in a :class:`.SmartList` and then in a :class:`.Wikicode` object. """ - return Wikicode(SmartList(self._stacks.pop())) + return Wikicode(smart_list(self._stacks.pop())) def _write(self, item): """Append a node to the current node list.""" @@ -92,7 +92,7 @@ def _handle_parameter(self, default): self._tokens.append(token) value = self._pop() if key is None: - key = Wikicode(SmartList([Text(str(default))])) + key = Wikicode(smart_list([Text(str(default))])) return Parameter(key, value, showkey) else: self._write(self._handle_token(token)) diff --git a/mwparserfromhell/smart_list/ListProxy.py b/mwparserfromhell/smart_list/ListProxy.py deleted file mode 100644 index 6d4b85cf..00000000 --- a/mwparserfromhell/smart_list/ListProxy.py +++ /dev/null @@ -1,263 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2012-2016 Ben Kurtovic -# Copyright (C) 2019-2020 Yuri Astrakhan -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -# SmartList has to be a full import in order to avoid cyclical import errors -import mwparserfromhell.smart_list.SmartList -from .utils import _SliceNormalizerMixIn, inheritdoc -from ..compat import py3k - - -class _ListProxy(_SliceNormalizerMixIn, list): - """Implement the ``list`` interface by getting elements from a parent. - - This is created by a :class:`.SmartList` object when slicing. It does not - actually store the list at any time; instead, whenever the list is needed, - it builds it dynamically using the :meth:`_render` method. - """ - - def __init__(self, parent, sliceinfo): - super(_ListProxy, self).__init__() - self._parent = parent - self._sliceinfo = sliceinfo - - def __repr__(self): - return repr(self._render()) - - def __lt__(self, other): - if isinstance(other, _ListProxy): - return self._render() < list(other) - return self._render() < other - - def __le__(self, other): - if isinstance(other, _ListProxy): - return self._render() <= list(other) - return self._render() <= other - - def __eq__(self, other): - if isinstance(other, _ListProxy): - return self._render() == list(other) - return self._render() == other - - def __ne__(self, other): - if isinstance(other, _ListProxy): - return self._render() != list(other) - return self._render() != other - - def __gt__(self, other): - if isinstance(other, _ListProxy): - return self._render() > list(other) - return self._render() > other - - def __ge__(self, other): - if isinstance(other, _ListProxy): - return self._render() >= list(other) - return self._render() >= other - - if py3k: - def __bool__(self): - return bool(self._render()) - else: - def __nonzero__(self): - return bool(self._render()) - - def __len__(self): - return max((self._stop - self._start) // self._step, 0) - - def __getitem__(self, key): - if isinstance(key, slice): - key = self._normalize_slice(key, clamp=True) - keystart = min(self._start + key.start, self._stop) - keystop = min(self._start + key.stop, self._stop) - adjusted = slice(keystart, keystop, key.step) - return self._parent[adjusted] - else: - return self._render()[key] - - def __setitem__(self, key, item): - if isinstance(key, slice): - key = self._normalize_slice(key, clamp=True) - keystart = min(self._start + key.start, self._stop) - keystop = min(self._start + key.stop, self._stop) - adjusted = slice(keystart, keystop, key.step) - self._parent[adjusted] = item - else: - length = len(self) - if key < 0: - key = length + key - if key < 0 or key >= length: - raise IndexError("list assignment index out of range") - self._parent[self._start + key] = item - - def __delitem__(self, key): - if isinstance(key, slice): - key = self._normalize_slice(key, clamp=True) - keystart = min(self._start + key.start, self._stop) - keystop = min(self._start + key.stop, self._stop) - adjusted = slice(keystart, keystop, key.step) - del self._parent[adjusted] - else: - length = len(self) - if key < 0: - key = length + key - if key < 0 or key >= length: - raise IndexError("list assignment index out of range") - del self._parent[self._start + key] - - def __iter__(self): - i = self._start - while i < self._stop: - yield self._parent[i] - i += self._step - - def __reversed__(self): - i = self._stop - 1 - while i >= self._start: - yield self._parent[i] - i -= self._step - - def __contains__(self, item): - return item in self._render() - - if not py3k: - def __getslice__(self, start, stop): - return self.__getitem__(slice(start, stop)) - - def __setslice__(self, start, stop, iterable): - self.__setitem__(slice(start, stop), iterable) - - def __delslice__(self, start, stop): - self.__delitem__(slice(start, stop)) - - def __add__(self, other): - return mwparserfromhell.smart_list.SmartList(list(self) + other) - - def __radd__(self, other): - return mwparserfromhell.smart_list.SmartList(other + list(self)) - - def __iadd__(self, other): - self.extend(other) - return self - - def __mul__(self, other): - return mwparserfromhell.smart_list.SmartList(list(self) * other) - - def __rmul__(self, other): - return mwparserfromhell.smart_list.SmartList(other * list(self)) - - def __imul__(self, other): - self.extend(list(self) * (other - 1)) - return self - - @property - def _start(self): - """The starting index of this list, inclusive.""" - return self._sliceinfo[0] - - @property - def _stop(self): - """The ending index of this list, exclusive.""" - if self._sliceinfo[1] is None: - return len(self._parent) - return self._sliceinfo[1] - - @property - def _step(self): - """The number to increase the index by between items.""" - return self._sliceinfo[2] - - def _render(self): - """Return the actual list from the stored start/stop/step.""" - return list(self._parent)[self._start:self._stop:self._step] - - @inheritdoc - def append(self, item): - self._parent.insert(self._stop, item) - - @inheritdoc - def count(self, item): - return self._render().count(item) - - @inheritdoc - def index(self, item, start=None, stop=None): - if start is not None: - if stop is not None: - return self._render().index(item, start, stop) - return self._render().index(item, start) - return self._render().index(item) - - @inheritdoc - def extend(self, item): - self._parent[self._stop:self._stop] = item - - @inheritdoc - def insert(self, index, item): - if index < 0: - index = len(self) + index - self._parent.insert(self._start + index, item) - - @inheritdoc - def pop(self, index=None): - length = len(self) - if index is None: - index = length - 1 - elif index < 0: - index = length + index - if index < 0 or index >= length: - raise IndexError("pop index out of range") - return self._parent.pop(self._start + index) - - @inheritdoc - def remove(self, item): - index = self.index(item) - del self._parent[self._start + index] - - @inheritdoc - def reverse(self): - item = self._render() - item.reverse() - self._parent[self._start:self._stop:self._step] = item - - if py3k: - @inheritdoc - def sort(self, key=None, reverse=None): - item = self._render() - kwargs = {} - if key is not None: - kwargs["key"] = key - if reverse is not None: - kwargs["reverse"] = reverse - item.sort(**kwargs) - self._parent[self._start:self._stop:self._step] = item - else: - @inheritdoc - def sort(self, cmp=None, key=None, reverse=None): - item = self._render() - kwargs = {} - if cmp is not None: - kwargs["cmp"] = cmp - if key is not None: - kwargs["key"] = key - if reverse is not None: - kwargs["reverse"] = reverse - item.sort(**kwargs) - self._parent[self._start:self._stop:self._step] = item diff --git a/mwparserfromhell/smart_list/SmartList.py b/mwparserfromhell/smart_list/SmartList.py deleted file mode 100644 index 30d2b1e2..00000000 --- a/mwparserfromhell/smart_list/SmartList.py +++ /dev/null @@ -1,185 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2012-2016 Ben Kurtovic -# Copyright (C) 2019-2020 Yuri Astrakhan -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from _weakref import ref - -from .ListProxy import _ListProxy -from .utils import _SliceNormalizerMixIn, inheritdoc -from ..compat import py3k - - -class SmartList(_SliceNormalizerMixIn, list): - """Implements the ``list`` interface with special handling of sublists. - - When a sublist is created (by ``list[i:j]``), any changes made to this - list (such as the addition, removal, or replacement of elements) will be - reflected in the sublist, or vice-versa, to the greatest degree possible. - This is implemented by having sublists - instances of the - :class:`._ListProxy` type - dynamically determine their elements by storing - their slice info and retrieving that slice from the parent. Methods that - change the size of the list also change the slice info. For example:: - - >>> parent = SmartList([0, 1, 2, 3]) - >>> parent - [0, 1, 2, 3] - >>> child = parent[2:] - >>> child - [2, 3] - >>> child.append(4) - >>> child - [2, 3, 4] - >>> parent - [0, 1, 2, 3, 4] - """ - - def __init__(self, iterable=None): - if iterable: - super(SmartList, self).__init__(iterable) - else: - super(SmartList, self).__init__() - self._children = {} - - def __getitem__(self, key): - if not isinstance(key, slice): - return super(SmartList, self).__getitem__(key) - key = self._normalize_slice(key, clamp=False) - sliceinfo = [key.start, key.stop, key.step] - child = _ListProxy(self, sliceinfo) - child_ref = ref(child, self._delete_child) - self._children[id(child_ref)] = (child_ref, sliceinfo) - return child - - def __setitem__(self, key, item): - if not isinstance(key, slice): - return super(SmartList, self).__setitem__(key, item) - item = list(item) - super(SmartList, self).__setitem__(key, item) - key = self._normalize_slice(key, clamp=True) - diff = len(item) + (key.start - key.stop) // key.step - if not diff: - return - values = self._children.values if py3k else self._children.itervalues - for child, (start, stop, step) in values(): - if start > key.stop: - self._children[id(child)][1][0] += diff - if stop is not None and stop >= key.stop: - self._children[id(child)][1][1] += diff - - def __delitem__(self, key): - super(SmartList, self).__delitem__(key) - if isinstance(key, slice): - key = self._normalize_slice(key, clamp=True) - else: - key = slice(key, key + 1, 1) - diff = (key.stop - key.start) // key.step - values = self._children.values if py3k else self._children.itervalues - for child, (start, stop, step) in values(): - if start > key.start: - self._children[id(child)][1][0] -= diff - if stop is not None and stop >= key.stop: - self._children[id(child)][1][1] -= diff - - if not py3k: - def __getslice__(self, start, stop): - return self.__getitem__(slice(start, stop)) - - def __setslice__(self, start, stop, iterable): - self.__setitem__(slice(start, stop), iterable) - - def __delslice__(self, start, stop): - self.__delitem__(slice(start, stop)) - - def __add__(self, other): - return SmartList(list(self) + other) - - def __radd__(self, other): - return SmartList(other + list(self)) - - def __iadd__(self, other): - self.extend(other) - return self - - def _delete_child(self, child_ref): - """Remove a child reference that is about to be garbage-collected.""" - del self._children[id(child_ref)] - - def _detach_children(self): - """Remove all children and give them independent parent copies.""" - children = [val[0] for val in self._children.values()] - for child in children: - child()._parent = list(self) - self._children.clear() - - @inheritdoc - def append(self, item): - head = len(self) - self[head:head] = [item] - - @inheritdoc - def extend(self, item): - head = len(self) - self[head:head] = item - - @inheritdoc - def insert(self, index, item): - self[index:index] = [item] - - @inheritdoc - def pop(self, index=None): - if index is None: - index = len(self) - 1 - item = self[index] - del self[index] - return item - - @inheritdoc - def remove(self, item): - del self[self.index(item)] - - @inheritdoc - def reverse(self): - self._detach_children() - super(SmartList, self).reverse() - - if py3k: - @inheritdoc - def sort(self, key=None, reverse=None): - self._detach_children() - kwargs = {} - if key is not None: - kwargs["key"] = key - if reverse is not None: - kwargs["reverse"] = reverse - super(SmartList, self).sort(**kwargs) - else: - @inheritdoc - def sort(self, cmp=None, key=None, reverse=None): - self._detach_children() - kwargs = {} - if cmp is not None: - kwargs["cmp"] = cmp - if key is not None: - kwargs["key"] = key - if reverse is not None: - kwargs["reverse"] = reverse - super(SmartList, self).sort(**kwargs) diff --git a/mwparserfromhell/smart_list/__init__.py b/mwparserfromhell/smart_list/__init__.py index 81d4fb19..4a30cf16 100644 --- a/mwparserfromhell/smart_list/__init__.py +++ b/mwparserfromhell/smart_list/__init__.py @@ -27,4 +27,4 @@ reflect changes made to the main list, and vice-versa. """ -from .SmartList import SmartList +from .smart_list import smart_list diff --git a/mwparserfromhell/smart_list/smart_list.py b/mwparserfromhell/smart_list/smart_list.py new file mode 100644 index 00000000..8b51c631 --- /dev/null +++ b/mwparserfromhell/smart_list/smart_list.py @@ -0,0 +1,409 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2012-2016 Ben Kurtovic +# Copyright (C) 2019-2020 Yuri Astrakhan +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from _weakref import ref +from typing import Tuple + +from .utils import inheritdoc +from ..compat import py3k + + +class ListStore: + def __init__(self, iterator=None, attach=False): + if attach: + self.data = iterator + else: + self.data = [] if iterator is None else list(iterator) + self.slices = {} + + def attach_slice(self, slc): + slc_ref = ref(slc, self.detach_slice_ref) + self.slices[id(slc_ref)] = slc_ref + + def detach_slice_ref(self, slice_ref): + """Remove a child reference that is about to be garbage-collected.""" + del self.slices[id(slice_ref)] + + def detach_slice(self, slc_to_delete): + """ + :type slc_to_delete: SmartSlice + """ + for sid, slc_ref in self.slices.items(): + # slc: Slice + slc = slc_ref() + if slc is not None and slc is slc_to_delete: + del self.slices[sid] + break + + +class SmartSlice: + def __init__(self, store, start, stop, step): + assert step is not None and step != 0 + assert start is None or stop is None or start <= stop + # self._store: Store + self._store = store + # self._start: Union[int, None] + self._start = start + # self._stop: Union[int, None] + self._stop = stop + # self._step: int + self._step = step + + self._store.attach_slice(self) + + def _update_indexes(self, start, stop, shift): + if shift: + for slc_ref in self._store.slices.values(): + # slc: SmartSlice + slc = slc_ref() + if slc is not None: + if slc._start is not None and stop < slc._start: + slc._start += shift + if slc._stop is not None and start <= slc._stop: + slc._stop += shift + + def _render(self): + """Return the actual list from the stored start/stop/step.""" + if self._start is None: + if self._stop is None: + return self._store.data[::self._step] + else: + return self._store.data[:self._stop:self._step] + elif self._stop is None: + return self._store.data[self._start::self._step] + else: + return self._store.data[self._start:self._stop:self._step] + + def _adjust(self, start, stop, step, materialize=False): + """ + :rtype: Tuple[int, int, int, int, int] + """ + int_start = 0 if self._start is None else self._start + int_stop = len(self._store.data) if self._stop is None else self._stop + + if self._step > 0: + if start is None: + _start = self._start + else: + _start = min(int_stop, max(int_start, ( + int_start if start >= 0 else int_stop) + start)) + + if stop is None: + _stop = self._stop + else: + _stop = min(int_stop, max(int_start, ( + int_start if stop >= 0 else int_stop) + stop)) + + _step = self._step if step is None else (self._step * step) + else: + raise ValueError("Not implemented") + # _start = stop if self._start is None else ( + # self._stop if stop is None else (self._stop + stop)) + # _stop = stop if self._stop is None else ( + # self._stop if stop is None else (self._stop + stop)) + # _step = self._step if step is None else (self._step * step) + + if materialize: + if _start is None: + _start = int_start + if _stop is None: + _stop = int_stop + + rng_start = _start if self._start is None else int_start + rng_stop = _stop if self._stop is None else int_stop + return _start, _stop, _step, rng_start, rng_stop + + def _adjust_pos(self, pos, validate): + """ + :type pos: int + :type validate: bool + :rtype: int + """ + assert isinstance(pos, int) + int_start = 0 if self._start is None else self._start + int_stop = len(self._store.data) if self._stop is None else self._stop + + if self._step > 0: + _pos = (int_start if pos >= 0 else int_stop) + pos + else: + raise ValueError("Not implemented") + + if validate and not (int_start <= _pos < int_stop): + raise IndexError('list index out of range') + + return _pos + + # def _delete_child(self, child_ref): + # """Remove a child reference that is about to be garbage-collected.""" + # del self._children[id(child_ref)] + # + # def _detach_children(self): + # """Remove all children and give them independent parent copies.""" + # children = [val[0] for val in self._children.values()] + # for child in children: + # child()._parent = list(self) + # self._children.clear() + + def __getitem__(self, key): + if isinstance(key, slice): + start, stop, step, _, _ = self._adjust(key.start, key.stop, key.step) + if start is not None and stop is not None and start > stop: + stop = start + return SmartSlice(self._store, start, stop, step) + elif isinstance(key, int): + return self._store.data[self._adjust_pos(key, True)] + else: + raise TypeError('list indices must be integers or slices, not ' + + type(key).__name__) + + def __setitem__(self, key, item): + old_size = len(self._store.data) + + if isinstance(key, slice): + start, stop, step, rng_start, rng_stop = self._adjust(key.start, key.stop, + key.step, True) + self._store.data[start:stop:step] = item + self._update_indexes(start, stop, len(self._store.data) - old_size) + elif isinstance(key, int): + self._store.data[self._adjust_pos(key, True)] = item + else: + raise TypeError('list indices must be integers or slices, not ' + + type(key).__name__) + + def __delitem__(self, key): + old_size = len(self._store.data) + if isinstance(key, slice): + start, stop, step, _, _ = self._adjust(key.start, key.stop, key.step, True) + del self._store.data[start:stop:step] + elif isinstance(key, int): + start = stop = self._adjust_pos(key, True) + del self._store.data[start] + else: + raise TypeError('list indices must be integers or slices, not ' + + type(key).__name__) + + self._update_indexes(start, stop, len(self._store.data) - old_size) + + if not py3k: + def __getslice__(self, start, stop): + return self.__getitem__(slice(start, stop)) + + def __setslice__(self, start, stop, iterable): + self.__setitem__(slice(start, stop), iterable) + + def __delslice__(self, start, stop): + self.__delitem__(slice(start, stop)) + + def __iter__(self): + start = self._start + stop = self._stop + if start is None: + start = 0 if self._step > 0 else (len(self._store.data) - 1) + slc = SmartSlice(self._store, start, stop, self._step) + while True: + i = slc._start + if self._step > 0: + if i >= (len(self._store.data) if self._stop is None else self._stop): + break + elif i <= (-1 if self._stop is None else self._stop): + break + value = self._store.data[i] + slc._start += self._step + yield value + + def __reversed__(self): + start = self._start + stop = self._stop + if stop is None: + stop = len(self._store.data) + slc = SmartSlice(self._store, start, stop, self._step) + while True: + i = slc._stop + if self._step > 0: + if i <= (0 if self._start is None else self._start): + break + elif i >= (len(self._store.data) if self._start is None else self._start): + break + value = self._store.data[i - 1] + slc._stop -= self._step + yield value + + def __repr__(self): + return repr(self._render()) + + def __lt__(self, other): + if isinstance(other, SmartSlice): + return self._render() < other._render() + return self._render() < other + + def __le__(self, other): + if isinstance(other, SmartSlice): + return self._render() <= other._render() + return self._render() <= other + + def __eq__(self, other): + if isinstance(other, SmartSlice): + return self._render() == other._render() + return self._render() == other + + def __ne__(self, other): + if isinstance(other, SmartSlice): + return self._render() != other._render() + return self._render() != other + + def __gt__(self, other): + if isinstance(other, SmartSlice): + return self._render() > other._render() + return self._render() > other + + def __ge__(self, other): + if isinstance(other, SmartSlice): + return self._render() >= other._render() + return self._render() >= other + + def __bool__(self): + return bool(self._render()) + + def __len__(self): + size = len(self._store.data) if self._stop is None else self._stop + if self._start is not None: + size -= self._start + return max(0, size // abs(self._step)) + + def __mul__(self, other): + return smart_list(self._render() * other) + + def __rmul__(self, other): + return smart_list(other * self._render()) + + def __imul__(self, other): + self.extend(self._render() * (other - 1)) + return self + + def __contains__(self, item): + return item in self._render() + + def __add__(self, other): + return smart_list(self._render() + other) + + def __radd__(self, other): + return smart_list(other + self._render()) + + def __iadd__(self, other): + self.extend(other) + return self + + @inheritdoc + def append(self, item): + size = len(self) + self[size:size] = [item] + + @inheritdoc + def extend(self, item): + size = len(self) + self[size:size] = item + + @inheritdoc + def insert(self, index, item): + start, stop, step, rng_start, rng_stop = self._adjust(index, index, 1, True) + self._store.data.insert(start, item) + self._update_indexes(start, stop, 1) + + @inheritdoc + def pop(self, index=None): + start = 0 if self._start is None else self._start + size = len(self) + if index is None: + index = size - 1 + elif index < 0: + index = size + index + if index < 0 or index >= size: + raise IndexError("pop index out of range") + pos = start + index + result = self._store.data.pop(pos) + self._update_indexes(pos, pos, -1) + return result + + @inheritdoc + def remove(self, item): + index = self.index(item) + del self[index] + + @inheritdoc + def reverse(self): + if self._start is None and self._stop is None and self._step == 1: + self._store.data.reverse() + else: + vals = self._render() + vals.reverse() + self[:] = vals + + # values = self._render() + # self._store.detach_slice(self) + # self._store = Store(values, attach=True) + # self._store.attach_slice(self) + # self._start = None + # self._stop = None + # self._step = 1 + + @inheritdoc + def count(self, item): + return self._render().count(item) + + @inheritdoc + def index(self, item, start=None, stop=None): + if start is None: + return self._render().index(item) + elif stop is None: + return self._render().index(item, start) + else: + return self._render().index(item, start, stop) + + if py3k: + @inheritdoc + def sort(self, key=None, reverse=None): + item = self._render() + kwargs = {} + if key is not None: + kwargs["key"] = key + if reverse is not None: + kwargs["reverse"] = reverse + item.sort(**kwargs) + self._store.data[self._start:self._stop:self._step] = item + else: + @inheritdoc + def sort(self, cmp=None, key=None, reverse=None): + item = self._render() + kwargs = {} + if cmp is not None: + kwargs["cmp"] = cmp + if key is not None: + kwargs["key"] = key + if reverse is not None: + kwargs["reverse"] = reverse + item.sort(**kwargs) + self._store.data[self._start:self._stop:self._step] = item + + +def smart_list(iterator=None): + return SmartSlice(ListStore(iterator), None, None, 1) diff --git a/mwparserfromhell/smart_list/utils.py b/mwparserfromhell/smart_list/utils.py index 609b095c..aed6bca8 100644 --- a/mwparserfromhell/smart_list/utils.py +++ b/mwparserfromhell/smart_list/utils.py @@ -23,8 +23,6 @@ from __future__ import unicode_literals -from sys import maxsize - __all__ = [] @@ -36,19 +34,3 @@ def inheritdoc(method): """ method.__doc__ = getattr(list, method.__name__).__doc__ return method - - -class _SliceNormalizerMixIn(object): - """MixIn that provides a private method to normalize slices.""" - - def _normalize_slice(self, key, clamp=False): - """Return a slice equivalent to the input *key*, standardized.""" - if key.start is None: - start = 0 - else: - start = (len(self) + key.start) if key.start < 0 else key.start - if key.stop is None or key.stop == maxsize: - stop = len(self) if clamp else None - else: - stop = (len(self) + key.stop) if key.stop < 0 else key.stop - return slice(start, stop, key.step or 1) diff --git a/mwparserfromhell/utils.py b/mwparserfromhell/utils.py index d30a2dad..6b32d3bd 100644 --- a/mwparserfromhell/utils.py +++ b/mwparserfromhell/utils.py @@ -29,7 +29,7 @@ from .compat import bytes, str from .nodes import Node -from .smart_list import SmartList +from .smart_list import smart_list __all__ = ["parse_anything"] @@ -53,7 +53,7 @@ def parse_anything(value, context=0, skip_style_tags=False): if isinstance(value, Wikicode): return value elif isinstance(value, Node): - return Wikicode(SmartList([value])) + return Wikicode(smart_list([value])) elif isinstance(value, str): return Parser().parse(value, context, skip_style_tags) elif isinstance(value, bytes): @@ -61,11 +61,11 @@ def parse_anything(value, context=0, skip_style_tags=False): elif isinstance(value, int): return Parser().parse(str(value), context, skip_style_tags) elif value is None: - return Wikicode(SmartList()) + return Wikicode(smart_list()) elif hasattr(value, "read"): return parse_anything(value.read(), context, skip_style_tags) try: - nodelist = SmartList() + nodelist = smart_list() for item in value: nodelist += parse_anything(item, context, skip_style_tags).nodes return Wikicode(nodelist) diff --git a/mwparserfromhell/wikicode.py b/mwparserfromhell/wikicode.py index 1a966e2e..f7406bce 100644 --- a/mwparserfromhell/wikicode.py +++ b/mwparserfromhell/wikicode.py @@ -28,7 +28,6 @@ from .compat import bytes, py3k, range, str from .nodes import (Argument, Comment, ExternalLink, Heading, HTMLEntity, Node, Tag, Template, Text, Wikilink) -from .smart_list.ListProxy import _ListProxy from .string_mixin import StringMixIn from .utils import parse_anything @@ -113,8 +112,8 @@ def getter(i, node): def _is_child_wikicode(self, obj, recursive=True): """Return whether the given :class:`.Wikicode` is a descendant.""" def deref(nodes): - if isinstance(nodes, _ListProxy): - return nodes._parent # pylint: disable=protected-access + # if isinstance(nodes, SmartSlice) and (nodes._start is not None and nodes._stop is not None: + # return nodes._parent # pylint: disable=protected-access return nodes target = deref(obj.nodes) diff --git a/tests/_test_tree_equality.py b/tests/_test_tree_equality.py index aba54d1c..1c00bdd8 100644 --- a/tests/_test_tree_equality.py +++ b/tests/_test_tree_equality.py @@ -27,10 +27,10 @@ from mwparserfromhell.nodes import (Argument, Comment, Heading, HTMLEntity, Tag, Template, Text, Wikilink) from mwparserfromhell.nodes.extras import Attribute, Parameter -from mwparserfromhell.smart_list import SmartList +from mwparserfromhell.smart_list import smart_list from mwparserfromhell.wikicode import Wikicode -wrap = lambda L: Wikicode(SmartList(L)) +wrap = lambda L: Wikicode(smart_list(L)) wraptext = lambda *args: wrap([Text(t) for t in args]) class TreeEqualityTestCase(TestCase): diff --git a/tests/test_smart_list.py b/tests/test_smart_list.py index 8deddd51..496f92a5 100644 --- a/tests/test_smart_list.py +++ b/tests/test_smart_list.py @@ -25,10 +25,11 @@ import unittest from mwparserfromhell.compat import py3k, range -from mwparserfromhell.smart_list import SmartList -from mwparserfromhell.smart_list.ListProxy import _ListProxy +from mwparserfromhell.smart_list import smart_list +from mwparserfromhell.smart_list.smart_list import SmartSlice +# noinspection DuplicatedCode class TestSmartList(unittest.TestCase): """Test cases for the SmartList class and its child, _ListProxy.""" @@ -277,10 +278,10 @@ def _test_list_methods(self, builder): def _dispatch_test_for_children(self, meth): """Run a test method on various different types of children.""" - meth(lambda L: SmartList(list(L))[:]) - meth(lambda L: SmartList([999] + list(L))[1:]) - meth(lambda L: SmartList(list(L) + [999])[:-1]) - meth(lambda L: SmartList([101, 102] + list(L) + [201, 202])[2:-2]) + meth(lambda lst: smart_list(list(lst))[:]) + meth(lambda lst: smart_list([999] + list(lst))[1:]) + meth(lambda lst: smart_list(list(lst) + [999])[:-1]) + meth(lambda lst: smart_list([101, 102] + list(lst) + [201, 202])[2:-2]) def test_docs(self): """make sure the methods of SmartList/_ListProxy have docstrings""" @@ -288,14 +289,12 @@ def test_docs(self): "remove", "reverse", "sort"] for meth in methods: expected = getattr(list, meth).__doc__ - smartlist_doc = getattr(SmartList, meth).__doc__ - listproxy_doc = getattr(_ListProxy, meth).__doc__ + smartlist_doc = getattr(SmartSlice, meth).__doc__ self.assertEqual(expected, smartlist_doc) - self.assertEqual(expected, listproxy_doc) def test_doctest(self): """make sure the test embedded in SmartList's docstring passes""" - parent = SmartList([0, 1, 2, 3]) + parent = smart_list([0, 1, 2, 3]) self.assertEqual([0, 1, 2, 3], parent) child = parent[2:] self.assertEqual([2, 3], child) @@ -305,19 +304,19 @@ def test_doctest(self): def test_parent_get_set_del(self): """make sure SmartList's getitem/setitem/delitem work""" - self._test_get_set_del_item(SmartList) + self._test_get_set_del_item(smart_list) def test_parent_add(self): """make sure SmartList's add/radd/iadd work""" - self._test_add_radd_iadd(SmartList) + self._test_add_radd_iadd(smart_list) def test_parent_other_magics(self): """make sure SmartList's other magically implemented features work""" - self._test_other_magic_methods(SmartList) + self._test_other_magic_methods(smart_list) def test_parent_methods(self): """make sure SmartList's non-magic methods work, like append()""" - self._test_list_methods(SmartList) + self._test_list_methods(smart_list) def test_child_get_set_del(self): """make sure _ListProxy's getitem/setitem/delitem work""" @@ -337,13 +336,12 @@ def test_child_methods(self): def test_influence(self): """make sure changes are propagated from parents to children""" - parent = SmartList([0, 1, 2, 3, 4, 5]) + parent = smart_list([0, 1, 2, 3, 4, 5]) child1 = parent[2:] child2 = parent[2:5] self.assertEqual([0, 1, 2, 3, 4, 5], parent) self.assertEqual([2, 3, 4, 5], child1) self.assertEqual([2, 3, 4], child2) - self.assertEqual(2, len(parent._children)) parent.append(6) child1.append(7) @@ -403,12 +401,10 @@ def test_influence(self): self.assertEqual([1, 4, 3, 2, 1.9, 1.8, 5, 6], parent) self.assertEqual([4, 3, 2, 1.9, 1.8], child2) self.assertEqual([], child3) - self.assertEqual(2, len(parent._children)) del child3 self.assertEqual([1, 4, 3, 2, 1.9, 1.8, 5, 6], parent) self.assertEqual([4, 3, 2, 1.9, 1.8], child2) - self.assertEqual(1, len(parent._children)) parent.remove(1.9) parent.remove(1.8) @@ -417,8 +413,8 @@ def test_influence(self): parent.reverse() self.assertEqual([6, 5, 2, 3, 4, 1], parent) - self.assertEqual([4, 3, 2], child2) - self.assertEqual(0, len(parent._children)) + self.assertEqual([5, 2, 3], child2) + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/tests/test_wikicode.py b/tests/test_wikicode.py index 307ee9ae..7d9dfcb0 100644 --- a/tests/test_wikicode.py +++ b/tests/test_wikicode.py @@ -29,7 +29,7 @@ from mwparserfromhell.compat import py3k, str from mwparserfromhell.nodes import (Argument, Comment, Heading, HTMLEntity, Node, Tag, Template, Text, Wikilink) -from mwparserfromhell.smart_list import SmartList +from mwparserfromhell.smart_list import smart_list from mwparserfromhell.wikicode import Wikicode from mwparserfromhell import parse @@ -49,7 +49,7 @@ def test_nodes(self): """test getter/setter for the nodes attribute""" code = parse("Have a {{template}}") self.assertEqual(["Have a ", "{{template}}"], code.nodes) - L1 = SmartList([Text("foobar"), Template(wraptext("abc"))]) + L1 = smart_list([Text("foobar"), Template(wraptext("abc"))]) L2 = [Text("barfoo"), Template(wraptext("cba"))] L3 = "abc{{def}}" code.nodes = L1