Skip to content

Commit

Permalink
Switch to using sized ranges to fix __getitem__ performance issue
Browse files Browse the repository at this point in the history 
Fixing this required forking ranges-v3 in order to fix ericniebler/range-v3#824.
  • Loading branch information
@hkalodner
hkalodner committed May 4, 2018
1 parent 7a29cb0 commit 53ce267
Show file tree
Hide file tree
Showing 13 changed files with 53 additions and 38 deletions.
2 changes: 1 addition & 1 deletion .gitmodules
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
url = https://github.com/mpark/variant.git
[submodule "libs/range-v3"]
path = external/range-v3
url = https://github.com/ericniebler/range-v3.git
url = https://github.com/hkalodner/range-v3.git
[submodule "Notebooks/blocksci/Blockchain-Known-Pools"]
path = blockscipy/blocksci/Blockchain-Known-Pools
url = https://github.com/blockchain/Blockchain-Known-Pools
Expand Down
8 changes: 4 additions & 4 deletions blockscipy/src/blocksci_range.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,15 @@
template <typename T>
struct RangeClasses {
pybind11::class_<ranges::any_view<T>> iterator;
pybind11::class_<ranges::any_view<T, ranges::category::random_access>> range;
pybind11::class_<ranges::any_view<T, ranges::category::random_access | ranges::category::sized>> range;
pybind11::class_<ranges::any_view<ranges::optional<T>>> optionalIterator;
pybind11::class_<ranges::any_view<ranges::optional<T>, ranges::category::random_access>> optionalRange;
pybind11::class_<ranges::any_view<ranges::optional<T>, ranges::category::random_access | ranges::category::sized>> optionalRange;

RangeClasses(pybind11::module &m) :
iterator(m, strdup(PythonTypeName<ranges::any_view<T>>::name().c_str())),
range(m, strdup(PythonTypeName<ranges::any_view<T, ranges::category::random_access>>::name().c_str())),
range(m, strdup(PythonTypeName<ranges::any_view<T, ranges::category::random_access | ranges::category::sized>>::name().c_str())),
optionalIterator(m, strdup(PythonTypeName<ranges::any_view<ranges::optional<T>>>::name().c_str())),
optionalRange(m, strdup(PythonTypeName<ranges::any_view<ranges::optional<T>, ranges::category::random_access>>::name().c_str())) {}
optionalRange(m, strdup(PythonTypeName<ranges::any_view<ranges::optional<T>, ranges::category::random_access | ranges::category::sized>>::name().c_str())) {}
};

#endif /* blocksci_blocksci_range_h */
2 changes: 1 addition & 1 deletion blockscipy/src/chain/block/block_py.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ struct AddBlockMethods {
using namespace blocksci;
namespace py = pybind11;

func(property_tag, "txes", [](const Block &block) -> ranges::any_view<Transaction, ranges::category::random_access> {
func(property_tag, "txes", [](const Block &block) -> ranges::any_view<Transaction, ranges::category::random_access | ranges::category::sized> {
return block;
}, "A range of all of the txes in the block");
func(property_tag, "inputs", [](const Block &block) -> ranges::any_view<Input> {
Expand Down
2 changes: 1 addition & 1 deletion blockscipy/src/chain/blockchain_py.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ void init_blockchain(py::class_<Blockchain> &cl) {
}
return chain[posIndex];
}, py::arg("index"))
.def("__getitem__", [](Blockchain &chain, pybind11::slice slice) -> ranges::any_view<decltype(chain[0]), ranges::category::random_access> {
.def("__getitem__", [](Blockchain &chain, pybind11::slice slice) -> ranges::any_view<decltype(chain[0]), ranges::category::random_access | ranges::category::sized> {
size_t start, stop, step, slicelength;
if (!slice.compute(chain.size(), &start, &stop, &step, &slicelength))
throw pybind11::error_already_set();
Expand Down
2 changes: 1 addition & 1 deletion blockscipy/src/cluster/cluster/cluster_py.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ void init_cluster_manager(pybind11::module &s) {
.def("cluster_with_address", [](const ClusterManager &cm, const Address &address) -> Cluster {
return cm.getCluster(address);
}, py::arg("address"), "Return the cluster containing the given address")
.def("clusters", [](ClusterManager &cm) -> ranges::any_view<Cluster, ranges::category::random_access> {
.def("clusters", [](ClusterManager &cm) -> ranges::any_view<Cluster, ranges::category::random_access | ranges::category::sized> {
return cm.getClusters();
}, "Get a list of all clusters (The list is lazy so there is no cost to calling this method)")
.def("tagged_clusters", [](ClusterManager &cm, const std::unordered_map<blocksci::Address, std::string> &tags) -> ranges::any_view<TaggedCluster> {
Expand Down
4 changes: 2 additions & 2 deletions blockscipy/src/method_types.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ struct PythonTypeName<ranges::optional<T>> {
};

template <typename T>
struct PythonTypeName<ranges::any_view<T, ranges::category::random_access>> {
struct PythonTypeName<ranges::any_view<T, ranges::category::random_access | ranges::category::sized>> {
static std::string name() {
return PythonTypeName<T>::name() + "Range";
}
Expand All @@ -237,7 +237,7 @@ struct PythonTypeName<ranges::any_view<T>> {
};

template <typename T>
struct PythonTypeName<ranges::any_view<ranges::optional<T>, ranges::category::random_access>> {
struct PythonTypeName<ranges::any_view<ranges::optional<T>, ranges::category::random_access | ranges::category::sized>> {
static std::string name() {
return PythonTypeName<T>::name() + "OptionalRange";
}
Expand Down
10 changes: 5 additions & 5 deletions blockscipy/src/range_conversion.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,8 @@ typename NumpyConverter<blocksci::uint160>::type NumpyConverter<blocksci::uint16

template <typename T>
pybind11::list convertRangeToPythonPy(T && t) {
if constexpr (ranges::RandomAccessRange<T>()) {
auto rangeSize = static_cast<size_t>(ranges::distance(t));
if constexpr (ranges::SizedRange<T>()) {
auto rangeSize = static_cast<size_t>(ranges::size(t));
pybind11::list list{rangeSize};
RANGES_FOR(auto && a, t) {
list.append(std::forward<decltype(a)>(a));
Expand All @@ -80,8 +80,8 @@ pybind11::array_t<typename NumpyConverter<ranges::range_value_type_t<T>>::type>
using numpy_value_type = typename numpy_converter::type;

auto numpy_converted = std::forward<T>(t) | ranges::view::transform(numpy_converter{});
if constexpr (ranges::RandomAccessRange<T>()) {
auto rangeSize = static_cast<size_t>(ranges::distance(numpy_converted));
if constexpr (ranges::SizedRange<T>()) {
auto rangeSize = static_cast<size_t>(ranges::size(numpy_converted));
pybind11::array_t<numpy_value_type> ret{rangeSize};
auto retPtr = ret.mutable_data();
ranges::copy(numpy_converted, retPtr);
Expand Down Expand Up @@ -110,7 +110,7 @@ converted_range_t<T> convertAnyRangeToPython(T && t) {
}

template <typename T>
using random_range = ranges::any_view<T, ranges::category::random_access>;
using random_range = ranges::any_view<T, ranges::category::random_access | ranges::category::sized>;

#define createConvertedRangeTag(functype) \
template converted_range_t<functype> convertAnyRangeToPython<functype>(functype &&);
Expand Down
10 changes: 6 additions & 4 deletions blockscipy/src/range_conversion.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@

#include <pybind11/numpy.h>

#include <range/v3/distance.hpp>
#include <range/v3/range_for.hpp>
#include <range/v3/range_traits.hpp>
#include <range/v3/view/any_view.hpp>
Expand Down Expand Up @@ -183,9 +182,12 @@ struct make_optional<ranges::optional<T>> { using type = ranges::optional<T>; };
template <typename T>
using make_optional_t = typename make_optional<T>::type;

template<class T> struct dependent_false : std::false_type {};

constexpr ranges::category getBlockSciCategory(ranges::category cat) {
if ((cat & ranges::category::random_access) == ranges::category::random_access) {
return ranges::category::random_access;
constexpr auto randomSized = ranges::category::random_access | ranges::category::sized;
if ((cat & randomSized) == randomSized) {
return randomSized;
} else {
return ranges::category::input;
}
Expand Down Expand Up @@ -277,7 +279,7 @@ template <typename T>
struct is_any_view : std::false_type {};

template <typename T>
struct is_any_view<ranges::any_view<T, ranges::category::random_access>> : std::true_type {};
struct is_any_view<ranges::any_view<T, ranges::category::random_access | ranges::category::sized>> : std::true_type {};

template <typename T>
struct is_any_view<ranges::any_view<T>> : std::true_type {};
Expand Down
2 changes: 1 addition & 1 deletion blockscipy/src/range_filter_apply_py.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ auto applyRangeMethodsToRange(Class &cl, Applier applier) {
template <template<typename> class Applier, typename T>
void applyRangeFiltersToRange(RangeClasses<T> &cls) {
applyRangeMethodsToRange(cls.iterator, Applier<ranges::any_view<T>>{});
applyRangeMethodsToRange(cls.range, Applier<ranges::any_view<T, ranges::category::random_access>>{});
applyRangeMethodsToRange(cls.range, Applier<ranges::any_view<T, ranges::category::random_access | ranges::category::sized>>{});
}

#endif /* range_filter_apply_py_h */
13 changes: 7 additions & 6 deletions blockscipy/src/ranges_py.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,52 +14,53 @@ namespace py = pybind11;
using namespace blocksci;

void init_ranges(py::module &m) {
constexpr auto rangeCat = ranges::category::random_access | ranges::category::sized;
{
py::class_<ranges::any_view<ranges::optional<int64_t>>> cl(m, "IntOptionalIterator");
addRangeMethods(cl);
}
{
py::class_<ranges::any_view<ranges::optional<int64_t>, ranges::category::random_access>> cl(m, "IntOptionalRange");
py::class_<ranges::any_view<ranges::optional<int64_t>, rangeCat>> cl(m, "IntOptionalRange");
addRangeMethods(cl);
}
{
py::class_<ranges::any_view<ranges::optional<std::chrono::system_clock::time_point>>> cl(m, "DateOptionalIterator");
addRangeMethods(cl);
}
{
py::class_<ranges::any_view<ranges::optional<std::chrono::system_clock::time_point>, ranges::category::random_access>> cl(m, "DateOptionalRange");
py::class_<ranges::any_view<ranges::optional<std::chrono::system_clock::time_point>, rangeCat>> cl(m, "DateOptionalRange");
addRangeMethods(cl);
}
{
py::class_<ranges::any_view<ranges::optional<uint256>>> cl(m, "UInt256OptionalIterator");
addRangeMethods(cl);
}
{
py::class_<ranges::any_view<ranges::optional<uint256>, ranges::category::random_access>> cl(m, "UInt256OptionalRange");
py::class_<ranges::any_view<ranges::optional<uint256>, rangeCat>> cl(m, "UInt256OptionalRange");
addRangeMethods(cl);
}
{
py::class_<ranges::any_view<ranges::optional<bool>>> cl(m, "BoolOptionalIterator");
addRangeMethods(cl);
}
{
py::class_<ranges::any_view<ranges::optional<bool>, ranges::category::random_access>> cl(m, "BoolOptionalRange");
py::class_<ranges::any_view<ranges::optional<bool>, rangeCat>> cl(m, "BoolOptionalRange");
addRangeMethods(cl);
}
{
py::class_<ranges::any_view<ranges::optional<AddressType::Enum>>> cl(m, "AddressTypeOptionalIterator");
addRangeMethods(cl);
}
{
py::class_<ranges::any_view<ranges::optional<AddressType::Enum>, ranges::category::random_access>> cl(m, "AddressTypeOptionalRange");
py::class_<ranges::any_view<ranges::optional<AddressType::Enum>, rangeCat>> cl(m, "AddressTypeOptionalRange");
addRangeMethods(cl);
}
{
py::class_<ranges::any_view<ranges::optional<py::bytes>>> cl(m, "BytesOptionalIterator");
addRangeMethods(cl);
}
{
py::class_<ranges::any_view<ranges::optional<py::bytes>, ranges::category::random_access>> cl(m, "BytesOptionalRange");
py::class_<ranges::any_view<ranges::optional<py::bytes>, rangeCat>> cl(m, "BytesOptionalRange");
addRangeMethods(cl);
}
}
32 changes: 22 additions & 10 deletions blockscipy/src/ranges_py.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#include <pybind11/functional.h>

#include <range/v3/range_for.hpp>
#include <range/v3/size.hpp>
#include <range/v3/view/any_view.hpp>
#include <range/v3/view/filter.hpp>
#include <range/v3/view/slice.hpp>
Expand Down Expand Up @@ -58,9 +59,14 @@ auto addRangeMethods(Class &cl) {
if constexpr (ranges::ForwardRange<Range>()) {
cl
.def("__bool__", [](Range &range) { return !ranges::empty(range); })
.def("__len__", [](Range &range) { return ranges::distance(range); })
;
}

if constexpr (ranges::BidirectionalRange<Range>()) {
cl
.def("__len__", [](Range &range) { return range.size(); })
.def("__getitem__", [](Range &range, int64_t posIndex) {
auto chainSize = static_cast<int64_t>(ranges::distance(range));
auto chainSize = static_cast<int64_t>(range.size());
if (posIndex < 0) {
posIndex += chainSize;
}
Expand All @@ -69,21 +75,27 @@ auto addRangeMethods(Class &cl) {
}
return range[posIndex];
}, pybind11::arg("index"))
;
}

if constexpr (ranges::BidirectionalRange<Range>()) {
cl.def("__getitem__", [](Range &range, pybind11::slice slice) {
.def("__getitem__", [](Range &range, pybind11::slice slice) {
size_t start, stop, step, slicelength;
auto chainSize = ranges::distance(range);
const auto &constRange = range;
auto chainSize = ranges::size(constRange);
// static_assert(std::is_integral<decltype(chainSize)>::value, "Size is not integral");
// static_assert(!ranges::disable_sized_range<Range>::value, "Size is disabled");
// static_assert(ranges::SizedRange<Range>(), "range is not sized");
// static_assert(ranges::SizedRange<const Range>(), "const range is not sized");
if (!slice.compute(chainSize, &start, &stop, &step, &slicelength))
throw pybind11::error_already_set();

auto subset = ranges::view::slice(range,
static_cast<ranges::range_difference_type_t<Range>>(start),
static_cast<ranges::range_difference_type_t<Range>>(stop));
return convertRangeToPython(subset | ranges::view::stride(step));
}, pybind11::arg("slice"));
// static_assert(ranges::SizedRange<const decltype(subset)>(), "slice is not sized");
auto strided = subset | ranges::view::stride(step);
// static_assert(ranges::SizedRange<const decltype(strided)>(), "strided is not sized");
return convertRangeToPython(strided);
}, pybind11::arg("slice"))
;
}


Expand All @@ -95,7 +107,7 @@ auto addRangeMethods(Class &cl) {
ss << "\n\n:type: :class:`" << PythonTypeName<ranges::any_view<WrappedType, getBlockSciCategory(ranges::get_categories<Range>())>>::name() << "`";

std::stringstream ss2;
ss2 << "\n\n:type: :class:` numpy.ndarray[bool]`";
ss2 << "\n\n:type: :class:`numpy.ndarray[bool]`";
cl
.def_property_readonly("has_value", [](Range &range) {
return convertRangeToPython(range | ranges::view::transform([](auto && val) { return val.has_value(); }));
Expand Down
2 changes: 1 addition & 1 deletion external/range-v3
Submodule range-v3 updated 57 files
+3 −1 .vscode/c_cpp_properties.json
+2 −6 CMakeLists.txt
+1 −1 conanfile.py
+0 −25 conanfile.py.in
+20 −0 doc/index.md
+63 −0 example/.clang-format
+104 −0 example/BUCK
+22 −0 example/CMakeLists.txt
+44 −0 example/any_all_none_of.cpp
+220 −149 example/calendar.cpp
+21 −26 example/comprehensions.cpp
+15 −15 example/count.cpp
+11 −10 example/count_if.cpp
+7 −8 example/fibonacci.cpp
+118 −0 example/find.cpp
+58 −0 example/for_each_assoc.cpp
+59 −0 example/for_each_sequence.cpp
+8 −8 example/hello.cpp
+35 −0 example/is_sorted.cpp
+1 −1 example/recursive_range.hpp
+16 −10 include/meta/meta.hpp
+1 −1 include/range/v3/action/action.hpp
+4 −4 include/range/v3/algorithm/generate.hpp
+2 −2 include/range/v3/algorithm/generate_n.hpp
+2 −2 include/range/v3/algorithm/transform.hpp
+2 −2 include/range/v3/detail/variant.hpp
+1 −1 include/range/v3/numeric/accumulate.hpp
+16 −0 include/range/v3/range_concepts.hpp
+2 −1 include/range/v3/range_fwd.hpp
+18 −8 include/range/v3/span.hpp
+5 −5 include/range/v3/utility/functional.hpp
+17 −7 include/range/v3/utility/invoke.hpp
+21 −21 include/range/v3/utility/iterator_concepts.hpp
+1 −1 include/range/v3/utility/random.hpp
+14 −5 include/range/v3/view/cartesian_product.hpp
+47 −36 include/range/v3/view/chunk.hpp
+4 −2 include/range/v3/view/drop.hpp
+192 −0 include/range/v3/view/exclusive_scan.hpp
+1 −1 include/range/v3/view/for_each.hpp
+5 −5 include/range/v3/view/generate.hpp
+1 −1 include/range/v3/view/generate_n.hpp
+4 −4 include/range/v3/view/partial_sum.hpp
+20 −12 include/range/v3/view/split.hpp
+10 −5 include/range/v3/view/stride.hpp
+4 −4 include/range/v3/view/transform.hpp
+49 −15 include/range/v3/view_interface.hpp
+33 −0 test/span.cpp
+4 −0 test/view/CMakeLists.txt
+18 −0 test/view/cartesian_product.cpp
+17 −0 test/view/chunk.cpp
+7 −0 test/view/drop.cpp
+54 −0 test/view/exclusive_scan.cpp
+7 −0 test/view/remove_if.cpp
+8 −1 test/view/split.cpp
+18 −0 test/view/stride.cpp
+3 −8 test_package/conanfile.py
+0 −38 test_package/conanfile.py.in
2 changes: 1 addition & 1 deletion include/blocksci/chain/blockchain.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ namespace blocksci {
class DataAccess;

template <AddressType::Enum type>
using ScriptRange = ranges::any_view<ScriptAddress<type>, ranges::category::random_access>;
using ScriptRange = ranges::any_view<ScriptAddress<type>, ranges::category::random_access | ranges::category::sized>;
using ScriptRangeVariant = to_variadic_t<to_address_tuple_t<ScriptRange>, mpark::variant>;

namespace internal {
Expand Down

0 comments on commit 53ce267

Please sign in to comment.