diff --git a/dart-if/include/dash/dart/if/dart_communication.h b/dart-if/include/dash/dart/if/dart_communication.h index 8670be2ef..247d20b84 100644 --- a/dart-if/include/dash/dart/if/dart_communication.h +++ b/dart-if/include/dash/dart/if/dart_communication.h @@ -165,6 +165,60 @@ dart_ret_t dart_allgatherv( const size_t * recvdispls, dart_team_t teamid) DART_NOTHROW; +/** + * DART Equivalent to MPI alltoall. + * + * \param sendbuf The buffer containing the data to be sent by each unit. + * \param recvbuf The buffer to hold the received data. + * \param nelem Number of values sent by each process and received from + * each unit. + * \param dtype The data type of values in \c sendbuf and \c recvbuf. + * \param team The team to participate in the allgather. + * + * \return \c DART_OK on success, any other of \ref dart_ret_t otherwise. + * + * \threadsafe_data{team} + * \ingroup DartCommunication + */ +dart_ret_t dart_alltoall( + const void * sendbuf, + void * recvbuf, + size_t nelem, + dart_datatype_t dtype, + dart_team_t team) DART_NOTHROW; + +/** + * DART Equivalent to MPI alltoallv. + * + * \param sendbuf The buffer containing the data to be sent by each unit. + * \param nsendcounts Array containing the number of values to be sent by + * each unit. + * \param senddispls Array containing the displacements of data to be sent + * by each unit. + * \param dtype The data type of values in \c sendbuf and \c recvbuf. + * \param recvbuf The buffer to hold the received data. + * \param nrecvelem Array containing the number of values to receive from + * each unit. + * \param recvdispls Array containing the displacements of data received + * from each unit in \c recvbuf. + * \param teamid The team to participate in the allgatherv. + * + * \return \c DART_OK on success, any other of \ref dart_ret_t otherwise. + * + * \threadsafe_data{team} + * \ingroup DartCommunication + */ +dart_ret_t dart_alltoallv( + const void * sendbuf, + const size_t * nsendcounts, + const size_t * senddispls, + dart_datatype_t dtype, + void * recvbuf, + const size_t * nrecvelem, + const size_t * recvdispls, + dart_team_t teamid) DART_NOTHROW; + + /** * DART Equivalent to MPI allreduce. * diff --git a/dart-impl/mpi/src/dart_communication.c b/dart-impl/mpi/src/dart_communication.c index b4da40d73..f0ea1c5cf 100644 --- a/dart-impl/mpi/src/dart_communication.c +++ b/dart-impl/mpi/src/dart_communication.c @@ -2125,7 +2125,7 @@ dart_ret_t dart_allgatherv( DART_LOG_ERROR("dart_allgatherv ! unknown teamid %d", teamid); return DART_ERR_INVAL; } - if (sendbuf == recvbuf || NULL == sendbuf) { + if (sendbuf == recvbuf) { sendbuf = MPI_IN_PLACE; } MPI_Comm comm = team_data->comm; @@ -2173,6 +2173,150 @@ dart_ret_t dart_allgatherv( return DART_OK; } +#if 0 +/* + * Implementation from branch feat-graph, to be discussed + */ +dart_ret_t dart_alltoall( + const void * sendbuf, + void * recvbuf, + size_t nelem, + dart_datatype_t dtype, + dart_team_t teamid) +{ + MPI_Datatype mpi_dtype = dart__mpi__datatype_struct(dtype)->basic.mpi_type; + MPI_Comm comm; + DART_LOG_TRACE("dart_alltoall() team:%d nelem:%"PRIu64"", + teamid, nelem); + + if (teamid == DART_UNDEFINED_TEAM_ID) { + DART_LOG_ERROR("dart_alltoall ! failed: team may not be DART_UNDEFINED_TEAM_ID"); + return DART_ERR_INVAL; + } + + /* + * MPI uses offset type int, do not copy more than INT_MAX elements: + */ + if (nelem > INT_MAX) { + DART_LOG_ERROR("dart_alltoall ! failed: nelem > INT_MAX"); + return DART_ERR_INVAL; + } + + dart_team_data_t *team_data = dart_adapt_teamlist_get(teamid); + if (team_data == NULL) { + DART_LOG_ERROR("dart_alltoall ! team:%d " + "dart_adapt_teamlist_convert failed", teamid); + return DART_ERR_INVAL; + } + if (sendbuf == recvbuf || NULL == sendbuf) { + sendbuf = MPI_IN_PLACE; + } + comm = team_data->comm; + if (MPI_Alltoall( + sendbuf, + nelem, + mpi_dtype, + recvbuf, + nelem, + mpi_dtype, + comm) != MPI_SUCCESS) { + DART_LOG_ERROR("dart_alltoall ! team:%d nelem:%"PRIu64" failed", + teamid, nelem); + return DART_ERR_INVAL; + } + DART_LOG_TRACE("dart_alltoall > team:%d nelem:%"PRIu64"", + teamid, nelem); + return DART_OK; +} +#endif + +/* + * Implementation from branch feat-graph, to be discussed + */ +dart_ret_t dart_alltoallv( + const void * sendbuf, + const size_t * nsendcounts, + const size_t * senddispls, + dart_datatype_t dtype, + void * recvbuf, + const size_t * nrecvcounts, + const size_t * recvdispls, + dart_team_t teamid) +{ + MPI_Datatype mpi_dtype = dart__mpi__datatype_struct(dtype)->contiguous.mpi_type; + MPI_Comm comm; + int comm_size; + DART_LOG_TRACE("dart_alltoallv() team:%d nsendelem:%"PRIu64"", + teamid, nsendcounts); + + if (teamid == DART_UNDEFINED_TEAM_ID) { + DART_LOG_ERROR( + "dart_alltoallv ! failed: team may not be DART_UNDEFINED_TEAM_ID"); + return DART_ERR_INVAL; + } + + dart_team_data_t *team_data = dart_adapt_teamlist_get(teamid); + if (team_data == NULL) { + DART_LOG_ERROR("dart_alltoallv ! team:%d " + "dart_adapt_teamlist_convert failed", teamid); + return DART_ERR_INVAL; + } + comm = team_data->comm; + + // convert nrecvcounts and recvdispls + MPI_Comm_size(comm, &comm_size); + int *insendcounts = malloc(sizeof(int) * comm_size); + int *isenddispls = malloc(sizeof(int) * comm_size); + int *inrecvcounts = malloc(sizeof(int) * comm_size); + int *irecvdispls = malloc(sizeof(int) * comm_size); + for (int i = 0; i < comm_size; i++) { + if (nsendcounts[i] > INT_MAX || senddispls[i] > INT_MAX) { + DART_LOG_ERROR( + "dart_alltoallv ! failed: " + "nsendcounts[%i] > INT_MAX || senddispls[%i] > INT_MAX", i, i); + free(insendcounts); + free(isenddispls); + return DART_ERR_INVAL; + } + if (nrecvcounts[i] > INT_MAX || recvdispls[i] > INT_MAX) { + DART_LOG_ERROR( + "dart_alltoallv ! failed: " + "nrecvcounts[%i] > INT_MAX || recvdispls[%i] > INT_MAX", i, i); + free(inrecvcounts); + free(irecvdispls); + return DART_ERR_INVAL; + } + insendcounts[i] = nsendcounts[i]; + isenddispls[i] = senddispls[i]; + inrecvcounts[i] = nrecvcounts[i]; + irecvdispls[i] = recvdispls[i]; + } + + if (MPI_Alltoallv( + sendbuf, + insendcounts, + isenddispls, + mpi_dtype, + recvbuf, + inrecvcounts, + irecvdispls, + mpi_dtype, + comm) != MPI_SUCCESS) { + DART_LOG_ERROR("dart_alltoallv ! team:%d failed", teamid); + free(insendcounts); + free(isenddispls); + free(inrecvcounts); + free(irecvdispls); + return DART_ERR_INVAL; + } + free(insendcounts); + free(isenddispls); + free(inrecvcounts); + free(irecvdispls); + DART_LOG_TRACE("dart_alltoallv > team:%d", teamid); + return DART_OK; +} + dart_ret_t dart_allreduce( const void * sendbuf, void * recvbuf, diff --git a/dash/examples/ex.02.graph-deref/main.cpp b/dash/examples/ex.02.graph-deref/main.cpp new file mode 100644 index 000000000..e44512464 --- /dev/null +++ b/dash/examples/ex.02.graph-deref/main.cpp @@ -0,0 +1,121 @@ +#include +#include +#include +#include +// #include "rmatrandom.h" + +struct vprop { + int comp; +}; + +struct eprop { + int comp; +}; + +typedef dash::Graph graph_t; + +int main(int argc, char* argv[]) { + dash::init(&argc, &argv); + + int n_unit_edges = 1; + int n_vertices_full = 448; + int n_edges_full = n_vertices_full * n_unit_edges; + // not exactly n_vertices_full are generated due to rounding + int n_vertices_start = n_vertices_full / dash::size(); + int n_size_rounds = 5; + int n_rounds = 1; + for(int rounds = 0; rounds < n_size_rounds; ++rounds) { + for(int i = 0; i < n_rounds; ++i) { + int scale = rounds + 1; + int n_vertices = n_vertices_start * scale; + graph_t g(n_vertices, n_unit_edges); + + for(int j = 0; j < n_vertices; ++j) { + g.add_vertex(); + } + + g.commit(); + + int n_edges = n_vertices * n_unit_edges; + auto src = g.vertices().lbegin(); + auto trg = src + 1; + auto end = g.vertices().lend(); + for(int j = 0; j < n_edges; ++j) { + g.add_edge(src, trg); + ++src; + ++trg; + if(trg == end) { + src = g.vertices().lbegin(); + trg = src + 1; + } + } + + g.commit(); + + std::clock_t g_begin_time = clock(); + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto v = g[it]; + v.attributes(); + } + std::clock_t g_end_time = clock(); + double time = double(g_end_time - g_begin_time) / CLOCKS_PER_SEC; + + double all_time; + dart_reduce(&time, &all_time, 1, DART_TYPE_DOUBLE, DART_OP_SUM, 0, + g.team().dart_id()); + + if(dash::myid() == 0) { + std::cout << "[round " << i << "] " << n_vertices_full * scale << + " vertices per node dereferenced (local): " << all_time << + std::endl; + } + + if(dash::myid() == 0) { + std::clock_t g_begin_time = clock(); + for(auto it = g.vertices().begin(); it != g.vertices().end(); ++it) { + auto v = g[it]; + v.attributes(); + } + std::clock_t g_end_time = clock(); + std::cout << "[round " << i << "] " << n_vertices_full * scale << + " vertices per node dereferenced (global): " << + double(g_end_time - g_begin_time) / CLOCKS_PER_SEC << std::endl; + } + + g_begin_time = clock(); + for(auto it = g.out_edges().lbegin(); it != g.out_edges().lend(); ++it) { + auto e = g[it]; + e.attributes(); + } + g_end_time = clock(); + time = double(g_end_time - g_begin_time) / CLOCKS_PER_SEC; + + dart_reduce(&time, &all_time, 1, DART_TYPE_DOUBLE, DART_OP_SUM, 0, + g.team().dart_id()); + + if(dash::myid() == 0) { + std::cout << "[round " << i << "] " << n_edges_full * scale << + " edges per node dereferenced (local): " << all_time << std::endl; + } + + if(dash::myid() == 0) { + std::clock_t g_begin_time = clock(); + for(auto it = g.out_edges().begin(); it != g.out_edges().end(); ++it) { + auto e = g[it]; + e.attributes(); + } + std::clock_t g_end_time = clock(); + std::cout << "[round " << i << "] " << n_edges_full * scale << + " edges per node dereferenced (global): " << + double(g_end_time - g_begin_time) / CLOCKS_PER_SEC << std::endl; + } + dash::barrier(); + } + if(dash::myid() == 0) std::cout << "-----------------" << std::endl; + } + + dash::finalize(); + return 0; +} + + diff --git a/dash/examples/ex.02.graph-iter/main.cpp b/dash/examples/ex.02.graph-iter/main.cpp new file mode 100644 index 000000000..1a195ae8f --- /dev/null +++ b/dash/examples/ex.02.graph-iter/main.cpp @@ -0,0 +1,142 @@ +#include +#include +#include +#include + +#include "rmatrandom.h" + +struct vprop { + int comp; +}; + +struct eprop { + int comp; +}; + +typedef dash::Graph graph_t; + +int main(int argc, char* argv[]) { + dash::init(&argc, &argv); + + int n_unit_edges = 1; + int n_vertices_full = 100000; + int n_edges_full = n_vertices_full * n_unit_edges; + // not exactly n_vertices_full are generated due to rounding + int n_vertices_start = n_vertices_full / dash::size(); + int n_size_rounds = 3; + int n_rounds = 5; + int n_iter_rounds = 10; + for(int rounds = 0; rounds < n_size_rounds; ++rounds) { + for(int i = 0; i < n_rounds; ++i) { + int scale = pow(10, rounds); + int n_vertices = n_vertices_start * scale; + graph_t g(n_vertices, n_unit_edges); + + for(int j = 0; j < n_vertices; ++j) { + g.add_vertex(); + } + + std::clock_t g_begin_time = clock(); + g.commit(); + std::clock_t g_end_time = clock(); + if(dash::myid() == 0) { + std::cout << "[round " << i << "] " << n_vertices_full * scale << + " vertices per node iterated (commit): " << + double(g_end_time - g_begin_time) / CLOCKS_PER_SEC << std::endl; + } + + int n_edges = n_vertices * n_unit_edges; + auto src = g.vertices().lbegin(); + auto trg = src + 1; + auto end = g.vertices().lend(); + for(int j = 0; j < n_edges; ++j) { + g.add_edge(src, trg); + ++src; + ++trg; + if(trg == end) { + src = g.vertices().lbegin(); + trg = src + 1; + } + } + + g_begin_time = clock(); + g.commit(); + g_end_time = clock(); + if(dash::myid() == 0) { + std::cout << "[round " << i << "] " << n_edges_full * scale << + " edges per node iterated (commit): " << + double(g_end_time - g_begin_time) / CLOCKS_PER_SEC << std::endl; + } + + g_begin_time = clock(); + for(int j = 0; j < n_iter_rounds; ++j) { + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + g[it]; + } + } + g_end_time = clock(); + double time = double(g_end_time - g_begin_time) / CLOCKS_PER_SEC; + double all_time = 0; + + dart_reduce(&time, &all_time, 1, DART_TYPE_DOUBLE, DART_OP_SUM, 0, + g.team().dart_id()); + + if(dash::myid() == 0) { + std::cout << "[round " << i << "] " << n_vertices_full * scale << + " vertices per node iterated (local): " << all_time << std::endl; + } + + if(dash::myid() == 0) { + std::clock_t g_begin_time = clock(); + for(int j = 0; j < n_iter_rounds; ++j) { + for(auto it = g.vertices().begin(); it != g.vertices().end(); ++it) { + g[it]; + } + } + std::clock_t g_end_time = clock(); + std::cout << "[round " << i << "] " << n_vertices_full * scale << + " vertices per node iterated (global): " << + double(g_end_time - g_begin_time) / CLOCKS_PER_SEC << std::endl; + } + + g_begin_time = clock(); + for(int j = 0; j < n_iter_rounds; ++j) { + for(auto it = g.out_edges().lbegin(); it != g.out_edges().lend(); ++it) { + g[it]; + } + } + g_end_time = clock(); + time = double(g_end_time - g_begin_time) / CLOCKS_PER_SEC; + + dart_reduce(&time, &all_time, 1, DART_TYPE_DOUBLE, DART_OP_SUM, 0, + g.team().dart_id()); + + if(dash::myid() == 0) { + std::cout << "[round " << i << "] " << n_edges_full * scale << + " edges per node iterated (local): " << all_time << std::endl; + } + + if(dash::myid() == 0) { + int count = 0; + std::clock_t g_begin_time = clock(); + for(int j = 0; j < n_iter_rounds; ++j) { + for(auto it = g.out_edges().begin(); it != g.out_edges().end(); ++it) { + g[it]; + ++count; + } + } + std::clock_t g_end_time = clock(); + std::cout << "[round " << i << "] " << n_edges_full * scale << + " edges per node iterated (global): " << + double(g_end_time - g_begin_time) / CLOCKS_PER_SEC << std::endl; + } + dash::barrier(); + } + if(dash::myid() == 0) std::cout << "-----------------" << std::endl; + } + + dash::finalize(); + return 0; +} + + diff --git a/dash/examples/ex.02.graph-iter/rmatrandom.h b/dash/examples/ex.02.graph-iter/rmatrandom.h new file mode 100644 index 000000000..ef30897da --- /dev/null +++ b/dash/examples/ex.02.graph-iter/rmatrandom.h @@ -0,0 +1,210 @@ +#ifndef DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED +#define DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED + +#include +#include +#include +#include + +class xorshift { + +public: + typedef unsigned long result_type; + + template + xorshift(SeedGenerator &s) : x(s()), y(s()), z(s()) {} + + result_type operator()() { + result_type t; + x ^= x << 16; + x ^= x >> 5; + x ^= x << 1; + + t = x; + x = y; + y = z; + z = t ^ x ^ y; + + return z; + } + + // std::uniform distribution too slow + double dist() { return static_cast((*this)()) / max(); } + + result_type min() { return 0; } + + result_type max() { return std::numeric_limits::max() - 1; } + +private: + result_type x, y, z; +}; + +template class RMATRandomGenerator { + +public: + typedef GraphType graph_type; + typedef RMATRandomGenerator self_t; + typedef typename graph_type::vertex_size_type vertex_size_type; + typedef typename graph_type::edge_size_type edge_size_type; + typedef std::pair value_type; + typedef std::uniform_int_distribution dist_int_type; + +private: +public: + /** + * Begin iterator + */ + template + RMATRandomGenerator(vertex_size_type n, edge_size_type m, int n_units, + dash::team_unit_t myid, VertexMapperFunction owner, + double a, double b, double c, double d) { + std::random_device rd; + xorshift gen(rd); + + // the intel compiler on SUPERMUC falls into a "internal error loop" with + // the end constructor. therefore, this constructor is misused as end + // constructor + if (n == 0) { + _done = true; + } else { + // generate 50% of the edges with RMAT + m /= 2; + edge_size_type m_unit_random = m / n_units; + int SCALE = int(floor(log(double(n)) / log(2.))); + + std::map edge_map; + + // generate whole graph on each unit, but only use edges belonging to this + // unit + edge_size_type generated = 0; + edge_size_type local_edges = 0; + do { + edge_size_type rejected = 0; + do { + vertex_size_type u, v; + std::tie(u, v) = generate_rmat_edge(gen, n, SCALE, a, b, c, d); + + if (owner(u, n, n_units, myid) == myid) { + // reject loop edges and multi-edges + if (u != v && + edge_map.find(std::make_pair(u, v)) == edge_map.end()) { + edge_map[std::make_pair(u, v)] = true; + ++local_edges; + } else { + ++rejected; + } + } + ++generated; + } while (generated < m); + // generate more edges based on the amount of edges rejected on each + // unit + int rejected_all; + dart_allreduce(&rejected, &rejected_all, 1, DART_TYPE_INT, DART_OP_SUM, + dash::Team::All().dart_id()); + generated -= rejected_all; + } while (generated < m); + + // reserve space for generated rmat edges and for coming random edges + _values.reserve(local_edges + m_unit_random); + typename std::map::reverse_iterator em_end = + edge_map.rend(); + for (typename std::map::reverse_iterator em_i = + edge_map.rbegin(); + em_i != em_end; ++em_i) { + _values.push_back(em_i->first); + } + + // generate 50% random edges + // source edge has to belong to this unit + int start = 0; + int end = 0; + if (myid == 0) { + end = owner.size(myid) - 1; + } else { + for (int i = 0; i < myid; ++i) { + dash::team_unit_t unit{i}; + start += owner.size(unit); + } + end = start + owner.size(myid) - 1; + } + dist_int_type dist_u(start, end); + dist_int_type dist_v(0, n - 1); + for (int i = 0; i < m_unit_random; ++i) { + vertex_size_type u = dist_u(gen); + vertex_size_type v = dist_v(gen); + _values.push_back(std::make_pair(u, v)); + } + + _current = _values.size() - 1; + } + } + + /** + * End iterator + * + * This constructor somehow results in an "internal error loop" with the + * intel compiler of the SUPERMUC system. + */ + RMATRandomGenerator() : _done(true) {} + + self_t &operator++() { + if (_current > 0) { + --_current; + } else { + _done = true; + } + + return *this; + } + + const value_type &operator*() const { return _values[_current]; } + + const value_type *operator->() const { return &_values[_current]; } + + bool operator!=(const self_t &other) { return !(_done && other._done); } + +private: + value_type generate_rmat_edge(xorshift &gen, vertex_size_type n, + unsigned int SCALE, double a, double b, + double c, double d) { + vertex_size_type u = 0; + vertex_size_type v = 0; + vertex_size_type step = n / 2; + + for (unsigned int j = 0; j < SCALE; ++j) { + double p = gen.dist(); + if (p < a) + ; + else if (p >= a && p < a + b) + v += step; + else if (p >= a + b && p < a + b + c) + u += step; + else { + u += step; + v += step; + } + + step /= 2; + + a *= 0.9 + 0.2 * gen.dist(); + b *= 0.9 + 0.2 * gen.dist(); + c *= 0.9 + 0.2 * gen.dist(); + d *= 0.9 + 0.2 * gen.dist(); + + double S = a + b + c + d; + + a /= S; + b /= S; + c /= S; + d = 1. - a - b - c; + } + + return std::make_pair(u, v); + } + + std::vector _values; + int _current; + bool _done = false; +}; + +#endif // DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED diff --git a/dash/examples/ex.02.graph/Makefile b/dash/examples/ex.02.graph/Makefile new file mode 100644 index 000000000..42043aca7 --- /dev/null +++ b/dash/examples/ex.02.graph/Makefile @@ -0,0 +1,5 @@ +# +# In-place makefile for use side-by-side with the +# CMake build system +# +include ../Makefile_cpp diff --git a/dash/examples/ex.02.graph/main.cpp b/dash/examples/ex.02.graph/main.cpp new file mode 100644 index 000000000..6fe08f57f --- /dev/null +++ b/dash/examples/ex.02.graph/main.cpp @@ -0,0 +1,58 @@ +#include +#include +#include +#include + +#include "rmatrandom.h" + +struct vprop { + dash::default_index_t comp; + dash::global_unit_t unit; +}; + +typedef dash::Graph graph_t; + +int main(int argc, char* argv[]) { + if(argc == 3) { + dash::init(&argc, &argv); + int n_vertices = 1000 * atoi(argv[1]); + int n_edges = 4000 * atoi(argv[2]); + auto & team = dash::Team::All(); + dash::LogarithmicVertexMapper mapper(n_vertices, team.size()); + for(int i = 0; i < 5; ++i) { + RMATRandomGenerator begin(n_vertices, n_edges, team.size(), + team.myid(), mapper, 0.25, 0.25, 0.25, 0.25); + RMATRandomGenerator end(0, 0, team.size(), team.myid(), mapper, + 0, 0, 0, 0); + + std::clock_t g_begin_time = clock(); + graph_t g(begin, end, n_vertices, team, mapper); + std::clock_t g_end_time = clock(); + if(dash::myid() == 0) { + std::cout << "[round " << i + 1 << "] construction: " << + double(g_end_time - g_begin_time) / CLOCKS_PER_SEC << std::endl; + } + + dash::barrier(); + std::clock_t begin_time = clock(); + + //dash::util::TraceStore::on(); + //dash::util::TraceStore::clear(); + + dash::connected_components(g); + + //dash::barrier(); + //dash::util::TraceStore::off(); + //dash::util::TraceStore::write(std::cout); + + std::clock_t end_time = clock(); + if(dash::myid() == 0) { + std::cout << "[round " << i + 1 << "] algorithm: " << + double(end_time - begin_time) / CLOCKS_PER_SEC << std::endl; + } + } + dash::finalize(); + } + return 0; +} + diff --git a/dash/examples/ex.02.graph/rmatrandom.h b/dash/examples/ex.02.graph/rmatrandom.h new file mode 100644 index 000000000..ef30897da --- /dev/null +++ b/dash/examples/ex.02.graph/rmatrandom.h @@ -0,0 +1,210 @@ +#ifndef DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED +#define DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED + +#include +#include +#include +#include + +class xorshift { + +public: + typedef unsigned long result_type; + + template + xorshift(SeedGenerator &s) : x(s()), y(s()), z(s()) {} + + result_type operator()() { + result_type t; + x ^= x << 16; + x ^= x >> 5; + x ^= x << 1; + + t = x; + x = y; + y = z; + z = t ^ x ^ y; + + return z; + } + + // std::uniform distribution too slow + double dist() { return static_cast((*this)()) / max(); } + + result_type min() { return 0; } + + result_type max() { return std::numeric_limits::max() - 1; } + +private: + result_type x, y, z; +}; + +template class RMATRandomGenerator { + +public: + typedef GraphType graph_type; + typedef RMATRandomGenerator self_t; + typedef typename graph_type::vertex_size_type vertex_size_type; + typedef typename graph_type::edge_size_type edge_size_type; + typedef std::pair value_type; + typedef std::uniform_int_distribution dist_int_type; + +private: +public: + /** + * Begin iterator + */ + template + RMATRandomGenerator(vertex_size_type n, edge_size_type m, int n_units, + dash::team_unit_t myid, VertexMapperFunction owner, + double a, double b, double c, double d) { + std::random_device rd; + xorshift gen(rd); + + // the intel compiler on SUPERMUC falls into a "internal error loop" with + // the end constructor. therefore, this constructor is misused as end + // constructor + if (n == 0) { + _done = true; + } else { + // generate 50% of the edges with RMAT + m /= 2; + edge_size_type m_unit_random = m / n_units; + int SCALE = int(floor(log(double(n)) / log(2.))); + + std::map edge_map; + + // generate whole graph on each unit, but only use edges belonging to this + // unit + edge_size_type generated = 0; + edge_size_type local_edges = 0; + do { + edge_size_type rejected = 0; + do { + vertex_size_type u, v; + std::tie(u, v) = generate_rmat_edge(gen, n, SCALE, a, b, c, d); + + if (owner(u, n, n_units, myid) == myid) { + // reject loop edges and multi-edges + if (u != v && + edge_map.find(std::make_pair(u, v)) == edge_map.end()) { + edge_map[std::make_pair(u, v)] = true; + ++local_edges; + } else { + ++rejected; + } + } + ++generated; + } while (generated < m); + // generate more edges based on the amount of edges rejected on each + // unit + int rejected_all; + dart_allreduce(&rejected, &rejected_all, 1, DART_TYPE_INT, DART_OP_SUM, + dash::Team::All().dart_id()); + generated -= rejected_all; + } while (generated < m); + + // reserve space for generated rmat edges and for coming random edges + _values.reserve(local_edges + m_unit_random); + typename std::map::reverse_iterator em_end = + edge_map.rend(); + for (typename std::map::reverse_iterator em_i = + edge_map.rbegin(); + em_i != em_end; ++em_i) { + _values.push_back(em_i->first); + } + + // generate 50% random edges + // source edge has to belong to this unit + int start = 0; + int end = 0; + if (myid == 0) { + end = owner.size(myid) - 1; + } else { + for (int i = 0; i < myid; ++i) { + dash::team_unit_t unit{i}; + start += owner.size(unit); + } + end = start + owner.size(myid) - 1; + } + dist_int_type dist_u(start, end); + dist_int_type dist_v(0, n - 1); + for (int i = 0; i < m_unit_random; ++i) { + vertex_size_type u = dist_u(gen); + vertex_size_type v = dist_v(gen); + _values.push_back(std::make_pair(u, v)); + } + + _current = _values.size() - 1; + } + } + + /** + * End iterator + * + * This constructor somehow results in an "internal error loop" with the + * intel compiler of the SUPERMUC system. + */ + RMATRandomGenerator() : _done(true) {} + + self_t &operator++() { + if (_current > 0) { + --_current; + } else { + _done = true; + } + + return *this; + } + + const value_type &operator*() const { return _values[_current]; } + + const value_type *operator->() const { return &_values[_current]; } + + bool operator!=(const self_t &other) { return !(_done && other._done); } + +private: + value_type generate_rmat_edge(xorshift &gen, vertex_size_type n, + unsigned int SCALE, double a, double b, + double c, double d) { + vertex_size_type u = 0; + vertex_size_type v = 0; + vertex_size_type step = n / 2; + + for (unsigned int j = 0; j < SCALE; ++j) { + double p = gen.dist(); + if (p < a) + ; + else if (p >= a && p < a + b) + v += step; + else if (p >= a + b && p < a + b + c) + u += step; + else { + u += step; + v += step; + } + + step /= 2; + + a *= 0.9 + 0.2 * gen.dist(); + b *= 0.9 + 0.2 * gen.dist(); + c *= 0.9 + 0.2 * gen.dist(); + d *= 0.9 + 0.2 * gen.dist(); + + double S = a + b + c + d; + + a /= S; + b /= S; + c /= S; + d = 1. - a - b - c; + } + + return std::make_pair(u, v); + } + + std::vector _values; + int _current; + bool _done = false; +}; + +#endif // DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED diff --git a/dash/examples/ex.02.graph_insert/main.cpp b/dash/examples/ex.02.graph_insert/main.cpp new file mode 100644 index 000000000..04bf70c13 --- /dev/null +++ b/dash/examples/ex.02.graph_insert/main.cpp @@ -0,0 +1,77 @@ +#include +#include +#include +#include + +#include "rmatrandom.h" + +struct vprop { + int comp; +}; + +struct eprop { + int comp; +}; + +typedef dash::Graph graph_t; + +int main(int argc, char* argv[]) { + dash::init(&argc, &argv); + + int n_vertices_start = 1000; + if(dash::myid() != 0) { + n_vertices_start = 0; + } + int n_unit_edges = 1; + int n_rounds = 1; + int n_size_rounds = 4; + for(int rounds = 3; rounds < n_size_rounds; ++rounds) { + for(int i = 0; i < n_rounds; ++i) { + int n_vertices = n_vertices_start * pow(10, rounds); + graph_t g(n_vertices, n_unit_edges); + + std::clock_t g_begin_time = clock(); + for(int j = 0; j < n_vertices; ++j) { + g.add_vertex(); + } + std::clock_t g_end_time = clock(); + if(dash::myid() == 0) { + std::cout << "[round " << i << "] " << n_vertices << + " vertices added: " << double(g_end_time - g_begin_time) + / CLOCKS_PER_SEC << std::endl; + } + + g.commit(); +/* + int n_edges = n_vertices * n_unit_edges; + if(dash::myid() != 0) { + n_edges = 0; + } + auto src = g.vertices().lbegin(); + auto trg = src + 1; + auto end = g.vertices().lend(); + g_begin_time = clock(); + for(int j = 0; j < n_edges; ++j) { + g.add_edge(src, trg); + ++src; + ++trg; + if(trg == end) { + src = g.vertices().lbegin(); + trg = src + 1; + } + } + g_end_time = clock(); + if(dash::myid() == 0) { + std::cout << "[round " << i << "] " << n_edges << " edges added: " << + double(g_end_time - g_begin_time) / CLOCKS_PER_SEC << std::endl; + } +*/ + } + dash::barrier(); + if(dash::myid() == 0) std::cout << "----------------------" << std::endl; + } + + dash::finalize(); + return 0; +} + diff --git a/dash/examples/ex.02.graph_insert/rmatrandom.h b/dash/examples/ex.02.graph_insert/rmatrandom.h new file mode 100644 index 000000000..ef30897da --- /dev/null +++ b/dash/examples/ex.02.graph_insert/rmatrandom.h @@ -0,0 +1,210 @@ +#ifndef DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED +#define DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED + +#include +#include +#include +#include + +class xorshift { + +public: + typedef unsigned long result_type; + + template + xorshift(SeedGenerator &s) : x(s()), y(s()), z(s()) {} + + result_type operator()() { + result_type t; + x ^= x << 16; + x ^= x >> 5; + x ^= x << 1; + + t = x; + x = y; + y = z; + z = t ^ x ^ y; + + return z; + } + + // std::uniform distribution too slow + double dist() { return static_cast((*this)()) / max(); } + + result_type min() { return 0; } + + result_type max() { return std::numeric_limits::max() - 1; } + +private: + result_type x, y, z; +}; + +template class RMATRandomGenerator { + +public: + typedef GraphType graph_type; + typedef RMATRandomGenerator self_t; + typedef typename graph_type::vertex_size_type vertex_size_type; + typedef typename graph_type::edge_size_type edge_size_type; + typedef std::pair value_type; + typedef std::uniform_int_distribution dist_int_type; + +private: +public: + /** + * Begin iterator + */ + template + RMATRandomGenerator(vertex_size_type n, edge_size_type m, int n_units, + dash::team_unit_t myid, VertexMapperFunction owner, + double a, double b, double c, double d) { + std::random_device rd; + xorshift gen(rd); + + // the intel compiler on SUPERMUC falls into a "internal error loop" with + // the end constructor. therefore, this constructor is misused as end + // constructor + if (n == 0) { + _done = true; + } else { + // generate 50% of the edges with RMAT + m /= 2; + edge_size_type m_unit_random = m / n_units; + int SCALE = int(floor(log(double(n)) / log(2.))); + + std::map edge_map; + + // generate whole graph on each unit, but only use edges belonging to this + // unit + edge_size_type generated = 0; + edge_size_type local_edges = 0; + do { + edge_size_type rejected = 0; + do { + vertex_size_type u, v; + std::tie(u, v) = generate_rmat_edge(gen, n, SCALE, a, b, c, d); + + if (owner(u, n, n_units, myid) == myid) { + // reject loop edges and multi-edges + if (u != v && + edge_map.find(std::make_pair(u, v)) == edge_map.end()) { + edge_map[std::make_pair(u, v)] = true; + ++local_edges; + } else { + ++rejected; + } + } + ++generated; + } while (generated < m); + // generate more edges based on the amount of edges rejected on each + // unit + int rejected_all; + dart_allreduce(&rejected, &rejected_all, 1, DART_TYPE_INT, DART_OP_SUM, + dash::Team::All().dart_id()); + generated -= rejected_all; + } while (generated < m); + + // reserve space for generated rmat edges and for coming random edges + _values.reserve(local_edges + m_unit_random); + typename std::map::reverse_iterator em_end = + edge_map.rend(); + for (typename std::map::reverse_iterator em_i = + edge_map.rbegin(); + em_i != em_end; ++em_i) { + _values.push_back(em_i->first); + } + + // generate 50% random edges + // source edge has to belong to this unit + int start = 0; + int end = 0; + if (myid == 0) { + end = owner.size(myid) - 1; + } else { + for (int i = 0; i < myid; ++i) { + dash::team_unit_t unit{i}; + start += owner.size(unit); + } + end = start + owner.size(myid) - 1; + } + dist_int_type dist_u(start, end); + dist_int_type dist_v(0, n - 1); + for (int i = 0; i < m_unit_random; ++i) { + vertex_size_type u = dist_u(gen); + vertex_size_type v = dist_v(gen); + _values.push_back(std::make_pair(u, v)); + } + + _current = _values.size() - 1; + } + } + + /** + * End iterator + * + * This constructor somehow results in an "internal error loop" with the + * intel compiler of the SUPERMUC system. + */ + RMATRandomGenerator() : _done(true) {} + + self_t &operator++() { + if (_current > 0) { + --_current; + } else { + _done = true; + } + + return *this; + } + + const value_type &operator*() const { return _values[_current]; } + + const value_type *operator->() const { return &_values[_current]; } + + bool operator!=(const self_t &other) { return !(_done && other._done); } + +private: + value_type generate_rmat_edge(xorshift &gen, vertex_size_type n, + unsigned int SCALE, double a, double b, + double c, double d) { + vertex_size_type u = 0; + vertex_size_type v = 0; + vertex_size_type step = n / 2; + + for (unsigned int j = 0; j < SCALE; ++j) { + double p = gen.dist(); + if (p < a) + ; + else if (p >= a && p < a + b) + v += step; + else if (p >= a + b && p < a + b + c) + u += step; + else { + u += step; + v += step; + } + + step /= 2; + + a *= 0.9 + 0.2 * gen.dist(); + b *= 0.9 + 0.2 * gen.dist(); + c *= 0.9 + 0.2 * gen.dist(); + d *= 0.9 + 0.2 * gen.dist(); + + double S = a + b + c + d; + + a /= S; + b /= S; + c /= S; + d = 1. - a - b - c; + } + + return std::make_pair(u, v); + } + + std::vector _values; + int _current; + bool _done = false; +}; + +#endif // DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED diff --git a/dash/include/dash/Algorithm.h b/dash/include/dash/Algorithm.h index 0ac8331dd..a16ca8f43 100644 --- a/dash/include/dash/Algorithm.h +++ b/dash/include/dash/Algorithm.h @@ -29,4 +29,7 @@ #include +#include +#include + #endif // DASH__ALGORITHM_H_ diff --git a/dash/include/dash/Container.h b/dash/include/dash/Container.h index 58c3ef697..4f103b0bd 100644 --- a/dash/include/dash/Container.h +++ b/dash/include/dash/Container.h @@ -75,5 +75,6 @@ // Dynamic containers: #include #include +#include #endif // DASH__CONTAINER_H_ diff --git a/dash/include/dash/DynamicPattern.h b/dash/include/dash/DynamicPattern.h new file mode 100644 index 000000000..02cf9fb5d --- /dev/null +++ b/dash/include/dash/DynamicPattern.h @@ -0,0 +1,27 @@ +#ifndef DASH__PARTITION_H__INCLUDED +#define DASH__PARTITION_H__INCLUDED + +namespace dash { + + /** + * \defgroup DashPartitionConcept Graph Partition Concept + * Partitioning concept for distributed graph data (vertices and edges). + * + * \ingroup DashConcept + * \{ + * \par Description + * + * Description here. + * + * \par Methods + * + * \} + */ + +} + +#endif // DASH__PARTITION_H_INCLUDED + +#include +//#include +#include diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h new file mode 100644 index 000000000..2d117e649 --- /dev/null +++ b/dash/include/dash/Graph.h @@ -0,0 +1,876 @@ +#ifndef DASH__GRAPH_H__INCLUDED +#define DASH__GRAPH_H__INCLUDED + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace dash { + +/** + * \defgroup DashGraphConcept Graph Concept + * Concept of a distributed graph container. + * + * \ingroup DashContainerConcept + * \{ + * \psr Description + * + * Description here. + * + * Extends Concepts: + * + * Container properties: + * + * Iterator validity: + * + * \par Member types + * + * \par Member functions + * + * \} + * + * Usage examples: + * + */ + +/** + * Distributed, dynamic graph container for sparse graphs. + */ +template< + GraphDirection Direction = DirectedGraph, + typename VertexProperties = EmptyProperties, // user-defined struct + typename EdgeProperties = EmptyProperties, // user-defined struct + typename VertexSizeType = int, + typename EdgeSizeType = int, +//template typename EdgeContainer = std::vector, +//template typename VertexContainer = std::vector + template class _EdgeContainer = std::vector, + template class _VertexContainer = std::vector +> +class Graph { + +public: + typedef Graph self_t; + + typedef Vertex vertex_type; + typedef Edge edge_type; + + typedef std::vector vertex_container_type; + typedef std::vector edge_container_type; + +//typedef VertexContainer vertex_container_type; +//typedef EdgeContainer edge_container_type; + +private: + + // TODO: add wrapper for all iterator types + typedef VertexIteratorWrapper vertex_it_wrapper; + typedef EdgeIteratorWrapper edge_it_wrapper; + typedef InEdgeIteratorWrapper in_edge_it_wrapper; + typedef OutEdgeIteratorWrapper out_edge_it_wrapper; + + friend vertex_it_wrapper; + friend edge_it_wrapper; + friend in_edge_it_wrapper; + friend out_edge_it_wrapper; + + typedef GlobHeapContiguousMem< + vertex_container_type> glob_mem_vert_type; + typedef GlobHeapContiguousMem< + edge_container_type> glob_mem_edge_type; + typedef + GlobHeapCombinedMem glob_mem_edge_comb_type; + typedef std::vector> edge_list_type; + +public: + + typedef typename + glob_mem_vert_type::bucket_index_type vertex_cont_ref_type; + typedef typename + glob_mem_edge_type::bucket_index_type edge_cont_ref_type; + typedef VertexSizeType vertex_size_type; + typedef EdgeSizeType edge_size_type; + + typedef VertexProperties vertex_properties_type; + typedef EdgeProperties edge_properties_type; + + typedef GlobRef reference; + + typedef VertexIndex vertex_index_type; + + typedef typename + glob_mem_vert_type::local_iterator local_vertex_iterator; + typedef typename + glob_mem_edge_type::local_iterator local_in_edge_iterator; + typedef typename + glob_mem_edge_type::local_iterator local_out_edge_iterator; + typedef local_out_edge_iterator local_inout_edge_iterator; + typedef typename + glob_mem_edge_comb_type::local_iterator local_edge_iterator; + + typedef typename + glob_mem_vert_type::global_iterator global_vertex_iterator; + typedef typename + glob_mem_edge_type::global_iterator global_in_edge_iterator; + typedef typename + glob_mem_edge_type::global_iterator global_out_edge_iterator; + typedef global_out_edge_iterator global_inout_edge_iterator; + typedef typename + glob_mem_edge_comb_type::global_iterator global_edge_iterator; + + + typedef VertexProxy + local_vertex_proxy_type; + typedef VertexProxy + global_vertex_proxy_type; + typedef EdgeProxy + local_inout_edge_proxy_type; + typedef EdgeProxy + global_inout_edge_proxy_type; + typedef EdgeProxy + local_edge_proxy_type; + typedef EdgeProxy + global_edge_proxy_type; + + friend local_vertex_proxy_type; + friend global_vertex_proxy_type; + friend local_inout_edge_proxy_type; + friend global_inout_edge_proxy_type; + friend local_edge_proxy_type; + friend global_edge_proxy_type; + friend vertex_type; + +public: + + /** + * Constructs an empty graph. + */ + Graph( + const vertex_size_type n_vertices = 0, + const edge_size_type n_vertex_edges = 0, + Team & team = dash::Team::All() + ) + : _team(&team), + _myid(team.myid()), + _size(team.size()), + _remote_edges(team.size()) + { + allocate(n_vertices, n_vertex_edges); + } + + /** + * Constructs a graph from an iterator range pointing to elements of type + * std::pair. + * Assumes vertex IDs are taken from a contiguous range [0...n] and n is + * divisible by the number of units in the Team of the container. + * Partitions vertices based on their id: + * vertex_id / (n / num_units) = owner + * + */ + template< + typename ForwardIterator, + typename VertexMapFunction = BlockedVertexMapper + > + Graph( + ForwardIterator begin, + ForwardIterator end, + vertex_size_type n_vertices, + Team & team = dash::Team::All(), + VertexMapFunction vertex_owner = BlockedVertexMapper() + ) + : _team(&team), + _myid(team.myid()), + _size(team.size()), + _remote_edges(team.size()) + { + // TODO: find heuristic to allocate a reasonable amount of edge memory + // per vertex + allocate(n_vertices, 0); + //VertexMapFunction vertex_owner; + + std::map lvertices; + std::map gvertices; + // TODO: find method that uses less memory + std::unordered_set remote_vertices_set; + std::vector> remote_vertices(_team->size()); + for(auto it = begin; it != end; ++it) { + auto item = *it; + auto v = edgelist_source(item); + if(vertex_owner(v, n_vertices, _size, _myid) == _myid) { + auto u = edgelist_target(item); + + if(lvertices.find(v) == lvertices.end()) { + // add dummy first, more vertices are added later and they have + // to be in order + lvertices[v] = local_vertex_iterator(); + } + auto target_owner = vertex_owner(u, n_vertices, _size, _myid); + if(target_owner == _myid) { + if(lvertices.find(u) == lvertices.end()) { + // add dummy first, more vertices are added later and they have + // to be in order + lvertices[u] = local_vertex_iterator(); + } + } else { + // collect vertices for remote nodes and prevent adding vertices more + // than once + bool inserted; + std::tie(std::ignore, inserted) = remote_vertices_set.insert(u); + if(inserted) { + remote_vertices[target_owner].push_back(u); + } + } + } + } + + // send vertices to their owner units and receive their local index + { + std::vector sizes_send(remote_vertices.size()); + std::vector sizes_send_n(remote_vertices.size()); + std::vector displs_send(remote_vertices.size()); + std::vector remote_vertices_send; + int total_send = 0; + for(int i = 0; i < remote_vertices.size(); ++i) { + sizes_send[i] = remote_vertices[i].size() * sizeof(vertex_size_type); + sizes_send_n[i] = remote_vertices[i].size(); + displs_send[i] = total_send * sizeof(vertex_size_type); + total_send += remote_vertices[i].size(); + } + remote_vertices_send.reserve(total_send); + for(auto & vertex_set : remote_vertices) { + remote_vertices_send.insert(remote_vertices_send.end(), + vertex_set.begin(), vertex_set.end()); + } + std::vector sizes_recv(remote_vertices.size()); + std::vector displs_recv(remote_vertices.size()); + dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), + DART_TYPE_BYTE, _team->dart_id()); + int total_recv = 0; + for(int i = 0; i < sizes_recv.size(); ++i) { + displs_recv[i] = total_recv; + total_recv += sizes_recv[i]; + } + std::vector remote_vertices_recv(total_recv / + sizeof(vertex_size_type)); + if(total_send > 0 || total_recv > 0) { + dart_alltoallv(remote_vertices_send.data(), sizes_send.data(), + displs_send.data(), DART_TYPE_BYTE, remote_vertices_recv.data(), + sizes_recv.data(), displs_recv.data(), _team->dart_id()); + } + + // exchange data + for(auto & index : remote_vertices_recv) { + lvertices[index] = local_vertex_iterator(); + } + // add vertices in order + for(auto & lvertex : lvertices) { + lvertex.second = add_vertex(); + } + // send local position of added vertices that are referenced by edges + // on other units + for(auto & index : remote_vertices_recv) { + if(lvertices.find(index) != lvertices.end()) { + index = lvertices[index].pos(); + } + } + if(total_send > 0 || total_recv > 0) { + dart_alltoallv(remote_vertices_recv.data(), sizes_recv.data(), + displs_recv.data(), DART_TYPE_BYTE, remote_vertices_send.data(), + sizes_send.data(), displs_send.data(), _team->dart_id()); + } + // all vertices have been added - commit changes to global memory space + commit(); + // remote_vertices_send now contains the local indices in the iteration + // space of the corresponding unit + team_unit_t unit { 0 }; + int index = 0; + for(auto & lindex : remote_vertices_send) { + while(index >= sizes_send_n[unit]) { + ++unit; + index = 0; + if(unit >= _team->size()) { + break; + } + } + if(unit >= _team->size()) { + break; + } + gvertices[remote_vertices[unit][index]] = global_vertex_iterator( + _glob_mem_vertex, unit, lindex); + ++index; + } + } + + // finally add edges with the vertex iterators gained from the previous + // steps + for(auto it = begin; it != end; ++it) { + auto item = *it; + auto v = edgelist_source(item); + if(vertex_owner(v, n_vertices, _size, _myid) == _myid) { + auto v_it = lvertices[v]; + auto u = edgelist_target(item); + + if(vertex_owner(u, n_vertices, _size, _myid) == _myid) { + auto u_it = lvertices[u]; + edgelist_add_edge(v_it, u_it, item); + } else { + auto u_it = gvertices[u]; + edgelist_add_edge(v_it, u_it, item); + } + } + } + // commit edges + commit(); + } + + /** Destructs the graph. + */ + ~Graph() { + deallocate(); + } + + /** + * Copy-constructor. Explicitly deleted. + */ + Graph(const self_t &) = delete; + + /** + * Move-constructor. Explicitly deleted. + */ + Graph(self_t &&) = delete; + + /** + * Copy-assignment operator. Explicitly deleted. + */ + self_t & operator=(const self_t &) = delete; + + /** + * Move-assignment operator. Explicitly deleted. + */ + self_t & operator=(self_t &&) = delete; + + /** + * Returns an object handling interactions with a vertex pointed to by + * the given iterator. + */ + local_vertex_proxy_type operator[](local_vertex_iterator it) { + return local_vertex_proxy_type(it, this); + } + + /** + * Returns an object handling interactions with a vertex pointed to by + * the given iterator. + */ + global_vertex_proxy_type operator[](global_vertex_iterator it) { + return global_vertex_proxy_type(it, this); + } + + /** + * Returns an object handling interactions with an edge pointed to by + * the given iterator. + */ + local_inout_edge_proxy_type operator[](local_inout_edge_iterator it) { + return local_inout_edge_proxy_type(it, this); + } + + /** + * Returns an object handling interactions with an edge pointed to by + * the given iterator. + */ + global_inout_edge_proxy_type operator[](global_inout_edge_iterator it) { + return global_inout_edge_proxy_type(it, this); + } + + /** + * Returns an object handling interactions with an edge pointed to by + * the given iterator. + */ + global_edge_proxy_type operator[](global_edge_iterator it) { + return global_edge_proxy_type(it, this); + } + + /** + * Returns a vertex iterator range object. + */ + vertex_it_wrapper & vertices() { + return _vertices; + } + + /** + * Returns an edge iterator range object. + */ + edge_it_wrapper & edges() { + return _edges; + } + + /** + * Returns an in-edge iterator range object. + */ + in_edge_it_wrapper & in_edges() { + return _in_edges; + } + + /** + * Returns an out-edge iterator range object. + */ + out_edge_it_wrapper & out_edges() { + return _out_edges; + } + + /** + * Returns the number of vertices in the whole graph. + * + * \return The amount of vertices in the whole graph. + */ + vertex_size_type num_vertices() const { + return _glob_mem_vertex->size(); + } + + /** + * Returns the number of edges in the whole graph. + * + * \return The amount of edges in the whole graph. + */ + edge_size_type num_edges() const { + return _glob_mem_edge->size(); + } + + /** + * Returns, whether the graph is empty. + * + * \return True, if the graph holds 0 vertices. Fals otherwise. + */ + bool empty() const { + return _glob_mem_vertex->size() == 0 ? true : false; + } + + /** + * Adds a vertex with the given properties locally. + * + * \return Index of the newly created vertex. + */ + local_vertex_iterator add_vertex(const VertexProperties & prop) { + _glob_mem_out_edge->add_container(_alloc_edges_per_vertex); + if(_glob_mem_in_edge != _glob_mem_out_edge) { + _glob_mem_in_edge->add_container(_alloc_edges_per_vertex); + } + vertex_type v(prop); + return _glob_mem_vertex->push_back(_vertex_container_ref, v); + } + + /** + * Adds a vertex locally. + * + * \return Index of the newly created vertex. + */ + local_vertex_iterator add_vertex() { + VertexProperties prop; + // : Reference to temporary! + return add_vertex(prop); + } + + /** + * Removes a given vertex. + */ + void remove_vertex(const local_vertex_iterator & v) { + + } + + /** + * Removes a given vertex. + */ + void remove_vertex(const global_vertex_iterator & v) { + + } + + /** + * Adds an edge between two given vertices with the given properties + * locally. + * + * \return Pair, with pair::first set to the index of the newly created edge + * and pair::second set to a boolean indicating whether the edge has + * actually been added. + */ + std::pair add_edge( + const local_vertex_iterator & source, + const local_vertex_iterator & target, + const EdgeProperties & prop + ) { + local_out_edge_iterator l_it; + l_it = add_local_edge(source, target, prop, _glob_mem_out_edge); + add_local_edge(target, source, prop, _glob_mem_in_edge); + + // currently, double edges are allowed for all cases, and vertex deletion + // is not implemented so we always return true + return std::make_pair(l_it, true); + } + + /** + * Adds an edge between two given vertices with the given properties + * locally. + * Edges that belong to vertices held on a different unit are marked for + * transfer. These edges will be transferred after calling \c commit(). + * + * \return Pair, with pair::first set to the index of the newly created edge + * and pair::second set to a boolean indicating whether the edge has + * actually been added. + */ + std::pair add_edge( + const local_vertex_iterator & source, + const global_vertex_iterator & target, + const EdgeProperties & prop + ) { + local_out_edge_iterator l_it; + l_it = add_local_edge(source, target, prop, _glob_mem_out_edge); + if(target.is_local()) { + // _glob_mem_in_edge == _glob_mem_out_edge for undirected graph types + add_local_edge(target.local(), source, prop, _glob_mem_in_edge); + // do not double-send edges + } else { + edge_type edge(source, target, prop, _myid); + _remote_edges[target.lpos().unit].push_back(edge); + } + + // currently, double edges are allowed for all cases, and vertex deletion + // is not implemented so we always return true + return std::make_pair(l_it, true); + } + + /** + * Adds an edge between two given vertices locally. + * Edges that belong to vertices held on a different unit are marked for + * transfer. These edges will be transferred after calling \c commit(). + * + * \return Pair, with pair::first set to the index of the newly created edge + * and pair::second set to a boolean indicating whether the edge has + * actually been added. + */ + std::pair add_edge( + const local_vertex_iterator & source, + const local_vertex_iterator & target + ) { + EdgeProperties prop; + return add_edge(source, target, prop); + } + + /** + * Adds an edge between two given vertices locally. + * Edges that belong to vertices held on a different unit are marked for + * transfer. These edges will be transferred after calling \c commit(). + * + * \return Pair, with pair::first set to the index of the newly created edge + * and pair::second set to a boolean indicating whether the edge has + * actually been added. + */ + std::pair add_edge( + const local_vertex_iterator & source, + const global_vertex_iterator & target + ) { + EdgeProperties prop; + return add_edge(source, target, prop); + } + + /** + * Removes the edges between two given vertices. + */ + void remove_edge(const local_vertex_iterator & v1, + const local_vertex_iterator & v2) { + + } + + /** + * Removes the edges between two given vertices. + */ + void remove_edge(const global_vertex_iterator & v1, + const global_vertex_iterator & v2) { + + } + + /** + * Removes a given edge. + */ + void remove_edge(const local_out_edge_iterator & e) { + + } + + /** + * Removes a given edge. + */ + void remove_edge(const global_out_edge_iterator & e) { + + } + + /** + * Commits local changes of the graph to global memory space since the last + * call of this method. + */ + void commit() { + // move all edges that have to be added by other units in a contiguous + // memory region + std::vector remote_edges; + std::vector remote_edges_count(_team->size()); + std::vector remote_edges_displs(_team->size()); + for(int i = 0; i < _remote_edges.size(); ++i) { + for(auto & remote_edge : _remote_edges[i]) { + remote_edges.push_back(remote_edge); + remote_edges_count[i] += sizeof(edge_type); + } + for(int j = i + 1; j < remote_edges_displs.size(); ++j) { + remote_edges_displs[j] += remote_edges_count[i]; + } + _remote_edges[i].clear(); + } + // exchange amount of edges to be transferred with other units + std::vector edge_count(_team->size()); + DASH_ASSERT_RETURNS( + dart_alltoall(remote_edges_count.data(), edge_count.data(), + sizeof(std::size_t), DART_TYPE_BYTE, _team->dart_id()), + DART_OK + ); + int total_count = 0; + std::vector edge_displs(_team->size()); + for(int i = 0; i < edge_count.size(); ++i) { + total_count += edge_count[i]; + for(int j = i + 1; j < edge_displs.size(); ++j) { + edge_displs[j] += edge_count[i]; + } + } + // exchange edges + std::vector edges(total_count / sizeof(edge_type)); + DASH_ASSERT_RETURNS( + dart_alltoallv(remote_edges.data(), + remote_edges_count.data(), + remote_edges_displs.data(), + DART_TYPE_BYTE, + edges.data(), + edge_count.data(), + edge_displs.data(), + _team->dart_id() + ), + DART_OK + ); + // add missing edges to local memory space + for(auto edge : edges) { + if(edge.source.unit == _myid) { + add_local_edge(edge.source, edge.target, edge.properties, + _glob_mem_out_edge); + } + if(edge.target.unit == _myid) { + // _glob_mem_in_edge == _glob_mem_out_edge for undirected graph types + add_local_edge(edge.target, edge.source, edge.properties, + _glob_mem_in_edge); + //TODO: for directed graphs, should target and source really be mutated + // in the in-edge list? + } + } + // commit changes in local memory space globally + _glob_mem_vertex->commit(); + _glob_mem_out_edge->commit(); + if(_glob_mem_out_edge != _glob_mem_in_edge) { + _glob_mem_in_edge->commit(); + } + _glob_mem_edge->commit(); + + for(auto it = vertices().lbegin(); it != vertices().lend(); ++it) { + auto & v = *it; + v.in_edge_list.index = _glob_mem_in_edge->container_begin(it.pos()); + v.in_edge_list.size = _glob_mem_in_edge->container_size(it.pos()); + v.out_edge_list.index = _glob_mem_out_edge->container_begin(it.pos()); + v.out_edge_list.size = _glob_mem_out_edge->container_size(it.pos()); + } + } + + /** + * Globally allocates memory for vertex and edge storage. + */ + bool allocate(vertex_size_type n_vertices, edge_size_type n_vertex_edges) { + auto vertex_lcap = dash::math::div_ceil(n_vertices, _team->size()); + _glob_mem_vertex = new glob_mem_vert_type(*_team); + _vertex_container_ref = _glob_mem_vertex->add_container(vertex_lcap); + // no edge list allocation yet, this will happen once the vertices are + // created. Each edge list will have n_vertex_edges elements reserved. + _alloc_edges_per_vertex = n_vertex_edges; + _glob_mem_out_edge = new glob_mem_edge_type(*_team); + if(Direction == DirectedGraph) { + _glob_mem_in_edge = new glob_mem_edge_type(*_team); + } else { + // there is no distinction between in- and out-edges in an undirected + // graph + _glob_mem_in_edge = _glob_mem_out_edge; + } + _glob_mem_edge = new glob_mem_edge_comb_type(*_team); + _glob_mem_edge->add_globmem(*_glob_mem_out_edge); + _glob_mem_edge->add_globmem(*_glob_mem_in_edge); + // Register deallocator at the respective team instance + _team->register_deallocator(this, std::bind(&Graph::deallocate, this)); + + _vertices = vertex_it_wrapper(this); + _in_edges = in_edge_it_wrapper(this); + _out_edges = out_edge_it_wrapper(this); + _edges = edge_it_wrapper(this); + return true; + } + + /** + * Deallocates global memory. + */ + void deallocate() { + if(_glob_mem_vertex != nullptr) { + delete _glob_mem_vertex; + _glob_mem_vertex = nullptr; + } + if(_glob_mem_in_edge != nullptr) { + delete _glob_mem_in_edge; + if(_glob_mem_in_edge == _glob_mem_out_edge) { + _glob_mem_out_edge = nullptr; + } + _glob_mem_in_edge = nullptr; + } + if(_glob_mem_out_edge != nullptr + && _glob_mem_out_edge != _glob_mem_in_edge) { + delete _glob_mem_out_edge; + _glob_mem_out_edge = nullptr; + } + if(_glob_mem_edge != nullptr) { + delete _glob_mem_edge; + _glob_mem_edge = nullptr; + } + if(_team != nullptr) { + // Remove deallocator from the respective team instance + _team->unregister_deallocator(this, std::bind(&Graph::deallocate, this)); + } + } + + /** + * Returns the team containing all units associated with this container + */ + Team & team() const { + return *_team; + } + +private: + + /** + * Inserts an edge locally. The source vertex must belong to this unit. + * + * \return Local iterator to the created edge. + */ + template + typename glob_mem_edge_type::local_iterator add_local_edge( + const local_vertex_iterator source, + const TargetIterator target, + const EdgeProperties & prop, + glob_mem_edge_type * glob_mem_edge + ) { + auto edge = edge_type(source, target, prop, _myid); + return glob_mem_edge->push_back(source.pos(), edge); + } + + /** + * Inserts an edge locally. The source vertex must belong to this unit. + * + * \return Local iterator to the created edge. + */ + typename glob_mem_edge_type::local_iterator add_local_edge( + const vertex_index_type source, + const vertex_index_type target, + const EdgeProperties & prop, + glob_mem_edge_type * glob_mem_edge + ) { + auto edge = edge_type(source, target, prop); + return glob_mem_edge->push_back(source.offset, edge); + } + + /* + team_unit_t vertex_owner(vertex_size_type v, vertex_size_type n_vertices) { + int owner_id = static_cast(v) / (static_cast(n_vertices) / + _team->size()); + int size = _team->size(); + int owner_id = static_cast(v) / (n_vertices + * ((static_cast(_myid) + 1) / (size * (size + 1) / 2))); + team_unit_t owner { owner_id }; + return owner; + } + */ + + // TODO: Generalize following methods for other edge attributes than { int } + + vertex_size_type edgelist_source(std::pair e) { + return e.first; + } + + vertex_size_type edgelist_source(std::pair, int> e) { + return e.first.first; + } + + vertex_size_type edgelist_target(std::pair e) { + return e.second; + } + + vertex_size_type edgelist_target(std::pair, int> e) { + return e.first.second; + } + + template + void edgelist_add_edge(SourceIterator s, TargetIterator t, + std::pair) { + add_edge(s, t); + } + + template + void edgelist_add_edge(SourceIterator s, TargetIterator t, + std::pair, int> item) { + edge_properties_type prop { item.second }; + add_edge(s, t, prop); + } + +private: + + /** the team containing all units using the container */ + Team * _team = nullptr; + /** Global memory allocation and access to vertices */ + glob_mem_vert_type * _glob_mem_vertex = nullptr; + /** Global memory allocation and access to inbound edges */ + glob_mem_edge_type * _glob_mem_in_edge = nullptr; + /** Global memory allocation and access to outbound edges */ + glob_mem_edge_type * _glob_mem_out_edge = nullptr; + /** Access to inbound and outbound edges */ + glob_mem_edge_comb_type * _glob_mem_edge = nullptr; + /** Unit ID of the current unit */ + team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; + /** Amount of units in the team */ + std::size_t _size; + /** Index of the vertex container in _glob_mem_vertex */ + vertex_cont_ref_type _vertex_container_ref; + /** Amount of edge elements to be pre-allocated for every vertex */ + edge_size_type _alloc_edges_per_vertex; + /** Edges that have to be added to vertices on another unit in next commit */ + edge_list_type _remote_edges; + /** wrapper for vertex iterator ranges */ + vertex_it_wrapper _vertices; + /** wrapper for edge iterator ranges */ + edge_it_wrapper _edges; + /** wrapper for in-edge iterator ranges */ + in_edge_it_wrapper _in_edges; + /** wrapper for out-edge iterator ranges */ + out_edge_it_wrapper _out_edges; + +}; + +} + +#endif // DASH__GRAPH_H__INCLUDED + diff --git a/dash/include/dash/algorithm/graph/ConnectedComponents.h b/dash/include/dash/algorithm/graph/ConnectedComponents.h new file mode 100644 index 000000000..348025498 --- /dev/null +++ b/dash/include/dash/algorithm/graph/ConnectedComponents.h @@ -0,0 +1,421 @@ +/** + * Author: Steffan Effenberger (github: @stiefn) + * via https://github.com/stiefn/dyn-data-structures-thesis + * + */ +#ifndef DASH__ALGORITHM____GRAPH__CONNECTED_COMPONENTS_H__ +#define DASH__ALGORITHM____GRAPH__CONNECTED_COMPONENTS_H__ + +#include +#include + +namespace dash { + +typedef std::vector> matrix_t; +template +using matrix_pair_t = std::vector>>; + +namespace internal { +// TODO: merge overlapping code of the following functions with the functions +// provided in MinimumSpanningTree.h + +template +auto cc_get_data( + matrix_t & indices, + matrix_t & permutations, + GraphType & graph, + dash::util::Trace & trace +) -> std::vector { + typedef typename GraphType::vertex_properties_type vprop_t; + trace.enter_state("send indices"); + // exchange indices to get + std::vector sizes_send(indices.size()); + std::vector displs_send(indices.size()); + std::vector sizes_recv_data(indices.size()); + std::vector displs_recv_data(indices.size()); + std::vector indices_send; + int total_send = 0; + for(int i = 0; i < indices.size(); ++i) { + sizes_send[i] = indices[i].size(); + displs_send[i] = total_send; + sizes_recv_data[i] = indices[i].size() * sizeof(vprop_t); + displs_recv_data[i] = total_send * sizeof(vprop_t); + total_send += indices[i].size(); + } + indices_send.reserve(total_send); + for(auto & index_set : indices) { + indices_send.insert(indices_send.end(), index_set.begin(), + index_set.end()); + } + std::vector sizes_recv(indices.size()); + std::vector displs_recv(indices.size()); + std::vector sizes_send_data(indices.size()); + std::vector displs_send_data(indices.size()); + dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), + DART_TYPE_BYTE, graph.team().dart_id()); + int total_recv = 0; + for(int i = 0; i < sizes_recv.size(); ++i) { + sizes_send_data[i] = sizes_recv[i] * sizeof(vprop_t); + displs_send_data[i] = total_recv * sizeof(vprop_t); + displs_recv[i] = total_recv; + total_recv += sizes_recv[i]; + } + std::vector indices_recv(total_recv); + dart_alltoallv(indices_send.data(), sizes_send.data(), displs_send.data(), + DART_TYPE_INT, indices_recv.data(), sizes_recv.data(), + displs_recv.data(), graph.team().dart_id()); + + std::vector data_send; + data_send.reserve(total_recv); + std::vector data_recv(total_send); + + trace.exit_state("send indices"); + // exchange data + trace.enter_state("get components"); + for(auto & index : indices_recv) { + data_send.push_back(graph.vertices().attributes(index)); + } + trace.exit_state("get components"); + trace.enter_state("send components"); + dart_alltoallv(data_send.data(), sizes_send_data.data(), + displs_send_data.data(), DART_TYPE_BYTE, data_recv.data(), + sizes_recv_data.data(), displs_recv_data.data(), + graph.team().dart_id()); + trace.exit_state("send components"); + + trace.enter_state("restore order"); + // restore original order + std::vector output(data_recv.size()); + int index = 0; + for(int i = 0; i < permutations.size(); ++i) { + for(int j = 0; j < permutations[i].size(); ++j) { + output[permutations[i][j]] = data_recv[index]; + ++index; + } + } + trace.exit_state("restore order"); + return output; +} + +template +auto cc_get_components( + matrix_t & indices, + int start, + GraphType & graph, + dash::util::Trace & trace +) -> std::pair< + std::unordered_map, + std::vector>> +{ + typedef typename GraphType::vertex_properties_type vprop_t; + trace.enter_state("send indices"); + auto myid = graph.team().myid(); + std::size_t lsize = graph.vertices().size(myid); + std::vector thresholds(indices.size()); + for(int i = 0; i < thresholds.size(); ++i) { + team_unit_t unit { i }; + // TODO: find a threshold that provides an optimal tradeoff for any + // amount of units + thresholds[i] = graph.vertices().size(unit) * 20; + } + + std::vector total_sizes(indices.size()); + std::vector sizes_send(indices.size()); + std::vector sizes_send_longdt(indices.size()); + std::vector displs_send(indices.size()); + std::vector sizes_recv_data(indices.size()); + std::vector cumul_sizes_recv_data(indices.size()); + std::vector displs_recv_data(indices.size()); + for(int i = 0; i < indices.size(); ++i) { + sizes_send[i] = indices[i].size(); + sizes_send_longdt[i] = indices[i].size(); + sizes_recv_data[i] = indices[i].size() * sizeof(vprop_t); + } + dart_allreduce(sizes_send_longdt.data(), total_sizes.data(), indices.size(), + DART_TYPE_LONGLONG, DART_OP_SUM, graph.team().dart_id()); + int total_send = 0; + int total_recv_data = 0; + for(int i = 0; i < indices.size(); ++i) { + if(total_sizes[i] > thresholds[i]) { + sizes_send[i] = 0; + team_unit_t unit { i }; + sizes_recv_data[i] = graph.vertices().size(unit) * sizeof(vprop_t); + indices[i].clear(); + } else { + sizes_send[i] = indices[i].size(); + sizes_recv_data[i] = indices[i].size() * sizeof(vprop_t); + } + displs_send[i] = total_send; + displs_recv_data[i] = total_recv_data; + total_send += sizes_send[i]; + total_recv_data += sizes_recv_data[i]; + cumul_sizes_recv_data[i] = total_recv_data / sizeof(vprop_t); + } + std::vector indices_send; + indices_send.reserve(total_send); + for(auto & index_set : indices) { + indices_send.insert(indices_send.end(), index_set.begin(), + index_set.end()); + } + std::vector sizes_recv(indices.size()); + std::vector displs_recv(indices.size()); + std::vector sizes_send_data(indices.size()); + std::vector displs_send_data(indices.size()); + dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), + DART_TYPE_BYTE, graph.team().dart_id()); + int total_recv = 0; + int total_send_data = 0; + for(int i = 0; i < sizes_recv.size(); ++i) { + if(total_sizes[myid] > thresholds[myid]) { + sizes_send_data[i] = lsize * sizeof(vprop_t); + displs_send_data[i] = 0; + } else { + sizes_send_data[i] = sizes_recv[i] * sizeof(vprop_t); + displs_send_data[i] = total_send_data; + displs_recv[i] = total_recv; + } + total_recv += sizes_recv[i]; + total_send_data += sizes_send_data[i]; + } + std::vector indices_recv(total_recv); + dart_alltoallv(indices_send.data(), sizes_send.data(), displs_send.data(), + DART_TYPE_INT, indices_recv.data(), sizes_recv.data(), + displs_recv.data(), graph.team().dart_id()); + trace.exit_state("send indices"); + trace.enter_state("get components"); + std::vector data_send; + std::vector data_recv(total_recv_data / sizeof(vprop_t)); + if(total_sizes[myid] > thresholds[myid]) { + data_send.reserve(lsize); + for(int i = 0; i < lsize; ++i) { + data_send.push_back(graph.vertices().attributes(i)); + } + } else { + data_send.reserve(total_recv); + int i = 0; + for(auto & index : indices_recv) { + data_send.push_back(graph.vertices().attributes(index - start)); + ++i; + } + } + trace.exit_state("get components"); + trace.enter_state("send components"); + dart_alltoallv(data_send.data(), sizes_send_data.data(), + displs_send_data.data(), DART_TYPE_BYTE, data_recv.data(), + sizes_recv_data.data(), displs_recv_data.data(), + graph.team().dart_id()); + trace.exit_state("send components"); + trace.enter_state("create map"); + std::unordered_map output_regular; + std::vector> output_contiguous(indices.size()); + output_regular.reserve(total_send); + int current_unit = 0; + int j = 0; + for(int i = 0; i < data_recv.size(); ++i) { + if(i >= cumul_sizes_recv_data[current_unit]) { + ++current_unit; + } + if(total_sizes[current_unit] > thresholds[current_unit]) { + output_contiguous[current_unit].push_back(data_recv[i]); + } else { + output_regular[indices_send[j]] = data_recv[i]; + ++j; + } + } + trace.exit_state("create map"); + return std::make_pair(output_regular, output_contiguous); +} + +template +void cc_set_data( + matrix_pair_t & data_pairs, + int start, + GraphType & graph, + dash::util::Trace & trace +) { + typedef typename GraphType::vertex_properties_type vprop_t; + trace.enter_state("send pairs"); + std::vector sizes_send(data_pairs.size()); + std::vector displs_send(data_pairs.size()); + std::vector> pairs_send; + int total_send = 0; + for(int i = 0; i < data_pairs.size(); ++i) { + sizes_send[i] = data_pairs[i].size() * sizeof(std::pair); + displs_send[i] = total_send * sizeof(std::pair); + total_send += data_pairs[i].size(); + } + pairs_send.reserve(total_send); + for(auto & pair_set : data_pairs) { + pairs_send.insert(pairs_send.end(), pair_set.begin(), pair_set.end()); + } + std::vector sizes_recv(data_pairs.size()); + std::vector displs_recv(data_pairs.size()); + dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), + DART_TYPE_BYTE, graph.team().dart_id()); + std::size_t total_recv = 0; + for(int i = 0; i < sizes_recv.size(); ++i) { + displs_recv[i] = total_recv; + total_recv += sizes_recv[i]; + } + std::vector> pairs_recv(total_recv / + sizeof(std::pair)); + dart_alltoallv(pairs_send.data(), sizes_send.data(), displs_send.data(), + DART_TYPE_BYTE, pairs_recv.data(), sizes_recv.data(), + displs_recv.data(), graph.team().dart_id()); + trace.exit_state("send pairs"); + trace.enter_state("set components"); + for(auto & pair : pairs_recv) { + int ind = pair.first - start; + graph.vertices().set_attributes(pair.first - start, pair.second); + } + trace.exit_state("set components"); +} + +} // namespace internal + +// TODO: component type should be Graph::vertex_size_type +/** + * Computes connected components on a graph. + * Requires the graph's vertices to store the following attributes: + * - (int) comp + * - (int) unit + * Requires the graph's edges to store the following attributep: + * none + * + * Output: + * The vertex component attributes store the respective components after + */ +template +void connected_components(GraphType & g) { + typedef typename GraphType::vertex_properties_type vprop_t; + dash::util::Trace trace("ConnectedComponents"); + // set component to global index in iteration space + // TODO: find faster method for this + trace.enter_state("vertex setup"); + int i = 0; + auto myid = dash::myid(); + std::vector unit_sizes(g.team().size()); + std::size_t total_size = 0; + for(int i = 0; i < g.team().size(); ++i) { + unit_sizes[i] = total_size; + team_unit_t unit { i }; + total_size += g.vertices().size(unit); + } + + auto start = unit_sizes[myid]; + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto index = it.pos(); + auto gindex = index + start; + vprop_t prop { gindex, myid }; + g.vertices().set_attributes(index, prop); + ++i; + } + trace.exit_state("vertex setup"); + + trace.enter_state("barrier"); + dash::barrier(); + trace.exit_state("barrier"); + + matrix_t indices(g.team().size()); + matrix_t permutations(g.team().size()); + { + trace.enter_state("compute indices"); + int i = 0; + for(auto it = g.out_edges().lbegin(); it != g.out_edges().lend(); ++it) { + auto trg = g[it].target(); + auto lpos = trg.lpos(); + // TODO: use more sophsticated sorting mechanism + indices[lpos.unit].push_back(lpos.index); + permutations[lpos.unit].push_back(i); + ++i; + } + trace.exit_state("compute indices"); + } + + while(1) { + int gr = 0; + { + std::vector data = internal::cc_get_data(indices, permutations, + g, trace); + trace.enter_state("compute pairs"); + matrix_pair_t data_pairs(g.team().size()); + { + std::vector> pair_set(g.team().size()); + int i = 0; + for(auto it = g.out_edges().lbegin(); it != g.out_edges().lend(); ++it) { + if(data[i].comp != 0) { + auto e = g[it]; + auto src = g[e.source()]; + auto src_comp = src.attributes(); + auto trg_comp = data[i]; + if(src_comp.comp < trg_comp.comp) { + auto & set = pair_set[trg_comp.unit]; + if(set.find(trg_comp.comp) == set.end()) { + data_pairs[trg_comp.unit].emplace_back(trg_comp.comp, src_comp); + set.insert(trg_comp.comp); + } + gr = 1; + } + } + ++i; + } + } + trace.exit_state("compute pairs"); + internal::cc_set_data(data_pairs, start, g, trace); + } + int gr_all = 0; + trace.enter_state("allreduce data"); + dart_allreduce(&gr, &gr_all, 1, DART_TYPE_INT, DART_OP_SUM, + g.team().dart_id()); + trace.exit_state("allreduce data"); + if(gr_all == 0) break; + while(1) { + int pj = 0; + matrix_t indices(g.team().size()); + { + std::vector> comp_set(g.team().size()); + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto c = g[it].attributes(); + auto & set = comp_set[c.unit]; + if(set.find(c.comp) == set.end()) { + indices[c.unit].push_back(c.comp); + set.insert(c.comp); + } + } + } + auto components = internal::cc_get_components(indices, start, g, trace); + + trace.enter_state("set data (pj)"); + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto v = g[it]; + auto comp = v.attributes(); + if(comp.comp != 0) { + vprop_t comp_next; + if(components.second[comp.unit].size() > 0) { + comp_next = + components.second[comp.unit][comp.comp - unit_sizes[comp.unit]]; + } else { + comp_next = components.first[comp.comp]; + } + if(comp.comp != comp_next.comp) { + v.set_attributes(comp_next); + pj = 1; + } + } + } + trace.exit_state("set data (pj)"); + int pj_all = 0; + trace.enter_state("allreduce pointerjumping"); + dart_allreduce(&pj, &pj_all, 1, DART_TYPE_INT, DART_OP_SUM, + g.team().dart_id()); + trace.exit_state("allreduce pointerjumping"); + if(pj_all == 0) break; + } + } + dash::barrier(); +} + +} // namespace dash + +#endif // DASH__ALGORITHM____GRAPH__CONNECTED_COMPONENTS_H__ + diff --git a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h new file mode 100644 index 000000000..31d78fc08 --- /dev/null +++ b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h @@ -0,0 +1,535 @@ +/** + * Author: Steffan Effenberger (github: @stiefn) + * via https://github.com/stiefn/dyn-data-structures-thesis + * + */ +#ifndef DASH__ALGORITHM____GRAPH__MINIMUM_SPANNING_TREE_H__ +#define DASH__ALGORITHM____GRAPH__MINIMUM_SPANNING_TREE_H__ + +#include +#include +#include + +namespace dash { + +typedef std::vector>> pair_t; +typedef std::vector> matrix_t; + +// (supervertex, new component, weight, owner, local offset) +template +using tuple_t = std::tuple; +template +using matrix_min_pairs_t = std::vector>>; + +template +using matrix_pair_t = std::vector>>; + +namespace internal { +// TODO: merge overlapping code of the following functions with the functions +// provided in ConnectedComponents.h + +template +auto mst_get_data( + matrix_t & indices, + matrix_t & permutations, + GraphType & graph, + dash::util::Trace & trace +) -> std::vector { + typedef typename GraphType::vertex_properties_type vprop_t; + trace.enter_state("send indices"); + // exchange indices to get + std::vector sizes_send(indices.size()); + std::vector displs_send(indices.size()); + std::vector sizes_recv_data(indices.size()); + std::vector displs_recv_data(indices.size()); + std::vector indices_send; + std::size_t total_send = 0; + for(int i = 0; i < indices.size(); ++i) { + sizes_send[i] = indices[i].size(); + displs_send[i] = total_send; + sizes_recv_data[i] = indices[i].size() * sizeof(vprop_t); + displs_recv_data[i] = total_send * sizeof(vprop_t); + total_send += indices[i].size(); + } + indices_send.reserve(total_send); + for(auto & index_set : indices) { + indices_send.insert(indices_send.end(), index_set.begin(), + index_set.end()); + } + std::vector sizes_recv(indices.size()); + std::vector displs_recv(indices.size()); + std::vector sizes_send_data(indices.size()); + std::vector displs_send_data(indices.size()); + dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), + DART_TYPE_BYTE, graph.team().dart_id()); + std::size_t total_recv = 0; + for(int i = 0; i < sizes_recv.size(); ++i) { + sizes_send_data[i] = sizes_recv[i] * sizeof(vprop_t); + displs_send_data[i] = total_recv * sizeof(vprop_t); + displs_recv[i] = total_recv; + total_recv += sizes_recv[i]; + } + std::vector indices_recv(total_recv); + dart_alltoallv(indices_send.data(), sizes_send.data(), displs_send.data(), + DART_TYPE_INT, indices_recv.data(), sizes_recv.data(), + displs_recv.data(), graph.team().dart_id()); + + std::vector data_send; + data_send.reserve(total_recv); + std::vector data_recv(total_send); + + trace.exit_state("send indices"); + // exchange data + trace.enter_state("get components"); + for(auto & index : indices_recv) { + data_send.push_back(graph.vertices().attributes(index)); + } + trace.exit_state("get components"); + trace.enter_state("send components"); + dart_alltoallv(data_send.data(), sizes_send_data.data(), + displs_send_data.data(), DART_TYPE_BYTE, data_recv.data(), + sizes_recv_data.data(), displs_recv_data.data(), + graph.team().dart_id()); + trace.exit_state("send components"); + + trace.enter_state("restore order"); + // restore original order + // TODO: use more sophisticated ordering mechanism + std::vector output(data_recv.size()); + int index = 0; + for(int i = 0; i < permutations.size(); ++i) { + for(int j = 0; j < permutations[i].size(); ++j) { + output[permutations[i][j]] = data_recv[index]; + ++index; + } + } + trace.exit_state("restore order"); + return output; +} + +template +auto mst_get_components( + matrix_t & indices, + int start, + GraphType & graph, + dash::util::Trace & trace +) -> std::pair< + std::unordered_map, + std::vector>> +{ + typedef typename GraphType::vertex_properties_type vprop_t; + trace.enter_state("send indices"); + auto myid = graph.team().myid(); + std::size_t lsize = graph.vertices().size(myid); + std::vector thresholds(indices.size()); + for(int i = 0; i < thresholds.size(); ++i) { + team_unit_t unit { i }; + // TODO: find a threshold that provides an optimal tradeoff for any + // amount of units + thresholds[i] = graph.vertices().size(unit) * 20; + } + + std::vector total_sizes(indices.size()); + std::vector sizes_send(indices.size()); + std::vector sizes_send_longdt(indices.size()); + std::vector displs_send(indices.size()); + std::vector sizes_recv_data(indices.size()); + std::vector cumul_sizes_recv_data(indices.size()); + std::vector displs_recv_data(indices.size()); + for(int i = 0; i < indices.size(); ++i) { + sizes_send[i] = indices[i].size(); + sizes_send_longdt[i] = indices[i].size(); + sizes_recv_data[i] = indices[i].size() * sizeof(vprop_t); + } + dart_allreduce(sizes_send_longdt.data(), total_sizes.data(), indices.size(), + DART_TYPE_LONGLONG, DART_OP_SUM, graph.team().dart_id()); + int total_send = 0; + int total_recv_data = 0; + for(int i = 0; i < indices.size(); ++i) { + if(total_sizes[i] > thresholds[i]) { + sizes_send[i] = 0; + team_unit_t unit { i }; + sizes_recv_data[i] = graph.vertices().size(unit) * sizeof(vprop_t); + indices[i].clear(); + } else { + sizes_send[i] = indices[i].size(); + sizes_recv_data[i] = indices[i].size() * sizeof(vprop_t); + } + displs_send[i] = total_send; + displs_recv_data[i] = total_recv_data; + total_send += sizes_send[i]; + total_recv_data += sizes_recv_data[i]; + cumul_sizes_recv_data[i] = total_recv_data / sizeof(vprop_t); + } + std::vector indices_send; + indices_send.reserve(total_send); + for(auto & index_set : indices) { + indices_send.insert(indices_send.end(), index_set.begin(), + index_set.end()); + } + std::vector sizes_recv(indices.size()); + std::vector displs_recv(indices.size()); + std::vector sizes_send_data(indices.size()); + std::vector displs_send_data(indices.size()); + dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), + DART_TYPE_BYTE, graph.team().dart_id()); + int total_recv = 0; + int total_send_data = 0; + for(int i = 0; i < sizes_recv.size(); ++i) { + if(total_sizes[myid] > thresholds[myid]) { + sizes_send_data[i] = lsize * sizeof(vprop_t); + displs_send_data[i] = 0; + } else { + sizes_send_data[i] = sizes_recv[i] * sizeof(vprop_t); + displs_send_data[i] = total_send_data; + displs_recv[i] = total_recv; + } + total_recv += sizes_recv[i]; + total_send_data += sizes_send_data[i]; + } + std::vector indices_recv(total_recv); + dart_alltoallv(indices_send.data(), sizes_send.data(), displs_send.data(), + DART_TYPE_INT, indices_recv.data(), sizes_recv.data(), + displs_recv.data(), graph.team().dart_id()); + trace.exit_state("send indices"); + trace.enter_state("get components"); + std::vector data_send; + std::vector data_recv(total_recv_data / sizeof(vprop_t)); + if(total_sizes[myid] > thresholds[myid]) { + data_send.reserve(lsize); + for(int i = 0; i < lsize; ++i) { + data_send.push_back(graph.vertices().attributes(i)); + } + } else { + data_send.reserve(total_recv); + for(auto & index : indices_recv) { + data_send.push_back(graph.vertices().attributes(index - start)); + } + } + trace.exit_state("get components"); + trace.enter_state("send components"); + dart_alltoallv(data_send.data(), sizes_send_data.data(), + displs_send_data.data(), DART_TYPE_BYTE, data_recv.data(), + sizes_recv_data.data(), displs_recv_data.data(), + graph.team().dart_id()); + trace.exit_state("send components"); + trace.enter_state("create map"); + std::unordered_map output_regular; + std::vector> output_contiguous(indices.size()); + output_regular.reserve(total_send); + int current_unit = 0; + int j = 0; + for(int i = 0; i < data_recv.size(); ++i) { + if(i >= cumul_sizes_recv_data[current_unit]) { + ++current_unit; + } + if(total_sizes[current_unit] > thresholds[current_unit]) { + output_contiguous[current_unit].push_back(data_recv[i]); + } else { + output_regular[indices_send[j]] = data_recv[i]; + ++j; + } + } + trace.exit_state("create map"); + return std::make_pair(output_regular, output_contiguous); +} + +template +void mst_set_data_min( + matrix_min_pairs_t & data_pairs, + std::vector> & remote_edges, + int start, + GraphType & graph, + dash::util::Trace & trace +) { + trace.enter_state("send pairs"); + typedef typename GraphType::vertex_properties_type vprop_t; + auto myid = graph.team().myid(); + std::vector sizes_send(data_pairs.size()); + std::vector displs_send(data_pairs.size()); + std::vector> pairs_send; + std::size_t total_send = 0; + for(int i = 0; i < data_pairs.size(); ++i) { + sizes_send[i] = data_pairs[i].size() * sizeof(tuple_t); + displs_send[i] = total_send * sizeof(tuple_t); + total_send += data_pairs[i].size(); + } + pairs_send.reserve(total_send); + for(auto & pair_set : data_pairs) { + pairs_send.insert(pairs_send.end(), pair_set.begin(), pair_set.end()); + } + std::vector sizes_recv(data_pairs.size()); + std::vector displs_recv(data_pairs.size()); + dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), + DART_TYPE_BYTE, graph.team().dart_id()); + std::size_t total_recv = 0; + for(int i = 0; i < sizes_recv.size(); ++i) { + displs_recv[i] = total_recv; + total_recv += sizes_recv[i]; + } + std::vector> pairs_recv(total_recv / + sizeof(tuple_t)); + dart_alltoallv(pairs_send.data(), sizes_send.data(), displs_send.data(), + DART_TYPE_BYTE, pairs_recv.data(), sizes_recv.data(), + displs_recv.data(), graph.team().dart_id()); + std::unordered_map> mapping; + for(auto & pair : pairs_recv) { + auto it = mapping.find(std::get<0>(pair)); + if(it != mapping.end()) { + auto weight = std::get<2>(it->second); + if(weight > std::get<2>(pair)) { + it->second = pair; + } + } else { + mapping[std::get<0>(pair)] = pair; + } + } + trace.exit_state("send pairs"); + trace.enter_state("set components"); + for(auto & pair : mapping) { + auto trg_comp = std::get<1>(pair.second); + graph.vertices().set_attributes(pair.first - start, trg_comp); + if(myid == get<3>(pair.second)) { + auto edge = std::get<4>(pair.second); + auto prop = graph.out_edges().attributes(edge); + prop.is_min = true; + graph.out_edges().set_attributes(edge, prop); + } else if(std::get<3>(pair.second) != std::get<1>(pair.second).unit) { + // if the edge belongs to a unit that does not hold any of the component + // vertices + remote_edges[std::get<3>(pair.second)].push_back( + std::get<4>(pair.second)); + } + } + trace.exit_state("set components"); +} + +template +void mst_set_edges( + std::vector> remote_edges, + GraphType & graph, + dash::util::Trace & trace +) { + typedef typename GraphType::edge_properties_type eprop_t; + trace.enter_state("send edges"); + std::vector sizes_send(remote_edges.size()); + std::vector displs_send(remote_edges.size()); + std::vector send; + std::size_t total_send = 0; + for(int i = 0; i < remote_edges.size(); ++i) { + sizes_send[i] = remote_edges[i].size(); + displs_send[i] = total_send; + total_send += sizes_send[i]; + } + send.reserve(total_send); + for(auto & set : remote_edges) { + send.insert(send.end(), set.begin(), set.end()); + } + std::vector sizes_recv(remote_edges.size()); + std::vector displs_recv(remote_edges.size()); + dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), + DART_TYPE_BYTE, graph.team().dart_id()); + std::size_t total_recv = 0; + for(int i = 0; i < sizes_recv.size(); ++i) { + displs_recv[i] = total_recv; + total_recv += sizes_recv[i]; + } + std::vector recv(total_recv); + dart_alltoallv(send.data(), sizes_send.data(), displs_send.data(), + DART_TYPE_INT, recv.data(), sizes_recv.data(), + displs_recv.data(), graph.team().dart_id()); + for(auto & edge : recv) { + auto prop = graph.out_edges().attributes(edge); + prop.is_min = true; + graph.out_edges().set_attributes(edge, prop); + } + trace.exit_state("send edges"); +} + +} // namespace internal + +// TODO: component type should be Graph::vertex_size_type +/** + * Computes minimum spanning tree on a graph. + * Requires the graph's vertices to store the following attributes: + * - (int) comp + * Requires the graph's edges to store the following attributep: + * - (int) weight + * - (bool) is_min + * + * Output: + * TODO: store the tree information in the edges OR in a separate data + * structure + */ +template +void minimum_spanning_tree(GraphType & g) { + typedef typename GraphType::vertex_properties_type vprop_t; + dash::util::Trace trace("MinimumSpanningTree"); + // set component to global index in iteration space + trace.enter_state("vertex setup"); + int i = 0; + auto myid = dash::myid(); + std::vector unit_sizes(g.team().size()); + std::size_t total_size = 0; + for(int i = 0; i < g.team().size(); ++i) { + unit_sizes[i] = total_size; + team_unit_t unit { i }; + total_size += g.vertices().size(unit); + } + + auto start = unit_sizes[myid]; + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto index = it.pos(); + auto gindex = index + start; + vprop_t prop { gindex, myid }; + g.vertices().set_attributes(index, prop); + ++i; + } + trace.exit_state("vertex setup"); + + trace.enter_state("barrier"); + dash::barrier(); + trace.exit_state("barrier"); + + matrix_t indices(g.team().size()); + matrix_t permutations(g.team().size()); + { + trace.enter_state("compute indices"); + int i = 0; + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto v = g[it]; + for(auto e_it = v.out_edges().lbegin(); e_it != v.out_edges().lend(); + ++e_it) { + auto trg = g[e_it].target(); + auto lpos = trg.lpos(); + indices[lpos.unit].push_back(lpos.index); + permutations[lpos.unit].push_back(i); + ++i; + } + } + trace.exit_state("compute indices"); + } + + std::vector> remote_edges(g.team().size()); + while(1) { + int gr = 0; + { + std::vector data = internal::mst_get_data(indices, permutations, + g, trace); + trace.enter_state("compute pairs"); + matrix_min_pairs_t data_pairs(g.team().size()); + std::vector>> pair_map(g.team().size()); + int i = 0; + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto v = g[it]; + auto src_comp = v.attributes(); + int min_weight = std::numeric_limits::max(); + vprop_t trg_comp_min; + int ledgepos = -1; + int src = 0; + int trg = 0; + for(auto e_it = v.out_edges().lbegin(); e_it != v.out_edges().lend(); + ++e_it) { + auto e = g[e_it]; + auto e_weight = e.attributes().weight; + auto trg_comp = data[i]; + if(src_comp.comp != trg_comp.comp && e_weight >= 0 && + min_weight > e_weight) { + min_weight = e_weight; + trg_comp_min = trg_comp; + ledgepos = e_it.pos(); + src = e.source().pos(); + trg = e.target().pos(); + } + ++i; + } + if(ledgepos >= 0) { + auto & map = pair_map[src_comp.unit]; + auto m_it = map.find(src_comp.comp); + if(m_it == map.end()) { + map[src_comp.comp] = std::make_tuple(src_comp, trg_comp_min, + min_weight, ledgepos); + } else { + if(std::get<2>(m_it->second) > min_weight) { + m_it->second = std::make_tuple(src_comp, trg_comp_min, min_weight, + ledgepos); + } + } + gr = 1; + } + } + for(int i = 0; i < pair_map.size(); ++i) { + for(auto & pair : pair_map[i]) { + data_pairs[i].emplace_back(std::get<0>(pair.second).comp, + std::get<1>(pair.second), std::get<2>(pair.second), myid, + std::get<3>(pair.second)); + auto trg_unit = std::get<1>(pair.second).unit; + if(std::get<0>(pair.second).unit != trg_unit) { + data_pairs[trg_unit].emplace_back(std::get<1>(pair.second).comp, + std::get<0>(pair.second), std::get<2>(pair.second), myid, + std::get<3>(pair.second)); + } + } + } + trace.exit_state("compute pairs"); + internal::mst_set_data_min(data_pairs, remote_edges, start, g, trace); + } + + int gr_all = 0; + trace.enter_state("allreduce data"); + dart_allreduce(&gr, &gr_all, 1, DART_TYPE_INT, DART_OP_SUM, + g.team().dart_id()); + trace.exit_state("allreduce data"); + if(gr_all == 0) break; + while(1) { + int pj = 0; + matrix_t indices(g.team().size()); + { + std::vector> comp_set(g.team().size()); + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto c = g[it].attributes(); + auto & set = comp_set[c.unit]; + if(set.find(c.comp) == set.end()) { + indices[c.unit].push_back(c.comp); + set.insert(c.comp); + } + } + } + auto components = internal::mst_get_components(indices, start, g, trace); + + trace.enter_state("set data (pj)"); + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto v = g[it]; + auto comp = v.attributes(); + if(comp.comp != 0) { + vprop_t comp_next; + if(components.second[comp.unit].size() > 0) { + comp_next = + components.second[comp.unit][comp.comp - unit_sizes[comp.unit]]; + } else { + comp_next = components.first[comp.comp]; + } + if(comp.comp > comp_next.comp) { + v.set_attributes(comp_next); + pj = 1; + } + } + } + trace.exit_state("set data (pj)"); + int pj_all = 0; + trace.enter_state("allreduce pointerjumping"); + dart_allreduce(&pj, &pj_all, 1, DART_TYPE_INT, DART_OP_SUM, + g.team().dart_id()); + trace.exit_state("allreduce pointerjumping"); + if(pj_all == 0) break; + } + } + internal::mst_set_edges(remote_edges, g, trace); + + dash::barrier(); + +} +} // namespace dash + +#endif // DASH__ALGORITHM____GRAPH__MINIMUM_SPANNING_TREE_H__ diff --git a/dash/include/dash/graph/EdgeIterator.h b/dash/include/dash/graph/EdgeIterator.h new file mode 100644 index 000000000..0ce497fb4 --- /dev/null +++ b/dash/include/dash/graph/EdgeIterator.h @@ -0,0 +1,138 @@ +#ifndef DASH__GRAPH__EDGE_ITERATOR_H__INCLUDED +#define DASH__GRAPH__EDGE_ITERATOR_H__INCLUDED + +namespace dash { + +/** + * Wrapper for the edge iterators of the graph. + */ +template +struct EdgeIteratorWrapper { + + typedef Graph graph_type; + typedef typename graph_type::glob_mem_edge_comb_type glob_mem_type; + typedef typename Graph::global_edge_iterator iterator; + typedef const iterator const_iterator; + typedef typename Graph::local_edge_iterator local_iterator; + typedef const local_iterator const_local_iterator; + typedef typename graph_type::edge_properties_type properties_type; + typedef typename graph_type::edge_size_type size_type; + + /** + * Constructs the wrapper. + */ + EdgeIteratorWrapper(graph_type * graph) + : _graph(graph), + _gmem(graph->_glob_mem_edge) + { } + + /* + * Default constructor. + */ + EdgeIteratorWrapper() = default; + + /** + * Returns global iterator to the beginning of the edge list. + */ + iterator begin() { + return _gmem->begin(); + //return iterator(_graph->_glob_mem_out_edge, 0); + } + + /** + * Returns global iterator to the beginning of the edge list. + */ + const_iterator begin() const { + return _gmem->begin(); + //return iterator(_graph->_glob_mem_out_edge, 0); + } + + /** + * Returns global iterator to the end of the edge list. + */ + iterator end() { + return _gmem->end(); + } + + /** + * Returns global iterator to the end of the edge list. + */ + const_iterator end() const { + return _gmem->end(); + } + + /** + * Returns local iterator to the beginning of the edge list. + */ + local_iterator lbegin() { + return _gmem->lbegin(); + } + + /** + * Returns local iterator to the beginning of the edge list. + */ + const_local_iterator lbegin() const { + return _gmem->lbegin(); + } + + /** + * Returns local iterator to the end of the edge list. + */ + local_iterator lend() { + return _gmem->lend(); + } + + /** + * Returns local iterator to the end of the edge list. + */ + const_local_iterator lend() const { + return _gmem->lend(); + } + + /** + * Returns the amount of edges in the whole graph. + */ + size_type size() { + return _gmem->size(); + } + + /** + * Returns the amount of in-edges the specified unit currently holds in + * global memory space. + */ + size_type size(team_unit_t unit) { + return _gmem->size(unit); + } + + /** + * Returns the amount of edges this unit currently holds in local memory + * space. + */ + size_type lsize() { + return _gmem->lsize(); + } + + /* + * Returns whether theare are edges in global memory space. + */ + bool empty() { + return size() == 0; + } + + /** + * Returns the maximum number of edges the graph can store. + */ + size_type max_size() { + return std::numeric_limits::max(); + } + +private: + + graph_type * _graph; + glob_mem_type * _gmem; + +}; + +} + +#endif // DASH__GRAPH__EDGE_ITERATOR_H__INCLUDED diff --git a/dash/include/dash/graph/InEdgeIterator.h b/dash/include/dash/graph/InEdgeIterator.h new file mode 100644 index 000000000..80cfb7984 --- /dev/null +++ b/dash/include/dash/graph/InEdgeIterator.h @@ -0,0 +1,155 @@ +#ifndef DASH__GRAPH__IN_EDGE_ITERATOR_H__INCLUDED +#define DASH__GRAPH__IN_EDGE_ITERATOR_H__INCLUDED + +namespace dash { + +/** + * Wrapper for the edge iterators of the graph. + */ +template +struct InEdgeIteratorWrapper { + + typedef Graph graph_type; + typedef typename graph_type::glob_mem_edge_type glob_mem_type; + typedef typename Graph::global_in_edge_iterator iterator; + typedef const iterator const_iterator; + typedef typename Graph::local_in_edge_iterator local_iterator; + typedef const local_iterator const_local_iterator; + typedef typename graph_type::edge_properties_type properties_type; + typedef typename graph_type::edge_size_type size_type; + + /** + * Constructs the wrapper. + */ + InEdgeIteratorWrapper(graph_type * graph) + : _graph(graph), + _gmem(graph->_glob_mem_in_edge) + { } + + /** + * Default constructor. + */ + InEdgeIteratorWrapper() = default; + + /** + * Returns global iterator to the beginning of the edge list. + */ + iterator begin() { + return _gmem->begin(); + //return iterator(_graph->_glob_mem_in_edge, 0); + } + + /** + * Returns global iterator to the beginning of the edge list. + */ + const_iterator begin() const { + return _gmem->begin(); + //return iterator(_graph->_glob_mem_in_edge, 0); + } + + /** + * Returns global iterator to the end of the edge list. + */ + iterator end() { + return _gmem->end(); + } + + /** + * Returns global iterator to the end of the edge list. + */ + const_iterator end() const { + return _gmem->end(); + } + + /** + * Returns local iterator to the beginning of the edge list. + */ + local_iterator lbegin() { + return _gmem->lbegin(); + } + + /** + * Returns local iterator to the beginning of the edge list. + */ + const_local_iterator lbegin() const { + return _gmem->lbegin(); + } + + /** + * Returns local iterator to the end of the edge list. + */ + local_iterator lend() { + return _gmem->lend(); + } + + /** + * Returns local iterator to the end of the edge list. + */ + const_local_iterator lend() const { + return _gmem->lend(); + } + + /** + * Directly gets the attributes of an in-edge identified by a local offset + */ + properties_type & attributes(size_type local_index) { + // TODO: Allow this only when there are no uncommitted changes + return _gmem->get(local_index).properties; + } + + /** + * Directly sets the attributes of an in-edge identified by a local offset + */ + void set_attributes(size_type local_index, properties_type prop) { + // TODO: Allow this only when there are no uncommitted changes + auto & v = _gmem->get(local_index); + v.properties = prop; + } + + /** + * Returns the amount of in-edges in the whole graph. + */ + size_type size() { + return _gmem->size(); + } + + /** + * Returns the amount of in-edges the specified unit currently holds in + * global memory space. + */ + size_type size(team_unit_t unit) { + return _gmem->size(unit); + } + + /** + * Returns the amount of in-edges this unit currently holds in local memory + * space. + */ + size_type lsize() { + return _gmem->lsize(); + } + + /* + * Returns whether theare are in-edges in global memory space. + */ + bool empty() { + return size() == 0; + } + + /** + * Returns the maximum number of in-edges the graph can store. + */ + size_type max_size() { + return std::numeric_limits::max(); + } + +private: + + graph_type * _graph; + glob_mem_type * _gmem; + +}; + +} + +#endif // DASH__GRAPH__IN_EDGE_ITERATOR_H__INCLUDED diff --git a/dash/include/dash/graph/OutEdgeIterator.h b/dash/include/dash/graph/OutEdgeIterator.h new file mode 100644 index 000000000..e3000ddcf --- /dev/null +++ b/dash/include/dash/graph/OutEdgeIterator.h @@ -0,0 +1,154 @@ +#ifndef DASH__GRAPH__OUT_EDGE_ITERATOR_H__INCLUDED +#define DASH__GRAPH__OUT_EDGE_ITERATOR_H__INCLUDED + +namespace dash { + +/** + * Wrapper for the edge iterators of the graph. + */ +template +struct OutEdgeIteratorWrapper { + + typedef Graph graph_type; + typedef typename graph_type::glob_mem_edge_type glob_mem_type; + typedef typename Graph::global_out_edge_iterator iterator; + typedef const iterator const_iterator; + typedef typename Graph::local_out_edge_iterator local_iterator; + typedef const local_iterator const_local_iterator; + typedef typename graph_type::edge_properties_type properties_type; + typedef typename graph_type::edge_size_type size_type; + /** + * Constructs the wrapper. + */ + OutEdgeIteratorWrapper(graph_type * graph) + : _graph(graph), + _gmem(graph->_glob_mem_out_edge) + { } + + /* + * Default constructor. + */ + OutEdgeIteratorWrapper() = default; + + /** + * Returns global iterator to the beginning of the edge list. + */ + iterator begin() { + return _gmem->begin(); + //return iterator(_graph->_glob_mem_out_edge, 0); + } + + /** + * Returns global iterator to the beginning of the edge list. + */ + const_iterator begin() const { + return _gmem->begin(); + //return iterator(_graph->_glob_mem_out_edge, 0); + } + + /** + * Returns global iterator to the end of the edge list. + */ + iterator end() { + return _gmem->end(); + } + + /** + * Returns global iterator to the end of the edge list. + */ + const_iterator end() const { + return _gmem->end(); + } + + /** + * Returns local iterator to the beginning of the edge list. + */ + local_iterator lbegin() { + return _gmem->lbegin(); + } + + /** + * Returns local iterator to the beginning of the edge list. + */ + const_local_iterator lbegin() const { + return _gmem->lbegin(); + } + + /** + * Returns local iterator to the end of the edge list. + */ + local_iterator lend() { + return _gmem->lend(); + } + + /** + * Returns local iterator to the end of the edge list. + */ + const_local_iterator lend() const { + return _gmem->lend(); + } + + /** + * Directly gets the attributes of an out-edge identified by a local offset + */ + properties_type & attributes(size_type local_index) { + // TODO: Allow this only when there are no uncommitted changes + return _gmem->get(local_index).properties; + } + + /** + * Directly sets the attributes of an out-edge identified by a local offset + */ + void set_attributes(size_type local_index, properties_type prop) { + // TODO: Allow this only when there are no uncommitted changes + auto & v = _gmem->get(local_index); + v.properties = prop; + } + + /** + * Returns the amount of out-edges in the whole graph. + */ + size_type size() { + return _gmem->size(); + } + + /** + * Returns the amount of out-edges the specified unit currently holds in + * global memory space. + */ + size_type size(team_unit_t unit) { + return _gmem->size(unit); + } + + /** + * Returns the amount of out-edges this unit currently holds in local memory + * space. + */ + size_type lsize() { + return _gmem->lsize(); + } + + /* + * Returns whether theare are out-edges in global memory space. + */ + bool empty() { + return size() == 0; + } + + /** + * Returns the maximum number of out-edges the graph can store. + */ + size_type max_size() { + return std::numeric_limits::max(); + } + +private: + + graph_type * _graph; + glob_mem_type * _gmem; + +}; + +} + +#endif // DASH__GRAPH__OUT_EDGE_ITERATOR_H__INCLUDED diff --git a/dash/include/dash/graph/VertexIterator.h b/dash/include/dash/graph/VertexIterator.h new file mode 100644 index 000000000..c02d75783 --- /dev/null +++ b/dash/include/dash/graph/VertexIterator.h @@ -0,0 +1,154 @@ +#ifndef DASH__GRAPH__VERTEX_ITERATOR_H__INCLUDED +#define DASH__GRAPH__VERTEX_ITERATOR_H__INCLUDED + +namespace dash { + +/** + * Wrapper for the vertex iterators of the graph. + */ +template +struct VertexIteratorWrapper { + + typedef Graph graph_type; + typedef typename graph_type::glob_mem_vert_type glob_mem_type; + typedef typename Graph::global_vertex_iterator iterator; + typedef const iterator const_iterator; + typedef typename Graph::local_vertex_iterator local_iterator; + typedef const local_iterator const_local_iterator; + typedef typename graph_type::vertex_properties_type properties_type; + typedef typename graph_type::vertex_size_type size_type; + + /** + * Constructs the wrapper. + */ + VertexIteratorWrapper(graph_type * graph) + : _graph(graph), + _gmem(graph->_glob_mem_vertex) + { } + + /* + * Default constructor. + */ + VertexIteratorWrapper() = default; + + /** + * Returns global iterator to the beginning of the vertex list. + */ + iterator begin() { + return _gmem->begin(); + } + + /** + * Returns global iterator to the beginning of the vertex list. + */ + const_iterator begin() const { + return _gmem->begin(); + } + + /** + * Returns global iterator to the end of the vertex list. + */ + iterator end() { + return _gmem->end(); + } + + /** + * Returns global iterator to the end of the vertex list. + */ + const_iterator end() const { + return _gmem->end(); + } + + /** + * Returns local iterator to the beginning of the vertex list. + */ + local_iterator lbegin() { + return _gmem->lbegin(); + } + + /** + * Returns local iterator to the beginning of the vertex list. + */ + const_local_iterator lbegin() const { + return _gmem->lbegin(); + } + + /** + * Returns local iterator to the end of the vertex list. + */ + local_iterator lend() { + return _gmem->lend(); + } + + /** + * Returns local iterator to the end of the vertex list. + */ + const_local_iterator lend() const { + return _gmem->lend(); + } + + + /** + * Directly gets the attributes of a vertex identified by a local offset + */ + properties_type & attributes(size_type local_index) { + // TODO: Allow this only when there are no uncommitted changes + return _gmem->get(local_index).properties; + } + + /** + * Directly sets the attributes of a vertex identified by a local offset + */ + void set_attributes(size_type local_index, properties_type prop) { + // TODO: Allow this only when there are no uncommitted changes + auto & v = _gmem->get(local_index); + v.properties = prop; + } + + /** + * Returns the amount of vertices in the whole graph. + */ + size_type size() { + return _gmem->size(); + } + + /** + * Returns the amount of vertices the specified unit currently holds in + * global memory space. + */ + size_type size(team_unit_t unit) { + return _gmem->size(unit); + } + + /** + * Returns the amount of vertices this unit currently holds in local memory + * space. + */ + size_type lsize() { + return _gmem->lsize(); + } + + /* + * Returns whether theare are vertices in global memory space. + */ + bool empty() { + return size() == 0; + } + + /** + * Returns the maximum number of vertices the graph can store. + */ + size_type max_size() { + return std::numeric_limits::max(); + } + +private: + + graph_type * _graph; + glob_mem_type * _gmem; + +}; + +} + +#endif // DASH__GRAPH__VERTEX_ITERATOR_H__INCLUDED diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h new file mode 100644 index 000000000..52ac414df --- /dev/null +++ b/dash/include/dash/graph/internal/Graph.h @@ -0,0 +1,758 @@ +#ifndef DASH__GRAPH__INTERNAL__GRAPH_H__INCLUDED +#define DASH__GRAPH__INTERNAL__GRAPH_H__INCLUDED + +namespace dash { + +/** + * Enum declaring the different graph types. + */ +enum GraphDirection { + UndirectedGraph, + DirectedGraph + //BidirectionalGraph +}; + +/** + * Maps vertices to units with equal block sizes. + */ +template +class BlockedVertexMapper { + +public: + + typedef GraphType graph_type; + typedef typename graph_type::vertex_size_type vertex_size_type; + + /** + * Default constructor. + */ + BlockedVertexMapper() = default; + + /** + * Constructs the blocked vertex mapper. + */ + BlockedVertexMapper(vertex_size_type n_vertices, std::size_t n_units) + : _blocksize(n_vertices / n_units) + { } + + /** + * Returns the unit a vertex is mapped to. + */ + dash::team_unit_t operator()(vertex_size_type v, vertex_size_type n_vertices, + std::size_t n_units, dash::team_unit_t myid) { + int owner = static_cast(v) / (static_cast(n_vertices) / + n_units); + dash::team_unit_t unit { owner }; + return unit; + } + + /** + * Returns the blocksize of a given unit. + */ + std::size_t size(team_unit_t unit) { + return _blocksize; + } + +private: + + std::size_t _blocksize; + +}; + +/** + * Maps vertices to units using a logarithmic function. + */ +template +class LogarithmicVertexMapper { + +public: + + typedef GraphType graph_type; + typedef typename graph_type::vertex_size_type vertex_size_type; + + /** + * Constructs the logarithmic vertex mapper. + * Factors are calculated with the following formula: + * + * factor[unit] = log10((unit + start) * scale) + */ + LogarithmicVertexMapper( + vertex_size_type n_vertices, + std::size_t n_units, + double start = 2, + double scale = 1) + : _blocks(n_units) + { + double factor_sum = 0; + std::vector factors(n_units); + for(double i = 0; i < n_units; ++i) { + factors[i] = log10((i + start) * scale); + factor_sum += factors[i]; + } + int total_vertices = 0; + for(int i = 0; i < n_units; ++i) { + if(i == n_units - 1) { + // avoid double errors + _blocks[i] = n_vertices; + } else { + total_vertices += n_vertices * (factors[i] / factor_sum); + _blocks[i] = total_vertices; + } + } + } + + /** + * Returns the unit a vertex is mapped to. + */ + dash::team_unit_t operator()( + vertex_size_type v, + vertex_size_type n_vertices, + std::size_t n_units, + dash::team_unit_t myid) + { + int owner = n_units - 1; + // TODO: can this be done in O(c)? + for(int i = 0; i < n_units; ++i) { + if(v < _blocks[i]) { + owner = i; + break; + } + } + dash::team_unit_t unit { owner }; + return unit; + } + + /** + * Returns the blocksize of a given unit. + */ + std::size_t size(team_unit_t unit) { + if(unit == 0) { + return _blocks[0]; + } + return _blocks[unit] - _blocks[unit - 1]; + } + +private: + std::vector _blocks; + +}; + +/** + * Index type for vertices. + */ +template +struct VertexIndex { + + typedef VertexIndex self_t; + + /** + * Default constructor. + */ + VertexIndex() = default; + ~VertexIndex() = default; + VertexIndex(const self_t &) = default; + VertexIndex(self_t &&) = default; + self_t & operator=(const self_t &) = default; + + /** + * Index Constructor. + */ + VertexIndex(team_unit_t u, IndexType o) + : unit(u), + offset(o) + { } + + /** The unit holding the referenced vertex */ + team_unit_t unit; + /** The offset of the vertex in local memory space of unit */ + IndexType offset; +}; + +/** + * Vertex type holding properties and references to its corresponding edge + * lists. + */ +template +class Vertex { + + typedef typename GraphType::vertex_properties_type properties_type; + typedef typename GraphType::glob_mem_vert_type gmem_type; + typedef typename gmem_type::index_type index_type; + + template + struct EdgeListIndex { + IndexType index; + IndexType size; + }; + +public: + + /** + * Default constructor needed for GlobRef / GlobSharedRef dereference op. + */ + Vertex() = default; + + /** + * Creates a vertex with given properties. + */ + Vertex( + const properties_type & properties + ) + : properties(properties) + { } + +public: + + /** Properties of this vertex */ + properties_type properties; + EdgeListIndex in_edge_list; + EdgeListIndex out_edge_list; + +}; + +/** + * Proxy type for vertices. Encapsulates logic for data retrieval, data manipulation + * and iterator retrieval. + */ +template +class VertexProxy { + + typedef GraphType graph_type; + typedef IteratorType iterator_type; + typedef VertexProxy self_t; + typedef typename graph_type::vertex_type vertex_type; + typedef typename graph_type::vertex_properties_type properties_type; + typedef typename graph_type::global_vertex_iterator global_vertex_iterator; + typedef typename graph_type::local_vertex_iterator local_vertex_iterator; + + /** + * Handles the iterator ranges for adjacency iteration of the respective + * vertex. + */ + template + class edge_range { + + typedef Parent parent_type; + typedef GlobMemType glob_mem_type; + typedef typename glob_mem_type::index_type index_type; + typedef typename glob_mem_type::global_iterator global_iterator; + typedef typename glob_mem_type::local_iterator local_iterator; + typedef typename parent_type::vertex_type vertex_type; + typedef typename parent_type::graph_type::local_vertex_iterator + local_vertex_iterator; + typedef typename parent_type::graph_type::glob_mem_edge_comb_type + glob_mem_edge_comb_type; + + public: + + /* + * Constructs the range handler object. + */ + edge_range(parent_type & p, glob_mem_type * gmem, + bool is_in_edge_list = false) + : _parent(p), + _glob_mem(gmem), + _is_in_edge_list(is_in_edge_list) + { } + + /** + * Returns global iterator to the first element in the edge list of + * the given vertex. + */ + global_iterator begin() { + return begin(_parent._iterator); + } + + /** + * Returns global iterator past the last element in the edge list of + * the given vertex. + */ + global_iterator end() { + return end(_parent._iterator); + } + + local_iterator lbegin() { + return lbegin(_parent._iterator); + } + + local_iterator lend() { + return lend(_parent._iterator); + } + + private: + + /** + * Begin iterator for global vertex iterators. + */ + template + global_iterator begin(VertexIteratorType it) { + _parent.lazy_load(); + auto lpos = _parent._iterator.lpos(); + auto index = g_it_position(_glob_mem, _parent._vertex); + return global_iterator( + _glob_mem, + lpos.unit, + index + ); + } + + /** + * Begin iterator for local vertex iterators. + */ + global_iterator begin(local_vertex_iterator it) { + _parent.lazy_load(); + auto index = g_it_position(_glob_mem, _parent._vertex); + return global_iterator( + _glob_mem, + _parent._graph->_myid, + index + ); + } + + /** + * End iterator for global vertex iterators. + */ + template + global_iterator end(VertexIteratorType it) { + _parent.lazy_load(); + auto lpos = _parent._iterator.lpos(); + auto index = g_it_position(_glob_mem, _parent._vertex); + auto size = g_it_size(_glob_mem, _parent._vertex); + return global_iterator( + _glob_mem, + lpos.unit, + index + size + ); + } + + /** + * End iterator for local vertex iterators. + */ + global_iterator end(local_vertex_iterator it) { + _parent.lazy_load(); + auto index = g_it_position(_glob_mem, _parent._vertex); + auto size = g_it_size(_glob_mem, _parent._vertex); + return global_iterator( + _glob_mem, + _parent._graph->_myid, + index + size + ); + } + + /** + * Local begin iterator for global vertex iterators. + * Not implemented because a VertexProxy referencing a remote vertex + * does not have any local data. + */ + template + local_iterator lbegin(VertexIteratorType it) { + //TODO: Show "not implemented" message + } + + /** + * Local begin iterator for local vertex iterators. + */ + local_iterator lbegin(local_vertex_iterator it) { + auto index = it_position(_glob_mem, it.pos()); + auto lindex = index * 2; + auto bucket_it = _glob_mem->_buckets.begin(); + std::advance(bucket_it, lindex); + // if bucket empty, advance to end iterator + if(bucket_it->size == 0) { + std::advance(bucket_it, 2); + } + auto position = 0; + // TODO: position WRONG, if there are uncommitted changes + // save cumulated sizes of unattached containers somewhere + if(index > 0) { + position = _glob_mem->_local_bucket_cumul_sizes[index - 1]; + } + // use lightweight constructor to avoid costly increment operations + return local_iterator( + _glob_mem->_buckets.begin(), + _glob_mem->_buckets.end(), + position, + bucket_it, + 0 + ); + } + + /** + * Local end iterator for global vertex iterators. + * Not implemented because a VertexProxy referencing a remote vertex + * does not have any local data. + */ + template + local_iterator lend(VertexIteratorType it) { + //TODO: Show "not implemented" message + } + + /** + * Local end iterator for local vertex iterators. + */ + local_iterator lend(local_vertex_iterator it) { + auto index = it_position(_glob_mem, it.pos()) + 1; + auto lindex = index * 2; + auto bucket_it = _glob_mem->_buckets.begin(); + //std::advance(bucket_it, index); + auto position = 0; + // TODO: position WRONG, if there are uncommitted changes + // save cumulated sizes of unattached containers somewhere + if(index > 0) { + position = _glob_mem->_local_bucket_cumul_sizes[index - 1]; + } + // use lightweight constructor to avoid costly increment operations + return local_iterator( + _glob_mem->_buckets.begin(), + _glob_mem->_buckets.end(), + position, + bucket_it, + 0 + ); + } + + /** + * Determines the position of the edge-list for inbound and outbound + * edge memory spaces. + */ + template + index_type it_position(_GMem * gmem, index_type pos) { + return pos; + } + + /** + * Determines the position of the edge-list for the combined edge memory + * space. + */ + index_type it_position(glob_mem_edge_comb_type * gmem, index_type pos) { + return pos * 2; + } + + /** + * Determines the position of the edge-list for inbound and outbound + * edge memory spaces. + */ + template + index_type g_it_position(_GMem * gmem, vertex_type v) { + //indices for in- and out-edge lists are the same + return v.out_edge_list.index; + } + + /** + * Determines the position of the edge-list for the combined edge memory + * space. + */ + index_type g_it_position(glob_mem_edge_comb_type * gmem, vertex_type v) { + return v.out_edge_list.index * 2; + } + + /** + * Determines the position of the edge-list for inbound and outbound + * edge memory spaces. + */ + template + index_type g_it_size(_GMem * gmem, vertex_type v) { + if(_is_in_edge_list) { + return v.in_edge_list.size; + } + return v.out_edge_list.size; + } + + /** + * Determines the position of the edge-list for the combined edge memory + * space. + */ + index_type g_it_size(glob_mem_edge_comb_type * gmem, vertex_type v) { + return v.out_edge_list.size + v.in_edge_list.size; + } + + private: + + /** Reference to the corresponding VertexProxy object */ + parent_type & _parent; + /** Reference to the GlobMem object of the targeted memory space */ + glob_mem_type * _glob_mem; + bool _is_in_edge_list; + + }; + + typedef edge_range + inout_edge_range_type; + typedef edge_range + edge_range_type; + +public: + + /** + * Default constructor. Explicitly deleted. + */ + VertexProxy() = delete; + + /** + * Constructs the object with a vertex iterator. + */ + VertexProxy(iterator_type it, graph_type * g) + : _iterator(it), + _graph(g), + _out_edges(*this, g->_glob_mem_out_edge), + _in_edges(*this, g->_glob_mem_in_edge, true), + _edges(*this, g->_glob_mem_edge) + { } + + /** + * Returns a range for adjacent outbound edges of the referenced vertex. + */ + inout_edge_range_type & out_edges() { + return _out_edges; + } + + /** + * Returns a range for adjacent inbound edges of the referenced vertex. + */ + inout_edge_range_type & in_edges() { + return _in_edges; + } + + /** + * Returns a range for adjacent edges of the referenced vertex. + */ + edge_range_type & edges() { + return _edges; + } + + /** + * Returns the properties of the referenced vertex. Data is loaded lazily. + */ + properties_type & attributes() { + lazy_load(); + return _vertex.properties; + } + + /** + * Sets the attribute data for the referenced vertex. + */ + void set_attributes(properties_type & prop) { + set_attributes(prop, _iterator); + } + +private: + + /** + * Loads vertex data lazily. + */ + void lazy_load() { + if(!_vertex_loaded) { + _vertex = *_iterator; + _vertex_loaded = true; + } + } + + /** + * Overload of set_aatributes for global pointers. + */ + void set_attributes(properties_type & prop, global_vertex_iterator it) { + _vertex.properties = prop; + auto ref = *_iterator; + ref = _vertex; + } + + /** + * Overload of set_aatributes for local pointers. + */ + void set_attributes(properties_type & prop, local_vertex_iterator it) { + _vertex.properties = prop; + auto ptr = _iterator; + *ptr = _vertex; + } + +private: + + /** Iterator to the referenced vertex */ + iterator_type _iterator; + /** data of the referenced vertex */ + vertex_type _vertex; + /** Whether the vertex data has already been loaded */ + bool _vertex_loaded = false; + /** Pointer to the graph container */ + graph_type * _graph; + /** Range object for outbound edges */ + inout_edge_range_type _out_edges; + /** Range object for inbbound edges */ + inout_edge_range_type _in_edges; + /** Range object for edges */ + edge_range_type _edges; + +}; + +/** + * Edge type holding properties and references to the vertices it belongs to. + */ +template +class Edge { + + typedef typename GraphType::local_vertex_iterator local_vertex_iterator; + typedef typename GraphType::global_vertex_iterator global_vertex_iterator; + typedef typename GraphType::vertex_index_type vertex_index_type; + typedef typename GraphType::edge_properties_type properties_type; + +public: + + /** + * Default constructor needed for GlobRef / GlobSharedRef dereference op. + */ + Edge() = default; + + /** + * Creates an edge. + */ + Edge( + const local_vertex_iterator & source, + const local_vertex_iterator & target, + const properties_type & properties, + const team_unit_t owner + ) + : source(owner, source.pos()), + target(owner, target.pos()), + properties(properties) + { } + + /** + * Creates an edge. + */ + Edge( + const local_vertex_iterator & source, + const global_vertex_iterator & target, + const properties_type & properties, + const team_unit_t owner + ) + : source(owner, source.pos()), + target(target.lpos().unit, target.lpos().index), + properties(properties) + { } + + /** + * Creates an edge. + */ + Edge( + const vertex_index_type & source, + const vertex_index_type & target, + const properties_type & properties + ) + : source(source), + target(target), + properties(properties) + { } + +public: + + /** Properties of this edge */ + properties_type properties; + /** Source vertex the edge is pointing from */ + vertex_index_type source; + /** Target vertex the edge is pointing to */ + vertex_index_type target; + +}; + +/** + * Proxy type for edges. Encapsulates logic for data retrieval and + * manipulation. + */ +template +class EdgeProxy { + + typedef GraphType graph_type; + typedef IteratorType iterator_type; + typedef typename graph_type::edge_type edge_type; + typedef typename graph_type::edge_properties_type properties_type; + typedef typename graph_type::global_vertex_iterator vertex_iterator; + +public: + /** + * Default constructor. Explicitly deleted. + */ + EdgeProxy() = delete; + + /** + * Constructs the edge proxy from a given edge iterator. + */ + EdgeProxy(iterator_type it, graph_type * g) + : _iterator(it), + _graph(g) + { } + + /** + * Returns the properties of the referenced edge. Data is loaded lazily. + */ + properties_type & attributes() { + lazy_load(); + return _edge.properties; + } + + /** + * Returns iglobal iterator to the source vertex. + */ + vertex_iterator source() { + lazy_load(); + return vertex_iterator( + _graph->_glob_mem_vertex, + _edge.source.unit, + _edge.source.offset + ); + } + + /** + * Returns iglobal iterator to the target vertex. + */ + vertex_iterator target() { + lazy_load(); + return vertex_iterator( + _graph->_glob_mem_vertex, + _edge.target.unit, + _edge.target.offset + ); + } + + /** + * Sets the attribute data for the referenced edge. + */ + void set_attributes(properties_type & prop) { + lazy_load(); + _edge.properties = prop; + auto ptr = _iterator; + *ptr = _edge; + } + +private: + + /** + * Loads edge data lazily. + */ + void lazy_load() { + if(!_edge_loaded) { + _edge = *_iterator; + _edge_loaded = true; + } + } + +private: + + /** Iterator to the referenced edge */ + iterator_type _iterator; + /** Data of the referenced edge */ + edge_type _edge; + /** Whether edge data has already been loaded */ + bool _edge_loaded = false; + /** Pointer to the graph container */ + graph_type * _graph; + +}; + +/** + * Default property type holding no data. + */ +struct EmptyProperties { }; + +} + +#endif // DASH__GRAPH__INTERNAL__GRAPH_H__INCLUDED diff --git a/dash/include/dash/memory/GlobHeapCombinedMem.h b/dash/include/dash/memory/GlobHeapCombinedMem.h new file mode 100644 index 000000000..ff5cb7f73 --- /dev/null +++ b/dash/include/dash/memory/GlobHeapCombinedMem.h @@ -0,0 +1,333 @@ +#ifndef DASH__MEMORY__GLOB_HEAP_COMBINED_MEM_H +#define DASH__MEMORY__GLOB_HEAP_COMBINED_MEM_H + +#include +#include + +namespace dash { + +template +class GlobHeapCombinedMem { + +public: + + typedef GlobHeapCombinedMem self_t; + typedef GlobMemType glob_mem_type; + typedef typename glob_mem_type::index_type index_type; + typedef typename glob_mem_type::size_type size_type; + typedef typename glob_mem_type::value_type value_type; + typedef std::vector> bucket_cumul_sizes_map; + typedef std::list glob_mem_list; + typedef GlobPtr global_iterator; + typedef typename glob_mem_type::local_iterator local_iterator; + typedef typename local_iterator::bucket_type bucket_type; + typedef typename std::vector bucket_list; + typedef local_iterator local_pointer; + typedef local_iterator const_local_pointer; + + template + friend class GlobPtr; + +public: + + /** + * Constructor. + */ + GlobHeapCombinedMem(Team & team = dash::Team::All()) + : _bucket_cumul_sizes(team.size()), + _team(&team), + _buckets() + { } + + /** + * Default constructor. Explicitly deleted. + */ + GlobHeapCombinedMem() = delete; + + /** + * Copy constructor. + */ + GlobHeapCombinedMem(const self_t & other) = default; + + /** + * Move constructor. + */ + GlobHeapCombinedMem(self_t && other) = default; + + /** + * Copy-assignment operator. + */ + self_t & operator=(const self_t & other) = default; + + /** + * Move-assignment operator. + */ + self_t & operator=(self_t && other) = default; + + /** + * Adds a GlobMem object to the unifier. + */ + void add_globmem(glob_mem_type & glob_mem) { + // only GlobMem objects with the same Team can be used + if(*_team == glob_mem.team()) { + _glob_mem_list.push_back(&glob_mem); + //update_bucket_sizes(); + } + } + + /** + * Updates memory spaces of underlying GlobMem objects. + * + * Non-collective operation. + */ + void commit() { + update_bucket_sizes(); + // TODO: The bucket list should normally update on every element insertion, + // to always be consistent with the current local memory space + update_bucket_list(); + update_size(); + + _begin = global_iterator(this, 0); + _end = global_iterator(this, _size); + } + + dart_gptr_t dart_gptr_at(team_unit_t unit, + index_type bucket_index, index_type bucket_phase) const { + auto gmem_index = bucket_index % _glob_mem_list.size(); + auto gmem_it = _glob_mem_list.begin(); + std::advance(gmem_it, gmem_index); + auto gmem = *gmem_it; + // translate combined bucket index to index for particular GlobMem object + int gmem_bucket_index = bucket_index / _glob_mem_list.size(); + return gmem->dart_gptr_at(unit, gmem_bucket_index, bucket_phase); + } + + /** + * Iterator to the beginning of the memory space. + */ + global_iterator begin() const { + return _begin; + } + + /** + * Iterator past the end of the memory space. + */ + global_iterator end() const { + return _end; + } + + /** + * Iterator to the beginning of the mamory space's local portion. + */ + local_iterator lbegin() const { + return _lbegin; + } + + /** + * Iterator to the end of the memory space's local portion. + */ + local_iterator lend() const { + return _lend; + } + + /** + * Returns the amount of elements currently available in global memory space. + */ + size_type size() const { + return _size; + } + + /** + * Returns the team containing all units associated with this memory space. + */ + Team & team() const { + return (_team != nullptr) ? *_team : dash::Team::Null(); + } + + /** + * Returns the size of a the memory space belonging to a certain bucket. + */ + size_type container_size(team_unit_t unit, size_type index) const { + size_type bucket_size = _bucket_cumul_sizes[unit][index + + _glob_mem_list.size() - 1]; + if(index > 0) { + bucket_size -= _bucket_cumul_sizes[unit][index - 1]; + } + return bucket_size; + } + + size_type lsize() { + return _local_size; + } + + size_type size(team_unit_t unit) { + return _bucket_cumul_sizes[unit][1]; + } + + size_type size() { + return _size; + } + +private: + + /** + * Combines bucket sizes of all currently added GlobMem objects. + * Resulting order for gmem_0 & gmem_1: + * [unit_0] : [gmem_0 b_0][gmem_1 b_0] ... [gmem_0 b_n][gmem_1 b_n] + * : : : : : + * [unit_n] : [gmem_0 b_0][gmem_1 b_0] ... [gmem_0 b_n][gmem_1 b_n] + */ + void update_bucket_sizes() { + // the index of a bucket directly translates to its corresponding GlobMem + // object: + // example for 2 GlobMem objects: + // bucket 0 -> GlobMem 0 + // bucket 1 -> GlobMem 1 + // bucket 2 -> GlobMem 0 + // ... + if(_glob_mem_list.size() > 0) { + for(int i = 0; i < _bucket_cumul_sizes.size(); ++i) { + int max = 0; + for(auto gmem : _glob_mem_list) { + if(gmem->_bucket_cumul_sizes[i].size() > max) { + max = gmem->_bucket_cumul_sizes[i].size(); + } + } + _bucket_cumul_sizes[i].clear(); + _bucket_cumul_sizes[i].resize(max * _glob_mem_list.size()); + } + int offset = 0; + for(auto gmem : _glob_mem_list) { + auto & gmem_buckets = gmem->_bucket_cumul_sizes; + int pos = offset; + for(int i = 0; i < gmem_buckets.size(); ++i) { + size_type last = 0; + for(int j = 0; j + offset < _bucket_cumul_sizes[i].size(); ++j) { + int gmem_index = j / _glob_mem_list.size(); + // if number of buckets shorter than the maximum, repeat the last + // bucket size for full accumulation + if(gmem_index < gmem_buckets[i].size()) { + last = gmem_buckets[i][gmem_index]; + } + _bucket_cumul_sizes[i][j + offset] += last; + } + } + ++offset; + } + } + } + + /** + * Updates the bucket list. + */ + void update_bucket_list() { + _buckets.clear(); + std::vector> iters; + int max = 0; + if(_glob_mem_list.size() > 0) { + for(auto gmem : _glob_mem_list) { + iters.push_back(std::make_pair(gmem->_buckets.begin(), + gmem->_buckets.end())); + if(max < gmem->_buckets.size()) { + max = gmem->_buckets.size(); + } + } + // TODO: Currently onlys working for GlobMemContiguousMem because of + // the double buckets + for(int i = 0; i < max; i += 2) { + for(auto & it : iters) { + if(it.first != it.second) { + _buckets.push_back(*(it.first)); + ++(it.first); + } + if(it.first != it.second) { + _buckets.push_back(*(it.first)); + ++(it.first); + } + } + } + } + update_local_size(); + update_lbegin(); + update_lend(); + } + +private: + + /** + * Updates the global size. + */ + void update_size() { + _size = 0; + for(auto gmem : _glob_mem_list) { + _size += gmem->_size; + } + } + + /** + * Updates the local size. + */ + void update_local_size() { + _local_size = 0; + for(auto gmem : _glob_mem_list) { + _local_size += gmem->_local_size; + } + } + + // NOTE: copied from GlobHeapMem.h + /** + * Native pointer of the initial address of the local memory of + * a unit. + * + */ + void update_lbegin() noexcept + { + local_iterator unit_lbegin( + // iteration space + _buckets.begin(), _buckets.end(), + // position in iteration space + 0, + // bucket at position in iteration space, + // offset in bucket + _buckets.begin(), 0); + _lbegin = unit_lbegin; + } + + // NOTE: copied from GlobHeapMem.h + /** + * Update internal native pointer of the final address of the local memory + * of a unit. + */ + void update_lend() noexcept + { + local_iterator unit_lend( + // iteration space + _buckets.begin(), _buckets.end(), + // position in iteration space + _local_size, + // bucket at position in iteration space, + // offset in bucket + _buckets.end(), 0); + _lend = unit_lend; + } + +private: + + bucket_list _buckets; + bucket_cumul_sizes_map _bucket_cumul_sizes; + glob_mem_list _glob_mem_list; + Team * _team = nullptr; + size_type _size = 0; + global_iterator _begin; + global_iterator _end; + local_iterator _lbegin; + local_iterator _lend; + size_type _local_size = 0; + +}; + +} + +#endif //DASH__MEMORY__GLOB_HEAP_COMBINED_MEM_H + diff --git a/dash/include/dash/memory/GlobHeapCombinedPtr.h b/dash/include/dash/memory/GlobHeapCombinedPtr.h new file mode 100644 index 000000000..f88c397e1 --- /dev/null +++ b/dash/include/dash/memory/GlobHeapCombinedPtr.h @@ -0,0 +1,755 @@ +#ifndef DASH__MEMORY__GLOB_HEAP_COMBINED_PTR_H__INCLUDED +#define DASH__MEMORY__GLOB_HEAP_COMBINED_PTR_H__INCLUDED + +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + + +namespace dash { + +// Forward-declaration +template +class GlobHeapCombinedMem; + +/** + * Iterator on global buckets. Represents global pointer type. + */ +template +class GlobPtr< + ElementType, + GlobHeapCombinedMem + > +{ + typedef GlobHeapCombinedMem GlobHeapMemType; + typedef GlobPtr self_t; + + template< + typename ElementType_, + class ContainerType_ > + friend std::ostream & operator<<( + std::ostream & os, + const dash::GlobPtr< + ElementType_, + GlobHeapCombinedMem + > & gptr); + + friend GlobPtr; + +public: + typedef typename GlobHeapMemType::index_type index_type; + typedef typename std::make_unsigned::type size_type; + + typedef ElementType value_type; + + typedef GlobSharedRef< value_type, self_t> reference; + typedef GlobSharedRef const_reference; + + typedef value_type * raw_pointer; + + typedef GlobHeapMemType globmem_type; + typedef typename GlobHeapMemType::local_pointer local_pointer; + + typedef struct { + team_unit_t unit; + index_type index; + } local_index; + + /* + * TODO: Implement rebind to allow for the allocation of different types + * than ElementType + * + template + struct rebind { + typedef GlobPtr< + U, + GlobHeapContiguousMem< + typename AllocatorType::template rebind< + typename std::remove_const::type + >::other + > + > other; + }; + */ + +private: + typedef std::vector > + bucket_cumul_sizes_map; + +private: + /// Global memory used to dereference iterated values. + const globmem_type * _globmem = nullptr; + /// Mapping unit id to buckets in the unit's attached local storage. + const bucket_cumul_sizes_map * _bucket_cumul_sizes = nullptr; + /// Pointer to first element in local data space. + local_pointer _lbegin; + /// Current position of the pointer in global canonical index space. + index_type _idx = 0; + /// Maximum position allowed for this pointer. + index_type _max_idx = 0; + /// Unit id of the active unit. + team_unit_t _myid; + /// Unit id at the pointer's current position. + team_unit_t _idx_unit_id; + /// Logical offset in local index space at the pointer's current position. + index_type _idx_local_idx = -1; + /// Local bucket index at the pointer's current position. + index_type _idx_bucket_idx = -1; + /// Element offset in bucket at the pointer's current position. + index_type _idx_bucket_phase = -1; + +public: + /** + * Default constructor. + */ + GlobPtr() + : _globmem(nullptr), + _bucket_cumul_sizes(nullptr), + _idx(0), + _max_idx(0), + _myid(dash::Team::GlobalUnitID()), + _idx_unit_id(DART_UNDEFINED_UNIT_ID), + _idx_local_idx(-1), + _idx_bucket_idx(-1), + _idx_bucket_phase(-1) + { + DASH_LOG_TRACE_VAR("GlobPtr()", _idx); + DASH_LOG_TRACE_VAR("GlobPtr()", _max_idx); + } + + /** + * Constructor, creates a global pointer on global memory from global + * offset in logical storage order. + */ + template + GlobPtr( + const MemSpaceT * gmem, + index_type position = 0) + : _globmem(reinterpret_cast(gmem)), + _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), + _lbegin(_globmem->lbegin()), + _idx(position), + _max_idx(gmem->size() - 1), + _myid(gmem->team().myid()), + _idx_unit_id(0), + _idx_local_idx(0), + _idx_bucket_idx(0), + _idx_bucket_phase(0) + { + DASH_LOG_TRACE("GlobPtr(gmem,idx)", "gidx:", position); + for (auto unit_bucket_cumul_sizes : *_bucket_cumul_sizes) { + DASH_LOG_TRACE_VAR("GlobPtr(gmem,idx)", + unit_bucket_cumul_sizes); + size_type bucket_cumul_size_prev = 0; + for (auto bucket_cumul_size : unit_bucket_cumul_sizes) { + DASH_LOG_TRACE_VAR("GlobPtr(gmem,idx)", bucket_cumul_size); + if (position < bucket_cumul_size) { + DASH_LOG_TRACE_VAR("GlobPtr(gmem,idx)", position); + _idx_bucket_phase -= bucket_cumul_size; + position = 0; + _idx_local_idx = position; + _idx_bucket_phase = position - bucket_cumul_size_prev; + break; + } + bucket_cumul_size_prev = bucket_cumul_size; + ++_idx_bucket_idx; + } + if (position == 0) { + break; + } + // Advance to next unit, adjust position relative to next unit's + // local index space: + position -= unit_bucket_cumul_sizes.back(); + ++_idx_unit_id; + } + DASH_LOG_TRACE("GlobPtr(gmem,idx)", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bucket:", _idx_bucket_idx, + "phase:", _idx_bucket_phase); + } + + /** + * Constructor, creates a global pointer on global memory from unit and + * local offset in logical storage order. + */ + template + GlobPtr( + const MemSpaceT * gmem, + team_unit_t unit, + index_type local_index) + : _globmem(reinterpret_cast(gmem)), + _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), + _lbegin(_globmem->lbegin()), + _idx(0), + _max_idx(gmem->size() - 1), + _myid(gmem->team().myid()), + _idx_unit_id(unit), + _idx_local_idx(0), + _idx_bucket_idx(0), + _idx_bucket_phase(0) + { + DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx)", + "unit:", unit, + "lidx:", local_index); + DASH_ASSERT_LT(unit, _bucket_cumul_sizes->size(), "invalid unit id"); + + for (size_type unit = 0; unit < _idx_unit_id; ++unit) { + auto prec_unit_local_size = (*_bucket_cumul_sizes)[unit].back(); + _idx += prec_unit_local_size; + } + increment(local_index); + DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx) >", + "gidx:", _idx, + "maxidx:", _max_idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bucket:", _idx_bucket_idx, + "phase:", _idx_bucket_phase); + } + + /** + * Constructor, creates a global pointer on global memory from unit, bucket + * index and local offset in logical storage order. + */ + template + GlobPtr( + const MemSpaceT * gmem, + team_unit_t unit, + index_type bucket_index, + index_type local_index) + : _globmem(reinterpret_cast(gmem)), + _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), + _lbegin(_globmem->lbegin()), + _idx(0), + _max_idx(gmem->size() - 1), + _myid(gmem->team().myid()), + _idx_unit_id(unit), + _idx_local_idx(0), + _idx_bucket_idx(0), + _idx_bucket_phase(0) + { + DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx)", + "unit:", unit, + "lidx:", local_index); + DASH_ASSERT_LT(unit, _bucket_cumul_sizes->size(), "invalid unit id"); + + for (size_type unit = 0; unit < _idx_unit_id; ++unit) { + auto prec_unit_local_size = (*_bucket_cumul_sizes)[unit].back(); + _idx += prec_unit_local_size; + } + if(bucket_index > 0) { + local_index += (*_bucket_cumul_sizes)[_idx_unit_id.id][bucket_index - 1]; + } + increment(local_index); + DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx) >", + "gidx:", _idx, + "maxidx:", _max_idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bucket:", _idx_bucket_idx, + "phase:", _idx_bucket_phase); + } + + /** + * Copy constructor. + */ + template + GlobPtr( + const GlobPtr & other) + : _globmem(other._globmem), + _bucket_cumul_sizes(other._bucket_cumul_sizes), + _lbegin(other._lbegin), + _idx(other._idx), + _max_idx(other._max_idx), + _idx_unit_id(other._idx_unit_id), + _idx_local_idx(other._idx_local_idx), + _idx_bucket_idx(other._idx_bucket_idx), + _idx_bucket_phase(other._idx_bucket_phase) + { } + + /** + * Assignment operator. + */ + template + self_t & operator=( + const GlobPtr & other) + { + _globmem = other._globmem; + _bucket_cumul_sizes = other._bucket_cumul_sizes; + _lbegin = other._lbegin; + _idx = other._idx; + _max_idx = other._max_idx; + _idx_unit_id = other._idx_unit_id; + _idx_local_idx = other._idx_local_idx; + _idx_bucket_idx = other._idx_bucket_idx; + _idx_bucket_phase = other._idx_bucket_phase; + } + + /** + * Explicit conversion to \c dart_gptr_t. + * + * \return A DART global pointer to the element at the pointer's + * position + */ + dart_gptr_t dart_gptr() const + { + DASH_LOG_TRACE_VAR("GlobPtr.dart_gptr()", _idx); + // Create global pointer from unit, bucket and phase: + dart_gptr_t dart_gptr = _globmem->dart_gptr_at( + _idx_unit_id, + _idx_bucket_idx, + _idx_bucket_phase); + DASH_LOG_TRACE_VAR("GlobPtr.dart_gptr >", dart_gptr); + return dart_gptr; + } + + /** + * Dereference operator. + * + * \return A global reference to the element at the pointer's position. + */ + reference operator*() const + { + auto lptr = local(); + if (lptr != nullptr) { + return reference(static_cast(lptr)); + } else { + return reference(dart_gptr()); + } + } + + /** + * Subscript operator, returns global reference to element at given + * global index. + */ + reference operator[]( + /// The global position of the element + index_type g_index) const + { + DASH_LOG_TRACE_VAR("GlobPtr.[]()", g_index); + auto git = *this; + git += g_index; + auto lptr = git.local(); + if (lptr != nullptr) { + return reference(lptr); + } else { + auto gref = *git; + DASH_LOG_TRACE_VAR("GlobPtr.[] >", gref); + return gref; + } + } + + /** + * Checks whether the element referenced by this global pointer is in + * the calling unit's local memory. + */ + inline bool is_local() const + { + return (_myid == _idx_unit_id); + } + + /** + * Conversion to local bucket pointer. + */ + local_pointer local() const + { + if (_myid != _idx_unit_id) { + // Iterator position does not point to local element + return nullptr; + } + return (_lbegin + _idx_local_idx); + } + + /** + * Unit and local offset at the pointer's position. + */ + inline local_index lpos() const + { + local_index local_pos; + local_pos.unit = _idx_unit_id; + local_pos.index = _idx_local_idx; + return local_pos; + } + + /** + * Map pointer to global index domain. + */ + inline self_t global() const + { + return *this; + } + + /** + * Position of the pointer in global index space. + */ + inline index_type pos() const + { + return _idx; + } + + /** + * Position of the pointer in global index range. + */ + inline index_type gpos() const + { + return _idx; + } + + /** + * The instance of \c GlobStaticMem used by this pointer to resolve + * addresses in global memory. + */ + inline const globmem_type & globmem() const + { + return *_globmem; + } + + /** + * The instance of \c GlobStaticMem used by this pointer to resolve + * addresses in global memory. + */ + inline globmem_type & globmem() + { + return *_globmem; + } + + /** + * Prefix increment operator. + */ + inline self_t & operator++() + { + increment(1); + return *this; + } + + /** + * Prefix decrement operator. + */ + inline self_t & operator--() + { + decrement(1); + return *this; + } + + /** + * Postfix increment operator. + */ + inline self_t operator++(int) + { + auto result = *this; + increment(1); + return result; + } + + /** + * Postfix decrement operator. + */ + inline self_t operator--(int) + { + auto result = *this; + decrement(1); + return result; + } + + inline self_t & operator+=(index_type offset) + { + increment(offset); + return *this; + } + + inline self_t & operator-=(index_type offset) + { + increment(offset); + return *this; + } + + inline self_t operator+(index_type offset) const + { + auto res = *this; + res.increment(offset); + return res; + } + + inline self_t operator-(index_type offset) const + { + auto res = *this; + res.decrement(offset); + return res; + } + + inline index_type operator+( + const self_t & other) const + { + return _idx + other._idx; + } + + inline index_type operator-( + const self_t & other) const + { + return _idx - other._idx; + } + + template + inline bool operator<(const GlobPtr & other) const + { + return (_idx < other._idx); + } + + template + inline bool operator<=(const GlobPtr & other) const + { + return (_idx <= other._idx); + } + + template + inline bool operator>(const GlobPtr & other) const + { + return (_idx > other._idx); + } + + template + inline bool operator>=(const GlobPtr & other) const + { + return (_idx >= other._idx); + } + + template + inline bool operator==(const GlobPtr & other) const + { + return _idx == other._idx; + } + + template + inline bool operator!=(const GlobPtr & other) const + { + return _idx != other._idx; + } + +private: + /** + * Advance pointer by specified position offset. + */ + void increment(int offset) + { + DASH_LOG_TRACE("GlobPtr.increment()", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase, + "offset:", offset); + _idx += offset; + auto current_bucket_size = + (*_bucket_cumul_sizes)[_idx_unit_id][_idx_bucket_idx]; + if (_idx_local_idx + offset < current_bucket_size) { + DASH_LOG_TRACE("GlobPtr.increment", "position current bucket"); + // element is in bucket currently referenced by this pointer: + _idx_bucket_phase += offset; + _idx_local_idx += offset; + } else { + DASH_LOG_TRACE("GlobPtr.increment", + "position in succeeding bucket"); + // iterate units: + auto unit_id_max = _bucket_cumul_sizes->size() - 1; + for (; _idx_unit_id <= unit_id_max; ++_idx_unit_id) { + if (offset == 0) { + break; + } + auto unit_bkt_sizes = (*_bucket_cumul_sizes)[_idx_unit_id]; + auto unit_bkt_sizes_total = unit_bkt_sizes.back(); + auto unit_num_bkts = unit_bkt_sizes.size(); + DASH_LOG_TRACE("GlobPtr.increment", + "unit:", _idx_unit_id, + "remaining offset:", offset, + "total local bucket size:", unit_bkt_sizes_total); + if (_idx_local_idx + offset >= unit_bkt_sizes_total) { + // offset refers to next unit: + DASH_LOG_TRACE("GlobPtr.increment", + "position in remote range"); + // subtract remaining own local size from remaining offset: + offset -= (unit_bkt_sizes_total - _idx_local_idx); + if (_idx_unit_id == unit_id_max) { + // end pointer, offset exceeds iteration space: + _idx_bucket_idx = unit_num_bkts - 1; + auto last_bkt_size = unit_bkt_sizes.back(); + if (unit_num_bkts > 1) { + last_bkt_size -= unit_bkt_sizes[_idx_bucket_idx-1]; + } + _idx_bucket_phase = last_bkt_size + offset; + _idx_local_idx += unit_bkt_sizes_total + offset; + break; + } + _idx_local_idx = 0; + _idx_bucket_idx = 0; + _idx_bucket_phase = 0; + } else { + // offset refers to current unit: + DASH_LOG_TRACE("GlobPtr.increment", + "position in local range", + "current bucket phase:", _idx_bucket_phase, + "cumul. bucket sizes:", unit_bkt_sizes); + _idx_local_idx += offset; + // iterate the unit's bucket sizes: + for (; _idx_bucket_idx < unit_num_bkts; ++_idx_bucket_idx) { + auto cumul_bucket_size = unit_bkt_sizes[_idx_bucket_idx]; + if (_idx_local_idx < cumul_bucket_size) { + auto cumul_prev = _idx_bucket_idx > 0 + ? unit_bkt_sizes[_idx_bucket_idx-1] + : 0; + // offset refers to current bucket: + _idx_bucket_phase = _idx_local_idx - cumul_prev; + offset = 0; + break; + } + } + if (offset == 0) { + break; + } + } + } + } + DASH_LOG_TRACE("GlobPtr.increment >", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase); + } + + /** + * Decrement pointer by specified position offset. + */ + void decrement(int offset) + { + DASH_LOG_TRACE("GlobPtr.decrement()", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase, + "offset:", -offset); + if (offset > _idx) { + DASH_THROW(dash::exception::OutOfRange, + "offset " << offset << " is out of range"); + } + _idx -= offset; + if (offset <= _idx_bucket_phase) { + // element is in bucket currently referenced by this pointer: + _idx_bucket_phase -= offset; + _idx_local_idx -= offset; + } else { + // iterate units: + auto first_unit = _idx_unit_id; + for (; _idx_unit_id >= 0; --_idx_unit_id) { + auto unit_bkt_sizes = (*_bucket_cumul_sizes)[_idx_unit_id]; + auto unit_bkt_sizes_total = unit_bkt_sizes.back(); + auto unit_num_bkts = unit_bkt_sizes.size(); + if (_idx_unit_id != first_unit) { + --offset; + _idx_bucket_idx = unit_num_bkts - 1; + _idx_local_idx = unit_bkt_sizes_total - 1; + auto last_bkt_size = unit_bkt_sizes.back(); + if (unit_num_bkts > 1) { + last_bkt_size -= unit_bkt_sizes[_idx_bucket_idx-1]; + } + _idx_bucket_phase = last_bkt_size - 1; + } + if (offset <= _idx_local_idx) { + // offset refers to current unit: + // iterate the unit's bucket sizes: + for (; _idx_bucket_idx >= 0; --_idx_bucket_idx) { + auto bucket_size = unit_bkt_sizes[_idx_bucket_idx]; + if (offset <= _idx_bucket_phase) { + // offset refers to current bucket: + _idx_local_idx -= offset; + _idx_bucket_phase -= offset; + offset = 0; + break; + } else { + // offset refers to preceeding bucket: + _idx_local_idx -= (_idx_bucket_phase + 1); + offset -= (_idx_bucket_phase + 1); + _idx_bucket_phase = unit_bkt_sizes[_idx_bucket_idx-1] - 1; + } + } + } else { + // offset refers to preceeding unit: + offset -= _idx_local_idx; + } + if (offset == 0) { + break; + } + } + } + DASH_LOG_TRACE("GlobPtr.decrement >", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase); + } + +}; // class GlobPtr + +/** + * Resolve the number of elements between two global bucket pointers. + * + * \complexity O(1) + * + * \ingroup Algorithms + */ +template< + typename ElementType, + class GlobMemType > +auto distance( + /// Global pointer to the first position in the global sequence + const dash::GlobPtr< + ElementType, GlobHeapCombinedMem + > & first, + /// Global pointer to the final position in the global sequence + const dash::GlobPtr< + ElementType, GlobHeapCombinedMem + > & last) +-> typename GlobHeapCombinedMem::index_type +{ + return last - first; +} + +template< + typename ElementType, + class GlobMemType > +std::ostream & operator<<( + std::ostream & os, + const dash::GlobPtr< + ElementType, + GlobHeapCombinedMem + > & gptr) +{ + std::ostringstream ss; + ss << "dash::GlobPtr<" << typeid(ElementType).name() << ">(" + << "gidx:" << gptr._idx << ", (" + << "unit:" << gptr._idx_unit_id << ", " + << "lidx:" << gptr._idx_local_idx << "), (" + << "bidx:" << gptr._idx_bucket_idx << ", " + << "bphase:" << gptr._idx_bucket_phase << ")" + << ")"; + return operator<<(os, ss.str()); +} + +} // namespace dash + +#endif // DASH__MEMORY__GLOB_HEAP_COMBINED_PTR_H__INCLUDED diff --git a/dash/include/dash/memory/GlobHeapContiguousLocalPtr.h b/dash/include/dash/memory/GlobHeapContiguousLocalPtr.h new file mode 100644 index 000000000..06e6dbcb4 --- /dev/null +++ b/dash/include/dash/memory/GlobHeapContiguousLocalPtr.h @@ -0,0 +1,509 @@ +#ifndef DASH__MEMORY__GLOB_HEAP_CONTIGUOUS_LOCAL_PTR_H__INCLUDED +#define DASH__MEMORY__GLOB_HEAP_CONTIGUOUS_LOCAL_PTR_H__INCLUDED + +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + + +namespace dash { + +// TODO: Remove this class and add template parameter for bucket list type to +// GlobHeapLocalPtr +/** + * Iterator on local buckets. Represents local pointer type. + */ +template< + typename ElementType, + typename IndexType, + typename PointerType = ElementType *, + typename ReferenceType = ElementType & > +class GlobHeapContiguousLocalPtr +: public std::iterator< + std::random_access_iterator_tag, + ElementType, + IndexType, + PointerType, + ReferenceType > +{ + template + friend class GlobHeapContiguousLocalPtr; + + template + friend std::ostream & dash::operator<<( + std::ostream & os, + const dash::GlobHeapContiguousLocalPtr & it); + +private: + typedef GlobHeapContiguousLocalPtr + self_t; + +public: + typedef IndexType index_type; + typedef typename std::make_unsigned::type size_type; + +/// Type definitions required for std::iterator_traits: +public: + typedef std::random_access_iterator_tag iterator_category; + typedef IndexType difference_type; + typedef ElementType value_type; + typedef ElementType * pointer; + typedef ElementType & reference; + + typedef internal::glob_dynamic_contiguous_mem_bucket_type + bucket_type; + + typedef typename std::vector + bucket_list; + +private: + + typedef typename bucket_list::iterator + bucket_iterator; + + typedef typename bucket_list::const_iterator + const_bucket_iterator; + +public: + template + GlobHeapContiguousLocalPtr( + const BucketIter & bucket_first, + const BucketIter & bucket_last, + index_type position, + const BucketIter & bucket_it, + index_type bucket_phase) + : _bucket_first(bucket_first), + _bucket_last(bucket_last), + _idx(position), + _bucket_it(bucket_it), + _bucket_phase(bucket_phase), + _is_nullptr(false) + { } + + template + GlobHeapContiguousLocalPtr( + const BucketIter & bucket_first, + const BucketIter & bucket_last, + index_type position) + : _bucket_first(bucket_first), + _bucket_last(bucket_last), + _idx(position), + _bucket_it(bucket_first), + _bucket_phase(0), + _is_nullptr(false) + { + DASH_LOG_TRACE_VAR("GlobHeapContiguousLocalPtr(idx)", position); +#ifdef DASH_ENABLE_TRACE_LOGGING + index_type bucket_idx = 0; +#endif + for (_bucket_it = _bucket_first; + _bucket_it != _bucket_last; ++_bucket_it) { + if (position >= _bucket_it->size) { + position -= _bucket_it->size; + } else { + _bucket_phase = position; + break; + } +#ifdef DASH_ENABLE_TRACE_LOGGING + ++bucket_idx; +#endif + } + DASH_LOG_TRACE("GlobHeapContiguousLocalPtr(idx) >", + "bucket:", bucket_idx, + "phase:", _bucket_phase); + } + + GlobHeapContiguousLocalPtr() = default; + + GlobHeapContiguousLocalPtr(const self_t & other) + : _bucket_first(other._bucket_first), + _bucket_last(other._bucket_last), + _idx(other._idx), + _bucket_it(other._bucket_it), + _bucket_phase(other._bucket_phase), + _is_nullptr(other._is_nullptr) + { } + + self_t & operator=(const self_t & rhs) + { + if (this != std::addressof(rhs)) { + _bucket_first = rhs._bucket_first; + _bucket_last = rhs._bucket_last; + _idx = rhs._idx; + _bucket_it = rhs._bucket_it; + _bucket_phase = rhs._bucket_phase; + _is_nullptr = rhs._is_nullptr; + } + return *this; + } + + /** + * Conversion to const iterator. + */ + template + operator GlobHeapContiguousLocalPtr() const + { + if (_is_nullptr) { + return GlobHeapContiguousLocalPtr(nullptr); + } + return GlobHeapContiguousLocalPtr( + _bucket_first, + _bucket_last, + _idx, + _bucket_it, + _bucket_phase); + } + + GlobHeapContiguousLocalPtr(std::nullptr_t) + : _is_nullptr(true) + { } + + self_t & operator=(std::nullptr_t) + { + _is_nullptr = true; + return *this; + } + + inline bool operator==(std::nullptr_t) const + { + return _is_nullptr; + } + + inline bool operator!=(std::nullptr_t) const + { + return !_is_nullptr; + } + + /** + * Dereference operator. + */ + reference operator*() + { + DASH_ASSERT(!_is_nullptr); + if (_bucket_phase > _bucket_it->size) { + DASH_THROW(dash::exception::OutOfRange, + "dereferenced position " << _idx << " is out of range: " << + "bucket phase: " << _bucket_phase << ", " << + "bucket size: " << _bucket_it->size); + } + return _bucket_it->lptr[_bucket_phase]; + } + + /** + * Random access operator. + */ + reference operator[](index_type offset) + { + DASH_ASSERT(!_is_nullptr); + if (_bucket_phase + offset < _bucket_it->size) { + // element is in bucket currently referenced by this iterator: + return _bucket_it->lptr[_bucket_phase + offset]; + } else { + // find bucket containing element at given offset: + for (auto b_it = _bucket_it; b_it != _bucket_last; ++b_it) { + if (offset >= b_it->size) { + offset -= b_it->size; + } else if (offset < b_it->size) { + return b_it->lptr[offset]; + } + } + } + DASH_THROW(dash::exception::OutOfRange, + "dereferenced position " << _idx + offset << " " << + "is out of range: pointer position: " << _idx << ", " << + "offset: " << offset); + } + + /** + * Conversion to native pointer. + * + * Use with caution: This conversion returns a pointer a that does not + * iterate over buckets, pointer arithmetics may lead to undefined + * behaviour. + */ + explicit operator pointer() const + { + DASH_LOG_TRACE("GlobHeapContiguousLocalPtr.pointer()"); + pointer lptr = nullptr; + if (_is_nullptr) { + DASH_LOG_TRACE("GlobHeapContiguousLocalPtr.pointer", "is nullptr"); + } else { + auto bucket_size = _bucket_it->size; + // This iterator type represents a local pointer so no bounds checks + // have to be performed in pointer arithmetics. + // Moving a pointer to out-of-bounds address is allowed, however + // dereferencing it will lead to segfault. This is a prerequisite for + // many common pointer arithmetic use cases. + // Example: + // value = *((globmem.lend() + 2) - 3); + // is a valid operation and equivalent to + // value = *(globmem.lend() + (2 - 3)); + // as it creates a temporary pointer to an address beyond _lend (+2) + // which is then moved back into valid memory range (-3). + if (_bucket_it == _bucket_last) { + DASH_LOG_TRACE("GlobHeapContiguousLocalPtr.pointer", "position at lend"); + } else if (_bucket_phase >= bucket_size) { + DASH_LOG_TRACE("GlobHeapContiguousLocalPtr.pointer", + "bucket size:", bucket_size, ",", + "bucket phase:", _bucket_phase); + DASH_LOG_TRACE("GlobHeapContiguousLocalPtr.pointer", + "note: iterator position out of bounds (lend?)"); + } + lptr = _bucket_it->lptr + _bucket_phase; + } + DASH_LOG_TRACE_VAR("GlobHeapContiguousLocalPtr.pointer >", lptr); + return lptr; + } + + self_t & operator++() + { + increment(1); + return *this; + } + + self_t & operator--() + { + decrement(1); + return *this; + } + + self_t & operator++(int) + { + auto res = *this; + increment(1); + return res; + } + + self_t & operator--(int) + { + auto res = *this; + decrement(1); + return res; + } + + self_t & operator+=(int offset) + { + increment(offset); + return *this; + } + + self_t & operator-=(int offset) + { + decrement(offset); + return *this; + } + + self_t operator+(int offset) const + { + auto res = *this; + res += offset; + return res; + } + + self_t operator-(int offset) const + { + auto res = *this; + res -= offset; + return res; + } + + inline index_type operator+( + const self_t & other) const + { + return _idx + other._idx; + } + + inline index_type operator-( + const self_t & other) const + { + return _idx - other._idx; + } + + template + inline bool operator<(const GlobHeapContiguousLocalPtr & other) const + { + return (_idx < other._idx); + } + + template + inline bool operator<=(const GlobHeapContiguousLocalPtr & other) const + { + return (_idx <= other._idx); + } + + template + inline bool operator>(const GlobHeapContiguousLocalPtr & other) const + { + return (_idx > other._idx); + } + + template + inline bool operator>=(const GlobHeapContiguousLocalPtr & other) const + { + return (_idx >= other._idx); + } + + template + inline bool operator==(const GlobHeapContiguousLocalPtr & other) const + { + return (this == std::addressof(other) || _idx == other._idx); + } + + template + inline bool operator!=(const GlobHeapContiguousLocalPtr & other) const + { + return !(*this == other); + } + + /** + * Whether the pointer references an element in local memory space. + * + * \return true + */ + constexpr bool is_local() const + { + return true; + } + + /** + * Position of the pointer relative to its referenced memory space. + */ + index_type pos() const + { + return _idx; + } + +private: + /** + * Advance pointer by specified position offset. + */ + void increment(int offset) + { + DASH_ASSERT(!_is_nullptr); + _idx += offset; + if (_bucket_phase + offset < _bucket_it->size) { + // element is in bucket currently referenced by this iterator: + _bucket_phase += offset; + } else { + // find bucket containing element at given offset: + for (; _bucket_it != _bucket_last; ++_bucket_it) { + if (offset + _bucket_phase >= _bucket_it->size) { + offset -= _bucket_it->size - _bucket_phase; + _bucket_phase = 0; + } else if (offset < _bucket_it->size) { + _bucket_phase = offset; + break; + } + } + } + // end iterator + if (_bucket_it == _bucket_last) { + _bucket_phase = offset; + } + } + + /** + * Decrement pointer by specified position offset. + */ + void decrement(int offset) + { + DASH_ASSERT(!_is_nullptr); + if (offset > _idx) { + DASH_THROW(dash::exception::OutOfRange, + "offset " << offset << " is out of range"); + } + _idx -= offset; + if (offset <= _bucket_phase) { + // element is in bucket currently referenced by this iterator: + _bucket_phase -= offset; + } else { + offset -= _bucket_phase; + // find bucket containing element at given offset: + for (; _bucket_it != _bucket_first; --_bucket_it) { + if (offset >= _bucket_it->size) { + offset -= _bucket_it->size; + } else if (offset < _bucket_it->size) { + _bucket_phase = _bucket_it->size - offset; + break; + } + } + } + if (_bucket_it == _bucket_first) { + _bucket_phase = _bucket_it->size - offset; + } + if (false) { + DASH_THROW(dash::exception::OutOfRange, + "offset " << offset << " is out of range"); + } + } + +private: + bucket_iterator _bucket_first; + bucket_iterator _bucket_last; + index_type _idx = 0; + bucket_iterator _bucket_it; + index_type _bucket_phase = 0; + bool _is_nullptr = false; + +}; // class GlobHeapContiguousLocalPtr + +/** + * Resolve the number of elements between two local bucket iterators. + * + * \complexity O(1) + * + * \ingroup Algorithms + */ +template< + typename ElementType, + typename IndexType, + class Pointer, + class Reference> +auto distance( + /// Global iterator to the first position in the global sequence + const dash::GlobHeapContiguousLocalPtr< + ElementType, IndexType, Pointer, Reference> & first, + /// Global iterator to the final position in the global sequence + const dash::GlobHeapContiguousLocalPtr< + ElementType, IndexType, Pointer, Reference> & last) +-> IndexType +{ + return last - first; +} + +template< + typename ElementType, + typename IndexType, + class Pointer, + class Reference> +std::ostream & operator<<( + std::ostream & os, + const dash::GlobHeapContiguousLocalPtr< + ElementType, IndexType, Pointer, Reference> & it) +{ + std::ostringstream ss; + ElementType * lptr = static_cast(it); + ss << "dash::GlobHeapContiguousLocalPtr<" + << typeid(ElementType).name() << ">" + << "(" + << "idx:" << it._idx << ", " + << "bp:" << it._bucket_phase << ", " + << "lptr:" << lptr + << ")"; + return operator<<(os, ss.str()); +} + +} // namespace dash + +#endif // DASH__MEMORY__GLOB_HEAP_CONTIGUOUS_LOCAL_PTR_H__INCLUDED diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h new file mode 100644 index 000000000..f8ede2a85 --- /dev/null +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -0,0 +1,499 @@ +#ifndef DASH__MEMORY__GLOB_HEAP_CONTIGUOUS_MEM_H_ +#define DASH__MEMORY__GLOB_HEAP_CONTIGUOUS_MEM_H_ + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include +#include + +namespace dash { + +namespace internal { + +template +struct container_data { + + typedef ContainerType container_type; + typedef typename ContainerType::value_type value_type; + typedef typename ContainerType::difference_type index_type; + typedef typename GlobHeapContiguousLocalPtr::bucket_list::difference_type bucket_index_type; + + container_data(bucket_index_type bkt) + : bucket(bkt) + { } + + container_type container; + bucket_index_type bucket; + +}; + +} + +//TODO: adapt to GlobMem concept +/** + * Global memory space for multiple, dynamically allocated contiguous memory + * regions. + */ +template +class GlobHeapContiguousMem { + +private: + typedef GlobHeapContiguousMem self_t; + +public: + typedef ContainerType container_type; + typedef internal::container_data data_type; + typedef std::vector container_list_type; + typedef typename ContainerType::value_type value_type; + typedef typename ContainerType::difference_type index_type; + typedef GlobHeapContiguousLocalPtr local_iterator; + typedef GlobPtr global_iterator; + typedef typename ContainerType::size_type size_type; + typedef typename local_iterator::bucket_type bucket_type; + // must be List because of GlobHeapLocalPtr + typedef typename std::vector bucket_list_type; + typedef typename container_list_type::difference_type bucket_index_type; + typedef local_iterator local_pointer; + typedef local_iterator const_local_pointer; + typedef std::vector unit_cumul_sizes_map; + typedef std::vector> bucket_cumul_sizes_map; + typedef std::vector + local_bucket_cumul_sizes_type; + + template + friend class dash::GlobPtr; + friend class GlobHeapCombinedMem; + template + friend class VertexProxy; + +public: + + /** + * Constructor. + */ + GlobHeapContiguousMem(Team & team = dash::Team::All()) + : _buckets(), + _team(&team), + _teamid(team.dart_id()), + _nunits(team.size()), + _myid(team.myid()), + _bucket_cumul_sizes(team.size()), + _unit_cumul_sizes(team.size()) + { + // set 1 bucket for every unit + for(auto & unit_bkt : _bucket_cumul_sizes) { + unit_bkt.resize(1); + } + } + + /** + * Adds a new bucket into memory space. + */ + bucket_index_type add_container(size_type n_elements) { + _local_bucket_cumul_sizes.resize(_local_bucket_cumul_sizes.size() + 1); + // create bucket data and add to bucket list + bucket_type cont_bucket { + 0, + nullptr + }; + bucket_type unattached_cont_bucket { + 0, + nullptr + }; + _buckets.push_back(cont_bucket); + // for global iteration, only _container's bucket is needed + + _unattached_containers.emplace_back(_buckets.size() - 1); + auto & unattached_container = _unattached_containers.back().container; + unattached_container.reserve(n_elements); + + _buckets.push_back(unattached_cont_bucket); + return _unattached_containers.size() - 1; + } + + /** + * Destructor, collectively frees underlying global memory. + */ + ~GlobHeapContiguousMem() { } + + /** + * Default constructor. Explicitly deleted. + */ + GlobHeapContiguousMem() = delete; + + /** + * Copy constructor. + */ + GlobHeapContiguousMem(const self_t & other) = default; + + /** + * Move constructor. + */ + GlobHeapContiguousMem(self_t && other) = default; + + /** + * Copy-assignment operator. + */ + self_t & operator=(const self_t & other) = default; + + /** + * Move-ssignment operator. + */ + self_t & operator=(self_t && other) = default; + + /** + * Commit local changes in memory to global memory space. + * + * Collective operation. + */ + void commit() { + // Gather information about the max amount of containers a single unit + // currently holds + std::vector bucket_count(_team->size()); + size_type my_bucket_count = _unattached_containers.size(); + DASH_ASSERT_RETURNS( + dart_allgather(&my_bucket_count, bucket_count.data(), + sizeof(size_type), DART_TYPE_BYTE, _team->dart_id()), + DART_OK + ); + auto max_buckets = std::max_element(bucket_count.begin(), + bucket_count.end()); + + container_type * new_container = new container_type(); + new_container->reserve(_local_size); + int count = 0; + int elements = 0; + int bucket_num = 0; + size_type bucket_cumul = 0; + auto unattached_container_it = _unattached_containers.begin(); + int elements_before = 0; + bucket_type * last_bucket; + for(auto & bucket : _buckets) { + elements_before = elements; + // move data to new range + if(bucket.lptr != nullptr) { + // TODO: optimize memory usage (delete already moved elements) + new_container->insert(new_container->end(), bucket.lptr, + bucket.lptr + bucket.size); + elements += bucket.size; + } + // bucket list alternates between attached and unattached buckets. + // if bucket is already attached: + if(count % 2 == 0) { + if(bucket.size > 0) { + bucket.lptr = new_container->data() + elements_before; + } + last_bucket = &bucket; + } else { + // if bucket is unattached: + if(bucket.size > 0) { + if(last_bucket->size == 0) { + last_bucket->lptr = new_container->data() + elements_before; + } + last_bucket->size += bucket.size; + bucket.size = 0; + bucket.lptr = nullptr; + unattached_container_it->container.clear(); + ++unattached_container_it; + } + // update cumulated bucket sizes + bucket_cumul += last_bucket->size; + _local_bucket_cumul_sizes[bucket_num] = bucket_cumul; + ++bucket_num; + } + ++count; + } + _bucket_cumul_sizes[_myid][0] = _local_size; + + // detach old container location from global memory space, if it has + // been attached before + if(_dart_gptr != DART_GPTR_NULL) { + DASH_ASSERT_RETURNS( + dart_team_memderegister(_dart_gptr), + DART_OK + ); + } + + _container.reset(new_container); + + // attach new container location to global memory space + dash::dart_storage ds(_container->size()); + DASH_ASSERT_RETURNS( + dart_team_memregister( + _team->dart_id(), + ds.nelem, + ds.dtype, + _container->data(), + &_dart_gptr), + DART_OK + ); + + // TODO: can now be simplified + // distribute bucket sizes between all units + auto bucket_amount = _bucket_cumul_sizes[_myid].size(); + std::vector bucket_sizes(bucket_amount * _team->size()); + std::vector local_buckets(_bucket_cumul_sizes[_myid]); + DASH_ASSERT_RETURNS( + dart_allgather(local_buckets.data(), bucket_sizes.data(), + sizeof(size_type) * local_buckets.size(), DART_TYPE_BYTE, _team->dart_id()), + DART_OK + ); + _size = 0; + auto begin = bucket_sizes.begin(); + for(int i = 0; i < _team->size(); ++i) { + auto end = begin + bucket_amount; + std::copy(begin, end, _bucket_cumul_sizes[i].begin()); + begin = end; + _size += *(end - 1); + } + + for(int i = 0; i < _team->size(); ++i) { + if(i == 0) { + _unit_cumul_sizes[i] = 0; + } else { + _unit_cumul_sizes[i] = _unit_cumul_sizes[i - 1] + + _bucket_cumul_sizes[i - 1].back(); + } + } + + // update local iterators + update_lbegin(); + update_lend(); + + // update global iterators + _begin = global_iterator(this, 0); + _end = global_iterator(this, _size); + } + + /** + * Iterator to the beginning of the memory space. + */ + global_iterator begin() const { + return _begin; + } + + /** + * Iterator to the end of the memory space. + */ + global_iterator end() const { + return _end; + } + + /** + * Iterator to the beginning of the memory space's local portion. + */ + local_iterator lbegin() const { + // TODO: use iterator of _container, if _unattached_containers do not + // contain any data + return _lbegin; + } + + /** + * Iterator to the end of the memory space's local portion. + */ + local_iterator lend() const { + return _lend; + } + + /** + * Insert value at the end of the given bucket. + */ + local_iterator push_back(bucket_index_type index, value_type & val) { + // we don't want a realloc of _container because this changes the memory + // location, which invalidates global pointers of other units + auto & unatt = _unattached_containers[index]; + unatt.container.push_back(val); + auto & bucket = _buckets[unatt.bucket]; + bucket.size = unatt.container.size(); + bucket.lptr = unatt.container.data(); + ++_local_size; + + update_lbegin(); + update_lend(); + return local_iterator( + _buckets.begin(), + _buckets.end(), + _local_size - 1, + _buckets.begin() + _local_size, + bucket.size - 1 + ); + } + + /** + * Returns the global size of a given bucket. + */ + size_type container_size(index_type index) const { + if(index <= 0) { + return _local_bucket_cumul_sizes[index]; + } + return _local_bucket_cumul_sizes[index] - + _local_bucket_cumul_sizes[index - 1]; + } + + index_type container_begin(index_type index) const { + if(index <= 0) { + return 0; + } + return _local_bucket_cumul_sizes[index - 1]; + } + + /** + * Returns the amount of elements in global memory space. + */ + size_type size() const { + return _size; + } + + /** + * Returns the team containing all units associated with this memory space. + */ + Team & team() const { + return *_team; + } + + // NOTE: method copied from GlobHeapMem.h + /** + * Global pointer referencing an element position in a unit's bucket. + */ + dart_gptr_t dart_gptr_at( + /// Unit id mapped to address in global memory space. + team_unit_t unit, + /// Index of bucket containing the referenced address. + bucket_index_type bucket_index, + /// Offset of the referenced address in the bucket's memory space. + index_type bucket_phase) const + { + DASH_LOG_DEBUG("GlobDynamicMem.dart_gptr_at(u,bi,bp)", + unit, bucket_index, bucket_phase); + if (_nunits == 0) { + DASH_THROW(dash::exception::RuntimeError, "No units in team"); + } + auto dart_gptr = _dart_gptr; + if (DART_GPTR_ISNULL(dart_gptr)) { + DASH_LOG_TRACE("GlobDynamicMem.dart_gptr_at", + "bucket.gptr is DART_GPTR_NULL"); + dart_gptr = DART_GPTR_NULL; + } else { + size_type bucket_start; + if(bucket_index > 0) { + bucket_start = _bucket_cumul_sizes[unit][bucket_index - 1]; + } else { + bucket_start = 0; + } + // Move dart_gptr to unit and local offset: + DASH_ASSERT_RETURNS( + dart_gptr_setunit(&dart_gptr, unit), + DART_OK); + DASH_ASSERT_RETURNS( + dart_gptr_incaddr( + &dart_gptr, + (bucket_start + bucket_phase) * sizeof(value_type)), + DART_OK); + } + DASH_LOG_DEBUG("GlobDynamicMem.dart_gptr_at >", dart_gptr); + return dart_gptr; + } + + value_type & get(index_type index) { + return (*_container)[index]; + } + + void set(index_type index, value_type value) { + (*_container)[index] = value; + } + + size_type lsize() { + return _local_size; + } + + size_type size(team_unit_t unit) { + return _bucket_cumul_sizes[unit][0]; + } + + size_type size() { + return _size; + } + +private: + + /** + * Native pointer of the initial address of the local memory of + * a unit. + * + */ + void update_lbegin() noexcept + { + // cannot use lightweight constructor here, because the first bucket might + // be empty + local_iterator unit_lbegin(_buckets.begin(), _buckets.end(), 0); + _lbegin = unit_lbegin; + } + + /** + * Update internal native pointer of the final address of the local memory + * of a unit. + */ + void update_lend() noexcept + { + local_iterator unit_lend( + _buckets.begin(), _buckets.end(), _local_size, + _buckets.end(), 0); + _lend = unit_lend; + } + +private: + + /** List of buckets for GlobHeapLocalPtr */ + bucket_list_type _buckets; + /** Container holding globally visible data */ + std::unique_ptr _container; + /** List of containers holding locally visible data of each bucket */ + container_list_type _unattached_containers; + /** DART gptr of _container */ + dart_gptr_t _dart_gptr = DART_GPTR_NULL; + /** Team associated with this memory space */ + Team * _team; + /** ID of the team */ + dart_team_t _teamid; + /** Number of units in the team */ + size_type _nunits = 0; + /** ID of this unit */ + team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; + /** Iterator to the beginning ot the memory space */ + global_iterator _begin; + /** Iterator to the end ot the memory space */ + global_iterator _end; + /** Iterator to the beginning of this memory space's local portion */ + local_iterator _lbegin; + /** Iterator to the end of this memory space's local portion */ + local_iterator _lend; + /** Accumulated sizes of the buckets of each unit. See GlobHeapMem */ + bucket_cumul_sizes_map _bucket_cumul_sizes; + local_bucket_cumul_sizes_type _local_bucket_cumul_sizes; + /** Accumulated sizes of elements per unit */ + unit_cumul_sizes_map _unit_cumul_sizes; + /** Global size of the memory space */ + size_type _size = 0; + /** Local size of the memory space */ + size_type _local_size = 0; + +}; + +} + +#endif // DASH_MEMORY__GLOB_HEAP_CONTIGUOUS_MEM_H_ + diff --git a/dash/include/dash/memory/GlobHeapContiguousPtr.h b/dash/include/dash/memory/GlobHeapContiguousPtr.h new file mode 100644 index 000000000..616f8aa46 --- /dev/null +++ b/dash/include/dash/memory/GlobHeapContiguousPtr.h @@ -0,0 +1,720 @@ +#ifndef DASH__MEMORY__GLOB_HEAP_CONTIGUOUS_PTR_H__INCLUDED +#define DASH__MEMORY__GLOB_HEAP_CONTIGUOUS_PTR_H__INCLUDED + +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + + +namespace dash { + +// Forward-declaration +template +class GlobHeapContiguousMem; + +/** + * Iterator on global buckets. Represents global pointer type. + */ +template +class GlobPtr< + ElementType, + GlobHeapContiguousMem + > +{ + typedef GlobHeapContiguousMem GlobHeapMemType; + typedef GlobPtr self_t; + + template< + typename ElementType_, + class ContainerType_ > + friend std::ostream & operator<<( + std::ostream & os, + const dash::GlobPtr< + ElementType_, + GlobHeapContiguousMem + > & gptr); + +public: + typedef typename GlobHeapMemType::index_type index_type; + typedef typename std::make_unsigned::type size_type; + + typedef ElementType value_type; + + typedef GlobSharedRef< value_type, self_t> reference; + typedef GlobSharedRef const_reference; + + typedef value_type * raw_pointer; + + typedef GlobHeapMemType globmem_type; + typedef typename GlobHeapMemType::local_pointer local_pointer; + + typedef struct { + team_unit_t unit; + index_type index; + } local_index; + + /* + * TODO: Implement rebind to allow for the allocation of different types + * than ElementType + * + template + struct rebind { + typedef GlobPtr< + U, + GlobHeapContiguousMem< + typename AllocatorType::template rebind< + typename std::remove_const::type + >::other + > + > other; + }; + */ + +private: + typedef std::vector> bucket_cumul_sizes_map; + typedef std::vector unit_cumul_sizes_map; + +private: + /// Global memory used to dereference iterated values. + const globmem_type * _globmem = nullptr; + /// Mapping unit id to buckets in the unit's attached local storage. + const bucket_cumul_sizes_map * _bucket_cumul_sizes = nullptr; + /// Mapping unit id element count + const unit_cumul_sizes_map * _unit_cumul_sizes = nullptr; + /// Pointer to first element in local data space. + local_pointer _lbegin; + /// Current position of the pointer in global canonical index space. + index_type _idx = 0; + /// Maximum position allowed for this pointer. + index_type _max_idx = 0; + /// Unit id of the active unit. + team_unit_t _myid; + /// Unit id at the pointer's current position. + team_unit_t _idx_unit_id; + /// Logical offset in local index space at the pointer's current position. + index_type _idx_local_idx = -1; + /// Local bucket index at the pointer's current position. + index_type _idx_bucket_idx = -1; + /// Element offset in bucket at the pointer's current position. + index_type _idx_bucket_phase = -1; + +public: + /** + * Default constructor. + */ + GlobPtr() + : _globmem(nullptr), + _bucket_cumul_sizes(nullptr), + _unit_cumul_sizes(nullptr), + _idx(0), + _max_idx(0), + _myid(dash::Team::GlobalUnitID()), + _idx_unit_id(DART_UNDEFINED_UNIT_ID), + _idx_local_idx(-1), + _idx_bucket_idx(-1), + _idx_bucket_phase(-1) + { + DASH_LOG_TRACE_VAR("GlobPtr()", _idx); + DASH_LOG_TRACE_VAR("GlobPtr()", _max_idx); + } + + /** + * Constructor, creates a global pointer on global memory from global + * offset in logical storage order. + */ + template + GlobPtr( + const MemSpaceT * gmem, + index_type position = 0) + : _globmem(reinterpret_cast(gmem)), + _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), + _unit_cumul_sizes(&_globmem->_unit_cumul_sizes), + _lbegin(_globmem->lbegin()), + _idx(position), + _max_idx(gmem->size() - 1), + _myid(gmem->team().myid()), + _idx_unit_id(0), + _idx_local_idx(0), + _idx_bucket_idx(0), + _idx_bucket_phase(0) + { + DASH_LOG_TRACE("GlobPtr(gmem,idx)", "gidx:", position); + for (auto unit_bucket_cumul_sizes : *_bucket_cumul_sizes) { + DASH_LOG_TRACE_VAR("GlobPtr(gmem,idx)", + unit_bucket_cumul_sizes); + size_type bucket_cumul_size_prev = 0; + for (auto bucket_cumul_size : unit_bucket_cumul_sizes) { + DASH_LOG_TRACE_VAR("GlobPtr(gmem,idx)", bucket_cumul_size); + if (position < bucket_cumul_size) { + DASH_LOG_TRACE_VAR("GlobPtr(gmem,idx)", position); + _idx_bucket_phase -= bucket_cumul_size; + position = 0; + _idx_local_idx = position; + _idx_bucket_phase = position - bucket_cumul_size_prev; + break; + } + bucket_cumul_size_prev = bucket_cumul_size; + ++_idx_bucket_idx; + } + if (position == 0) { + break; + } + // Advance to next unit, adjust position relative to next unit's + // local index space: + position -= unit_bucket_cumul_sizes.back(); + ++_idx_unit_id; + } + DASH_LOG_TRACE("GlobPtr(gmem,idx)", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bucket:", _idx_bucket_idx, + "phase:", _idx_bucket_phase); + } + + /** + * Constructor, creates a global pointer on global memory from unit and + * local offset in logical storage order. + */ + template + GlobPtr( + const MemSpaceT * gmem, + team_unit_t unit, + index_type local_index) + : _globmem(reinterpret_cast(gmem)), + _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), + _unit_cumul_sizes(&_globmem->_unit_cumul_sizes), + _lbegin(_globmem->lbegin()), + _idx(0), + _max_idx(gmem->size() - 1), + _myid(gmem->team().myid()), + _idx_unit_id(unit), + _idx_local_idx(0), + _idx_bucket_idx(0), + _idx_bucket_phase(0) + { + DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx)", + "unit:", unit, + "lidx:", local_index); + DASH_ASSERT_LT(unit, _bucket_cumul_sizes->size(), "invalid unit id"); + + /* + for (size_type unit = 0; unit < _idx_unit_id; ++unit) { + auto prec_unit_local_size = (*_bucket_cumul_sizes)[unit].back(); + _idx += prec_unit_local_size; + } + */ + _idx += (*_unit_cumul_sizes)[unit]; + increment(local_index); + DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx) >", + "gidx:", _idx, + "maxidx:", _max_idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bucket:", _idx_bucket_idx, + "phase:", _idx_bucket_phase); + } + + /** + * Copy constructor. + */ + template + GlobPtr( + const GlobPtr & other) + : _globmem(other._globmem), + _bucket_cumul_sizes(other._bucket_cumul_sizes), + _unit_cumul_sizes(&_globmem->_unit_cumul_sizes), + _lbegin(other._lbegin), + _idx(other._idx), + _max_idx(other._max_idx), + _idx_unit_id(other._idx_unit_id), + _idx_local_idx(other._idx_local_idx), + _idx_bucket_idx(other._idx_bucket_idx), + _idx_bucket_phase(other._idx_bucket_phase) + { } + + /** + * Assignment operator. + */ + template + self_t & operator=( + const GlobPtr & other) + { + _globmem = other._globmem; + _bucket_cumul_sizes = other._bucket_cumul_sizes; + _unit_cumul_sizes = other._unit_cumul_sizes; + _lbegin = other._lbegin; + _idx = other._idx; + _max_idx = other._max_idx; + _idx_unit_id = other._idx_unit_id; + _idx_local_idx = other._idx_local_idx; + _idx_bucket_idx = other._idx_bucket_idx; + _idx_bucket_phase = other._idx_bucket_phase; + } + + /** + * Explicit conversion to \c dart_gptr_t. + * + * \return A DART global pointer to the element at the pointer's + * position + */ + dart_gptr_t dart_gptr() const + { + DASH_LOG_TRACE_VAR("GlobPtr.dart_gptr()", _idx); + // Create global pointer from unit, bucket and phase: + dart_gptr_t dart_gptr = _globmem->dart_gptr_at( + _idx_unit_id, + _idx_bucket_idx, + _idx_bucket_phase); + DASH_LOG_TRACE_VAR("GlobPtr.dart_gptr >", dart_gptr); + return dart_gptr; + } + + /** + * Dereference operator. + * + * \return A global reference to the element at the pointer's position. + */ + reference operator*() const + { + auto lptr = local(); + if (lptr != nullptr) { + return reference(static_cast(lptr)); + } else { + return reference(dart_gptr()); + } + } + + /** + * Subscript operator, returns global reference to element at given + * global index. + */ + reference operator[]( + /// The global position of the element + index_type g_index) const + { + DASH_LOG_TRACE_VAR("GlobPtr.[]()", g_index); + auto git = *this; + git += g_index; + auto lptr = git.local(); + if (lptr != nullptr) { + return reference(lptr); + } else { + auto gref = *git; + DASH_LOG_TRACE_VAR("GlobPtr.[] >", gref); + return gref; + } + } + + /** + * Checks whether the element referenced by this global pointer is in + * the calling unit's local memory. + */ + inline bool is_local() const + { + return (_myid == _idx_unit_id); + } + + /** + * Conversion to local bucket pointer. + */ + local_pointer local() const + { + if (_myid != _idx_unit_id) { + // Iterator position does not point to local element + return nullptr; + } + return (_lbegin + _idx_local_idx); + } + + /** + * Unit and local offset at the pointer's position. + */ + inline local_index lpos() const + { + local_index local_pos; + local_pos.unit = _idx_unit_id; + local_pos.index = _idx_local_idx; + return local_pos; + } + + /** + * Map pointer to global index domain. + */ + inline self_t global() const + { + return *this; + } + + /** + * Position of the pointer in global index space. + */ + inline index_type pos() const + { + return _idx; + } + + /** + * Position of the pointer in global index range. + */ + inline index_type gpos() const + { + return _idx; + } + + /** + * The instance of \c GlobStaticMem used by this pointer to resolve + * addresses in global memory. + */ + inline const globmem_type & globmem() const + { + return *_globmem; + } + + /** + * The instance of \c GlobStaticMem used by this pointer to resolve + * addresses in global memory. + */ + inline globmem_type & globmem() + { + return *_globmem; + } + + /** + * Prefix increment operator. + */ + inline self_t & operator++() + { + increment(1); + return *this; + } + + /** + * Prefix decrement operator. + */ + inline self_t & operator--() + { + decrement(1); + return *this; + } + + /** + * Postfix increment operator. + */ + inline self_t operator++(int) + { + auto result = *this; + increment(1); + return result; + } + + /** + * Postfix decrement operator. + */ + inline self_t operator--(int) + { + auto result = *this; + decrement(1); + return result; + } + + inline self_t & operator+=(index_type offset) + { + increment(offset); + return *this; + } + + inline self_t & operator-=(index_type offset) + { + increment(offset); + return *this; + } + + inline self_t operator+(index_type offset) const + { + auto res = *this; + res.increment(offset); + return res; + } + + inline self_t operator-(index_type offset) const + { + auto res = *this; + res.decrement(offset); + return res; + } + + inline index_type operator+( + const self_t & other) const + { + return _idx + other._idx; + } + + inline index_type operator-( + const self_t & other) const + { + return _idx - other._idx; + } + + template + inline bool operator<(const GlobPtr & other) const + { + return (_idx < other._idx); + } + + template + inline bool operator<=(const GlobPtr & other) const + { + return (_idx <= other._idx); + } + + template + inline bool operator>(const GlobPtr & other) const + { + return (_idx > other._idx); + } + + template + inline bool operator>=(const GlobPtr & other) const + { + return (_idx >= other._idx); + } + + template + inline bool operator==(const GlobPtr & other) const + { + return _idx == other._idx; + } + + template + inline bool operator!=(const GlobPtr & other) const + { + return _idx != other._idx; + } + +private: + /** + * Advance pointer by specified position offset. + */ + void increment(int offset) + { + DASH_LOG_TRACE("GlobPtr.increment()", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase, + "offset:", offset); + _idx += offset; + auto current_bucket_size = + (*_bucket_cumul_sizes)[_idx_unit_id][_idx_bucket_idx]; + if (_idx_local_idx + offset < current_bucket_size) { + DASH_LOG_TRACE("GlobPtr.increment", "position current bucket"); + // element is in bucket currently referenced by this pointer: + _idx_bucket_phase += offset; + _idx_local_idx += offset; + } else { + DASH_LOG_TRACE("GlobPtr.increment", + "position in succeeding bucket"); + // iterate units: + auto unit_id_max = _bucket_cumul_sizes->size() - 1; + for (; _idx_unit_id <= unit_id_max; ++_idx_unit_id) { + if(offset == 0) { + break; + } + auto & unit_bkt_sizes = (*_bucket_cumul_sizes)[_idx_unit_id]; + auto unit_bkt_sizes_total = unit_bkt_sizes.back(); + auto unit_num_bkts = unit_bkt_sizes.size(); + DASH_LOG_TRACE("GlobPtr.increment", + "unit:", _idx_unit_id, + "remaining offset:", offset, + "total local bucket size:", unit_bkt_sizes_total); + if (_idx_local_idx + offset >= unit_bkt_sizes_total) { + // offset refers to next unit: + DASH_LOG_TRACE("GlobPtr.increment", + "position in remote range"); + // subtract remaining own local size from remaining offset: + offset -= (unit_bkt_sizes_total - _idx_local_idx); + if (_idx_unit_id == unit_id_max) { + // end pointer, offset exceeds iteration space: + _idx_bucket_idx = unit_num_bkts - 1; + auto last_bkt_size = unit_bkt_sizes.back(); + if (unit_num_bkts > 1) { + last_bkt_size -= unit_bkt_sizes[_idx_bucket_idx-1]; + } + _idx_bucket_phase = last_bkt_size + offset; + _idx_local_idx += unit_bkt_sizes_total + offset; + break; + } + _idx_local_idx = 0; + _idx_bucket_idx = 0; + _idx_bucket_phase = 0; + } else { + // offset refers to current unit: + DASH_LOG_TRACE("GlobPtr.increment", + "position in local range", + "current bucket phase:", _idx_bucket_phase, + "cumul. bucket sizes:", unit_bkt_sizes); + _idx_local_idx += offset; + // iterate the unit's bucket sizes: + for (; _idx_bucket_idx < unit_num_bkts; ++_idx_bucket_idx) { + auto cumul_bucket_size = unit_bkt_sizes[_idx_bucket_idx]; + if (_idx_local_idx < cumul_bucket_size) { + auto cumul_prev = _idx_bucket_idx > 0 + ? unit_bkt_sizes[_idx_bucket_idx-1] + : 0; + // offset refers to current bucket: + _idx_bucket_phase = _idx_local_idx - cumul_prev; + offset = 0; + break; + } + } + if (offset == 0) { + break; + } + } + } + } + DASH_LOG_TRACE("GlobPtr.increment >", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase); + } + + /** + * Decrement pointer by specified position offset. + */ + void decrement(int offset) + { + DASH_LOG_TRACE("GlobPtr.decrement()", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase, + "offset:", -offset); + if (offset > _idx) { + DASH_THROW(dash::exception::OutOfRange, + "offset " << offset << " is out of range"); + } + _idx -= offset; + if (offset <= _idx_bucket_phase) { + // element is in bucket currently referenced by this pointer: + _idx_bucket_phase -= offset; + _idx_local_idx -= offset; + } else { + // iterate units: + auto first_unit = _idx_unit_id; + for (; _idx_unit_id >= 0; --_idx_unit_id) { + auto unit_bkt_sizes = (*_bucket_cumul_sizes)[_idx_unit_id]; + auto unit_bkt_sizes_total = unit_bkt_sizes.back(); + auto unit_num_bkts = unit_bkt_sizes.size(); + if (_idx_unit_id != first_unit) { + --offset; + _idx_bucket_idx = unit_num_bkts - 1; + _idx_local_idx = unit_bkt_sizes_total - 1; + auto last_bkt_size = unit_bkt_sizes.back(); + if (unit_num_bkts > 1) { + last_bkt_size -= unit_bkt_sizes[_idx_bucket_idx-1]; + } + _idx_bucket_phase = last_bkt_size - 1; + } + if (offset <= _idx_local_idx) { + // offset refers to current unit: + // iterate the unit's bucket sizes: + for (; _idx_bucket_idx >= 0; --_idx_bucket_idx) { + auto bucket_size = unit_bkt_sizes[_idx_bucket_idx]; + if (offset <= _idx_bucket_phase) { + // offset refers to current bucket: + _idx_local_idx -= offset; + _idx_bucket_phase -= offset; + offset = 0; + break; + } else { + // offset refers to preceeding bucket: + _idx_local_idx -= (_idx_bucket_phase + 1); + offset -= (_idx_bucket_phase + 1); + _idx_bucket_phase = unit_bkt_sizes[_idx_bucket_idx-1] - 1; + } + } + } else { + // offset refers to preceeding unit: + offset -= _idx_local_idx; + } + if (offset == 0) { + break; + } + } + } + DASH_LOG_TRACE("GlobPtr.decrement >", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase); + } + +}; // class GlobPtr + +/** + * Resolve the number of elements between two global bucket pointers. + * + * \complexity O(1) + * + * \ingroup Algorithms + */ +template< + typename ElementType, + class ContainerType > +auto distance( + /// Global pointer to the first position in the global sequence + const dash::GlobPtr< + ElementType, GlobHeapContiguousMem + > & first, + /// Global pointer to the final position in the global sequence + const dash::GlobPtr< + ElementType, GlobHeapContiguousMem + > & last) +-> typename GlobHeapContiguousMem::index_type +{ + return last - first; +} + +template< + typename ElementType, + class ContainerType > +std::ostream & operator<<( + std::ostream & os, + const dash::GlobPtr< + ElementType, + GlobHeapContiguousMem + > & gptr) +{ + std::ostringstream ss; + ss << "dash::GlobPtr<" << typeid(ElementType).name() << ">(" + << "gidx:" << gptr._idx << ", (" + << "unit:" << gptr._idx_unit_id << ", " + << "lidx:" << gptr._idx_local_idx << "), (" + << "bidx:" << gptr._idx_bucket_idx << ", " + << "bphase:" << gptr._idx_bucket_phase << ")" + << ")"; + return operator<<(os, ss.str()); +} + +} // namespace dash + +#endif // DASH__MEMORY__GLOB_HEAP_CONTIGUOUS_PTR_H__INCLUDED diff --git a/dash/include/dash/memory/GlobHeapLocalPtr.h b/dash/include/dash/memory/GlobHeapLocalPtr.h index 66f1019c3..f68ad7a7e 100644 --- a/dash/include/dash/memory/GlobHeapLocalPtr.h +++ b/dash/include/dash/memory/GlobHeapLocalPtr.h @@ -395,8 +395,9 @@ class GlobHeapLocalPtr } else { // find bucket containing element at given offset: for (; _bucket_it != _bucket_last; ++_bucket_it) { - if (offset >= _bucket_it->size) { - offset -= _bucket_it->size; + if (offset + _bucket_phase >= _bucket_it->size) { + offset -= _bucket_it->size - _bucket_phase; + _bucket_phase = 0; } else if (offset < _bucket_it->size) { _bucket_phase = offset; break; diff --git a/dash/include/dash/memory/internal/GlobHeapMemTypes.h b/dash/include/dash/memory/internal/GlobHeapMemTypes.h index 83a6f4cac..056929962 100644 --- a/dash/include/dash/memory/internal/GlobHeapMemTypes.h +++ b/dash/include/dash/memory/internal/GlobHeapMemTypes.h @@ -18,6 +18,15 @@ struct glob_dynamic_mem_bucket_type bool attached; }; +template< + typename SizeType, + typename ElementType > +struct glob_dynamic_contiguous_mem_bucket_type +{ + SizeType size; + ElementType * lptr; +}; + } // namespace internal } // namespace dash diff --git a/dash/test/algorithm/ConnectedComponentsTest.cc b/dash/test/algorithm/ConnectedComponentsTest.cc new file mode 100644 index 000000000..5ca0382df --- /dev/null +++ b/dash/test/algorithm/ConnectedComponentsTest.cc @@ -0,0 +1,748 @@ +#include "ConnectedComponentsTest.h" +#include +#include + +struct vprop { + dash::default_index_t comp; + dash::global_unit_t unit; +}; + +typedef dash::Graph graph_t; + +TEST_F(ConnectedComponentsTest, AlgorithmRun) +{ + std::list> edge_list = { + {433, 280}, {471, 192}, {738, 907}, {128, 664}, {999, 933}, {984, 440}, + {247, 310}, {565, 909}, {700, 727}, {408, 406}, {265, 550}, {411, 899}, + {630, 170}, {179, 57}, {733, 727}, {543, 943}, {475, 141}, {632, 394}, + {859, 581}, {203, 609}, {796, 589}, {494, 784}, {947, 420}, {354, 889}, + {184, 492}, {939, 587}, {598, 623}, {110, 971}, {296, 560}, {507, 881}, + {310, 674}, {280, 274}, {829, 845}, {273, 673}, {851, 52}, {814, 902}, + {995, 527}, {389, 611}, {537, 387}, {521, 350}, {849, 934}, {208, 57}, + {227, 962}, {30, 617}, {617, 962}, {512, 167}, {325, 1}, {232, 645}, {676, + 963}, {89, 242}, {57, 237}, {118, 252}, {807, 913}, {75, 425}, {782, 63}, + {901, 394}, {917, 23}, {923, 686}, {569, 451}, {217, 309}, {54, 950}, {508, + 185}, {642, 135}, {548, 436}, {495, 332}, {93, 368}, {483, 300}, {537, + 504}, {330, 526}, {29, 220}, {705, 598}, {468, 861}, {140, 13}, {192, 598}, + {347, 504}, {859, 182}, {369, 826}, {405, 979}, {694, 603}, {138, 269}, + {24, 364}, {256, 442}, {983, 657}, {946, 896}, {449, 915}, {6, 838}, {673, + 773}, {24, 231}, {235, 574}, {226, 875}, {327, 722}, {474, 311}, {501, + 904}, {291, 525}, {410, 937}, {804, 981}, {367, 113}, {449, 281}, {390, + 841}, {148, 698}, {396, 122}, {470, 726}, {442, 23}, {627, 762}, {493, + 406}, {710, 914}, {600, 289}, {182, 684}, {800, 569}, {965, 896}, {953, + 509}, {173, 25}, {671, 289}, {682, 205}, {71, 683}, {666, 286}, {414, 381}, + {23, 568}, {951, 45}, {987, 833}, {196, 225}, {686, 689}, {708, 87}, {954, + 715}, {209, 238}, {735, 700}, {357, 571}, {352, 466}, {373, 730}, {441, + 701}, {349, 224}, {289, 890}, {234, 680}, {721, 974}, {698, 801}, {181, + 460}, {47, 444}, {446, 142}, {262, 236}, {27, 494}, {958, 229}, {283, 210}, + {191, 279}, {359, 6}, {166, 810}, {808, 281}, {11, 801}, {484, 134}, {207, + 78}, {97, 128}, {182, 655}, {658, 329}, {926, 225}, {677, 285}, {183, 468}, + {685, 750}, {200, 999}, {871, 48}, {832, 515}, {485, 859}, {485, 75}, {13, + 435}, {612, 658}, {138, 796}, {919, 949}, {910, 747}, {975, 691}, {410, + 123}, {143, 788}, {535, 650}, {289, 386}, {48, 466}, {625, 783}, {690, + 851}, {761, 689}, {638, 964}, {106, 327}, {886, 578}, {223, 840}, {83, + 572}, {484, 802}, {703, 337}, {139, 522}, {234, 937}, {73, 555}, {15, 789}, + {335, 444}, {159, 468}, {747, 900}, {202, 563}, {880, 741}, {227, 307}, + {37, 594}, {380, 88}, {827, 286}, {747, 679}, {351, 208}, {374, 482}, {989, + 684}, {157, 691}, {207, 681}, {213, 302}, {372, 741}, {260, 231}, {710, + 110}, {529, 723}, {876, 580}, {156, 344}, {503, 467}, {887, 439}, {798, + 53}, {312, 342}, {890, 237}, {973, 738}, {36, 726}, {581, 386}, {153, 333}, + {341, 992}, {210, 389}, {551, 194}, {402, 407}, {875, 538}, {16, 873}, + {180, 47}, {119, 360}, {402, 996}, {422, 30}, {380, 939}, {722, 24}, {663, + 827}, {621, 266}, {113, 815}, {887, 976}, {859, 285}, {527, 646}, {971, + 190}, {84, 915}, {110, 953}, {934, 716}, {304, 316}, {413, 419}, {32, 373}, + {730, 936}, {478, 235}, {976, 947}, {859, 384}, {94, 33}, {912, 253}, {277, + 39}, {980, 696}, {91, 692}, {160, 263}, {190, 707}, {459, 166}, {221, 537}, + {849, 375}, {997, 679}, {57, 591}, {601, 48}, {393, 78}, {771, 839}, {254, + 485}, {939, 809}, {478, 167}, {881, 502}, {81, 317}, {76, 31}, {614, 887}, + {578, 417}, {970, 582}, {237, 389}, {951, 228}, {783, 626}, {146, 834}, + {233, 138}, {458, 861}, {167, 957}, {391, 619}, {880, 742}, {769, 210}, + {595, 786}, {708, 823}, {574, 159}, {663, 907}, {383, 781}, {861, 971}, + {395, 450}, {942, 183}, {558, 883}, {827, 609}, {23, 561}, {479, 386}, {68, + 915}, {75, 926}, {874, 927}, {15, 344}, {308, 359}, {788, 210}, {771, 65}, + {18, 494}, {760, 102}, {218, 602}, {53, 478}, {577, 476}, {794, 736}, {633, + 394}, {102, 941}, {686, 188}, {19, 326}, {626, 42}, {580, 305}, {304, 691}, + {569, 294}, {184, 239}, {698, 274}, {84, 168}, {994, 35}, {415, 591}, {139, + 416}, {152, 790}, {414, 123}, {987, 740}, {230, 973}, {206, 337}, {8, 329}, + {661, 477}, {514, 582}, {646, 944}, {32, 887}, {79, 655}, {749, 541}, {831, + 479}, {697, 443}, {691, 497}, {531, 432}, {595, 315}, {440, 663}, {567, + 705}, {710, 788}, {659, 805}, {112, 585}, {633, 934}, {358, 108}, {170, + 381}, {360, 876}, {155, 383}, {496, 69}, {104, 259}, {579, 513}, {907, 90}, + {438, 791}, {296, 322}, {382, 380}, {788, 196}, {667, 833}, {632, 598}, + {834, 396}, {45, 579}, {809, 724}, {470, 139}, {122, 686}, {667, 858}, + {860, 724}, {971, 644}, {838, 708}, {525, 660}, {94, 971}, {248, 531}, + {381, 380}, {178, 225}, {388, 291}, {218, 876}, {893, 177}, {896, 985}, + {930, 37}, {718, 804}, {553, 972}, {789, 952}, {367, 343}, {674, 387}, + {556, 160}, {985, 256}, {443, 917}, {377, 25}, {113, 453}, {43, 271}, {802, + 567}, {79, 199}, {713, 108}, {793, 517}, {860, 488}, {232, 686}, {983, + 451}, {360, 106}, {328, 853}, {190, 654}, {853, 10}, {580, 896}, {125, + 334}, {570, 449}, {192, 702}, {884, 146}, {209, 918}, {383, 810}, {186, + 460}, {137, 104}, {144, 845}, {775, 91}, {296, 805}, {773, 742}, {2, 136}, + {204, 752}, {370, 227}, {638, 957}, {192, 298}, {471, 309}, {718, 2}, {964, + 584}, {770, 1}, {232, 333}, {836, 921}, {263, 900}, {414, 629}, {824, 304}, + {379, 575}, {233, 160}, {547, 70}, {978, 603}, {55, 803}, {89, 160}, {490, + 598}, {888, 100}, {475, 973}, {880, 544}, {231, 894}, {546, 77}, {612, + 810}, {410, 882}, {964, 539}, {816, 920}, {809, 528}, {848, 472}, {202, + 291}, {724, 5}, {594, 131}, {385, 738}, {95, 277}, {432, 829}, {218, 434}, + {21, 127}, {926, 930}, {522, 650}, {563, 734}, {940, 927}, {733, 830}, + {494, 847}, {193, 290}, {492, 456}, {961, 47}, {876, 891}, {711, 827}, + {715, 474}, {489, 253}, {678, 289}, {398, 385}, {433, 146}, {173, 189}, + {985, 296}, {401, 357}, {592, 386}, {725, 243}, {308, 332}, {244, 637}, + {253, 938}, {852, 905}, {366, 712}, {560, 854}, {669, 270}, {666, 915}, + {362, 933}, {344, 832}, {255, 526}, {735, 438}, {400, 804}, {144, 230}, + {304, 338}, {287, 577}, {842, 8}, {327, 383}, {700, 41}, {499, 713}, {343, + 629}, {969, 654}, {24, 916}, {686, 276}, {525, 722}, {769, 302}, {240, 56}, + {828, 236}, {554, 486}, {727, 969}, {775, 356}, {411, 369}, {42, 943}, {70, + 429}, {824, 292}, {747, 577}, {359, 193}, {869, 301}, {390, 377}, {763, + 507}, {489, 526}, {63, 869}, {504, 647}, {525, 315}, {330, 65}, {818, 970}, + {976, 690}, {593, 324}, {242, 378}, {734, 271}, {120, 111}, {891, 898}, + {70, 675}, {929, 823}, {742, 330}, {347, 904}, {142, 496}, {353, 505}, + {448, 352}, {325, 794}, {54, 485}, {543, 70}, {763, 423}, {142, 846}, {112, + 7}, {652, 82}, {999, 939}, {402, 293}, {378, 300}, {717, 731}, {412, 885}, + {827, 595}, {179, 823}, {389, 424}, {353, 533}, {229, 511}, {889, 743}, + {235, 394}, {717, 22}, {121, 962}, {31, 991}, {843, 687}, {904, 89}, {695, + 705}, {624, 573}, {869, 643}, {887, 495}, {340, 827}, {833, 219}, {746, + 804}, {547, 350}, {127, 206}, {755, 827}, {108, 539}, {586, 321}, {451, + 251}, {860, 409}, {505, 953}, {713, 909}, {177, 226}, {15, 708}, {425, + 339}, {80, 63}, {181, 26}, {419, 139}, {298, 680}, {805, 607}, {259, 545}, + {451, 440}, {880, 225}, {255, 259}, {337, 637}, {974, 363}, {849, 850}, + {905, 206}, {868, 820}, {972, 972}, {736, 724}, {305, 420}, {63, 848}, + {947, 836}, {39, 947}, {355, 636}, {418, 13}, {816, 641}, {845, 374}, {359, + 921}, {322, 65}, {165, 728}, {997, 813}, {77, 344}, {340, 390}, {264, 367}, + {261, 712}, {953, 838}, {204, 743}, {48, 398}, {801, 61}, {186, 700}, {854, + 109}, {624, 172}, {304, 915}, {426, 711}, {45, 117}, {321, 549}, {387, + 555}, {206, 970}, {791, 317}, {786, 20}, {343, 594}, {361, 382}, {278, + 277}, {331, 225}, {510, 807}, {441, 493}, {396, 271}, {797, 678}, {254, + 123}, {539, 172}, {380, 267}, {420, 211}, {96, 247}, {407, 644}, {869, + 814}, {202, 338}, {327, 759}, {734, 713}, {18, 699}, {827, 103}, {953, + 551}, {206, 734}, {499, 98}, {66, 784}, {346, 444}, {824, 654}, {3, 224}, + {519, 231}, {319, 627}, {816, 292}, {81, 9}, {926, 899}, {27, 659}, {432, + 183}, {170, 709}, {522, 394}, {165, 264}, {469, 54}, {838, 705}, {101, + 415}, {713, 54}, {828, 650}, {480, 333}, {899, 162}, {830, 709}, {93, 789}, + {192, 173}, {720, 943}, {921, 758}, {100, 616}, {611, 62}, {168, 702}, + {776, 468}, {417, 79}, {420, 73}, {415, 149}, {383, 193}, {258, 597}, {871, + 677}, {327, 545}, {693, 93}, {714, 7}, {313, 135}, {788, 457}, {976, 961}, + {680, 113}, {422, 998}, {162, 301}, {269, 440}, {834, 25}, {559, 853}, + {787, 947}, {919, 930}, {910, 932}, {970, 417}, {846, 238}, {252, 741}, + {119, 401}, {136, 438}, {829, 432}, {614, 245}, {890, 7}, {303, 367}, {498, + 31}, {363, 384}, {805, 632}, {476, 821}, {940, 345}, {526, 932}, {966, + 643}, {572, 542}, {504, 255}, {841, 653}, {63, 91}, {97, 440}, {885, 550}, + {156, 851}, {190, 241}, {105, 859}, {264, 18}, {704, 965}, {598, 654}, + {922, 197}, {18, 543}, {677, 871}, {168, 514}, {334, 157}, {294, 553}, + {466, 973}, {792, 673}, {541, 331}, {307, 943}, {640, 551}, {947, 250}, + {390, 780}, {343, 379}, {80, 340}, {838, 145}, {634, 45}, {410, 2}, {934, + 338}, {621, 206}, {606, 977}, {735, 258}, {373, 274}, {631, 574}, {120, + 947}, {882, 725}, {139, 294}, {694, 450}, {476, 201}, {815, 305}, {234, + 563}, {305, 214}, {498, 39}, {753, 93}, {863, 81}, {202, 617}, {341, 719}, + {118, 153}, {586, 342}, {624, 806}, {498, 730}, {64, 211}, {858, 569}, + {541, 202}, {204, 259}, {517, 187}, {3, 682}, {36, 760}, {300, 287}, {980, + 873}, {260, 430}, {427, 411}, {677, 520}, {119, 515}, {18, 823}, {530, 39}, + {728, 185}, {537, 856}, {190, 883}, {409, 157}, {442, 271}, {347, 185}, + {884, 749}, {757, 55}, {394, 98}, {662, 829}, {32, 751}, {685, 491}, {937, + 571}, {624, 613}, {3, 417}, {90, 932}, {48, 377}, {541, 231}, {498, 940}, + {573, 755}, {157, 40}, {888, 205}, {161, 489}, {344, 601}, {300, 543}, + {319, 378}, {257, 918}, {536, 295}, {738, 305}, {970, 797}, {609, 698}, + {90, 668}, {114, 527}, {116, 267}, {429, 953}, {273, 401}, {76, 772}, {806, + 158}, {411, 627}, {271, 128}, {177, 72}, {995, 652}, {574, 35}, {250, 284}, + {283, 339}, {154, 304}, {602, 904}, {465, 102}, {709, 747}, {952, 735}, + {12, 418}, {473, 24}, {24, 880}, {38, 195}, {34, 797}, {647, 687}, {34, + 367}, {165, 62}, {454, 192}, {949, 492}, {671, 677}, {868, 896}, {333, + 965}, {846, 833}, {486, 905}, {123, 273}, {495, 697}, {319, 687}, {20, + 827}, {863, 991}, {530, 940}, {934, 56}, {2, 922}, {423, 716}, {863, 560}, + {447, 208}, {300, 597}, {941, 567}, {652, 665}, {668, 666}, {962, 732}, + {221, 61}, {108, 983}, {359, 374}, {589, 882}, {545, 460}, {728, 281}, + {101, 286}, {913, 474}, {819, 438}, {189, 863}, {377, 662}, {715, 322}, + {851, 766}, {957, 400}, {948, 312}, {444, 219}, {708, 763}, {19, 939}, + {390, 433}, {629, 689}, {55, 92}, {854, 656}, {67, 592}, {104, 334}, {578, + 863}, {961, 905}, {607, 412}, {177, 53}, {799, 76}, {891, 446}, {343, 78}, + {112, 79}, {872, 242}, {604, 137}, {606, 267}, {158, 640}, {559, 979}, + {908, 154}, {532, 630}, {519, 234}, {647, 247}, {690, 376}, {193, 268}, + {617, 89}, {561, 501}, {719, 455}, {820, 890}, {568, 977}, {363, 416}, + {234, 833}, {851, 107}, {213, 609}, {140, 436}, {229, 40}, {438, 262}, {77, + 404}, {686, 9}, {42, 545}, {398, 433}, {759, 105}, {13, 808}, {727, 495}, + {918, 189}, {879, 817}, {45, 747}, {335, 886}, {266, 985}, {708, 643}, + {200, 613}, {855, 834}, {980, 575}, {756, 71}, {609, 504}, {711, 333}, + {798, 973}, {316, 946}, {444, 449}, {213, 929}, {860, 579}, {683, 38}, {8, + 863}, {992, 618}, {407, 715}, {791, 526}, {848, 85}, {30, 690}, {468, 208}, + {238, 404}, {989, 480}, {662, 156}, {670, 160}, {691, 172}, {16, 903}, + {620, 93}, {856, 96}, {308, 571}, {479, 881}, {44, 903}, {621, 540}, {9, + 168}, {481, 405}, {551, 995}, {172, 260}, {369, 659}, {110, 855}, {880, + 298}, {687, 388}, {95, 828}, {960, 512}, {968, 6}, {878, 909}, {138, 124}, + {751, 408}, {252, 243}, {683, 119}, {573, 282}, {768, 782}, {415, 691}, + {272, 643}, {108, 966}, {657, 186}, {486, 665}, {889, 927}, {565, 826}, + {901, 858}, {99, 999}, {748, 415}, {571, 472}, {910, 484}, {905, 735}, + {894, 229}, {381, 908}, {944, 320}, {815, 85}, {177, 915}, {669, 415}, + {686, 425}, {19, 242}, {985, 845}, {213, 277}, {955, 349}, {940, 804}, + {954, 198}, {403, 329}, {87, 855}, {239, 563}, {152, 898}, {544, 643}, + {760, 95}, {780, 501}, {470, 475}, {301, 646}, {298, 5}, {795, 899}, {82, + 558}, {778, 360}, {209, 706}, {589, 301}, {719, 798}, {268, 205}, {327, + 460}, {470, 22}, {660, 372}, {96, 497}, {485, 760}, {168, 821}, {215, 332}, + {276, 112}, {332, 804}, {808, 747}, {673, 35}, {605, 230}, {987, 466}, + {179, 383}, {705, 165}, {356, 404}, {943, 208}, {14, 50}, {776, 346}, {789, + 358}, {751, 233}, {303, 306}, {654, 843}, {841, 547}, {263, 35}, {218, + 227}, {462, 727}, {52, 64}, {220, 173}, {675, 635}, {856, 409}, {495, 696}, + {664, 132}, {388, 849}, {766, 515}, {781, 228}, {145, 633}, {405, 645}, + {426, 954}, {793, 318}, {46, 30}, {848, 998}, {907, 612}, {163, 173}, {670, + 712}, {356, 239}, {966, 345}, {341, 388}, {435, 237}, {459, 470}, {466, + 310}, {380, 128}, {174, 263}, {28, 118}, {119, 782}, {531, 8}, {930, 160}, + {604, 165}, {131, 773}, {949, 844}, {753, 316}, {740, 917}, {78, 608}, + {917, 51}, {923, 824}, {381, 895}, {61, 109}, {959, 224}, {357, 393}, {756, + 289}, {826, 231}, {23, 535}, {416, 756}, {812, 411}, {1, 12}, {340, 746}, + {472, 109}, {467, 595}, {218, 179}, {582, 756}, {801, 138}, {698, 895}, + {43, 359}, {999, 378}, {624, 595}, {37, 23}, {454, 226}, {70, 213}, {676, + 673}, {654, 62}, {249, 809}, {150, 776}, {192, 442}, {875, 325}, {706, + 628}, {212, 638}, {597, 149}, {205, 604}, {951, 189}, {641, 944}, {915, + 487}, {944, 532}, {460, 48}, {95, 482}, {735, 61}, {136, 970}, {788, 733}, + {533, 485}, {533, 660}, {758, 626}, {777, 313}, {702, 720}, {709, 926}, + {777, 297}, {332, 698}, {552, 27}, {465, 725}, {330, 473}, {938, 20}, {276, + 605}, {454, 844}, {950, 805}, {276, 408}, {391, 773}, {297, 949}, {64, 38}, + {177, 236}, {76, 498}, {218, 521}, {582, 428}, {205, 175}, {987, 756}, + {247, 115}, {815, 970}, {860, 775}, {319, 15}, {860, 36}, {326, 228}, {966, + 293}, {257, 159}, {935, 552}, {34, 735}, {39, 165}, {878, 215}, {405, 352}, + {72, 298}, {927, 771}, {896, 66}, {116, 677}, {36, 268}, {546, 11}, {742, + 766}, {42, 783}, {774, 864}, {245, 4}, {932, 149}, {77, 509}, {714, 65}, + {839, 333}, {204, 656}, {345, 375}, {855, 807}, {466, 109}, {727, 880}, + {948, 413}, {5, 541}, {355, 443}, {583, 629}, {350, 688}, {739, 416}, {329, + 182}, {185, 140}, {713, 911}, {950, 346}, {927, 204}, {375, 753}, {974, + 735}, {28, 783}, {706, 207}, {0, 21}, {842, 758}, {743, 924}, {966, 312}, + {329, 675}, {291, 490}, {329, 493}, {523, 862}, {916, 834}, {860, 424}, + {861, 46}, {617, 363}, {844, 174}, {309, 106}, {45, 899}, {353, 567}, {783, + 689}, {681, 155}, {524, 933}, {980, 142}, {69, 794}, {417, 89}, {562, 922}, + {741, 86}, {74, 996}, {813, 989}, {694, 73}, {680, 415}, {772, 797}, {982, + 101}, {283, 936}, {825, 226}, {945, 394}, {740, 599}, {68, 519}, {979, + 192}, {138, 4}, {201, 791}, {595, 210}, {215, 215}, {802, 488}, {539, 277}, + {919, 844}, {746, 495}, {604, 1}, {951, 456}, {51, 346}, {214, 108}, {587, + 565}, {464, 939}, {109, 945}, {691, 274}, {878, 197}, {311, 534}, {458, + 992}, {737, 696}, {853, 295}, {156, 139}, {285, 518}, {92, 169}, {590, + 142}, {263, 792}, {109, 656}, {406, 132}, {610, 734}, {359, 367}, {206, + 953}, {883, 324}, {669, 123}, {878, 598}, {569, 32}, {422, 527}, {957, + 964}, {700, 522}, {333, 719}, {647, 832}, {269, 457}, {703, 110}, {512, + 238}, {233, 121}, {710, 179}, {802, 93}, {715, 825}, {321, 307}, {866, + 980}, {231, 586}, {845, 456}, {26, 265}, {656, 883}, {221, 951}, {131, 80}, + {629, 876}, {736, 293}, {171, 526}, {121, 189}, {269, 962}, {709, 939}, + {800, 320}, {874, 235}, {728, 670}, {821, 325}, {149, 445}, {17, 593}, + {573, 443}, {879, 321}, {718, 148}, {282, 455}, {420, 144}, {449, 303}, + {404, 53}, {958, 594}, {855, 155}, {33, 537}, {179, 325}, {907, 621}, {26, + 951}, {722, 289}, {213, 846}, {141, 106}, {530, 683}, {905, 934}, {425, + 686}, {486, 256}, {47, 82}, {133, 45}, {526, 690}, {733, 856}, {144, 172}, + {69, 853}, {658, 302}, {64, 165}, {957, 613}, {29, 136}, {464, 346}, {114, + 87}, {832, 654}, {631, 900}, {96, 461}, {657, 458}, {863, 372}, {217, 614}, + {352, 705}, {791, 378}, {617, 550}, {222, 578}, {86, 907}, {163, 593}, + {410, 307}, {603, 403}, {335, 892}, {20, 926}, {400, 745}, {747, 190}, + {443, 698}, {705, 735}, {169, 30}, {921, 106}, {167, 498}, {11, 339}, {89, + 578}, {470, 594}, {325, 50}, {127, 952}, {767, 463}, {887, 49}, {193, 249}, + {69, 487}, {723, 280}, {294, 744}, {242, 384}, {685, 92}, {975, 970}, {824, + 253}, {592, 419}, {278, 481}, {361, 162}, {439, 17}, {881, 221}, {4, 615}, + {399, 211}, {546, 45}, {834, 438}, {553, 879}, {603, 895}, {412, 366}, {63, + 697}, {16, 246}, {808, 2}, {73, 225}, {288, 607}, {475, 616}, {428, 999}, + {560, 595}, {421, 404}, {563, 899}, {126, 138}, {388, 419}, {47, 376}, + {305, 1}, {420, 381}, {501, 447}, {378, 720}, {469, 301}, {584, 853}, {659, + 498}, {849, 517}, {970, 851}, {840, 212}, {612, 508}, {38, 471}, {98, 504}, + {143, 368}, {846, 492}, {760, 788}, {720, 353}, {115, 858}, {389, 911}, + {142, 719}, {960, 273}, {97, 714}, {306, 744}, {336, 321}, {684, 493}, + {951, 464}, {985, 894}, {627, 860}, {550, 559}, {503, 840}, {473, 170}, + {322, 672}, {521, 717}, {329, 566}, {703, 582}, {934, 969}, {313, 635}, + {698, 984}, {128, 880}, {658, 154}, {524, 145}, {9, 315}, {71, 425}, {565, + 104}, {817, 84}, {429, 976}, {601, 279}, {695, 146}, {210, 144}, {740, + 609}, {872, 201}, {163, 52}, {857, 27}, {699, 499}, {921, 545}, {730, 762}, + {850, 326}, {609, 673}, {699, 453}, {910, 238}, {592, 215}, {573, 850}, + {749, 217}, {449, 592}, {468, 331}, {476, 543}, {16, 800}, {361, 250}, + {995, 363}, {288, 858}, {971, 118}, {541, 583}, {561, 587}, {832, 293}, + {660, 60}, {247, 685}, {799, 279}, {169, 913}, {115, 746}, {91, 341}, {153, + 758}, {985, 628}, {116, 170}, {791, 712}, {449, 731}, {4, 719}, {975, 88}, + {292, 114}, {670, 607}, {4, 178}, {8, 389}, {492, 51}, {183, 606}, {314, + 888}, {571, 887}, {182, 275}, {572, 352}, {903, 459}, {131, 558}, {681, + 288}, {40, 333}, {587, 267}, {483, 764}, {966, 348}, {676, 279}, {141, + 409}, {904, 760}, {740, 406}, {474, 565}, {932, 428}, {40, 192}, {539, + 232}, {613, 528}, {207, 63}, {321, 714}, {872, 570}, {862, 809}, {882, + 348}, {594, 603}, {756, 279}, {531, 767}, {993, 556}, {666, 276}, {961, + 176}, {573, 800}, {417, 146}, {863, 327}, {627, 118}, {692, 250}, {272, + 244}, {587, 159}, {22, 704}, {263, 515}, {492, 19}, {728, 979}, {257, 917}, + {383, 409}, {58, 423}, {480, 417}, {399, 716}, {374, 633}, {408, 265}, + {762, 749}, {695, 629}, {893, 272}, {285, 719}, {673, 456}, {690, 491}, + {243, 595}, {425, 955}, {558, 459}, {485, 562}, {47, 996}, {902, 486}, + {581, 716}, {525, 812}, {519, 945}, {923, 918}, {607, 572}, {882, 719}, + {544, 313}, {136, 36}, {977, 567}, {23, 244}, {777, 198}, {515, 590}, {349, + 90}, {858, 183}, {271, 996}, {116, 821}, {267, 829}, {576, 681}, {718, + 619}, {614, 393}, {571, 478}, {647, 840}, {843, 691}, {387, 201}, {330, + 391}, {667, 494}, {504, 437}, {111, 560}, {800, 103}, {931, 618}, {322, + 595}, {634, 848}, {96, 443}, {804, 14}, {647, 636}, {496, 521}, {529, 633}, + {556, 117}, {600, 162}, {211, 432}, {550, 909}, {890, 45}, {725, 61}, {147, + 626}, {482, 894}, {755, 159}, {55, 731}, {352, 488}, {934, 163}, {93, 653}, + {524, 36}, {607, 528}, {341, 126}, {425, 87}, {354, 490}, {22, 72}, {869, + 869}, {671, 203}, {181, 372}, {105, 40}, {634, 342}, {229, 923}, {271, + 472}, {325, 837}, {929, 218}, {968, 410}, {144, 628}, {716, 344}, {983, + 457}, {237, 209}, {400, 562}, {271, 227}, {134, 461}, {587, 903}, {658, + 519}, {172, 354}, {530, 739}, {244, 175}, {883, 940}, {786, 638}, {725, + 141}, {316, 830}, {584, 514}, {729, 900}, {481, 737}, {800, 228}, {478, 8}, + {36, 395}, {684, 178}, {369, 397}, {824, 109}, {792, 796}, {756, 679}, + {699, 241}, {227, 789}, {510, 521}, {665, 375}, {107, 595}, {565, 386}, + {687, 129}, {498, 585}, {897, 718}, {275, 767}, {538, 469}, {849, 208}, + {15, 349}, {518, 750}, {416, 158}, {577, 390}, {751, 420}, {892, 233}, + {793, 895}, {870, 691}, {339, 125}, {41, 403}, {784, 7}, {426, 181}, {197, + 430}, {71, 474}, {950, 597}, {183, 147}, {186, 939}, {679, 571}, {209, + 878}, {836, 395}, {391, 472}, {934, 326}, {536, 151}, {238, 925}, {38, + 763}, {425, 185}, {308, 254}, {110, 47}, {599, 993}, {236, 918}, {64, 337}, + {41, 57}, {676, 439}, {378, 388}, {845, 335}, {130, 115}, {662, 594}, {281, + 557}, {867, 935}, {236, 982}, {896, 905}, {952, 302}, {440, 186}, {46, + 229}, {301, 727}, {634, 140}, {85, 411}, {256, 571}, {16, 616}, {729, 321}, + {316, 102}, {756, 885}, {482, 617}, {98, 90}, {550, 914}, {651, 19}, {174, + 642}, {484, 245}, {949, 551}, {130, 581}, {745, 75}, {799, 498}, {375, 30}, + {30, 333}, {195, 199}, {80, 717}, {133, 852}, {464, 906}, {137, 642}, {108, + 772}, {833, 352}, {975, 236}, {799, 557}, {571, 106}, {237, 379}, {698, + 712}, {618, 735}, {352, 682}, {754, 799}, {981, 502}, {976, 721}, {216, + 281}, {470, 956}, {322, 777}, {387, 286}, {682, 601}, {351, 208}, {662, + 912}, {812, 226}, {806, 450}, {560, 469}, {94, 346}, {350, 347}, {625, + 116}, {21, 708}, {246, 770}, {17, 194}, {696, 357}, {397, 824}, {337, 647}, + {347, 913}, {989, 966}, {865, 180}, {669, 562}, {996, 627}, {455, 299}, + {672, 664}, {127, 377}, {646, 630}, {688, 232}, {584, 944}, {359, 678}, + {669, 61}, {628, 693}, {407, 293}, {206, 220}, {921, 248}, {750, 96}, {635, + 432}, {197, 304}, {692, 814}, {90, 506}, {407, 172}, {999, 392}, {851, + 698}, {479, 678}, {559, 484}, {837, 77}, {625, 13}, {837, 672}, {990, 558}, + {176, 124}, {366, 101}, {624, 558}, {69, 197}, {715, 502}, {186, 413}, + {614, 999}, {994, 827}, {964, 576}, {520, 539}, {599, 5}, {501, 329}, {954, + 376}, {276, 766}, {586, 487}, {765, 221}, {381, 549}, {257, 476}, {267, + 772}, {44, 362}, {961, 300}, {811, 52}, {264, 264}, {365, 994}, {776, 246}, + {943, 689}, {53, 545}, {93, 179}, {586, 443}, {861, 552}, {677, 338}, {448, + 496}, {740, 410}, {488, 537}, {588, 939}, {988, 592}, {448, 853}, {939, + 535}, {481, 302}, {241, 570}, {257, 231}, {706, 565}, {161, 725}, {509, + 477}, {555, 367}, {892, 476}, {287, 492}, {756, 49}, {305, 257}, {976, + 210}, {630, 406}, {792, 899}, {566, 152}, {801, 657}, {707, 759}, {49, + 246}, {812, 444}, {800, 615}, {484, 520}, {912, 857}, {26, 649}, {699, + 806}, {219, 916}, {300, 9}, {972, 176}, {447, 722}, {622, 791}, {806, 7}, + {824, 747}, {572, 233}, {741, 433}, {201, 402}, {262, 236}, {685, 390}, + {246, 64}, {190, 960}, {199, 936}, {984, 415}, {671, 121}, {739, 895}, + {239, 647}, {439, 129}, {6, 796}, {623, 682}, {429, 487}, {348, 180}, {299, + 899}, {834, 206}, {657, 78}, {155, 359}, {9, 884}, {660, 712}, {622, 243}, + {861, 605}, {570, 666}, {202, 614}, {933, 742}, {946, 580}, {797, 961}, + {376, 382}, {159, 153}, {925, 179}, {639, 942}, {233, 640}, {895, 229}, + {37, 552}, {850, 532}, {133, 500}, {740, 928}, {418, 742}, {994, 330}, + {694, 468}, {9, 545}, {171, 926}, {369, 674}, {333, 905}, {675, 27}, {479, + 266}, {122, 397}, {724, 366}, {363, 633}, {974, 268}, {243, 38}, {519, + 977}, {183, 256}, {80, 356}, {688, 363}, {964, 704}, {39, 806}, {102, 218}, + {865, 619}, {406, 385}, {354, 605}, {694, 495}, {159, 269}, {425, 290}, + {858, 232}, {338, 45}, {420, 337}, {404, 153}, {471, 485}, {639, 532}, + {628, 607}, {980, 680}, {826, 492}, {382, 321}, {33, 366}, {709, 250}, + {644, 122}, {763, 175}, {86, 126}, {606, 304}, {689, 490}, {791, 427}, + {732, 659}, {443, 40}, {773, 967}, {223, 400}, {473, 851}, {294, 983}, {76, + 705}, {310, 932}, {908, 862}, {451, 940}, {660, 670}, {250, 128}, {315, + 347}, {383, 576}, {121, 152}, {373, 334}, {963, 908}, {912, 936}, {974, + 563}, {989, 207}, {76, 699}, {239, 737}, {921, 325}, {819, 508}, {313, + 871}, {640, 904}, {679, 409}, {272, 429}, {793, 494}, {281, 143}, {446, + 929}, {961, 471}, {688, 492}, {519, 461}, {492, 569}, {258, 430}, {46, + 387}, {265, 162}, {746, 960}, {729, 795}, {304, 753}, {691, 833}, {941, + 109}, {97, 570}, {26, 652}, {842, 661}, {510, 778}, {883, 982}, {711, 605}, + {209, 160}, {182, 364}, {149, 82}, {245, 435}, {247, 818}, {499, 301}, + {820, 669}, {381, 89}, {899, 165}, {398, 969}, {22, 147}, {0, 1}, {0, 3}, + {0, 4}, {0, 7}, {0, 8}, {0, 10}, {0, 15}, {0, 16}, {0, 18}, {0, 31}, {0, + 32}, {0, 34}, {0, 35}, {0, 38}, {0, 41}, {0, 46}, {0, 57}, {0, 62}, {0, + 63}, {0, 65}, {0, 69}, {0, 72}, {0, 77}, {0, 85}, {0, 88}, {0, 94}, {0, + 96}, {0, 97}, {0, 108}, {0, 125}, {0, 126}, {0, 128}, {0, 132}, {0, 140}, + {0, 147}, {0, 156}, {0, 157}, {0, 160}, {0, 163}, {0, 166}, {0, 187}, {0, + 191}, {0, 195}, {0, 203}, {0, 205}, {0, 213}, {0, 218}, {0, 221}, {0, 250}, + {0, 253}, {0, 257}, {0, 260}, {0, 265}, {0, 266}, {0, 268}, {0, 269}, {0, + 281}, {0, 285}, {0, 306}, {0, 312}, {0, 313}, {0, 316}, {0, 327}, {0, 343}, + {0, 378}, {0, 382}, {0, 401}, {0, 406}, {0, 444}, {0, 471}, {0, 483}, {0, + 484}, {0, 500}, {0, 501}, {0, 503}, {0, 504}, {0, 507}, {0, 515}, {0, 518}, + {0, 522}, {0, 532}, {0, 542}, {0, 562}, {0, 565}, {0, 569}, {0, 612}, {0, + 625}, {0, 626}, {0, 628}, {0, 632}, {0, 650}, {0, 697}, {0, 750}, {0, 751}, + {0, 754}, {0, 757}, {0, 760}, {0, 781}, {0, 784}, {0, 785}, {0, 797}, {0, + 813}, {0, 837}, {0, 898}, {1, 0}, {1, 3}, {1, 4}, {1, 7}, {1, 15}, {1, 18}, + {1, 19}, {1, 25}, {1, 31}, {1, 34}, {1, 35}, {1, 46}, {1, 62}, {1, 63}, {1, + 65}, {1, 69}, {1, 80}, {1, 93}, {1, 125}, {1, 132}, {1, 135}, {1, 144}, {1, + 156}, {1, 171}, {1, 194}, {1, 233}, {1, 250}, {1, 254}, {1, 266}, {1, 281}, + {1, 285}, {1, 313}, {1, 327}, {1, 344}, {1, 409}, {1, 437}, {1, 452}, {1, + 475}, {1, 500}, {1, 515}, {1, 531}, {1, 565}, {1, 593}, {1, 600}, {1, 628}, + {1, 632}, {3, 0}, {3, 1}, {3, 7}, {3, 15}, {3, 26}, {3, 31}, {3, 32}, {3, + 34}, {3, 46}, {3, 62}, {3, 63}, {3, 65}, {3, 69}, {3, 111}, {3, 125}, {3, + 132}, {3, 140}, {3, 141}, {3, 143}, {3, 156}, {3, 187}, {3, 250}, {3, 269}, + {3, 281}, {3, 312}, {3, 327}, {3, 328}, {3, 346}, {3, 437}, {3, 500}, {3, + 501}, {3, 508}, {3, 515}, {3, 519}, {3, 547}, {3, 562}, {3, 594}, {3, 663}, + {3, 698}, {3, 750}, {3, 765}, {3, 885}, {4, 0}, {4, 7}, {4, 16}, {4, 23}, + {4, 26}, {4, 31}, {4, 57}, {4, 125}, {4, 150}, {4, 229}, {4, 236}, {4, + 258}, {4, 625}, {4, 671}, {4, 688}, {4, 843}, {7, 0}, {7, 1}, {7, 3}, {7, + 15}, {7, 16}, {7, 18}, {7, 22}, {7, 31}, {7, 62}, {7, 69}, {7, 93}, {7, + 125}, {7, 132}, {7, 135}, {7, 140}, {7, 159}, {7, 187}, {7, 202}, {7, 236}, + {7, 257}, {7, 265}, {7, 273}, {7, 284}, {7, 313}, {7, 315}, {7, 382}, {7, + 383}, {7, 390}, {7, 452}, {7, 503}, {7, 515}, {7, 518}, {7, 531}, {7, 538}, + {7, 549}, {7, 562}, {7, 565}, {7, 628}, {7, 750}, {7, 843}, {7, 876}, {8, + 3}, {8, 4}, {8, 7}, {8, 15}, {8, 22}, {8, 31}, {8, 47}, {8, 49}, {8, 135}, + {8, 159}, {8, 228}, {8, 250}, {8, 437}, {8, 508}, {8, 566}, {10, 0}, {10, + 1}, {10, 15}, {10, 16}, {10, 31}, {10, 41}, {10, 62}, {10, 125}, {10, 140}, + {10, 205}, {10, 250}, {10, 375}, {10, 562}, {10, 656}, {10, 882}, {11, 0}, + {11, 62}, {11, 265}, {11, 281}, {11, 625}, {15, 0}, {15, 1}, {15, 4}, {15, + 7}, {15, 10}, {15, 16}, {15, 18}, {15, 31}, {15, 38}, {15, 46}, {15, 54}, + {15, 62}, {15, 63}, {15, 69}, {15, 73}, {15, 93}, {15, 94}, {15, 109}, {15, + 125}, {15, 175}, {15, 190}, {15, 225}, {15, 254}, {15, 281}, {15, 284}, + {15, 316}, {15, 346}, {15, 406}, {15, 422}, {15, 500}, {15, 501}, {15, + 510}, {15, 515}, {15, 523}, {15, 562}, {15, 625}, {15, 702}, {15, 753}, + {15, 781}, {15, 791}, {16, 0}, {16, 1}, {16, 3}, {16, 10}, {16, 18}, {16, + 22}, {16, 62}, {16, 65}, {16, 140}, {16, 167}, {16, 282}, {16, 330}, {16, + 500}, {16, 503}, {16, 516}, {16, 562}, {16, 570}, {16, 641}, {16, 750}, + {16, 784}, {18, 1}, {18, 7}, {18, 22}, {18, 65}, {18, 132}, {18, 147}, {18, + 171}, {18, 187}, {18, 222}, {18, 251}, {18, 253}, {18, 268}, {18, 312}, + {18, 413}, {18, 422}, {18, 515}, {18, 516}, {18, 633}, {18, 758}, {19, 0}, + {19, 3}, {19, 4}, {19, 46}, {19, 273}, {19, 289}, {19, 750}, {22, 0}, {22, + 31}, {22, 34}, {22, 38}, {22, 46}, {22, 69}, {22, 70}, {22, 187}, {22, + 222}, {22, 250}, {22, 291}, {22, 312}, {22, 313}, {22, 359}, {22, 382}, + {22, 390}, {22, 437}, {22, 626}, {22, 641}, {23, 0}, {23, 41}, {23, 78}, + {23, 100}, {23, 281}, {23, 375}, {25, 0}, {25, 62}, {25, 140}, {25, 187}, + {25, 531}, {25, 844}, {26, 108}, {26, 394}, {31, 0}, {31, 1}, {31, 3}, {31, + 7}, {31, 15}, {31, 18}, {31, 47}, {31, 53}, {31, 65}, {31, 93}, {31, 108}, + {31, 112}, {31, 126}, {31, 140}, {31, 156}, {31, 157}, {31, 163}, {31, + 187}, {31, 202}, {31, 253}, {31, 265}, {31, 268}, {31, 312}, {31, 375}, + {31, 393}, {31, 468}, {31, 500}, {31, 503}, {31, 504}, {31, 507}, {31, + 510}, {31, 516}, {31, 538}, {31, 569}, {31, 687}, {31, 750}, {31, 812}, + {31, 876}, {32, 0}, {32, 3}, {32, 4}, {32, 18}, {32, 25}, {32, 26}, {32, + 46}, {32, 62}, {32, 140}, {32, 257}, {32, 272}, {32, 327}, {32, 382}, {32, + 500}, {32, 587}, {34, 0}, {34, 15}, {34, 16}, {34, 38}, {34, 46}, {34, 63}, + {34, 115}, {34, 126}, {34, 141}, {34, 265}, {34, 501}, {34, 508}, {34, + 538}, {34, 597}, {34, 750}, {35, 0}, {35, 3}, {35, 31}, {35, 34}, {35, 62}, + {35, 69}, {35, 80}, {35, 171}, {35, 508}, {38, 0}, {38, 7}, {38, 16}, {38, + 18}, {38, 25}, {38, 63}, {38, 66}, {38, 70}, {38, 96}, {38, 125}, {38, + 251}, {38, 265}, {38, 291}, {38, 313}, {38, 315}, {38, 316}, {38, 376}, + {38, 569}, {38, 625}, {38, 750}, {38, 757}, {38, 785}, {38, 875}, {39, 34}, + {39, 69}, {39, 125}, {39, 132}, {39, 266}, {39, 319}, {41, 50}, {41, 250}, + {41, 750}, {41, 765}, {42, 3}, {42, 312}, {46, 0}, {46, 1}, {46, 3}, {46, + 4}, {46, 7}, {46, 10}, {46, 15}, {46, 16}, {46, 25}, {46, 77}, {46, 163}, + {46, 265}, {46, 500}, {46, 501}, {46, 515}, {47, 101}, {47, 143}, {47, + 265}, {49, 1}, {49, 7}, {49, 125}, {49, 202}, {49, 257}, {49, 656}, {49, + 907}, {50, 22}, {53, 11}, {53, 228}, {53, 688}, {54, 62}, {54, 554}, {56, + 0}, {56, 1}, {56, 10}, {56, 31}, {56, 128}, {62, 0}, {62, 1}, {62, 3}, {62, + 4}, {62, 7}, {62, 10}, {62, 15}, {62, 16}, {62, 18}, {62, 25}, {62, 31}, + {62, 32}, {62, 35}, {62, 38}, {62, 46}, {62, 47}, {62, 65}, {62, 66}, {62, + 69}, {62, 96}, {62, 108}, {62, 125}, {62, 126}, {62, 135}, {62, 140}, {62, + 144}, {62, 147}, {62, 159}, {62, 187}, {62, 188}, {62, 250}, {62, 251}, + {62, 253}, {62, 257}, {62, 265}, {62, 288}, {62, 312}, {62, 319}, {62, + 353}, {62, 375}, {62, 376}, {62, 500}, {62, 504}, {62, 508}, {62, 510}, + {62, 525}, {62, 534}, {62, 538}, {62, 569}, {62, 625}, {62, 635}, {62, + 750}, {62, 812}, {62, 850}, {62, 851}, {63, 0}, {63, 1}, {63, 8}, {63, 46}, + {63, 65}, {63, 140}, {63, 171}, {63, 202}, {63, 250}, {63, 281}, {63, 284}, + {63, 503}, {63, 515}, {63, 518}, {63, 538}, {63, 562}, {63, 593}, {63, + 734}, {65, 0}, {65, 3}, {65, 4}, {65, 15}, {65, 16}, {65, 38}, {65, 87}, + {65, 112}, {65, 125}, {65, 126}, {65, 132}, {65, 133}, {65, 140}, {65, + 174}, {65, 266}, {65, 312}, {65, 375}, {65, 437}, {65, 503}, {65, 508}, + {65, 562}, {65, 626}, {66, 0}, {66, 46}, {66, 93}, {66, 257}, {66, 500}, + {66, 772}, {69, 1}, {69, 7}, {69, 11}, {69, 18}, {69, 62}, {69, 132}, {69, + 234}, {69, 257}, {69, 281}, {69, 296}, {69, 504}, {69, 532}, {69, 562}, + {69, 572}, {69, 694}, {69, 750}, {69, 757}, {70, 0}, {70, 15}, {70, 18}, + {70, 63}, {70, 157}, {70, 171}, {70, 250}, {70, 757}, {72, 32}, {72, 190}, + {72, 218}, {72, 291}, {72, 659}, {73, 0}, {73, 187}, {73, 250}, {73, 281}, + {73, 328}, {73, 500}, {77, 1}, {77, 3}, {77, 31}, {77, 38}, {77, 78}, {77, + 129}, {77, 171}, {77, 198}, {77, 250}, {77, 281}, {77, 508}, {77, 510}, + {77, 538}, {77, 563}, {77, 750}, {77, 754}, {77, 765}, {77, 768}, {77, + 937}, {78, 0}, {78, 1}, {78, 31}, {78, 218}, {78, 269}, {78, 376}, {80, 0}, + {80, 8}, {80, 10}, {80, 31}, {80, 69}, {80, 250}, {80, 269}, {80, 285}, + {80, 500}, {80, 547}, {81, 3}, {81, 31}, {81, 515}, {81, 531}, {84, 10}, + {84, 15}, {84, 132}, {84, 190}, {84, 250}, {84, 507}, {84, 532}, {84, 534}, + {84, 878}, {85, 126}, {85, 312}, {87, 140}, {93, 0}, {93, 3}, {93, 7}, {93, + 15}, {93, 18}, {93, 46}, {93, 53}, {93, 63}, {93, 132}, {93, 228}, {93, + 250}, {93, 261}, {93, 379}, {93, 383}, {93, 444}, {93, 507}, {93, 515}, + {93, 534}, {93, 562}, {94, 3}, {94, 34}, {94, 62}, {94, 500}, {94, 515}, + {94, 632}, {96, 38}, {96, 46}, {96, 53}, {96, 128}, {96, 250}, {96, 313}, + {100, 1}, {100, 128}, {101, 7}, {108, 0}, {108, 70}, {108, 254}, {108, + 272}, {108, 515}, {108, 565}, {109, 656}, {112, 500}, {115, 77}, {115, + 250}, {125, 0}, {125, 1}, {125, 3}, {125, 4}, {125, 7}, {125, 8}, {125, + 11}, {125, 15}, {125, 16}, {125, 23}, {125, 31}, {125, 32}, {125, 38}, + {125, 49}, {125, 62}, {125, 126}, {125, 156}, {125, 164}, {125, 187}, {125, + 194}, {125, 198}, {125, 205}, {125, 250}, {125, 253}, {125, 266}, {125, + 281}, {125, 284}, {125, 296}, {125, 347}, {125, 382}, {125, 501}, {125, + 507}, {125, 515}, {125, 523}, {125, 538}, {125, 554}, {125, 569}, {125, + 577}, {125, 593}, {125, 608}, {125, 647}, {125, 710}, {125, 718}, {125, + 750}, {125, 765}, {125, 766}, {126, 0}, {126, 1}, {126, 15}, {126, 25}, + {126, 32}, {126, 63}, {126, 80}, {126, 125}, {126, 128}, {126, 147}, {126, + 268}, {126, 284}, {126, 312}, {126, 437}, {126, 515}, {126, 531}, {128, 3}, + {128, 7}, {128, 15}, {128, 22}, {128, 62}, {128, 63}, {128, 73}, {128, 77}, + {128, 140}, {128, 147}, {128, 209}, {128, 269}, {128, 272}, {128, 281}, + {128, 296}, {128, 331}, {128, 515}, {128, 626}, {128, 632}, {129, 0}, {129, + 250}, {132, 0}, {132, 7}, {132, 62}, {132, 70}, {132, 77}, {132, 125}, + {132, 141}, {132, 160}, {132, 281}, {132, 500}, {132, 510}, {132, 625}, + {132, 626}, {132, 635}, {133, 18}, {133, 25}, {133, 125}, {133, 150}, {133, + 156}, {133, 281}, {133, 343}, {133, 500}, {135, 0}, {135, 1}, {135, 10}, + {135, 250}, {135, 531}, {136, 219}, {136, 375}, {136, 625}, {140, 0}, {140, + 3}, {140, 7}, {140, 31}, {140, 46}, {140, 56}, {140, 87}, {140, 250}, {140, + 257}, {140, 265}, {140, 282}, {140, 312}, {140, 500}, {140, 501}, {140, + 562}, {140, 565}, {140, 609}, {140, 875}, {141, 0}, {141, 140}, {141, 143}, + {141, 254}, {141, 281}, {141, 500}, {141, 522}, {141, 882}, {143, 4}, {143, + 10}, {143, 15}, {143, 62}, {143, 93}, {143, 226}, {143, 350}, {143, 584}, + {143, 633}, {143, 690}, {144, 0}, {144, 34}, {144, 375}, {144, 501}, {147, + 0}, {147, 22}, {147, 257}, {150, 265}, {156, 1}, {156, 8}, {156, 15}, {156, + 23}, {156, 31}, {156, 39}, {156, 125}, {156, 140}, {156, 203}, {156, 289}, + {156, 327}, {156, 376}, {156, 566}, {156, 577}, {156, 578}, {156, 596}, + {156, 907}, {157, 132}, {157, 156}, {157, 163}, {157, 437}, {157, 503}, + {157, 515}, {157, 593}, {157, 702}, {159, 1}, {159, 7}, {159, 32}, {159, + 62}, {159, 65}, {159, 77}, {159, 108}, {159, 132}, {159, 140}, {159, 233}, + {159, 501}, {160, 0}, {160, 72}, {160, 77}, {163, 0}, {163, 1}, {163, 125}, + {163, 126}, {163, 133}, {163, 625}, {163, 626}, {164, 1}, {164, 156}, {164, + 250}, {164, 365}, {166, 0}, {166, 78}, {171, 0}, {171, 1}, {171, 54}, {171, + 65}, {171, 94}, {171, 261}, {171, 500}, {172, 319}, {174, 77}, {175, 3}, + {178, 1}, {178, 11}, {178, 62}, {179, 500}, {187, 0}, {187, 3}, {187, 4}, + {187, 80}, {187, 126}, {187, 250}, {187, 281}, {187, 500}, {187, 518}, + {187, 534}, {187, 632}, {187, 702}, {187, 750}, {187, 785}, {188, 8}, {188, + 125}, {188, 147}, {188, 179}, {188, 253}, {188, 688}, {190, 0}, {190, 1}, + {190, 38}, {190, 766}, {191, 0}, {191, 140}, {194, 0}, {194, 3}, {194, 7}, + {194, 10}, {194, 35}, {194, 47}, {194, 93}, {194, 265}, {194, 410}, {194, + 846}, {195, 500}, {197, 15}, {197, 500}, {198, 0}, {202, 7}, {202, 62}, + {202, 125}, {202, 187}, {202, 453}, {202, 503}, {205, 69}, {206, 0}, {210, + 0}, {210, 789}, {212, 751}, {218, 15}, {218, 46}, {218, 63}, {218, 125}, + {218, 174}, {218, 257}, {218, 261}, {218, 265}, {219, 7}, {221, 0}, {221, + 7}, {221, 281}, {221, 508}, {225, 46}, {225, 516}, {241, 31}, {241, 63}, + {250, 0}, {250, 1}, {250, 3}, {250, 7}, {250, 8}, {250, 16}, {250, 18}, + {250, 19}, {250, 31}, {250, 32}, {250, 49}, {250, 53}, {250, 62}, {250, + 63}, {250, 69}, {250, 70}, {250, 77}, {250, 84}, {250, 108}, {250, 125}, + {250, 128}, {250, 129}, {250, 135}, {250, 140}, {250, 156}, {250, 167}, + {250, 187}, {250, 190}, {250, 209}, {250, 234}, {250, 257}, {250, 281}, + {250, 285}, {250, 313}, {250, 315}, {250, 343}, {250, 376}, {250, 452}, + {250, 500}, {250, 503}, {250, 510}, {250, 515}, {250, 562}, {250, 563}, + {250, 565}, {250, 572}, {250, 577}, {250, 593}, {250, 608}, {250, 629}, + {250, 640}, {250, 753}, {250, 769}, {250, 812}, {250, 827}, {250, 890}, + {250, 921}, {251, 3}, {251, 10}, {251, 34}, {251, 38}, {251, 125}, {251, + 126}, {251, 140}, {251, 171}, {251, 190}, {251, 202}, {251, 222}, {251, + 241}, {251, 284}, {251, 441}, {251, 500}, {251, 515}, {251, 556}, {251, + 728}, {251, 751}, {251, 827}, {253, 0}, {253, 1}, {253, 10}, {253, 22}, + {253, 25}, {253, 31}, {253, 47}, {253, 62}, {253, 125}, {253, 132}, {253, + 163}, {253, 195}, {253, 222}, {253, 315}, {253, 320}, {253, 375}, {253, + 500}, {253, 501}, {253, 507}, {253, 635}, {253, 651}, {253, 812}, {254, 0}, + {254, 135}, {254, 266}, {254, 281}, {254, 959}, {257, 10}, {257, 22}, {257, + 31}, {257, 62}, {257, 63}, {257, 125}, {257, 147}, {257, 157}, {257, 250}, + {257, 288}, {257, 297}, {257, 334}, {257, 353}, {257, 375}, {257, 501}, + {257, 694}, {257, 757}, {258, 65}, {258, 522}, {258, 593}, {258, 625}, + {258, 827}, {260, 0}, {260, 3}, {260, 254}, {260, 312}, {260, 330}, {260, + 516}, {260, 781}, {261, 10}, {261, 251}, {265, 0}, {265, 1}, {265, 3}, + {265, 8}, {265, 10}, {265, 26}, {265, 65}, {265, 77}, {265, 125}, {265, + 188}, {265, 257}, {265, 312}, {265, 409}, {265, 500}, {265, 503}, {265, + 570}, {266, 3}, {266, 156}, {266, 233}, {266, 694}, {268, 3}, {268, 8}, + {268, 16}, {268, 77}, {268, 125}, {268, 140}, {268, 503}, {268, 584}, {269, + 8}, {269, 31}, {272, 3}, {272, 77}, {272, 500}, {272, 632}, {275, 69}, + {275, 534}, {281, 0}, {281, 1}, {281, 8}, {281, 11}, {281, 39}, {281, 46}, + {281, 63}, {281, 70}, {281, 93}, {281, 128}, {281, 251}, {281, 507}, {281, + 516}, {281, 569}, {281, 671}, {281, 687}, {281, 768}, {282, 1}, {282, 23}, + {282, 31}, {282, 96}, {282, 132}, {282, 205}, {282, 288}, {282, 390}, {282, + 503}, {282, 788}, {284, 0}, {284, 31}, {284, 258}, {284, 500}, {288, 0}, + {288, 80}, {288, 140}, {288, 538}, {291, 7}, {291, 31}, {296, 0}, {296, 4}, + {296, 7}, {296, 16}, {296, 194}, {296, 359}, {296, 500}, {296, 501}, {296, + 508}, {296, 629}, {296, 687}, {296, 751}, {296, 812}, {297, 500}, {299, 0}, + {299, 1}, {299, 539}, {303, 22}, {303, 32}, {303, 656}, {306, 42}, {312, + 0}, {312, 7}, {312, 31}, {312, 128}, {312, 156}, {312, 229}, {312, 378}, + {312, 385}, {312, 578}, {312, 751}, {312, 788}, {312, 910}, {313, 0}, {313, + 15}, {313, 100}, {313, 125}, {313, 128}, {315, 49}, {315, 202}, {315, 593}, + {315, 909}, {316, 31}, {316, 38}, {316, 327}, {316, 625}, {319, 0}, {319, + 34}, {319, 125}, {319, 500}, {319, 625}, {319, 718}, {320, 19}, {320, 225}, + {320, 609}, {322, 0}, {322, 10}, {322, 140}, {322, 315}, {322, 625}, {323, + 538}, {327, 1}, {327, 3}, {327, 31}, {327, 69}, {328, 0}, {328, 77}, {328, + 132}, {330, 0}, {334, 156}, {334, 313}, {334, 376}, {334, 508}, {334, 546}, + {334, 648}, {337, 500}, {337, 503}, {338, 3}, {338, 126}, {343, 0}, {343, + 534}, {344, 34}, {344, 226}, {346, 125}, {346, 500}, {350, 562}, {350, + 587}, {353, 15}, {353, 500}, {353, 750}, {354, 500}, {358, 1}, {358, 299}, + {359, 501}, {365, 7}, {375, 0}, {375, 1}, {375, 35}, {375, 190}, {375, + 312}, {376, 15}, {376, 69}, {376, 78}, {376, 93}, {376, 250}, {376, 781}, + {378, 63}, {378, 69}, {378, 382}, {379, 15}, {382, 31}, {382, 531}, {382, + 640}, {385, 35}, {386, 15}, {386, 62}, {386, 159}, {386, 515}, {390, 94}, + {390, 125}, {390, 140}, {390, 260}, {390, 569}, {390, 609}, {390, 753}, + {391, 0}, {391, 18}, {391, 78}, {391, 876}, {393, 251}, {393, 625}, {398, + 897}, {400, 3}, {406, 34}, {406, 65}, {406, 144}, {406, 327}, {406, 515}, + {406, 531}, {406, 785}, {407, 3}, {409, 15}, {409, 46}, {409, 437}, {410, + 3}, {413, 15}, {413, 656}, {414, 49}, {421, 109}, {421, 197}, {422, 100}, + {424, 315}, {437, 15}, {437, 62}, {437, 125}, {437, 500}, {440, 7}, {440, + 500}, {440, 562}, {440, 775}, {448, 0}, {448, 438}, {452, 16}, {452, 31}, + {452, 250}, {452, 562}, {459, 62}, {459, 268}, {459, 796}, {468, 0}, {471, + 7}, {471, 538}, {475, 109}, {476, 63}, {490, 7}, {500, 0}, {500, 1}, {500, + 4}, {500, 7}, {500, 10}, {500, 11}, {500, 15}, {500, 18}, {500, 22}, {500, + 25}, {500, 31}, {500, 34}, {500, 38}, {500, 42}, {500, 46}, {500, 47}, + {500, 50}, {500, 62}, {500, 63}, {500, 69}, {500, 73}, {500, 77}, {500, + 80}, {500, 93}, {500, 94}, {500, 125}, {500, 126}, {500, 144}, {500, 166}, + {500, 171}, {500, 178}, {500, 250}, {500, 257}, {500, 265}, {500, 281}, + {500, 315}, {500, 316}, {500, 376}, {500, 386}, {500, 413}, {500, 437}, + {500, 503}, {500, 507}, {500, 508}, {500, 510}, {500, 516}, {500, 542}, + {500, 674}, {500, 687}, {500, 718}, {500, 750}, {500, 819}, {500, 875}, + {500, 906}, {501, 0}, {501, 7}, {501, 18}, {501, 22}, {501, 31}, {501, 47}, + {501, 125}, {501, 141}, {501, 150}, {501, 171}, {501, 190}, {501, 203}, + {501, 209}, {501, 260}, {501, 288}, {501, 500}, {501, 510}, {501, 534}, + {501, 656}, {501, 875}, {503, 0}, {503, 3}, {503, 16}, {503, 42}, {503, + 46}, {503, 63}, {503, 66}, {503, 147}, {503, 250}, {503, 261}, {503, 322}, + {503, 350}, {503, 378}, {503, 468}, {503, 500}, {503, 625}, {504, 1}, {504, + 3}, {504, 22}, {504, 77}, {504, 115}, {504, 500}, {504, 600}, {504, 789}, + {504, 796}, {507, 0}, {507, 3}, {507, 15}, {507, 18}, {507, 35}, {507, 38}, + {507, 125}, {507, 250}, {507, 500}, {507, 501}, {507, 547}, {508, 0}, {508, + 32}, {508, 132}, {508, 140}, {508, 382}, {508, 501}, {510, 66}, {510, 210}, + {510, 303}, {511, 268}, {511, 299}, {511, 500}, {515, 0}, {515, 1}, {515, + 3}, {515, 53}, {515, 69}, {515, 93}, {515, 128}, {515, 132}, {515, 156}, + {515, 167}, {515, 171}, {515, 188}, {515, 206}, {515, 250}, {515, 257}, + {515, 281}, {515, 375}, {515, 410}, {515, 437}, {515, 516}, {515, 751}, + {516, 0}, {516, 15}, {516, 35}, {516, 77}, {516, 103}, {516, 156}, {516, + 296}, {516, 300}, {516, 315}, {516, 459}, {516, 656}, {516, 834}, {516, + 865}, {518, 0}, {518, 7}, {518, 15}, {518, 65}, {518, 194}, {518, 250}, + {518, 265}, {518, 268}, {518, 577}, {518, 625}, {518, 812}, {519, 62}, + {522, 0}, {522, 31}, {522, 65}, {522, 80}, {522, 315}, {522, 376}, {522, + 531}, {522, 781}, {522, 897}, {523, 150}, {523, 282}, {525, 22}, {525, 31}, + {525, 251}, {526, 49}, {531, 0}, {531, 1}, {531, 7}, {531, 10}, {531, 49}, + {531, 93}, {531, 126}, {531, 132}, {531, 156}, {531, 202}, {531, 218}, + {531, 258}, {531, 315}, {531, 375}, {531, 503}, {531, 593}, {531, 765}, + {531, 772}, {532, 0}, {532, 3}, {532, 15}, {532, 31}, {532, 32}, {532, 62}, + {532, 73}, {532, 132}, {532, 562}, {534, 15}, {534, 300}, {535, 0}, {535, + 163}, {538, 0}, {538, 7}, {538, 15}, {538, 629}, {539, 62}, {539, 468}, + {541, 891}, {546, 1}, {546, 125}, {546, 281}, {546, 288}, {546, 319}, {546, + 327}, {546, 500}, {546, 532}, {546, 539}, {546, 640}, {547, 1}, {547, 250}, + {547, 784}, {549, 128}, {550, 0}, {550, 221}, {556, 15}, {562, 0}, {562, + 3}, {562, 4}, {562, 8}, {562, 31}, {562, 34}, {562, 46}, {562, 69}, {562, + 93}, {562, 125}, {562, 143}, {562, 319}, {562, 500}, {562, 501}, {562, + 504}, {562, 508}, {562, 510}, {563, 0}, {563, 39}, {563, 73}, {563, 257}, + {563, 376}, {563, 500}, {565, 0}, {565, 15}, {565, 296}, {565, 390}, {566, + 0}, {566, 815}, {569, 3}, {569, 125}, {569, 126}, {569, 175}, {569, 190}, + {569, 313}, {569, 538}, {570, 65}, {572, 65}, {572, 251}, {573, 500}, {577, + 0}, {577, 1}, {577, 15}, {577, 18}, {577, 159}, {577, 501}, {577, 518}, + {577, 584}, {578, 0}, {578, 385}, {580, 500}, {584, 625}, {593, 0}, {593, + 1}, {593, 3}, {593, 31}, {593, 41}, {593, 53}, {593, 281}, {593, 633}, + {593, 878}, {593, 882}, {594, 32}, {594, 93}, {594, 784}, {596, 66}, {600, + 0}, {600, 253}, {601, 577}, {603, 62}, {608, 0}, {611, 18}, {616, 375}, + {625, 0}, {625, 1}, {625, 3}, {625, 7}, {625, 34}, {625, 108}, {625, 126}, + {625, 129}, {625, 190}, {625, 254}, {625, 275}, {625, 284}, {625, 344}, + {625, 346}, {625, 390}, {625, 425}, {625, 532}, {625, 538}, {625, 750}, + {625, 968}, {626, 31}, {626, 32}, {626, 34}, {626, 63}, {626, 100}, {626, + 111}, {626, 500}, {626, 508}, {626, 643}, {628, 0}, {628, 46}, {629, 62}, + {629, 625}, {632, 8}, {632, 81}, {632, 93}, {632, 156}, {632, 202}, {632, + 547}, {633, 15}, {633, 268}, {633, 516}, {640, 0}, {640, 23}, {640, 39}, + {640, 251}, {640, 319}, {640, 358}, {640, 522}, {641, 8}, {641, 343}, {641, + 751}, {643, 265}, {644, 18}, {644, 41}, {648, 19}, {656, 3}, {656, 7}, + {656, 62}, {656, 382}, {656, 507}, {656, 757}, {657, 0}, {657, 81}, {657, + 894}, {659, 65}, {659, 476}, {659, 750}, {663, 125}, {671, 78}, {671, 253}, + {672, 375}, {687, 0}, {687, 15}, {687, 18}, {687, 31}, {687, 32}, {687, + 108}, {687, 125}, {687, 296}, {687, 500}, {688, 0}, {688, 38}, {690, 265}, + {690, 757}, {694, 0}, {694, 65}, {694, 250}, {695, 0}, {697, 93}, {702, 3}, + {702, 156}, {702, 282}, {703, 281}, {705, 46}, {705, 250}, {706, 94}, {709, + 628}, {718, 7}, {718, 31}, {718, 62}, {725, 0}, {725, 18}, {733, 376}, + {736, 784}, {740, 77}, {740, 126}, {750, 0}, {750, 16}, {750, 22}, {750, + 31}, {750, 32}, {750, 62}, {750, 63}, {750, 115}, {750, 128}, {750, 159}, + {750, 250}, {750, 315}, {750, 343}, {750, 344}, {750, 424}, {750, 562}, + {750, 569}, {750, 626}, {751, 46}, {751, 93}, {751, 300}, {751, 522}, {753, + 0}, {753, 80}, {753, 171}, {753, 250}, {753, 406}, {753, 562}, {757, 46}, + {757, 47}, {757, 209}, {757, 265}, {757, 312}, {758, 266}, {760, 0}, {760, + 8}, {761, 62}, {761, 600}, {765, 62}, {765, 148}, {765, 516}, {765, 534}, + {765, 823}, {766, 141}, {766, 500}, {768, 0}, {768, 141}, {768, 251}, {768, + 781}, {772, 375}, {772, 500}, {773, 0}, {781, 62}, {781, 63}, {781, 147}, + {781, 157}, {781, 187}, {782, 3}, {782, 15}, {782, 34}, {782, 125}, {782, + 253}, {784, 516}, {784, 569}, {788, 111}, {788, 116}, {789, 7}, {789, 140}, + {789, 376}, {796, 906}, {797, 1}, {812, 18}, {812, 126}, {812, 750}, {812, + 796}, {813, 0}, {813, 7}, {813, 500}, {815, 49}, {815, 96}, {819, 565}, + {822, 0}, {827, 4}, {827, 125}, {827, 383}, {830, 31}, {843, 4}, {843, + 281}, {846, 0}, {846, 69}, {859, 260}, {875, 35}, {875, 80}, {875, 148}, + {875, 206}, {875, 250}, {875, 312}, {875, 500}, {875, 503}, {875, 702}, + {875, 782}, {875, 878}, {876, 0}, {878, 31}, {878, 78}, {879, 515}, {890, + 163}, {890, 816}, {891, 553}, {897, 46}, {897, 265}, {906, 0}, {906, 65}, + {906, 518}, {913, 69}, {913, 197}, {937, 413}, {937, 531}, {937, 565}, + {940, 16}, {941, 250}, {944, 53}, {944, 346}, {947, 1}, {952, 375}, {59, 371}, + {371, 431}, {431, 564}, {564, 779}, {779, 835}, {835, 877}, {877, 986} + }; + + std::vector results = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 774, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 774, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0 + }; + + DASH_LOG_DEBUG("ConnectedComponentsTest.Blocked.AlgorithmRun", + "construction started"); + graph_t g(edge_list.begin(), edge_list.end(), 1000); + DASH_LOG_DEBUG("ConnectedComponentsTest.Blocked.AlgorithmRun", + "construction finished"); + + dash::barrier(); + + DASH_LOG_DEBUG("ConnectedComponentsTest.Blocked.AlgorithmRun", + "algorithm started"); + dash::connected_components(g); + DASH_LOG_DEBUG("ConnectedComponentsTest.Blocked.AlgorithmRun", + "algorithm finished"); + + int wrong_components = 0; + if(dash::myid() == 0) { + int index = 0; + for(auto it = g.vertices().begin(); it != g.vertices().end(); ++it) { + auto v = g[it]; + if(v.attributes().comp != results[index]) { + ++wrong_components; + break; + } + ++index; + } + + // each component set according to results array + EXPECT_EQ_U(0, wrong_components); + } + + auto & team = dash::Team::All(); + dash::LogarithmicVertexMapper mapper(1000, team.size()); + DASH_LOG_DEBUG("ConnectedComponentsTest.Logarithmic.AlgorithmRun", + "construction started"); + // TODO: creating 2 graphs in their own scopes results in a segfault + // probably a problem with the destructor + // happens only in GTest though + graph_t g2(edge_list.begin(), edge_list.end(), 1000, team, mapper); + DASH_LOG_DEBUG("ConnectedComponentsTest.Logarithmic.AlgorithmRun", + "construction finished"); + + dash::barrier(); + + DASH_LOG_DEBUG("ConnectedComponentsTest.Logarithmic.AlgorithmRun", + "algorithm started"); + dash::connected_components(g2); + DASH_LOG_DEBUG("ConnectedComponentsTest.Logarithmic.AlgorithmRun", + "algorithm finished"); + + wrong_components = 0; + if(dash::myid() == 0) { + int index = 0; + for(auto it = g2.vertices().begin(); it != g2.vertices().end(); ++it) { + auto v = g2[it]; + if(v.attributes().comp != results[index]) { + ++wrong_components; + break; + } + ++index; + } + + // each component set according to results array + EXPECT_EQ_U(0, wrong_components); + } + dash::barrier(); + +} + diff --git a/dash/test/algorithm/ConnectedComponentsTest.h b/dash/test/algorithm/ConnectedComponentsTest.h new file mode 100644 index 000000000..426160e4b --- /dev/null +++ b/dash/test/algorithm/ConnectedComponentsTest.h @@ -0,0 +1,21 @@ +#ifndef DASH__TEST__CONNECTED_COMPONENTS_TEST_H_ +#define DASH__TEST__CONNECTED_COMPONENTS_TEST_H_ + +#include "../TestBase.h" + +/** + * Test fixture for class dash::List + */ +class ConnectedComponentsTest : public dash::test::TestBase { +protected: + + ConnectedComponentsTest() { + LOG_MESSAGE(">>> Test suite: ConnectedComponentsTest"); + } + + virtual ~ConnectedComponentsTest() { + LOG_MESSAGE("<<< Closing test suite: ConnectedComponentsTest"); + } +}; + +#endif // DASH__TEST__CONNECTED_COMPONENTS_TEST_H_ diff --git a/dash/test/algorithm/MinimumSpanningTreeTest.cc b/dash/test/algorithm/MinimumSpanningTreeTest.cc new file mode 100644 index 000000000..ef49d79e2 --- /dev/null +++ b/dash/test/algorithm/MinimumSpanningTreeTest.cc @@ -0,0 +1,115 @@ +/** + * Author: Steffan Effenberger (github: @stiefn) + * via https://github.com/stiefn/dyn-data-structures-thesis + * + */ + +#include "MinimumSpanningTreeTest.h" +#include +#include + +struct vprop { + dash::default_index_t comp; + dash::global_unit_t unit; +}; + +struct eprop { + int weight; + bool is_min; +}; + +typedef dash::Graph graph_t; + +TEST_F(MinimumSpanningTreeTest, AlgorithmRun) +{ + std::vector, int>> edge_list = { + {{12, 11}, 8}, {{10, 0}, 1}, {{2, 0}, 8}, {{15, 18}, 4}, {{11, 13}, 0}, + {{8, 18}, 3}, {{1, 9}, 10}, {{11, 1}, 9}, {{11, 13}, 9}, {{0, 19}, 7}, + {{19, 17}, 6}, {{2, 5}, 3}, {{18, 16}, 6}, {{10, 9}, 3}, {{16, 11}, 10}, + {{17, 1}, 1}, {{16, 13}, 4}, {{7, 7}, 1}, {{15, 19}, 0}, {{13, 14}, 6}, + {{10, 8}, 4}, {{10, 1}, 3}, {{7, 9}, 7}, {{8, 13}, 7}, {{14, 8}, 7}, + {{16, 11}, 4}, {{0, 3}, 10}, {{13, 10}, 7}, {{17, 7}, 7}, {{15, 10}, 8}, + {{0, 2}, 6}, {{12, 7}, 9}, {{5, 6}, 9}, {{3, 4}, 9}, {{14, 0}, 9}, + {{17, 14}, 6}, {{4, 4}, 5}, {{1, 13}, 2}, {{11, 15}, 6}, {{9, 2}, 2}, + {{0, 1}, 5}, {{0, 2}, 7}, {{0, 3}, 3}, {{0, 5}, 7}, {{0, 6}, 9}, {{0, 7}, + 6}, {{0, 10}, 9}, {{0, 13}, 7}, {{0, 15}, 4}, {{1, 0}, 3}, {{1, 2}, 7}, + {{1, 10}, 2}, {{2, 1}, 4}, {{2, 10}, 2}, {{2, 16}, 9}, {{3, 0}, 4}, {{3, + 2}, 0}, {{5, 0}, 1}, {{5, 2}, 3}, {{5, 7}, 5}, {{5, 10}, 4}, {{5, 12}, 6}, + {{6, 2}, 5}, {{6, 12}, 2}, {{8, 0}, 2}, {{10, 0}, 0}, {{10, 1}, 8}, {{10, + 3}, 10}, {{11, 0}, 9}, {{11, 2}, 8}, {{11, 7}, 10}, {{11, 10}, 10}, {{12, + 0}, 8}, {{12, 1}, 4}, {{13, 1}, 10}, {{13, 3}, 9}, {{15, 0}, 0}, {{15, 1}, + 10}, {{15, 5}, 7}, {{15, 10}, 8} + }; + + std::size_t TREE_SIZE = 39; + + DASH_LOG_DEBUG("MinimumSpanningTreeTest.Blocked.AlgorithmRun", + "construction started"); + graph_t g(edge_list.begin(), edge_list.end(), 20); + DASH_LOG_DEBUG("MinimumSpanningTreeTest.Blocked.AlgorithmRun", + "construction finished"); + + DASH_LOG_DEBUG("MinimumSpanningTreeTest.Blocked.AlgorithmRun", + "algorithm started"); + dash::minimum_spanning_tree(g); + DASH_LOG_DEBUG("MinimumSpanningTreeTest.Blocked.AlgorithmRun", + "algorithm finished"); + + std::unordered_set results; + std::size_t total_weight = 0; + if(dash::myid() == 0) { + for(auto it = g.out_edges().begin(); it != g.out_edges().end(); ++it) { + auto e = g[it]; + if(e.attributes().is_min) { + auto u = e.source().pos(); + auto v = e.target().pos(); + if(v < u) std::swap(u, v); + auto hash = std::hash()(u) ^ std::hash()(v); + if(results.find(hash) == results.end()) { + results.insert(hash); + total_weight += e.attributes().weight; + } + } + } + EXPECT_EQ_U(TREE_SIZE, total_weight); + } + + auto & team = dash::Team::All(); + dash::LogarithmicVertexMapper mapper(20, team.size()); + DASH_LOG_DEBUG("MinimumSpanningTreeTest.Logarithmic.AlgorithmRun", + "construction started"); + // TODO: creating 2 graphs in their own scopes results in a segfault + // probably a problem with the destructor + // happens only in GTest though + graph_t g2(edge_list.begin(), edge_list.end(), 20, team, mapper); + DASH_LOG_DEBUG("MinimumSpanningTreeTest.Logarithmic.AlgorithmRun", + "construction finished"); + + DASH_LOG_DEBUG("MinimumSpanningTreeTest.Logarithmic.AlgorithmRun", + "algorithm started"); + dash::minimum_spanning_tree(g2); + DASH_LOG_DEBUG("MinimumSpanningTreeTest.Logarithmic.AlgorithmRun", + "algorithm finished"); + + results.clear(); + total_weight = 0; + if(dash::myid() == 0) { + for(auto it = g2.out_edges().begin(); it != g2.out_edges().end(); ++it) { + auto e = g2[it]; + if(e.attributes().is_min) { + auto u = e.source().pos(); + auto v = e.target().pos(); + if(v < u) std::swap(u, v); + auto hash = std::hash()(u) ^ std::hash()(v); + if(results.find(hash) == results.end()) { + results.insert(hash); + total_weight += e.attributes().weight; + } + } + } + EXPECT_EQ_U(TREE_SIZE, total_weight); + } + + dash::barrier(); +} + diff --git a/dash/test/algorithm/MinimumSpanningTreeTest.h b/dash/test/algorithm/MinimumSpanningTreeTest.h new file mode 100644 index 000000000..cc4825155 --- /dev/null +++ b/dash/test/algorithm/MinimumSpanningTreeTest.h @@ -0,0 +1,21 @@ +#ifndef DASH__TEST__MINIMUM_SPANNING_TREE_TEST_H_ +#define DASH__TEST__MINIMUM_SPANNING_TREE_TEST_H_ + +#include "../TestBase.h" + +/** + * Test fixture for class dash::List + */ +class MinimumSpanningTreeTest : public dash::test::TestBase { +protected: + + MinimumSpanningTreeTest() { + LOG_MESSAGE(">>> Test suite: MinimumSpanningTreeTest"); + } + + virtual ~MinimumSpanningTreeTest() { + LOG_MESSAGE("<<< Closing test suite: MinimumSpanningTreeTest"); + } +}; + +#endif // DASH__TEST__MINIMUM_SPANNING_TREE_TEST_H_ diff --git a/dash/test/container/GraphTest.cc b/dash/test/container/GraphTest.cc new file mode 100644 index 000000000..3f2578977 --- /dev/null +++ b/dash/test/container/GraphTest.cc @@ -0,0 +1,674 @@ + +#include + +#include "GraphTest.h" +#include + +struct vprop { + int comp; +}; + +typedef dash::Graph + graph_t; + +TEST_F(GraphTest, Construction) +{ + std::list> edge_list = { + {433, 280}, {471, 192}, {738, 907}, {128, 664}, {999, 933}, {984, 440}, + {247, 310}, {565, 909}, {700, 727}, {408, 406}, {265, 550}, {411, 899}, + {630, 170}, {179, 57}, {733, 727}, {543, 943}, {475, 141}, {632, 394}, + {859, 581}, {203, 609}, {796, 589}, {494, 784}, {947, 420}, {354, 889}, + {184, 492}, {939, 587}, {598, 623}, {110, 971}, {296, 560}, {507, 881}, + {310, 674}, {280, 274}, {829, 845}, {273, 673}, {851, 52}, {814, 902}, + {995, 527}, {389, 611}, {537, 387}, {521, 350}, {849, 934}, {208, 57}, + {227, 962}, {30, 617}, {617, 962}, {512, 167}, {325, 1}, {232, 645}, {676, + 963}, {89, 242}, {57, 237}, {118, 252}, {807, 913}, {75, 425}, {782, 63}, + {901, 394}, {917, 23}, {923, 686}, {569, 451}, {217, 309}, {54, 950}, {508, + 185}, {642, 135}, {548, 436}, {495, 332}, {93, 368}, {483, 300}, {537, + 504}, {330, 526}, {29, 220}, {705, 598}, {468, 861}, {140, 13}, {192, 598}, + {347, 504}, {859, 182}, {369, 826}, {405, 979}, {694, 603}, {138, 269}, + {24, 364}, {256, 442}, {983, 657}, {946, 896}, {449, 915}, {6, 838}, {673, + 773}, {24, 231}, {235, 574}, {226, 875}, {327, 722}, {474, 311}, {501, + 904}, {291, 525}, {410, 937}, {804, 981}, {367, 113}, {449, 281}, {390, + 841}, {148, 698}, {396, 122}, {470, 726}, {442, 23}, {627, 762}, {493, + 406}, {710, 914}, {600, 289}, {182, 684}, {800, 569}, {965, 896}, {953, + 509}, {173, 25}, {671, 289}, {682, 205}, {71, 683}, {666, 286}, {414, 381}, + {23, 568}, {951, 45}, {987, 833}, {196, 225}, {686, 689}, {708, 87}, {954, + 715}, {209, 238}, {735, 700}, {357, 571}, {352, 466}, {373, 730}, {441, + 701}, {349, 224}, {289, 890}, {234, 680}, {721, 974}, {698, 801}, {181, + 460}, {47, 444}, {446, 142}, {262, 236}, {27, 494}, {958, 229}, {283, 210}, + {191, 279}, {359, 6}, {166, 810}, {808, 281}, {11, 801}, {484, 134}, {207, + 78}, {97, 128}, {182, 655}, {658, 329}, {926, 225}, {677, 285}, {183, 468}, + {685, 750}, {200, 999}, {871, 48}, {832, 515}, {485, 859}, {485, 75}, {13, + 435}, {612, 658}, {138, 796}, {919, 949}, {910, 747}, {975, 691}, {410, + 123}, {143, 788}, {535, 650}, {289, 386}, {48, 466}, {625, 783}, {690, + 851}, {761, 689}, {638, 964}, {106, 327}, {886, 578}, {223, 840}, {83, + 572}, {484, 802}, {703, 337}, {139, 522}, {234, 937}, {73, 555}, {15, 789}, + {335, 444}, {159, 468}, {747, 900}, {202, 563}, {880, 741}, {227, 307}, + {37, 594}, {380, 88}, {827, 286}, {747, 679}, {351, 208}, {374, 482}, {989, + 684}, {157, 691}, {207, 681}, {213, 302}, {372, 741}, {260, 231}, {710, + 110}, {529, 723}, {876, 580}, {156, 344}, {503, 467}, {887, 439}, {798, + 53}, {312, 342}, {890, 237}, {973, 738}, {36, 726}, {581, 386}, {153, 333}, + {341, 992}, {210, 389}, {551, 194}, {402, 407}, {875, 538}, {16, 873}, + {180, 47}, {119, 360}, {402, 996}, {422, 30}, {380, 939}, {722, 24}, {663, + 827}, {621, 266}, {113, 815}, {887, 976}, {859, 285}, {527, 646}, {971, + 190}, {84, 915}, {110, 953}, {934, 716}, {304, 316}, {413, 419}, {32, 373}, + {730, 936}, {478, 235}, {976, 947}, {859, 384}, {94, 33}, {912, 253}, {277, + 39}, {980, 696}, {91, 692}, {160, 263}, {190, 707}, {459, 166}, {221, 537}, + {849, 375}, {997, 679}, {57, 591}, {601, 48}, {393, 78}, {771, 839}, {254, + 485}, {939, 809}, {478, 167}, {881, 502}, {81, 317}, {76, 31}, {614, 887}, + {578, 417}, {970, 582}, {237, 389}, {951, 228}, {783, 626}, {146, 834}, + {233, 138}, {458, 861}, {167, 957}, {391, 619}, {880, 742}, {769, 210}, + {595, 786}, {708, 823}, {574, 159}, {663, 907}, {383, 781}, {861, 971}, + {395, 450}, {942, 183}, {558, 883}, {827, 609}, {23, 561}, {479, 386}, {68, + 915}, {75, 926}, {874, 927}, {15, 344}, {308, 359}, {788, 210}, {771, 65}, + {18, 494}, {760, 102}, {218, 602}, {53, 478}, {577, 476}, {794, 736}, {633, + 394}, {102, 941}, {686, 188}, {19, 326}, {626, 42}, {580, 305}, {304, 691}, + {569, 294}, {184, 239}, {698, 274}, {84, 168}, {994, 35}, {415, 591}, {139, + 416}, {152, 790}, {414, 123}, {987, 740}, {230, 973}, {206, 337}, {8, 329}, + {661, 477}, {514, 582}, {646, 944}, {32, 887}, {79, 655}, {749, 541}, {831, + 479}, {697, 443}, {691, 497}, {531, 432}, {595, 315}, {440, 663}, {567, + 705}, {710, 788}, {659, 805}, {112, 585}, {633, 934}, {358, 108}, {170, + 381}, {360, 876}, {155, 383}, {496, 69}, {104, 259}, {579, 513}, {907, 90}, + {438, 791}, {296, 322}, {382, 380}, {788, 196}, {667, 833}, {632, 598}, + {834, 396}, {45, 579}, {809, 724}, {470, 139}, {122, 686}, {667, 858}, + {860, 724}, {971, 644}, {838, 708}, {525, 660}, {94, 971}, {248, 531}, + {381, 380}, {178, 225}, {388, 291}, {218, 876}, {893, 177}, {896, 985}, + {930, 37}, {718, 804}, {553, 972}, {789, 952}, {367, 343}, {674, 387}, + {556, 160}, {985, 256}, {443, 917}, {377, 25}, {113, 453}, {43, 271}, {802, + 567}, {79, 199}, {713, 108}, {793, 517}, {860, 488}, {232, 686}, {983, + 451}, {360, 106}, {328, 853}, {190, 654}, {853, 10}, {580, 896}, {125, + 334}, {570, 449}, {192, 702}, {884, 146}, {209, 918}, {383, 810}, {186, + 460}, {137, 104}, {144, 845}, {775, 91}, {296, 805}, {773, 742}, {2, 136}, + {204, 752}, {370, 227}, {638, 957}, {192, 298}, {471, 309}, {718, 2}, {964, + 584}, {770, 1}, {232, 333}, {836, 921}, {263, 900}, {414, 629}, {824, 304}, + {379, 575}, {233, 160}, {547, 70}, {978, 603}, {55, 803}, {89, 160}, {490, + 598}, {888, 100}, {475, 973}, {880, 544}, {231, 894}, {546, 77}, {612, + 810}, {410, 882}, {964, 539}, {816, 920}, {809, 528}, {848, 472}, {202, + 291}, {724, 5}, {594, 131}, {385, 738}, {95, 277}, {432, 829}, {218, 434}, + {21, 127}, {926, 930}, {522, 650}, {563, 734}, {940, 927}, {733, 830}, + {494, 847}, {193, 290}, {492, 456}, {961, 47}, {876, 891}, {711, 827}, + {715, 474}, {489, 253}, {678, 289}, {398, 385}, {433, 146}, {173, 189}, + {985, 296}, {401, 357}, {592, 386}, {725, 243}, {308, 332}, {244, 637}, + {253, 938}, {852, 905}, {366, 712}, {560, 854}, {669, 270}, {666, 915}, + {362, 933}, {344, 832}, {255, 526}, {735, 438}, {400, 804}, {144, 230}, + {304, 338}, {287, 577}, {842, 8}, {327, 383}, {700, 41}, {499, 713}, {343, + 629}, {969, 654}, {24, 916}, {686, 276}, {525, 722}, {769, 302}, {240, 56}, + {828, 236}, {554, 486}, {727, 969}, {775, 356}, {411, 369}, {42, 943}, {70, + 429}, {824, 292}, {747, 577}, {359, 193}, {869, 301}, {390, 377}, {763, + 507}, {489, 526}, {63, 869}, {504, 647}, {525, 315}, {330, 65}, {818, 970}, + {976, 690}, {593, 324}, {242, 378}, {734, 271}, {120, 111}, {891, 898}, + {70, 675}, {929, 823}, {742, 330}, {347, 904}, {142, 496}, {353, 505}, + {448, 352}, {325, 794}, {54, 485}, {543, 70}, {763, 423}, {142, 846}, {112, + 7}, {652, 82}, {999, 939}, {402, 293}, {378, 300}, {717, 731}, {412, 885}, + {827, 595}, {179, 823}, {389, 424}, {353, 533}, {229, 511}, {889, 743}, + {235, 394}, {717, 22}, {121, 962}, {31, 991}, {843, 687}, {904, 89}, {695, + 705}, {624, 573}, {869, 643}, {887, 495}, {340, 827}, {833, 219}, {746, + 804}, {547, 350}, {127, 206}, {755, 827}, {108, 539}, {586, 321}, {451, + 251}, {860, 409}, {505, 953}, {713, 909}, {177, 226}, {15, 708}, {425, + 339}, {80, 63}, {181, 26}, {419, 139}, {298, 680}, {805, 607}, {259, 545}, + {451, 440}, {880, 225}, {255, 259}, {337, 637}, {974, 363}, {849, 850}, + {905, 206}, {868, 820}, {972, 972}, {736, 724}, {305, 420}, {63, 848}, + {947, 836}, {39, 947}, {355, 636}, {418, 13}, {816, 641}, {845, 374}, {359, + 921}, {322, 65}, {165, 728}, {997, 813}, {77, 344}, {340, 390}, {264, 367}, + {261, 712}, {953, 838}, {204, 743}, {48, 398}, {801, 61}, {186, 700}, {854, + 109}, {624, 172}, {304, 915}, {426, 711}, {45, 117}, {321, 549}, {387, + 555}, {206, 970}, {791, 317}, {786, 20}, {343, 594}, {361, 382}, {278, + 277}, {331, 225}, {510, 807}, {441, 493}, {396, 271}, {797, 678}, {254, + 123}, {539, 172}, {380, 267}, {420, 211}, {96, 247}, {407, 644}, {869, + 814}, {202, 338}, {327, 759}, {734, 713}, {18, 699}, {827, 103}, {953, + 551}, {206, 734}, {499, 98}, {66, 784}, {346, 444}, {824, 654}, {3, 224}, + {519, 231}, {319, 627}, {816, 292}, {81, 9}, {926, 899}, {27, 659}, {432, + 183}, {170, 709}, {522, 394}, {165, 264}, {469, 54}, {838, 705}, {101, + 415}, {713, 54}, {828, 650}, {480, 333}, {899, 162}, {830, 709}, {93, 789}, + {192, 173}, {720, 943}, {921, 758}, {100, 616}, {611, 62}, {168, 702}, + {776, 468}, {417, 79}, {420, 73}, {415, 149}, {383, 193}, {258, 597}, {871, + 677}, {327, 545}, {693, 93}, {714, 7}, {313, 135}, {788, 457}, {976, 961}, + {680, 113}, {422, 998}, {162, 301}, {269, 440}, {834, 25}, {559, 853}, + {787, 947}, {919, 930}, {910, 932}, {970, 417}, {846, 238}, {252, 741}, + {119, 401}, {136, 438}, {829, 432}, {614, 245}, {890, 7}, {303, 367}, {498, + 31}, {363, 384}, {805, 632}, {476, 821}, {940, 345}, {526, 932}, {966, + 643}, {572, 542}, {504, 255}, {841, 653}, {63, 91}, {97, 440}, {885, 550}, + {156, 851}, {190, 241}, {105, 859}, {264, 18}, {704, 965}, {598, 654}, + {922, 197}, {18, 543}, {677, 871}, {168, 514}, {334, 157}, {294, 553}, + {466, 973}, {792, 673}, {541, 331}, {307, 943}, {640, 551}, {947, 250}, + {390, 780}, {343, 379}, {80, 340}, {838, 145}, {634, 45}, {410, 2}, {934, + 338}, {621, 206}, {606, 977}, {735, 258}, {373, 274}, {631, 574}, {120, + 947}, {882, 725}, {139, 294}, {694, 450}, {476, 201}, {815, 305}, {234, + 563}, {305, 214}, {498, 39}, {753, 93}, {863, 81}, {202, 617}, {341, 719}, + {118, 153}, {586, 342}, {624, 806}, {498, 730}, {64, 211}, {858, 569}, + {541, 202}, {204, 259}, {517, 187}, {3, 682}, {36, 760}, {300, 287}, {980, + 873}, {260, 430}, {427, 411}, {677, 520}, {119, 515}, {18, 823}, {530, 39}, + {728, 185}, {537, 856}, {190, 883}, {409, 157}, {442, 271}, {347, 185}, + {884, 749}, {757, 55}, {394, 98}, {662, 829}, {32, 751}, {685, 491}, {937, + 571}, {624, 613}, {3, 417}, {90, 932}, {48, 377}, {541, 231}, {498, 940}, + {573, 755}, {157, 40}, {888, 205}, {161, 489}, {344, 601}, {300, 543}, + {319, 378}, {257, 918}, {536, 295}, {738, 305}, {970, 797}, {609, 698}, + {90, 668}, {114, 527}, {116, 267}, {429, 953}, {273, 401}, {76, 772}, {806, + 158}, {411, 627}, {271, 128}, {177, 72}, {995, 652}, {574, 35}, {250, 284}, + {283, 339}, {154, 304}, {602, 904}, {465, 102}, {709, 747}, {952, 735}, + {12, 418}, {473, 24}, {24, 880}, {38, 195}, {34, 797}, {647, 687}, {34, + 367}, {165, 62}, {454, 192}, {949, 492}, {671, 677}, {868, 896}, {333, + 965}, {846, 833}, {486, 905}, {123, 273}, {495, 697}, {319, 687}, {20, + 827}, {863, 991}, {530, 940}, {934, 56}, {2, 922}, {423, 716}, {863, 560}, + {447, 208}, {300, 597}, {941, 567}, {652, 665}, {668, 666}, {962, 732}, + {221, 61}, {108, 983}, {359, 374}, {589, 882}, {545, 460}, {728, 281}, + {101, 286}, {913, 474}, {819, 438}, {189, 863}, {377, 662}, {715, 322}, + {851, 766}, {957, 400}, {948, 312}, {444, 219}, {708, 763}, {19, 939}, + {390, 433}, {629, 689}, {55, 92}, {854, 656}, {67, 592}, {104, 334}, {578, + 863}, {961, 905}, {607, 412}, {177, 53}, {799, 76}, {891, 446}, {343, 78}, + {112, 79}, {872, 242}, {604, 137}, {606, 267}, {158, 640}, {559, 979}, + {908, 154}, {532, 630}, {519, 234}, {647, 247}, {690, 376}, {193, 268}, + {617, 89}, {561, 501}, {719, 455}, {820, 890}, {568, 977}, {363, 416}, + {234, 833}, {851, 107}, {213, 609}, {140, 436}, {229, 40}, {438, 262}, {77, + 404}, {686, 9}, {42, 545}, {398, 433}, {759, 105}, {13, 808}, {727, 495}, + {918, 189}, {879, 817}, {45, 747}, {335, 886}, {266, 985}, {708, 643}, + {200, 613}, {855, 834}, {980, 575}, {756, 71}, {609, 504}, {711, 333}, + {798, 973}, {316, 946}, {444, 449}, {213, 929}, {860, 579}, {683, 38}, {8, + 863}, {992, 618}, {407, 715}, {791, 526}, {848, 85}, {30, 690}, {468, 208}, + {238, 404}, {989, 480}, {662, 156}, {670, 160}, {691, 172}, {16, 903}, + {620, 93}, {856, 96}, {308, 571}, {479, 881}, {44, 903}, {621, 540}, {9, + 168}, {481, 405}, {551, 995}, {172, 260}, {369, 659}, {110, 855}, {880, + 298}, {687, 388}, {95, 828}, {960, 512}, {968, 6}, {878, 909}, {138, 124}, + {751, 408}, {252, 243}, {683, 119}, {573, 282}, {768, 782}, {415, 691}, + {272, 643}, {108, 966}, {657, 186}, {486, 665}, {889, 927}, {565, 826}, + {901, 858}, {99, 999}, {748, 415}, {571, 472}, {910, 484}, {905, 735}, + {894, 229}, {381, 908}, {944, 320}, {815, 85}, {177, 915}, {669, 415}, + {686, 425}, {19, 242}, {985, 845}, {213, 277}, {955, 349}, {940, 804}, + {954, 198}, {403, 329}, {87, 855}, {239, 563}, {152, 898}, {544, 643}, + {760, 95}, {780, 501}, {470, 475}, {301, 646}, {298, 5}, {795, 899}, {82, + 558}, {778, 360}, {209, 706}, {589, 301}, {719, 798}, {268, 205}, {327, + 460}, {470, 22}, {660, 372}, {96, 497}, {485, 760}, {168, 821}, {215, 332}, + {276, 112}, {332, 804}, {808, 747}, {673, 35}, {605, 230}, {987, 466}, + {179, 383}, {705, 165}, {356, 404}, {943, 208}, {14, 50}, {776, 346}, {789, + 358}, {751, 233}, {303, 306}, {654, 843}, {841, 547}, {263, 35}, {218, + 227}, {462, 727}, {52, 64}, {220, 173}, {675, 635}, {856, 409}, {495, 696}, + {664, 132}, {388, 849}, {766, 515}, {781, 228}, {145, 633}, {405, 645}, + {426, 954}, {793, 318}, {46, 30}, {848, 998}, {907, 612}, {163, 173}, {670, + 712}, {356, 239}, {966, 345}, {341, 388}, {435, 237}, {459, 470}, {466, + 310}, {380, 128}, {174, 263}, {28, 118}, {119, 782}, {531, 8}, {930, 160}, + {604, 165}, {131, 773}, {949, 844}, {753, 316}, {740, 917}, {78, 608}, + {917, 51}, {923, 824}, {381, 895}, {61, 109}, {959, 224}, {357, 393}, {756, + 289}, {826, 231}, {23, 535}, {416, 756}, {812, 411}, {1, 12}, {340, 746}, + {472, 109}, {467, 595}, {218, 179}, {582, 756}, {801, 138}, {698, 895}, + {43, 359}, {999, 378}, {624, 595}, {37, 23}, {454, 226}, {70, 213}, {676, + 673}, {654, 62}, {249, 809}, {150, 776}, {192, 442}, {875, 325}, {706, + 628}, {212, 638}, {597, 149}, {205, 604}, {951, 189}, {641, 944}, {915, + 487}, {944, 532}, {460, 48}, {95, 482}, {735, 61}, {136, 970}, {788, 733}, + {533, 485}, {533, 660}, {758, 626}, {777, 313}, {702, 720}, {709, 926}, + {777, 297}, {332, 698}, {552, 27}, {465, 725}, {330, 473}, {938, 20}, {276, + 605}, {454, 844}, {950, 805}, {276, 408}, {391, 773}, {297, 949}, {64, 38}, + {177, 236}, {76, 498}, {218, 521}, {582, 428}, {205, 175}, {987, 756}, + {247, 115}, {815, 970}, {860, 775}, {319, 15}, {860, 36}, {326, 228}, {966, + 293}, {257, 159}, {935, 552}, {34, 735}, {39, 165}, {878, 215}, {405, 352}, + {72, 298}, {927, 771}, {896, 66}, {116, 677}, {36, 268}, {546, 11}, {742, + 766}, {42, 783}, {774, 864}, {245, 4}, {932, 149}, {77, 509}, {714, 65}, + {839, 333}, {204, 656}, {345, 375}, {855, 807}, {466, 109}, {727, 880}, + {948, 413}, {5, 541}, {355, 443}, {583, 629}, {350, 688}, {739, 416}, {329, + 182}, {185, 140}, {713, 911}, {950, 346}, {927, 204}, {375, 753}, {974, + 735}, {28, 783}, {706, 207}, {0, 21}, {842, 758}, {743, 924}, {966, 312}, + {329, 675}, {291, 490}, {329, 493}, {523, 862}, {916, 834}, {860, 424}, + {861, 46}, {617, 363}, {844, 174}, {309, 106}, {45, 899}, {353, 567}, {783, + 689}, {681, 155}, {524, 933}, {980, 142}, {69, 794}, {417, 89}, {562, 922}, + {741, 86}, {74, 996}, {813, 989}, {694, 73}, {680, 415}, {772, 797}, {982, + 101}, {283, 936}, {825, 226}, {945, 394}, {740, 599}, {68, 519}, {979, + 192}, {138, 4}, {201, 791}, {595, 210}, {215, 215}, {802, 488}, {539, 277}, + {919, 844}, {746, 495}, {604, 1}, {951, 456}, {51, 346}, {214, 108}, {587, + 565}, {464, 939}, {109, 945}, {691, 274}, {878, 197}, {311, 534}, {458, + 992}, {737, 696}, {853, 295}, {156, 139}, {285, 518}, {92, 169}, {590, + 142}, {263, 792}, {109, 656}, {406, 132}, {610, 734}, {359, 367}, {206, + 953}, {883, 324}, {669, 123}, {878, 598}, {569, 32}, {422, 527}, {957, + 964}, {700, 522}, {333, 719}, {647, 832}, {269, 457}, {703, 110}, {512, + 238}, {233, 121}, {710, 179}, {802, 93}, {715, 825}, {321, 307}, {866, + 980}, {231, 586}, {845, 456}, {26, 265}, {656, 883}, {221, 951}, {131, 80}, + {629, 876}, {736, 293}, {171, 526}, {121, 189}, {269, 962}, {709, 939}, + {800, 320}, {874, 235}, {728, 670}, {821, 325}, {149, 445}, {17, 593}, + {573, 443}, {879, 321}, {718, 148}, {282, 455}, {420, 144}, {449, 303}, + {404, 53}, {958, 594}, {855, 155}, {33, 537}, {179, 325}, {907, 621}, {26, + 951}, {722, 289}, {213, 846}, {141, 106}, {530, 683}, {905, 934}, {425, + 686}, {486, 256}, {47, 82}, {133, 45}, {526, 690}, {733, 856}, {144, 172}, + {69, 853}, {658, 302}, {64, 165}, {957, 613}, {29, 136}, {464, 346}, {114, + 87}, {832, 654}, {631, 900}, {96, 461}, {657, 458}, {863, 372}, {217, 614}, + {352, 705}, {791, 378}, {617, 550}, {222, 578}, {86, 907}, {163, 593}, + {410, 307}, {603, 403}, {335, 892}, {20, 926}, {400, 745}, {747, 190}, + {443, 698}, {705, 735}, {169, 30}, {921, 106}, {167, 498}, {11, 339}, {89, + 578}, {470, 594}, {325, 50}, {127, 952}, {767, 463}, {887, 49}, {193, 249}, + {69, 487}, {723, 280}, {294, 744}, {242, 384}, {685, 92}, {975, 970}, {824, + 253}, {592, 419}, {278, 481}, {361, 162}, {439, 17}, {881, 221}, {4, 615}, + {399, 211}, {546, 45}, {834, 438}, {553, 879}, {603, 895}, {412, 366}, {63, + 697}, {16, 246}, {808, 2}, {73, 225}, {288, 607}, {475, 616}, {428, 999}, + {560, 595}, {421, 404}, {563, 899}, {126, 138}, {388, 419}, {47, 376}, + {305, 1}, {420, 381}, {501, 447}, {378, 720}, {469, 301}, {584, 853}, {659, + 498}, {849, 517}, {970, 851}, {840, 212}, {612, 508}, {38, 471}, {98, 504}, + {143, 368}, {846, 492}, {760, 788}, {720, 353}, {115, 858}, {389, 911}, + {142, 719}, {960, 273}, {97, 714}, {306, 744}, {336, 321}, {684, 493}, + {951, 464}, {985, 894}, {627, 860}, {550, 559}, {503, 840}, {473, 170}, + {322, 672}, {521, 717}, {329, 566}, {703, 582}, {934, 969}, {313, 635}, + {698, 984}, {128, 880}, {658, 154}, {524, 145}, {9, 315}, {71, 425}, {565, + 104}, {817, 84}, {429, 976}, {601, 279}, {695, 146}, {210, 144}, {740, + 609}, {872, 201}, {163, 52}, {857, 27}, {699, 499}, {921, 545}, {730, 762}, + {850, 326}, {609, 673}, {699, 453}, {910, 238}, {592, 215}, {573, 850}, + {749, 217}, {449, 592}, {468, 331}, {476, 543}, {16, 800}, {361, 250}, + {995, 363}, {288, 858}, {971, 118}, {541, 583}, {561, 587}, {832, 293}, + {660, 60}, {247, 685}, {799, 279}, {169, 913}, {115, 746}, {91, 341}, {153, + 758}, {985, 628}, {116, 170}, {791, 712}, {449, 731}, {4, 719}, {975, 88}, + {292, 114}, {670, 607}, {4, 178}, {8, 389}, {492, 51}, {183, 606}, {314, + 888}, {571, 887}, {182, 275}, {572, 352}, {903, 459}, {131, 558}, {681, + 288}, {40, 333}, {587, 267}, {483, 764}, {966, 348}, {676, 279}, {141, + 409}, {904, 760}, {740, 406}, {474, 565}, {932, 428}, {40, 192}, {539, + 232}, {613, 528}, {207, 63}, {321, 714}, {872, 570}, {862, 809}, {882, + 348}, {594, 603}, {756, 279}, {531, 767}, {993, 556}, {666, 276}, {961, + 176}, {573, 800}, {417, 146}, {863, 327}, {627, 118}, {692, 250}, {272, + 244}, {587, 159}, {22, 704}, {263, 515}, {492, 19}, {728, 979}, {257, 917}, + {383, 409}, {58, 423}, {480, 417}, {399, 716}, {374, 633}, {408, 265}, + {762, 749}, {695, 629}, {893, 272}, {285, 719}, {673, 456}, {690, 491}, + {243, 595}, {425, 955}, {558, 459}, {485, 562}, {47, 996}, {902, 486}, + {581, 716}, {525, 812}, {519, 945}, {923, 918}, {607, 572}, {882, 719}, + {544, 313}, {136, 36}, {977, 567}, {23, 244}, {777, 198}, {515, 590}, {349, + 90}, {858, 183}, {271, 996}, {116, 821}, {267, 829}, {576, 681}, {718, + 619}, {614, 393}, {571, 478}, {647, 840}, {843, 691}, {387, 201}, {330, + 391}, {667, 494}, {504, 437}, {111, 560}, {800, 103}, {931, 618}, {322, + 595}, {634, 848}, {96, 443}, {804, 14}, {647, 636}, {496, 521}, {529, 633}, + {556, 117}, {600, 162}, {211, 432}, {550, 909}, {890, 45}, {725, 61}, {147, + 626}, {482, 894}, {755, 159}, {55, 731}, {352, 488}, {934, 163}, {93, 653}, + {524, 36}, {607, 528}, {341, 126}, {425, 87}, {354, 490}, {22, 72}, {869, + 869}, {671, 203}, {181, 372}, {105, 40}, {634, 342}, {229, 923}, {271, + 472}, {325, 837}, {929, 218}, {968, 410}, {144, 628}, {716, 344}, {983, + 457}, {237, 209}, {400, 562}, {271, 227}, {134, 461}, {587, 903}, {658, + 519}, {172, 354}, {530, 739}, {244, 175}, {883, 940}, {786, 638}, {725, + 141}, {316, 830}, {584, 514}, {729, 900}, {481, 737}, {800, 228}, {478, 8}, + {36, 395}, {684, 178}, {369, 397}, {824, 109}, {792, 796}, {756, 679}, + {699, 241}, {227, 789}, {510, 521}, {665, 375}, {107, 595}, {565, 386}, + {687, 129}, {498, 585}, {897, 718}, {275, 767}, {538, 469}, {849, 208}, + {15, 349}, {518, 750}, {416, 158}, {577, 390}, {751, 420}, {892, 233}, + {793, 895}, {870, 691}, {339, 125}, {41, 403}, {784, 7}, {426, 181}, {197, + 430}, {71, 474}, {950, 597}, {183, 147}, {186, 939}, {679, 571}, {209, + 878}, {836, 395}, {391, 472}, {934, 326}, {536, 151}, {238, 925}, {38, + 763}, {425, 185}, {308, 254}, {110, 47}, {599, 993}, {236, 918}, {64, 337}, + {41, 57}, {676, 439}, {378, 388}, {845, 335}, {130, 115}, {662, 594}, {281, + 557}, {867, 935}, {236, 982}, {896, 905}, {952, 302}, {440, 186}, {46, + 229}, {301, 727}, {634, 140}, {85, 411}, {256, 571}, {16, 616}, {729, 321}, + {316, 102}, {756, 885}, {482, 617}, {98, 90}, {550, 914}, {651, 19}, {174, + 642}, {484, 245}, {949, 551}, {130, 581}, {745, 75}, {799, 498}, {375, 30}, + {30, 333}, {195, 199}, {80, 717}, {133, 852}, {464, 906}, {137, 642}, {108, + 772}, {833, 352}, {975, 236}, {799, 557}, {571, 106}, {237, 379}, {698, + 712}, {618, 735}, {352, 682}, {754, 799}, {981, 502}, {976, 721}, {216, + 281}, {470, 956}, {322, 777}, {387, 286}, {682, 601}, {351, 208}, {662, + 912}, {812, 226}, {806, 450}, {560, 469}, {94, 346}, {350, 347}, {625, + 116}, {21, 708}, {246, 770}, {17, 194}, {696, 357}, {397, 824}, {337, 647}, + {347, 913}, {989, 966}, {865, 180}, {669, 562}, {996, 627}, {455, 299}, + {672, 664}, {127, 377}, {646, 630}, {688, 232}, {584, 944}, {359, 678}, + {669, 61}, {628, 693}, {407, 293}, {206, 220}, {921, 248}, {750, 96}, {635, + 432}, {197, 304}, {692, 814}, {90, 506}, {407, 172}, {999, 392}, {851, + 698}, {479, 678}, {559, 484}, {837, 77}, {625, 13}, {837, 672}, {990, 558}, + {176, 124}, {366, 101}, {624, 558}, {69, 197}, {715, 502}, {186, 413}, + {614, 999}, {994, 827}, {964, 576}, {520, 539}, {599, 5}, {501, 329}, {954, + 376}, {276, 766}, {586, 487}, {765, 221}, {381, 549}, {257, 476}, {267, + 772}, {44, 362}, {961, 300}, {811, 52}, {264, 264}, {365, 994}, {776, 246}, + {943, 689}, {53, 545}, {93, 179}, {586, 443}, {861, 552}, {677, 338}, {448, + 496}, {740, 410}, {488, 537}, {588, 939}, {988, 592}, {448, 853}, {939, + 535}, {481, 302}, {241, 570}, {257, 231}, {706, 565}, {161, 725}, {509, + 477}, {555, 367}, {892, 476}, {287, 492}, {756, 49}, {305, 257}, {976, + 210}, {630, 406}, {792, 899}, {566, 152}, {801, 657}, {707, 759}, {49, + 246}, {812, 444}, {800, 615}, {484, 520}, {912, 857}, {26, 649}, {699, + 806}, {219, 916}, {300, 9}, {972, 176}, {447, 722}, {622, 791}, {806, 7}, + {824, 747}, {572, 233}, {741, 433}, {201, 402}, {262, 236}, {685, 390}, + {246, 64}, {190, 960}, {199, 936}, {984, 415}, {671, 121}, {739, 895}, + {239, 647}, {439, 129}, {6, 796}, {623, 682}, {429, 487}, {348, 180}, {299, + 899}, {834, 206}, {657, 78}, {155, 359}, {9, 884}, {660, 712}, {622, 243}, + {861, 605}, {570, 666}, {202, 614}, {933, 742}, {946, 580}, {797, 961}, + {376, 382}, {159, 153}, {925, 179}, {639, 942}, {233, 640}, {895, 229}, + {37, 552}, {850, 532}, {133, 500}, {740, 928}, {418, 742}, {994, 330}, + {694, 468}, {9, 545}, {171, 926}, {369, 674}, {333, 905}, {675, 27}, {479, + 266}, {122, 397}, {724, 366}, {363, 633}, {974, 268}, {243, 38}, {519, + 977}, {183, 256}, {80, 356}, {688, 363}, {964, 704}, {39, 806}, {102, 218}, + {865, 619}, {406, 385}, {354, 605}, {694, 495}, {159, 269}, {425, 290}, + {858, 232}, {338, 45}, {420, 337}, {404, 153}, {471, 485}, {639, 532}, + {628, 607}, {980, 680}, {826, 492}, {382, 321}, {33, 366}, {709, 250}, + {644, 122}, {763, 175}, {86, 126}, {606, 304}, {689, 490}, {791, 427}, + {732, 659}, {443, 40}, {773, 967}, {223, 400}, {473, 851}, {294, 983}, {76, + 705}, {310, 932}, {908, 862}, {451, 940}, {660, 670}, {250, 128}, {315, + 347}, {383, 576}, {121, 152}, {373, 334}, {963, 908}, {912, 936}, {974, + 563}, {989, 207}, {76, 699}, {239, 737}, {921, 325}, {819, 508}, {313, + 871}, {640, 904}, {679, 409}, {272, 429}, {793, 494}, {281, 143}, {446, + 929}, {961, 471}, {688, 492}, {519, 461}, {492, 569}, {258, 430}, {46, + 387}, {265, 162}, {746, 960}, {729, 795}, {304, 753}, {691, 833}, {941, + 109}, {97, 570}, {26, 652}, {842, 661}, {510, 778}, {883, 982}, {711, 605}, + {209, 160}, {182, 364}, {149, 82}, {245, 435}, {247, 818}, {499, 301}, + {820, 669}, {381, 89}, {899, 165}, {398, 969}, {22, 147}, {0, 1}, {0, 3}, + {0, 4}, {0, 7}, {0, 8}, {0, 10}, {0, 15}, {0, 16}, {0, 18}, {0, 31}, {0, + 32}, {0, 34}, {0, 35}, {0, 38}, {0, 41}, {0, 46}, {0, 57}, {0, 62}, {0, + 63}, {0, 65}, {0, 69}, {0, 72}, {0, 77}, {0, 85}, {0, 88}, {0, 94}, {0, + 96}, {0, 97}, {0, 108}, {0, 125}, {0, 126}, {0, 128}, {0, 132}, {0, 140}, + {0, 147}, {0, 156}, {0, 157}, {0, 160}, {0, 163}, {0, 166}, {0, 187}, {0, + 191}, {0, 195}, {0, 203}, {0, 205}, {0, 213}, {0, 218}, {0, 221}, {0, 250}, + {0, 253}, {0, 257}, {0, 260}, {0, 265}, {0, 266}, {0, 268}, {0, 269}, {0, + 281}, {0, 285}, {0, 306}, {0, 312}, {0, 313}, {0, 316}, {0, 327}, {0, 343}, + {0, 378}, {0, 382}, {0, 401}, {0, 406}, {0, 444}, {0, 471}, {0, 483}, {0, + 484}, {0, 500}, {0, 501}, {0, 503}, {0, 504}, {0, 507}, {0, 515}, {0, 518}, + {0, 522}, {0, 532}, {0, 542}, {0, 562}, {0, 565}, {0, 569}, {0, 612}, {0, + 625}, {0, 626}, {0, 628}, {0, 632}, {0, 650}, {0, 697}, {0, 750}, {0, 751}, + {0, 754}, {0, 757}, {0, 760}, {0, 781}, {0, 784}, {0, 785}, {0, 797}, {0, + 813}, {0, 837}, {0, 898}, {1, 0}, {1, 3}, {1, 4}, {1, 7}, {1, 15}, {1, 18}, + {1, 19}, {1, 25}, {1, 31}, {1, 34}, {1, 35}, {1, 46}, {1, 62}, {1, 63}, {1, + 65}, {1, 69}, {1, 80}, {1, 93}, {1, 125}, {1, 132}, {1, 135}, {1, 144}, {1, + 156}, {1, 171}, {1, 194}, {1, 233}, {1, 250}, {1, 254}, {1, 266}, {1, 281}, + {1, 285}, {1, 313}, {1, 327}, {1, 344}, {1, 409}, {1, 437}, {1, 452}, {1, + 475}, {1, 500}, {1, 515}, {1, 531}, {1, 565}, {1, 593}, {1, 600}, {1, 628}, + {1, 632}, {3, 0}, {3, 1}, {3, 7}, {3, 15}, {3, 26}, {3, 31}, {3, 32}, {3, + 34}, {3, 46}, {3, 62}, {3, 63}, {3, 65}, {3, 69}, {3, 111}, {3, 125}, {3, + 132}, {3, 140}, {3, 141}, {3, 143}, {3, 156}, {3, 187}, {3, 250}, {3, 269}, + {3, 281}, {3, 312}, {3, 327}, {3, 328}, {3, 346}, {3, 437}, {3, 500}, {3, + 501}, {3, 508}, {3, 515}, {3, 519}, {3, 547}, {3, 562}, {3, 594}, {3, 663}, + {3, 698}, {3, 750}, {3, 765}, {3, 885}, {4, 0}, {4, 7}, {4, 16}, {4, 23}, + {4, 26}, {4, 31}, {4, 57}, {4, 125}, {4, 150}, {4, 229}, {4, 236}, {4, + 258}, {4, 625}, {4, 671}, {4, 688}, {4, 843}, {7, 0}, {7, 1}, {7, 3}, {7, + 15}, {7, 16}, {7, 18}, {7, 22}, {7, 31}, {7, 62}, {7, 69}, {7, 93}, {7, + 125}, {7, 132}, {7, 135}, {7, 140}, {7, 159}, {7, 187}, {7, 202}, {7, 236}, + {7, 257}, {7, 265}, {7, 273}, {7, 284}, {7, 313}, {7, 315}, {7, 382}, {7, + 383}, {7, 390}, {7, 452}, {7, 503}, {7, 515}, {7, 518}, {7, 531}, {7, 538}, + {7, 549}, {7, 562}, {7, 565}, {7, 628}, {7, 750}, {7, 843}, {7, 876}, {8, + 3}, {8, 4}, {8, 7}, {8, 15}, {8, 22}, {8, 31}, {8, 47}, {8, 49}, {8, 135}, + {8, 159}, {8, 228}, {8, 250}, {8, 437}, {8, 508}, {8, 566}, {10, 0}, {10, + 1}, {10, 15}, {10, 16}, {10, 31}, {10, 41}, {10, 62}, {10, 125}, {10, 140}, + {10, 205}, {10, 250}, {10, 375}, {10, 562}, {10, 656}, {10, 882}, {11, 0}, + {11, 62}, {11, 265}, {11, 281}, {11, 625}, {15, 0}, {15, 1}, {15, 4}, {15, + 7}, {15, 10}, {15, 16}, {15, 18}, {15, 31}, {15, 38}, {15, 46}, {15, 54}, + {15, 62}, {15, 63}, {15, 69}, {15, 73}, {15, 93}, {15, 94}, {15, 109}, {15, + 125}, {15, 175}, {15, 190}, {15, 225}, {15, 254}, {15, 281}, {15, 284}, + {15, 316}, {15, 346}, {15, 406}, {15, 422}, {15, 500}, {15, 501}, {15, + 510}, {15, 515}, {15, 523}, {15, 562}, {15, 625}, {15, 702}, {15, 753}, + {15, 781}, {15, 791}, {16, 0}, {16, 1}, {16, 3}, {16, 10}, {16, 18}, {16, + 22}, {16, 62}, {16, 65}, {16, 140}, {16, 167}, {16, 282}, {16, 330}, {16, + 500}, {16, 503}, {16, 516}, {16, 562}, {16, 570}, {16, 641}, {16, 750}, + {16, 784}, {18, 1}, {18, 7}, {18, 22}, {18, 65}, {18, 132}, {18, 147}, {18, + 171}, {18, 187}, {18, 222}, {18, 251}, {18, 253}, {18, 268}, {18, 312}, + {18, 413}, {18, 422}, {18, 515}, {18, 516}, {18, 633}, {18, 758}, {19, 0}, + {19, 3}, {19, 4}, {19, 46}, {19, 273}, {19, 289}, {19, 750}, {22, 0}, {22, + 31}, {22, 34}, {22, 38}, {22, 46}, {22, 69}, {22, 70}, {22, 187}, {22, + 222}, {22, 250}, {22, 291}, {22, 312}, {22, 313}, {22, 359}, {22, 382}, + {22, 390}, {22, 437}, {22, 626}, {22, 641}, {23, 0}, {23, 41}, {23, 78}, + {23, 100}, {23, 281}, {23, 375}, {25, 0}, {25, 62}, {25, 140}, {25, 187}, + {25, 531}, {25, 844}, {26, 108}, {26, 394}, {31, 0}, {31, 1}, {31, 3}, {31, + 7}, {31, 15}, {31, 18}, {31, 47}, {31, 53}, {31, 65}, {31, 93}, {31, 108}, + {31, 112}, {31, 126}, {31, 140}, {31, 156}, {31, 157}, {31, 163}, {31, + 187}, {31, 202}, {31, 253}, {31, 265}, {31, 268}, {31, 312}, {31, 375}, + {31, 393}, {31, 468}, {31, 500}, {31, 503}, {31, 504}, {31, 507}, {31, + 510}, {31, 516}, {31, 538}, {31, 569}, {31, 687}, {31, 750}, {31, 812}, + {31, 876}, {32, 0}, {32, 3}, {32, 4}, {32, 18}, {32, 25}, {32, 26}, {32, + 46}, {32, 62}, {32, 140}, {32, 257}, {32, 272}, {32, 327}, {32, 382}, {32, + 500}, {32, 587}, {34, 0}, {34, 15}, {34, 16}, {34, 38}, {34, 46}, {34, 63}, + {34, 115}, {34, 126}, {34, 141}, {34, 265}, {34, 501}, {34, 508}, {34, + 538}, {34, 597}, {34, 750}, {35, 0}, {35, 3}, {35, 31}, {35, 34}, {35, 62}, + {35, 69}, {35, 80}, {35, 171}, {35, 508}, {38, 0}, {38, 7}, {38, 16}, {38, + 18}, {38, 25}, {38, 63}, {38, 66}, {38, 70}, {38, 96}, {38, 125}, {38, + 251}, {38, 265}, {38, 291}, {38, 313}, {38, 315}, {38, 316}, {38, 376}, + {38, 569}, {38, 625}, {38, 750}, {38, 757}, {38, 785}, {38, 875}, {39, 34}, + {39, 69}, {39, 125}, {39, 132}, {39, 266}, {39, 319}, {41, 50}, {41, 250}, + {41, 750}, {41, 765}, {42, 3}, {42, 312}, {46, 0}, {46, 1}, {46, 3}, {46, + 4}, {46, 7}, {46, 10}, {46, 15}, {46, 16}, {46, 25}, {46, 77}, {46, 163}, + {46, 265}, {46, 500}, {46, 501}, {46, 515}, {47, 101}, {47, 143}, {47, + 265}, {49, 1}, {49, 7}, {49, 125}, {49, 202}, {49, 257}, {49, 656}, {49, + 907}, {50, 22}, {53, 11}, {53, 228}, {53, 688}, {54, 62}, {54, 554}, {56, + 0}, {56, 1}, {56, 10}, {56, 31}, {56, 128}, {62, 0}, {62, 1}, {62, 3}, {62, + 4}, {62, 7}, {62, 10}, {62, 15}, {62, 16}, {62, 18}, {62, 25}, {62, 31}, + {62, 32}, {62, 35}, {62, 38}, {62, 46}, {62, 47}, {62, 65}, {62, 66}, {62, + 69}, {62, 96}, {62, 108}, {62, 125}, {62, 126}, {62, 135}, {62, 140}, {62, + 144}, {62, 147}, {62, 159}, {62, 187}, {62, 188}, {62, 250}, {62, 251}, + {62, 253}, {62, 257}, {62, 265}, {62, 288}, {62, 312}, {62, 319}, {62, + 353}, {62, 375}, {62, 376}, {62, 500}, {62, 504}, {62, 508}, {62, 510}, + {62, 525}, {62, 534}, {62, 538}, {62, 569}, {62, 625}, {62, 635}, {62, + 750}, {62, 812}, {62, 850}, {62, 851}, {63, 0}, {63, 1}, {63, 8}, {63, 46}, + {63, 65}, {63, 140}, {63, 171}, {63, 202}, {63, 250}, {63, 281}, {63, 284}, + {63, 503}, {63, 515}, {63, 518}, {63, 538}, {63, 562}, {63, 593}, {63, + 734}, {65, 0}, {65, 3}, {65, 4}, {65, 15}, {65, 16}, {65, 38}, {65, 87}, + {65, 112}, {65, 125}, {65, 126}, {65, 132}, {65, 133}, {65, 140}, {65, + 174}, {65, 266}, {65, 312}, {65, 375}, {65, 437}, {65, 503}, {65, 508}, + {65, 562}, {65, 626}, {66, 0}, {66, 46}, {66, 93}, {66, 257}, {66, 500}, + {66, 772}, {69, 1}, {69, 7}, {69, 11}, {69, 18}, {69, 62}, {69, 132}, {69, + 234}, {69, 257}, {69, 281}, {69, 296}, {69, 504}, {69, 532}, {69, 562}, + {69, 572}, {69, 694}, {69, 750}, {69, 757}, {70, 0}, {70, 15}, {70, 18}, + {70, 63}, {70, 157}, {70, 171}, {70, 250}, {70, 757}, {72, 32}, {72, 190}, + {72, 218}, {72, 291}, {72, 659}, {73, 0}, {73, 187}, {73, 250}, {73, 281}, + {73, 328}, {73, 500}, {77, 1}, {77, 3}, {77, 31}, {77, 38}, {77, 78}, {77, + 129}, {77, 171}, {77, 198}, {77, 250}, {77, 281}, {77, 508}, {77, 510}, + {77, 538}, {77, 563}, {77, 750}, {77, 754}, {77, 765}, {77, 768}, {77, + 937}, {78, 0}, {78, 1}, {78, 31}, {78, 218}, {78, 269}, {78, 376}, {80, 0}, + {80, 8}, {80, 10}, {80, 31}, {80, 69}, {80, 250}, {80, 269}, {80, 285}, + {80, 500}, {80, 547}, {81, 3}, {81, 31}, {81, 515}, {81, 531}, {84, 10}, + {84, 15}, {84, 132}, {84, 190}, {84, 250}, {84, 507}, {84, 532}, {84, 534}, + {84, 878}, {85, 126}, {85, 312}, {87, 140}, {93, 0}, {93, 3}, {93, 7}, {93, + 15}, {93, 18}, {93, 46}, {93, 53}, {93, 63}, {93, 132}, {93, 228}, {93, + 250}, {93, 261}, {93, 379}, {93, 383}, {93, 444}, {93, 507}, {93, 515}, + {93, 534}, {93, 562}, {94, 3}, {94, 34}, {94, 62}, {94, 500}, {94, 515}, + {94, 632}, {96, 38}, {96, 46}, {96, 53}, {96, 128}, {96, 250}, {96, 313}, + {100, 1}, {100, 128}, {101, 7}, {108, 0}, {108, 70}, {108, 254}, {108, + 272}, {108, 515}, {108, 565}, {109, 656}, {112, 500}, {115, 77}, {115, + 250}, {125, 0}, {125, 1}, {125, 3}, {125, 4}, {125, 7}, {125, 8}, {125, + 11}, {125, 15}, {125, 16}, {125, 23}, {125, 31}, {125, 32}, {125, 38}, + {125, 49}, {125, 62}, {125, 126}, {125, 156}, {125, 164}, {125, 187}, {125, + 194}, {125, 198}, {125, 205}, {125, 250}, {125, 253}, {125, 266}, {125, + 281}, {125, 284}, {125, 296}, {125, 347}, {125, 382}, {125, 501}, {125, + 507}, {125, 515}, {125, 523}, {125, 538}, {125, 554}, {125, 569}, {125, + 577}, {125, 593}, {125, 608}, {125, 647}, {125, 710}, {125, 718}, {125, + 750}, {125, 765}, {125, 766}, {126, 0}, {126, 1}, {126, 15}, {126, 25}, + {126, 32}, {126, 63}, {126, 80}, {126, 125}, {126, 128}, {126, 147}, {126, + 268}, {126, 284}, {126, 312}, {126, 437}, {126, 515}, {126, 531}, {128, 3}, + {128, 7}, {128, 15}, {128, 22}, {128, 62}, {128, 63}, {128, 73}, {128, 77}, + {128, 140}, {128, 147}, {128, 209}, {128, 269}, {128, 272}, {128, 281}, + {128, 296}, {128, 331}, {128, 515}, {128, 626}, {128, 632}, {129, 0}, {129, + 250}, {132, 0}, {132, 7}, {132, 62}, {132, 70}, {132, 77}, {132, 125}, + {132, 141}, {132, 160}, {132, 281}, {132, 500}, {132, 510}, {132, 625}, + {132, 626}, {132, 635}, {133, 18}, {133, 25}, {133, 125}, {133, 150}, {133, + 156}, {133, 281}, {133, 343}, {133, 500}, {135, 0}, {135, 1}, {135, 10}, + {135, 250}, {135, 531}, {136, 219}, {136, 375}, {136, 625}, {140, 0}, {140, + 3}, {140, 7}, {140, 31}, {140, 46}, {140, 56}, {140, 87}, {140, 250}, {140, + 257}, {140, 265}, {140, 282}, {140, 312}, {140, 500}, {140, 501}, {140, + 562}, {140, 565}, {140, 609}, {140, 875}, {141, 0}, {141, 140}, {141, 143}, + {141, 254}, {141, 281}, {141, 500}, {141, 522}, {141, 882}, {143, 4}, {143, + 10}, {143, 15}, {143, 62}, {143, 93}, {143, 226}, {143, 350}, {143, 584}, + {143, 633}, {143, 690}, {144, 0}, {144, 34}, {144, 375}, {144, 501}, {147, + 0}, {147, 22}, {147, 257}, {150, 265}, {156, 1}, {156, 8}, {156, 15}, {156, + 23}, {156, 31}, {156, 39}, {156, 125}, {156, 140}, {156, 203}, {156, 289}, + {156, 327}, {156, 376}, {156, 566}, {156, 577}, {156, 578}, {156, 596}, + {156, 907}, {157, 132}, {157, 156}, {157, 163}, {157, 437}, {157, 503}, + {157, 515}, {157, 593}, {157, 702}, {159, 1}, {159, 7}, {159, 32}, {159, + 62}, {159, 65}, {159, 77}, {159, 108}, {159, 132}, {159, 140}, {159, 233}, + {159, 501}, {160, 0}, {160, 72}, {160, 77}, {163, 0}, {163, 1}, {163, 125}, + {163, 126}, {163, 133}, {163, 625}, {163, 626}, {164, 1}, {164, 156}, {164, + 250}, {164, 365}, {166, 0}, {166, 78}, {171, 0}, {171, 1}, {171, 54}, {171, + 65}, {171, 94}, {171, 261}, {171, 500}, {172, 319}, {174, 77}, {175, 3}, + {178, 1}, {178, 11}, {178, 62}, {179, 500}, {187, 0}, {187, 3}, {187, 4}, + {187, 80}, {187, 126}, {187, 250}, {187, 281}, {187, 500}, {187, 518}, + {187, 534}, {187, 632}, {187, 702}, {187, 750}, {187, 785}, {188, 8}, {188, + 125}, {188, 147}, {188, 179}, {188, 253}, {188, 688}, {190, 0}, {190, 1}, + {190, 38}, {190, 766}, {191, 0}, {191, 140}, {194, 0}, {194, 3}, {194, 7}, + {194, 10}, {194, 35}, {194, 47}, {194, 93}, {194, 265}, {194, 410}, {194, + 846}, {195, 500}, {197, 15}, {197, 500}, {198, 0}, {202, 7}, {202, 62}, + {202, 125}, {202, 187}, {202, 453}, {202, 503}, {205, 69}, {206, 0}, {210, + 0}, {210, 789}, {212, 751}, {218, 15}, {218, 46}, {218, 63}, {218, 125}, + {218, 174}, {218, 257}, {218, 261}, {218, 265}, {219, 7}, {221, 0}, {221, + 7}, {221, 281}, {221, 508}, {225, 46}, {225, 516}, {241, 31}, {241, 63}, + {250, 0}, {250, 1}, {250, 3}, {250, 7}, {250, 8}, {250, 16}, {250, 18}, + {250, 19}, {250, 31}, {250, 32}, {250, 49}, {250, 53}, {250, 62}, {250, + 63}, {250, 69}, {250, 70}, {250, 77}, {250, 84}, {250, 108}, {250, 125}, + {250, 128}, {250, 129}, {250, 135}, {250, 140}, {250, 156}, {250, 167}, + {250, 187}, {250, 190}, {250, 209}, {250, 234}, {250, 257}, {250, 281}, + {250, 285}, {250, 313}, {250, 315}, {250, 343}, {250, 376}, {250, 452}, + {250, 500}, {250, 503}, {250, 510}, {250, 515}, {250, 562}, {250, 563}, + {250, 565}, {250, 572}, {250, 577}, {250, 593}, {250, 608}, {250, 629}, + {250, 640}, {250, 753}, {250, 769}, {250, 812}, {250, 827}, {250, 890}, + {250, 921}, {251, 3}, {251, 10}, {251, 34}, {251, 38}, {251, 125}, {251, + 126}, {251, 140}, {251, 171}, {251, 190}, {251, 202}, {251, 222}, {251, + 241}, {251, 284}, {251, 441}, {251, 500}, {251, 515}, {251, 556}, {251, + 728}, {251, 751}, {251, 827}, {253, 0}, {253, 1}, {253, 10}, {253, 22}, + {253, 25}, {253, 31}, {253, 47}, {253, 62}, {253, 125}, {253, 132}, {253, + 163}, {253, 195}, {253, 222}, {253, 315}, {253, 320}, {253, 375}, {253, + 500}, {253, 501}, {253, 507}, {253, 635}, {253, 651}, {253, 812}, {254, 0}, + {254, 135}, {254, 266}, {254, 281}, {254, 959}, {257, 10}, {257, 22}, {257, + 31}, {257, 62}, {257, 63}, {257, 125}, {257, 147}, {257, 157}, {257, 250}, + {257, 288}, {257, 297}, {257, 334}, {257, 353}, {257, 375}, {257, 501}, + {257, 694}, {257, 757}, {258, 65}, {258, 522}, {258, 593}, {258, 625}, + {258, 827}, {260, 0}, {260, 3}, {260, 254}, {260, 312}, {260, 330}, {260, + 516}, {260, 781}, {261, 10}, {261, 251}, {265, 0}, {265, 1}, {265, 3}, + {265, 8}, {265, 10}, {265, 26}, {265, 65}, {265, 77}, {265, 125}, {265, + 188}, {265, 257}, {265, 312}, {265, 409}, {265, 500}, {265, 503}, {265, + 570}, {266, 3}, {266, 156}, {266, 233}, {266, 694}, {268, 3}, {268, 8}, + {268, 16}, {268, 77}, {268, 125}, {268, 140}, {268, 503}, {268, 584}, {269, + 8}, {269, 31}, {272, 3}, {272, 77}, {272, 500}, {272, 632}, {275, 69}, + {275, 534}, {281, 0}, {281, 1}, {281, 8}, {281, 11}, {281, 39}, {281, 46}, + {281, 63}, {281, 70}, {281, 93}, {281, 128}, {281, 251}, {281, 507}, {281, + 516}, {281, 569}, {281, 671}, {281, 687}, {281, 768}, {282, 1}, {282, 23}, + {282, 31}, {282, 96}, {282, 132}, {282, 205}, {282, 288}, {282, 390}, {282, + 503}, {282, 788}, {284, 0}, {284, 31}, {284, 258}, {284, 500}, {288, 0}, + {288, 80}, {288, 140}, {288, 538}, {291, 7}, {291, 31}, {296, 0}, {296, 4}, + {296, 7}, {296, 16}, {296, 194}, {296, 359}, {296, 500}, {296, 501}, {296, + 508}, {296, 629}, {296, 687}, {296, 751}, {296, 812}, {297, 500}, {299, 0}, + {299, 1}, {299, 539}, {303, 22}, {303, 32}, {303, 656}, {306, 42}, {312, + 0}, {312, 7}, {312, 31}, {312, 128}, {312, 156}, {312, 229}, {312, 378}, + {312, 385}, {312, 578}, {312, 751}, {312, 788}, {312, 910}, {313, 0}, {313, + 15}, {313, 100}, {313, 125}, {313, 128}, {315, 49}, {315, 202}, {315, 593}, + {315, 909}, {316, 31}, {316, 38}, {316, 327}, {316, 625}, {319, 0}, {319, + 34}, {319, 125}, {319, 500}, {319, 625}, {319, 718}, {320, 19}, {320, 225}, + {320, 609}, {322, 0}, {322, 10}, {322, 140}, {322, 315}, {322, 625}, {323, + 538}, {327, 1}, {327, 3}, {327, 31}, {327, 69}, {328, 0}, {328, 77}, {328, + 132}, {330, 0}, {334, 156}, {334, 313}, {334, 376}, {334, 508}, {334, 546}, + {334, 648}, {337, 500}, {337, 503}, {338, 3}, {338, 126}, {343, 0}, {343, + 534}, {344, 34}, {344, 226}, {346, 125}, {346, 500}, {350, 562}, {350, + 587}, {353, 15}, {353, 500}, {353, 750}, {354, 500}, {358, 1}, {358, 299}, + {359, 501}, {365, 7}, {375, 0}, {375, 1}, {375, 35}, {375, 190}, {375, + 312}, {376, 15}, {376, 69}, {376, 78}, {376, 93}, {376, 250}, {376, 781}, + {378, 63}, {378, 69}, {378, 382}, {379, 15}, {382, 31}, {382, 531}, {382, + 640}, {385, 35}, {386, 15}, {386, 62}, {386, 159}, {386, 515}, {390, 94}, + {390, 125}, {390, 140}, {390, 260}, {390, 569}, {390, 609}, {390, 753}, + {391, 0}, {391, 18}, {391, 78}, {391, 876}, {393, 251}, {393, 625}, {398, + 897}, {400, 3}, {406, 34}, {406, 65}, {406, 144}, {406, 327}, {406, 515}, + {406, 531}, {406, 785}, {407, 3}, {409, 15}, {409, 46}, {409, 437}, {410, + 3}, {413, 15}, {413, 656}, {414, 49}, {421, 109}, {421, 197}, {422, 100}, + {424, 315}, {437, 15}, {437, 62}, {437, 125}, {437, 500}, {440, 7}, {440, + 500}, {440, 562}, {440, 775}, {448, 0}, {448, 438}, {452, 16}, {452, 31}, + {452, 250}, {452, 562}, {459, 62}, {459, 268}, {459, 796}, {468, 0}, {471, + 7}, {471, 538}, {475, 109}, {476, 63}, {490, 7}, {500, 0}, {500, 1}, {500, + 4}, {500, 7}, {500, 10}, {500, 11}, {500, 15}, {500, 18}, {500, 22}, {500, + 25}, {500, 31}, {500, 34}, {500, 38}, {500, 42}, {500, 46}, {500, 47}, + {500, 50}, {500, 62}, {500, 63}, {500, 69}, {500, 73}, {500, 77}, {500, + 80}, {500, 93}, {500, 94}, {500, 125}, {500, 126}, {500, 144}, {500, 166}, + {500, 171}, {500, 178}, {500, 250}, {500, 257}, {500, 265}, {500, 281}, + {500, 315}, {500, 316}, {500, 376}, {500, 386}, {500, 413}, {500, 437}, + {500, 503}, {500, 507}, {500, 508}, {500, 510}, {500, 516}, {500, 542}, + {500, 674}, {500, 687}, {500, 718}, {500, 750}, {500, 819}, {500, 875}, + {500, 906}, {501, 0}, {501, 7}, {501, 18}, {501, 22}, {501, 31}, {501, 47}, + {501, 125}, {501, 141}, {501, 150}, {501, 171}, {501, 190}, {501, 203}, + {501, 209}, {501, 260}, {501, 288}, {501, 500}, {501, 510}, {501, 534}, + {501, 656}, {501, 875}, {503, 0}, {503, 3}, {503, 16}, {503, 42}, {503, + 46}, {503, 63}, {503, 66}, {503, 147}, {503, 250}, {503, 261}, {503, 322}, + {503, 350}, {503, 378}, {503, 468}, {503, 500}, {503, 625}, {504, 1}, {504, + 3}, {504, 22}, {504, 77}, {504, 115}, {504, 500}, {504, 600}, {504, 789}, + {504, 796}, {507, 0}, {507, 3}, {507, 15}, {507, 18}, {507, 35}, {507, 38}, + {507, 125}, {507, 250}, {507, 500}, {507, 501}, {507, 547}, {508, 0}, {508, + 32}, {508, 132}, {508, 140}, {508, 382}, {508, 501}, {510, 66}, {510, 210}, + {510, 303}, {511, 268}, {511, 299}, {511, 500}, {515, 0}, {515, 1}, {515, + 3}, {515, 53}, {515, 69}, {515, 93}, {515, 128}, {515, 132}, {515, 156}, + {515, 167}, {515, 171}, {515, 188}, {515, 206}, {515, 250}, {515, 257}, + {515, 281}, {515, 375}, {515, 410}, {515, 437}, {515, 516}, {515, 751}, + {516, 0}, {516, 15}, {516, 35}, {516, 77}, {516, 103}, {516, 156}, {516, + 296}, {516, 300}, {516, 315}, {516, 459}, {516, 656}, {516, 834}, {516, + 865}, {518, 0}, {518, 7}, {518, 15}, {518, 65}, {518, 194}, {518, 250}, + {518, 265}, {518, 268}, {518, 577}, {518, 625}, {518, 812}, {519, 62}, + {522, 0}, {522, 31}, {522, 65}, {522, 80}, {522, 315}, {522, 376}, {522, + 531}, {522, 781}, {522, 897}, {523, 150}, {523, 282}, {525, 22}, {525, 31}, + {525, 251}, {526, 49}, {531, 0}, {531, 1}, {531, 7}, {531, 10}, {531, 49}, + {531, 93}, {531, 126}, {531, 132}, {531, 156}, {531, 202}, {531, 218}, + {531, 258}, {531, 315}, {531, 375}, {531, 503}, {531, 593}, {531, 765}, + {531, 772}, {532, 0}, {532, 3}, {532, 15}, {532, 31}, {532, 32}, {532, 62}, + {532, 73}, {532, 132}, {532, 562}, {534, 15}, {534, 300}, {535, 0}, {535, + 163}, {538, 0}, {538, 7}, {538, 15}, {538, 629}, {539, 62}, {539, 468}, + {541, 891}, {546, 1}, {546, 125}, {546, 281}, {546, 288}, {546, 319}, {546, + 327}, {546, 500}, {546, 532}, {546, 539}, {546, 640}, {547, 1}, {547, 250}, + {547, 784}, {549, 128}, {550, 0}, {550, 221}, {556, 15}, {562, 0}, {562, + 3}, {562, 4}, {562, 8}, {562, 31}, {562, 34}, {562, 46}, {562, 69}, {562, + 93}, {562, 125}, {562, 143}, {562, 319}, {562, 500}, {562, 501}, {562, + 504}, {562, 508}, {562, 510}, {563, 0}, {563, 39}, {563, 73}, {563, 257}, + {563, 376}, {563, 500}, {565, 0}, {565, 15}, {565, 296}, {565, 390}, {566, + 0}, {566, 815}, {569, 3}, {569, 125}, {569, 126}, {569, 175}, {569, 190}, + {569, 313}, {569, 538}, {570, 65}, {572, 65}, {572, 251}, {573, 500}, {577, + 0}, {577, 1}, {577, 15}, {577, 18}, {577, 159}, {577, 501}, {577, 518}, + {577, 584}, {578, 0}, {578, 385}, {580, 500}, {584, 625}, {593, 0}, {593, + 1}, {593, 3}, {593, 31}, {593, 41}, {593, 53}, {593, 281}, {593, 633}, + {593, 878}, {593, 882}, {594, 32}, {594, 93}, {594, 784}, {596, 66}, {600, + 0}, {600, 253}, {601, 577}, {603, 62}, {608, 0}, {611, 18}, {616, 375}, + {625, 0}, {625, 1}, {625, 3}, {625, 7}, {625, 34}, {625, 108}, {625, 126}, + {625, 129}, {625, 190}, {625, 254}, {625, 275}, {625, 284}, {625, 344}, + {625, 346}, {625, 390}, {625, 425}, {625, 532}, {625, 538}, {625, 750}, + {625, 968}, {626, 31}, {626, 32}, {626, 34}, {626, 63}, {626, 100}, {626, + 111}, {626, 500}, {626, 508}, {626, 643}, {628, 0}, {628, 46}, {629, 62}, + {629, 625}, {632, 8}, {632, 81}, {632, 93}, {632, 156}, {632, 202}, {632, + 547}, {633, 15}, {633, 268}, {633, 516}, {640, 0}, {640, 23}, {640, 39}, + {640, 251}, {640, 319}, {640, 358}, {640, 522}, {641, 8}, {641, 343}, {641, + 751}, {643, 265}, {644, 18}, {644, 41}, {648, 19}, {656, 3}, {656, 7}, + {656, 62}, {656, 382}, {656, 507}, {656, 757}, {657, 0}, {657, 81}, {657, + 894}, {659, 65}, {659, 476}, {659, 750}, {663, 125}, {671, 78}, {671, 253}, + {672, 375}, {687, 0}, {687, 15}, {687, 18}, {687, 31}, {687, 32}, {687, + 108}, {687, 125}, {687, 296}, {687, 500}, {688, 0}, {688, 38}, {690, 265}, + {690, 757}, {694, 0}, {694, 65}, {694, 250}, {695, 0}, {697, 93}, {702, 3}, + {702, 156}, {702, 282}, {703, 281}, {705, 46}, {705, 250}, {706, 94}, {709, + 628}, {718, 7}, {718, 31}, {718, 62}, {725, 0}, {725, 18}, {733, 376}, + {736, 784}, {740, 77}, {740, 126}, {750, 0}, {750, 16}, {750, 22}, {750, + 31}, {750, 32}, {750, 62}, {750, 63}, {750, 115}, {750, 128}, {750, 159}, + {750, 250}, {750, 315}, {750, 343}, {750, 344}, {750, 424}, {750, 562}, + {750, 569}, {750, 626}, {751, 46}, {751, 93}, {751, 300}, {751, 522}, {753, + 0}, {753, 80}, {753, 171}, {753, 250}, {753, 406}, {753, 562}, {757, 46}, + {757, 47}, {757, 209}, {757, 265}, {757, 312}, {758, 266}, {760, 0}, {760, + 8}, {761, 62}, {761, 600}, {765, 62}, {765, 148}, {765, 516}, {765, 534}, + {765, 823}, {766, 141}, {766, 500}, {768, 0}, {768, 141}, {768, 251}, {768, + 781}, {772, 375}, {772, 500}, {773, 0}, {781, 62}, {781, 63}, {781, 147}, + {781, 157}, {781, 187}, {782, 3}, {782, 15}, {782, 34}, {782, 125}, {782, + 253}, {784, 516}, {784, 569}, {788, 111}, {788, 116}, {789, 7}, {789, 140}, + {789, 376}, {796, 906}, {797, 1}, {812, 18}, {812, 126}, {812, 750}, {812, + 796}, {813, 0}, {813, 7}, {813, 500}, {815, 49}, {815, 96}, {819, 565}, + {822, 0}, {827, 4}, {827, 125}, {827, 383}, {830, 31}, {843, 4}, {843, + 281}, {846, 0}, {846, 69}, {859, 260}, {875, 35}, {875, 80}, {875, 148}, + {875, 206}, {875, 250}, {875, 312}, {875, 500}, {875, 503}, {875, 702}, + {875, 782}, {875, 878}, {876, 0}, {878, 31}, {878, 78}, {879, 515}, {890, + 163}, {890, 816}, {891, 553}, {897, 46}, {897, 265}, {906, 0}, {906, 65}, + {906, 518}, {913, 69}, {913, 197}, {937, 413}, {937, 531}, {937, 565}, + {940, 16}, {941, 250}, {944, 53}, {944, 346}, {947, 1}, {952, 375}, {59, 371}, + {371, 431}, {431, 564}, {564, 779}, {779, 835}, {835, 877}, {877, 986} + }; + + DASH_LOG_DEBUG("GraphTest.Construction", "construction started"); + graph_t g(edge_list.begin(), edge_list.end(), 1000); + DASH_LOG_DEBUG("GraphTest.Construction", "construction finished"); + + int unrecognized_edges = 0; + if (dash::myid() == 0) { + for (auto it = g.vertices().begin(); it != g.vertices().end(); ++it) { + auto v = g[it]; + for (auto e_it = v.out_edges().begin(); e_it != v.out_edges().end(); + ++e_it) { + auto e = g[e_it]; + //TODO: use std::map + bool unrecognized = true; + for (auto el_it = edge_list.begin(); + el_it != edge_list.end(); + ++el_it) { + if ( el_it->first == e.source().pos() + && el_it->second == e.target().pos()) { + edge_list.erase(el_it); + unrecognized = false; + break; + } + } + if(unrecognized) { + ++unrecognized_edges; + } + } + } + // all edges from the edge list in the graph + EXPECT_EQ_U(0, edge_list.size()); + // no edges in the graph that are not in the edge list + EXPECT_EQ_U(0, unrecognized_edges); + } + dash::barrier(); +} + diff --git a/dash/test/container/GraphTest.h b/dash/test/container/GraphTest.h new file mode 100644 index 000000000..3e5c060a9 --- /dev/null +++ b/dash/test/container/GraphTest.h @@ -0,0 +1,21 @@ +#ifndef DASH__TEST__GRAPH_TEST_H_ +#define DASH__TEST__GRAPH_TEST_H_ + +#include "../TestBase.h" + +/** + * Test fixture for class dash::List + */ +class GraphTest : public dash::test::TestBase { +protected: + + GraphTest() { + LOG_MESSAGE(">>> Test suite: GraphTest"); + } + + virtual ~GraphTest() { + LOG_MESSAGE("<<< Closing test suite: GraphTest"); + } +}; + +#endif // DASH__TEST__GRAPH_TEST_H_