From e76743aef1514b63b4ab96035d704ee11e6e8e4f Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 12 Dec 2016 17:26:41 +0100 Subject: [PATCH 001/102] added rough graph concept --- dash/include/dash/Graph.h | 250 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 dash/include/dash/Graph.h diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h new file mode 100644 index 000000000..9d258c128 --- /dev/null +++ b/dash/include/dash/Graph.h @@ -0,0 +1,250 @@ +#ifndef DASH__GRAPH_H__INCLUDED +#define DASH__GRAPH_H__INCLUDED + +namespace dash { + +/** + * \defgroup DashGraphConcept Graph Concept + * Concept of a statically distributed graph container. + * + * \ingroup DashContainerConcept + * \{ + * \psr Description + * + * Description here. + * + * Extends Concepts: + * + * Container properties: + * + * Iterator validity: + * + * \par Member types + * + * \par Member functions + * + * \} + * + * Usage examples: + * + */ + +namespace graph { + +enum GraphDirection { + DirectedGraph, + BidirectionalGraph, + UndirectedGraph +}; + +} + +namespace detail { + +class vertex { + // TODO: define vertex interface +} + +template< + GraphDirection Direction, + typename VertexDescriptor> +class edge { + // TODO: define edge interface +}; + +} + +template< + GraphDirection Direction = dash::graph::DirectedGraph, + typename Partition = dash::graph::VertexPartition, + typename VertexProperties = Empty, // user-defined struct + typename EdgeProperties = Empty, // user-defined struct + typename GraphProperties = Empty, // user-defined struct + typename VertexContainer = std::vector, + typename EdgeContainer = std::vector, + typename VertexSizeType = std::size_t, + typename EdgeSizeType = std::size_t> +class Graph { + +public: + + typedef detail::vertex vertex_descriptor; + typedef detail::edge edge_descriptor; + typedef VertexSizeType vertex_size_type; + typedef vertex_size_type vertex_index_type; + typedef EdgeSizeType edge_size_type; + typedef edge_size_type edge_index_type; + + typedef Direction direction_type; + typedef Partition partition_type; + typedef VertexContainer vertex_container_type; + typedef EdgeContainer edge_container_type; + typedef VertexProperties vertex_properties_type; + typedef EdgeProperties edge_properties_type; + + // iterators: + // TODO: Find types + // Things to consider: + // - Undifrected Graph: no difference betw. in- and out-edge + // iterators and edge_iterator + typedef vertex_iterator; + typedef edge_iterator; + typedef in_edge_iterator; + typedef out_edge_iterator; + typedef adjacency_iterator; + + /** + * Constructs a graph with given properties + * NOTE: GraphProperties() probably won't work, if the template argument + * is empty + */ + Graph(const GraphProperties & prop = GraphProperties()); + + /** + * Copy-constructs graph from another one. + */ + Graph(const Graph & other); + + /** + * Copy-assigns data from another graph. + */ + Graph & operator=(const Graph & other); + + /** + * Returns a property object for the given vertex. + */ + VertexProperties & operator[](const vertex_descriptor & v) const; + + /** + * Returns a property object for the given edge. + */ + EdgeProperties & operator[](const edge_descriptor & e) const; + + /** + * TODO: Return full property object for graph properties or + * use getter- and setter methods for individual fields? + * Also: Are graph properties rellay needed? The user could just + * save them elsewhere. + */ + + /** + * Returns the number of vertices in the whole graph. + */ + const vertex_size_type num_vertices() const; + + /** + * Returns the index of the vertex with the highest index in the whole graph. + */ + const vertex_index_type max_vertex_index() const; + + /** + * Returns the number of edges in the whole graph. + */ + const edge_size_type num_edges() const; + + /** + * Returns the index of the edge with the highest index in the whole graph. + */ + const edge_index_type max_edge_index() const; + + /** + * Returns, whether the graph is empty. + */ + bool empty() const; + + /** + * Adds a vertex with default properties. + * NOTE: global method. + */ + vertex_descriptor add_vertex(); + + /** + * Adds a vertex with the given properties. + * NOTE: global method. + */ + vertex_descriptor add_vertex(const VertexProperties & prop); + + /** + * Removes a given vertex. + * NOTE: global method. + */ + void remove_vertex(const vertex_descriptor & v); + + /** + * Removes all edges (in & out) from the given vertex). + * NOTE: global method. + */ + void clear_vertex(vertex_descriptor & v); + + /** + * Adds an edge between two given vertices with default properties. + * NOTE: global method. + * + * \return pair, with pair::first set to the edge_descriptor of the added + * edge + */ + std::pair add_edge(const vertex_descriptor & v1, + const vertex_descriptor & v2); + + /** + * Adds an edge between two given vertices with the given properties + * locally. + * NOTE: global method. + */ + std::pair add_edge(const vertex_descriptor & v1, + const vertex_descriptor & v2, const EdgeProperties & prop); + + /** + * Removes an edge between two given vertices. + * NOTE: global method. + */ + void remove_edge(const vertex_descriptor & v1, + const vertex_descriptor & v2); + + /** + * Removes a given edge. + * NOTE: global method. + */ + void remove_edge(const edge_descriptor & e); + + /** + * Commits local changes performed by methods classified as "global" to + * the whole data structure. + */ + void commit(); + + /** + * Returns iterator to the beginning of the vertex list. + */ + vertex_iterator v_begin(); + + /** + * TODO: Iterator method names are not really straightforward. + */ + vertex_iterator v_end(); + edge_iterator e_begin(); + edge_iterator e_end(); + in_edge_iterator ie_begin(); + in_edge_iterator ie_end(); + out_edge_iterator oe_begin(); + out_edge_iterator oe_end(); + adjacency_iterator a_begin(); + adjacency_iterator a_end(); + + vertex_iterator v_lbegin(); + vertex_iterator v_lend(); + edge_iterator e_lbegin(); + edge_iterator e_lend(); + in_edge_iterator ie_lbegin(); + in_edge_iterator ie_lend(); + out_edge_iterator oe_lbegin(); + out_edge_iterator oe_lend(); + adjacency_iterator a_lbegin(); + adjacency_iterator a_lend(); + +}; + +} + +#endif // DASH__GRAPH_H__INCLUDED + From e9a28577b0f044477bc92a84dd4e65c862c1ea68 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 12 Dec 2016 17:31:26 +0100 Subject: [PATCH 002/102] added empty partition concept --- dash/include/dash/Partition.h | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 dash/include/dash/Partition.h diff --git a/dash/include/dash/Partition.h b/dash/include/dash/Partition.h new file mode 100644 index 000000000..cd64b45ee --- /dev/null +++ b/dash/include/dash/Partition.h @@ -0,0 +1,27 @@ +#ifndef DASH__PARTITION_H_ +#define DASH__PARTITION_H_ + +namespace dash { + + /** + * \defgroup DashPartitionConcept Graph Partition Concept + * Partitioning concept for distributed graph data (vertices and edges). + * + * \ingroup DashConcept + * \{ + * \par Description + * + * Description here. + * + * \par Methods + * + * \} + */ + +} + +#endif // DASH__PARTITION_H_ + +#include +#include +#include From cbeedc907c0ae0040b2f41196922bc153413e8bd Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Wed, 14 Dec 2016 15:22:59 +0100 Subject: [PATCH 003/102] added starting point for vertex partitioning scheme concept --- .../dash/graph/partition/VertexPartition.h | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 dash/include/dash/graph/partition/VertexPartition.h diff --git a/dash/include/dash/graph/partition/VertexPartition.h b/dash/include/dash/graph/partition/VertexPartition.h new file mode 100644 index 000000000..d9e50f847 --- /dev/null +++ b/dash/include/dash/graph/partition/VertexPartition.h @@ -0,0 +1,67 @@ +#ifndef DASH__PARTITION_H__INCLUDED +#define DASH__PARTITION_H__INCLUDED + +namespace dash { + +/** + * \defgroup DashVertexPartitionConcept Graph Partition Concept (by vertex) + * Partitioning concept for graph data distributing the vertices according + * to some user-delivered function. + * + * \ingroup DashPartitionConcept + * \{ + * \par Description + * + * Description here. + * + * \par Methods + * + * \} + */ + +namespace detail { + +/** + * Functor that computes the unit ID for a given vertex ID in a team. + */ +class VertexHashDistribution { + + /** + * Constructs the Functor. + */ + VertexHashDistribution(Team & team); + + /** + * Maps a vertex ID to the corresponding unit. + * TODO: argument should be Graph::size_type + */ + dart_unit_t operator()(std::size_t id); + +}; + +} + +namespace graph { + +/** + * Partitions scheme for graph data. Partitions the data by vertex and + * distributes these vertices to units using some hash function. + * + * \concept{DashVertexPartitionConcept} + */ +template< + class DistributionMethod = dash::detail::VertexHashDistribution> +class VertexPartition { + + /** + * Constructs the partitioning scheme. + */ + VertexPartition(Team & team); + +}; + +} + +} + +#endif // DASH__PARTITION_H__INCLUDED From 3e2c0f6148ab77d8bdbaf2f55173bafcbd968ec4 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Wed, 14 Dec 2016 15:32:30 +0100 Subject: [PATCH 004/102] added starting point for load balanced vertex partitioning scheem concept --- .../graph/partition/LoadBalancePartition.h | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 dash/include/dash/graph/partition/LoadBalancePartition.h diff --git a/dash/include/dash/graph/partition/LoadBalancePartition.h b/dash/include/dash/graph/partition/LoadBalancePartition.h new file mode 100644 index 000000000..88f1be0c5 --- /dev/null +++ b/dash/include/dash/graph/partition/LoadBalancePartition.h @@ -0,0 +1,69 @@ +#ifndef DASH__PARTITION_H__INCLUDED +#define DASH__PARTITION_H__INCLUDED + +namespace dash { + +/** + * \defgroup DashLoadBalancedVertexPartitionConcept Graph Partition Concept (by + * vertex, load-balanced) + * Partitioning concept for graph data dynamically distributing the vertices + * according to some user-delivered cost function. Vertices might get re- + * distributed, when the addition of new vertices changes the cost function. + * + * \ingroup DashPartitionConcept + * \{ + * \par Description + * + * Description here. + * + * \par Methods + * + * \} + */ + +namespace detail { + +/** + * Functor that computes the unit ID for a given vertex ID in a team. + */ +class LoadBalanceDistribution{ + + /** + * Constructs the Functor. + */ + LoadBalanceDistribution(Team & team); + + /** + * Maps a vertex ID to the corresponding unit. + * TODO: argument should be Graph::size_type + */ + dart_unit_t operator()(std::size_t id); + +}; + +} + +namespace graph { + +/** + * Partitions scheme for graph data. Partitions the data by vertex and + * distributes these vertices to units using some hash function. + * + * \concept{DashLoadBalancedVertexPartitionConcept} + */ +template< + class CostMethod = dash::detail::LoadBalanceDistribution> +class LoadBalancedVertexPartition { + + /** + * Constructs the partitioning scheme. + */ + LoadBalancedVertexPartition(Team & team); + +}; + +} + +} + +#endif // DASH__PARTITION_H__INCLUDED From c4d54cc3debb5c2a5bca9ca5aaad249f6165b412 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Wed, 14 Dec 2016 15:34:52 +0100 Subject: [PATCH 005/102] renamed file --- dash/include/dash/Partition.h | 2 +- .../{LoadBalancePartition.h => LoadBalancedVertexPartition.h} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename dash/include/dash/graph/partition/{LoadBalancePartition.h => LoadBalancedVertexPartition.h} (100%) diff --git a/dash/include/dash/Partition.h b/dash/include/dash/Partition.h index cd64b45ee..a3697dbc2 100644 --- a/dash/include/dash/Partition.h +++ b/dash/include/dash/Partition.h @@ -24,4 +24,4 @@ namespace dash { #include #include -#include +#include diff --git a/dash/include/dash/graph/partition/LoadBalancePartition.h b/dash/include/dash/graph/partition/LoadBalancedVertexPartition.h similarity index 100% rename from dash/include/dash/graph/partition/LoadBalancePartition.h rename to dash/include/dash/graph/partition/LoadBalancedVertexPartition.h From 31af333030627ad58ca461f48d25a3a6ab779744 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Wed, 14 Dec 2016 15:35:49 +0100 Subject: [PATCH 006/102] changed define in Partition.h --- dash/include/dash/Partition.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dash/include/dash/Partition.h b/dash/include/dash/Partition.h index a3697dbc2..755902a15 100644 --- a/dash/include/dash/Partition.h +++ b/dash/include/dash/Partition.h @@ -1,5 +1,5 @@ -#ifndef DASH__PARTITION_H_ -#define DASH__PARTITION_H_ +#ifndef DASH__PARTITION_H__INCLUDED +#define DASH__PARTITION_H__INCLUDED namespace dash { @@ -20,7 +20,7 @@ namespace dash { } -#endif // DASH__PARTITION_H_ +#endif // DASH__PARTITION_H_INCLUDED #include #include From c4f1aa58d901967d239cb6d73b0eab5d9e0711bd Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Tue, 17 Jan 2017 10:50:53 +0100 Subject: [PATCH 007/102] refactored partitions and graph iterators --- .../dash/{Partition.h => DynamicPattern.h} | 6 +- dash/include/dash/Graph.h | 113 +++++++++++------- .../LoadBalancedDynamicPattern.h} | 4 +- .../VertexPartitionedDynamicPattern.h} | 4 +- 4 files changed, 77 insertions(+), 50 deletions(-) rename dash/include/dash/{Partition.h => DynamicPattern.h} (69%) rename dash/include/dash/graph/{partition/LoadBalancedVertexPartition.h => pattern/LoadBalancedDynamicPattern.h} (94%) rename dash/include/dash/graph/{partition/VertexPartition.h => pattern/VertexPartitionedDynamicPattern.h} (92%) diff --git a/dash/include/dash/Partition.h b/dash/include/dash/DynamicPattern.h similarity index 69% rename from dash/include/dash/Partition.h rename to dash/include/dash/DynamicPattern.h index 755902a15..02cf9fb5d 100644 --- a/dash/include/dash/Partition.h +++ b/dash/include/dash/DynamicPattern.h @@ -22,6 +22,6 @@ namespace dash { #endif // DASH__PARTITION_H_INCLUDED -#include -#include -#include +#include +//#include +#include diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 9d258c128..c59c09245 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -52,11 +52,64 @@ class edge { // TODO: define edge interface }; +// TODO: find template paramters +class VertexIterator { + +} + +struct VertexIteratorWrapper { + + typedef VertexIterator vertex_iterator; + + /** + * Returns global iterator to the beginning of the vertex list. + */ + vertex_iterator begin(); + + /** + * Returns global iterator to the beginning of the vertex list. + */ + const_vertex_iterator begin() const; + + /** + * Returns global iterator to the end of the vertex list. + */ + vertex_iterator end(); + + /** + * Returns global iterator to the end of the vertex list. + */ + const_vertex_iterator end() const; + + /** + * Returns local iterator to the beginning of the vertex list. + */ + vertex_iterator lbegin(); + + /** + * Returns local iterator to the beginning of the vertex list. + */ + const_vertex_iterator lbegin() const; + + /** + * Returns local iterator to the end of the vertex list. + */ + vertex_iterator lend(); + + /** + * Returns local iterator to the end of the vertex list. + */ + const_vertex_iterator lend() const; + +}; + +// TODO: add all iterator types + wrappers + } template< GraphDirection Direction = dash::graph::DirectedGraph, - typename Partition = dash::graph::VertexPartition, + typename DynamicPattern = dash::graph::VertexPartitionedDynamicPattern, typename VertexProperties = Empty, // user-defined struct typename EdgeProperties = Empty, // user-defined struct typename GraphProperties = Empty, // user-defined struct @@ -66,6 +119,8 @@ template< typename EdgeSizeType = std::size_t> class Graph { + typedef detail::VertexIteratorWrapper vertex_it_wrapper; + public: typedef detail::vertex vertex_descriptor; @@ -76,22 +131,23 @@ class Graph { typedef edge_size_type edge_index_type; typedef Direction direction_type; - typedef Partition partition_type; + typedef DynamicPattern pattern_type; typedef VertexContainer vertex_container_type; typedef EdgeContainer edge_container_type; typedef VertexProperties vertex_properties_type; typedef EdgeProperties edge_properties_type; - // iterators: - // TODO: Find types - // Things to consider: - // - Undifrected Graph: no difference betw. in- and out-edge - // iterators and edge_iterator - typedef vertex_iterator; - typedef edge_iterator; - typedef in_edge_iterator; - typedef out_edge_iterator; - typedef adjacency_iterator; + typedef vertex_it_wrapper::vertex_iterator vertex_iterator; + typedef edge_iterator; + typedef in_edge_iterator; + typedef out_edge_iterator; + typedef adjacency_iterator; + +public: + + vertex_it_wrapper vertices; + +public: /** * Constructs a graph with given properties @@ -123,7 +179,7 @@ class Graph { /** * TODO: Return full property object for graph properties or * use getter- and setter methods for individual fields? - * Also: Are graph properties rellay needed? The user could just + * Also: Are graph properties really needed? The user could just * save them elsewhere. */ @@ -211,36 +267,7 @@ class Graph { * Commits local changes performed by methods classified as "global" to * the whole data structure. */ - void commit(); - - /** - * Returns iterator to the beginning of the vertex list. - */ - vertex_iterator v_begin(); - - /** - * TODO: Iterator method names are not really straightforward. - */ - vertex_iterator v_end(); - edge_iterator e_begin(); - edge_iterator e_end(); - in_edge_iterator ie_begin(); - in_edge_iterator ie_end(); - out_edge_iterator oe_begin(); - out_edge_iterator oe_end(); - adjacency_iterator a_begin(); - adjacency_iterator a_end(); - - vertex_iterator v_lbegin(); - vertex_iterator v_lend(); - edge_iterator e_lbegin(); - edge_iterator e_lend(); - in_edge_iterator ie_lbegin(); - in_edge_iterator ie_lend(); - out_edge_iterator oe_lbegin(); - out_edge_iterator oe_lend(); - adjacency_iterator a_lbegin(); - adjacency_iterator a_lend(); + void barrier(); }; diff --git a/dash/include/dash/graph/partition/LoadBalancedVertexPartition.h b/dash/include/dash/graph/pattern/LoadBalancedDynamicPattern.h similarity index 94% rename from dash/include/dash/graph/partition/LoadBalancedVertexPartition.h rename to dash/include/dash/graph/pattern/LoadBalancedDynamicPattern.h index 88f1be0c5..68bcb5090 100644 --- a/dash/include/dash/graph/partition/LoadBalancedVertexPartition.h +++ b/dash/include/dash/graph/pattern/LoadBalancedDynamicPattern.h @@ -53,12 +53,12 @@ namespace graph { */ template< class CostMethod = dash::detail::LoadBalanceDistribution> -class LoadBalancedVertexPartition { +class LoadBalancedDynamicPattern { /** * Constructs the partitioning scheme. */ - LoadBalancedVertexPartition(Team & team); + LoadBalancedDynamicPattern(Team & team); }; diff --git a/dash/include/dash/graph/partition/VertexPartition.h b/dash/include/dash/graph/pattern/VertexPartitionedDynamicPattern.h similarity index 92% rename from dash/include/dash/graph/partition/VertexPartition.h rename to dash/include/dash/graph/pattern/VertexPartitionedDynamicPattern.h index d9e50f847..7b38013a2 100644 --- a/dash/include/dash/graph/partition/VertexPartition.h +++ b/dash/include/dash/graph/pattern/VertexPartitionedDynamicPattern.h @@ -51,12 +51,12 @@ namespace graph { */ template< class DistributionMethod = dash::detail::VertexHashDistribution> -class VertexPartition { +class VertexPartitionedDynamicPattern { /** * Constructs the partitioning scheme. */ - VertexPartition(Team & team); + VertexPartitionedDynamicPattern(Team & team); }; From 82d08f97b2ae2c5ad328d6bf26584733156b68ff Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 30 Jan 2017 15:26:43 +0100 Subject: [PATCH 008/102] added raw signature for graph patterns --- .../pattern/EdgePartitionedDynamicPattern.h | 83 +++++++++++++ .../pattern/LoadBalancedDynamicPattern.h | 117 +++++++++++++----- .../pattern/VertexPartitionedDynamicPattern.h | 84 +++++++++---- 3 files changed, 233 insertions(+), 51 deletions(-) create mode 100644 dash/include/dash/graph/pattern/EdgePartitionedDynamicPattern.h diff --git a/dash/include/dash/graph/pattern/EdgePartitionedDynamicPattern.h b/dash/include/dash/graph/pattern/EdgePartitionedDynamicPattern.h new file mode 100644 index 000000000..66ab20087 --- /dev/null +++ b/dash/include/dash/graph/pattern/EdgePartitionedDynamicPattern.h @@ -0,0 +1,83 @@ +#ifndef DASH__EDGE_PARTITIONED_DYNAMIC_PATTERN_H__INCLUDED +#define DASH__EDGE_PARTITIONED_DYNAMIC_PATTERN_H__INCLUDED +namespace dash { + +namespace graph { + +/** + * Pattern for graph data. Partitions the (logical) adjecency matrix of the + * graph into 2-dimensional blocks. Assigns one block to each unit. + * NOTE: The number of units in the assigned team must be a power of 2. + * NOTE: This pattern is intended for static graphs only. + * + * \concept{DashDynamicPatternConcept} + */ +template< + typename IndexType = std::size_t> +class EdgePartitionedDynamicPattern { + + typedef IndexType index_type; + typedef typename std::make_unsigned::type size_type; + + typedef struct { + dart_unit_t unit; + index_type index; + } local_index_type; + + /** + * Constructs the partitioning scheme. + */ + EdgePartitionedDynamicPattern(Team & team); + + /** + * Converts the given global vertex index to the owning unit. + */ + dart_unit_t unit_at(index_type global_index) const; + + /** + * Converts the given global vertex index to the owning unit and its + * local index. + */ + local_index_type local(index_type global_index) const; + + /** + * Converts the goiven local index of the calling unit to the respective + * global index. + */ + index_type global(index_type local_index) const; + + /** + * Returns, whether the element at the given global index is locak to + * the given unit. + */ + bool is_local(index_type global_index, dart_unit_t unit) const; + + /** + * Returns the amount of elements for which memory has been allocated. + */ + size_type capacity() const; + + /** + * Returns the amount of elements for which memory has been allocated at the + * calling unit. + */ + size_type local_capacity() const; + + /** + * Returns the amount of elements currently indexed in the pattern. + */ + size_type size() const; + + /** + * Returns the amount of elements currently indexed in the pattern local to + * the calling unit. + */ + size_type local_size() const; + +}; + +} + +} + +#endif // DASH__EDGE_PARTITIONED_DYNAMIC_PATTERN_H__INCLUDED diff --git a/dash/include/dash/graph/pattern/LoadBalancedDynamicPattern.h b/dash/include/dash/graph/pattern/LoadBalancedDynamicPattern.h index 68bcb5090..50a340311 100644 --- a/dash/include/dash/graph/pattern/LoadBalancedDynamicPattern.h +++ b/dash/include/dash/graph/pattern/LoadBalancedDynamicPattern.h @@ -1,37 +1,20 @@ -#ifndef DASH__PARTITION_H__INCLUDED -#define DASH__PARTITION_H__INCLUDED - +#ifndef DASH__LOAD_BALANCED_DYNAMIC_PATTERN_H__INCLUDED +#define DASH__LOAD_BALANCED_DYNAMIC_PATTERN_H__INCLUDED namespace dash { -/** - * \defgroup DashLoadBalancedVertexPartitionConcept Graph Partition Concept (by - * vertex, load-balanced) - * Partitioning concept for graph data dynamically distributing the vertices - * according to some user-delivered cost function. Vertices might get re- - * distributed, when the addition of new vertices changes the cost function. - * - * \ingroup DashPartitionConcept - * \{ - * \par Description - * - * Description here. - * - * \par Methods - * - * \} - */ +namespace graph { namespace detail { /** * Functor that computes the unit ID for a given vertex ID in a team. */ -class LoadBalanceDistribution{ +class VertexHashDistribution { /** * Constructs the Functor. */ - LoadBalanceDistribution(Team & team); + VertexHashDistribution(Team & team); /** * Maps a vertex ID to the corresponding unit. @@ -41,29 +24,107 @@ class LoadBalanceDistribution{ }; +/** + * Functor that computes an integer value depicting the current workload of + * the calling unit. + */ +class CostFunction { + + typedef local_edge_iterator; + + /** + * Constructs the functor. + */ + CostFunction(); + + /** + * Computes an integer value depicting the current workload of the calling + * uniti. + * TODO: What happens if the main workload source are vertices? + */ + int operator()(local_edge_iterator first, local_edge_iterator last); +}; + } -namespace graph { /** - * Partitions scheme for graph data. Partitions the data by vertex and - * distributes these vertices to units using some hash function. + * Pattern for graph data. Partitions the data by vertex and + * distributes these vertices to units using some hash function. + * Additionally redistributes vertices according to some cost funtion. + * NOTE: Redistribution allocates additional memory for a distributed + * directory. * - * \concept{DashLoadBalancedVertexPartitionConcept} + * \concept{DashDynamicPatternConcept} */ template< - class CostMethod = dash::detail::LoadBalanceDistribution> + class DistributionMethod = dash::graph::detail::VertexHashDistribution, + class CostFunction = dash::graph::detail::CostFunction, + typename IndexType = std::size_t> class LoadBalancedDynamicPattern { + typedef IndexType index_type; + typedef typename std::make_unsigned::type size_type; + + typedef struct { + dart_unit_t unit; + index_type index; + } local_index_type; + /** * Constructs the partitioning scheme. */ LoadBalancedDynamicPattern(Team & team); + /** + * Converts the given global vertex index to the owning unit. + */ + dart_unit_t unit_at(index_type global_index) const; + + /** + * Converts the given global vertex index to the owning unit and its + * local index. + */ + local_index_type local(index_type global_index) const; + + /** + * Converts the goiven local index of the calling unit to the respective + * global index. + */ + index_type global(index_type local_index) const; + + /** + * Returns, whether the element at the given global index is locak to + * the given unit. + */ + bool is_local(index_type global_index, dart_unit_t unit) const; + + /** + * Returns the amount of elements for which memory has been allocated. + */ + size_type capacity() const; + + /** + * Returns the amount of elements for which memory has been allocated at the + * calling unit. + */ + size_type local_capacity() const; + + /** + * Returns the amount of elements currently indexed in the pattern. + */ + size_type size() const; + + /** + * Returns the amount of elements currently indexed in the pattern local to + * the calling unit. + */ + size_type local_size() const; + }; } } -#endif // DASH__PARTITION_H__INCLUDED +#endif // DASH__LOAD_BALANCED_DYNAMIC_PATTERN_H__INCLUDED diff --git a/dash/include/dash/graph/pattern/VertexPartitionedDynamicPattern.h b/dash/include/dash/graph/pattern/VertexPartitionedDynamicPattern.h index 7b38013a2..895082735 100644 --- a/dash/include/dash/graph/pattern/VertexPartitionedDynamicPattern.h +++ b/dash/include/dash/graph/pattern/VertexPartitionedDynamicPattern.h @@ -1,23 +1,8 @@ -#ifndef DASH__PARTITION_H__INCLUDED -#define DASH__PARTITION_H__INCLUDED - +#ifndef DASH__VERTEX_PARTITIONED_DYNAMIC_PATTERN_H__INCLUDED +#define DASH__VERTEX_PARTITIONED_DYNAMIC_PATTERN_H__INCLUDED namespace dash { -/** - * \defgroup DashVertexPartitionConcept Graph Partition Concept (by vertex) - * Partitioning concept for graph data distributing the vertices according - * to some user-delivered function. - * - * \ingroup DashPartitionConcept - * \{ - * \par Description - * - * Description here. - * - * \par Methods - * - * \} - */ +namespace graph { namespace detail { @@ -41,27 +26,80 @@ class VertexHashDistribution { } -namespace graph { /** - * Partitions scheme for graph data. Partitions the data by vertex and + * Pattern for graph data. Partitions the data by vertex and * distributes these vertices to units using some hash function. * - * \concept{DashVertexPartitionConcept} + * \concept{DashDynamicPatternConcept} */ template< - class DistributionMethod = dash::detail::VertexHashDistribution> + class DistributionMethod = dash::graph::detail::VertexHashDistribution, + typename IndexType = std::size_t> class VertexPartitionedDynamicPattern { + typedef IndexType index_type; + typedef typename std::make_unsigned::type size_type; + + typedef struct { + dart_unit_t unit; + index_type index; + } local_index_type; + /** * Constructs the partitioning scheme. */ VertexPartitionedDynamicPattern(Team & team); + /** + * Converts the given global vertex index to the owning unit. + */ + dart_unit_t unit_at(index_type global_index) const; + + /** + * Converts the given global vertex index to the owning unit and its + * local index. + */ + local_index_type local(index_type global_index) const; + + /** + * Converts the goiven local index of the calling unit to the respective + * global index. + */ + index_type global(index_type local_index) const; + + /** + * Returns, whether the element at the given global index is locak to + * the given unit. + */ + bool is_local(index_type global_index, dart_unit_t unit) const; + + /** + * Returns the amount of elements for which memory has been allocated. + */ + size_type capacity() const; + + /** + * Returns the amount of elements for which memory has been allocated at the + * calling unit. + */ + size_type local_capacity() const; + + /** + * Returns the amount of elements currently indexed in the pattern. + */ + size_type size() const; + + /** + * Returns the amount of elements currently indexed in the pattern local to + * the calling unit. + */ + size_type local_size() const; + }; } } -#endif // DASH__PARTITION_H__INCLUDED +#endif // DASH__VERTEX_PARTITIONED_DYNAMIC_PATTERN_H__INCLUDED From 62354461887acb414f4bb6bad8ea7406046b3efc Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 30 Jan 2017 17:26:27 +0100 Subject: [PATCH 009/102] updated graph interface --- dash/include/dash/Graph.h | 92 +++++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 42 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index c59c09245..49a9bf202 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -5,7 +5,7 @@ namespace dash { /** * \defgroup DashGraphConcept Graph Concept - * Concept of a statically distributed graph container. + * Concept of a distributed graph container. * * \ingroup DashContainerConcept * \{ @@ -57,49 +57,55 @@ class VertexIterator { } +/** + * Wrapper for the vertex iterators of the graph. + */ struct VertexIteratorWrapper { - typedef VertexIterator vertex_iterator; + typedef VertexIterator iterator; + typedef const VertexIterator const_iterator; + typedef LocalVertexIterator local_iterator; + typedef const LocalVertexIterator const_local_iterator; /** * Returns global iterator to the beginning of the vertex list. */ - vertex_iterator begin(); + iterator begin(); /** * Returns global iterator to the beginning of the vertex list. */ - const_vertex_iterator begin() const; + const_iterator begin() const; /** * Returns global iterator to the end of the vertex list. */ - vertex_iterator end(); + iterator end(); /** * Returns global iterator to the end of the vertex list. */ - const_vertex_iterator end() const; + const_iterator end() const; /** * Returns local iterator to the beginning of the vertex list. */ - vertex_iterator lbegin(); + local_iterator lbegin(); /** * Returns local iterator to the beginning of the vertex list. */ - const_vertex_iterator lbegin() const; + const_local_iterator lbegin() const; /** * Returns local iterator to the end of the vertex list. */ - vertex_iterator lend(); + local_iterator lend(); /** * Returns local iterator to the end of the vertex list. */ - const_vertex_iterator lend() const; + const_local_iterator lend() const; }; @@ -107,12 +113,14 @@ struct VertexIteratorWrapper { } +/** + * Distributed, dynamic graph container for sparse graphs. + */ template< GraphDirection Direction = dash::graph::DirectedGraph, typename DynamicPattern = dash::graph::VertexPartitionedDynamicPattern, typename VertexProperties = Empty, // user-defined struct typename EdgeProperties = Empty, // user-defined struct - typename GraphProperties = Empty, // user-defined struct typename VertexContainer = std::vector, typename EdgeContainer = std::vector, typename VertexSizeType = std::size_t, @@ -120,15 +128,18 @@ template< class Graph { typedef detail::VertexIteratorWrapper vertex_it_wrapper; + typedef detail::EdgeIteratorWrapper edge_it_wrapper; + typedef detail::InEdgeIteratorWrapper in_edge_it_wrapper; + typedef detail::OutEdgeIteratorWrapper out_edge_it_wrapper; + typedef detail::AdjacencyIteratorWrapper adjacency_it_wrapper; public: - typedef detail::vertex vertex_descriptor; - typedef detail::edge edge_descriptor; + typedef typename detail::vertex::index_type vertex_index_type; + typedef typename detail::edge::index_type edge_index_type; typedef VertexSizeType vertex_size_type; - typedef vertex_size_type vertex_index_type; typedef EdgeSizeType edge_size_type; - typedef edge_size_type edge_index_type; typedef Direction direction_type; typedef DynamicPattern pattern_type; @@ -137,15 +148,19 @@ class Graph { typedef VertexProperties vertex_properties_type; typedef EdgeProperties edge_properties_type; - typedef vertex_it_wrapper::vertex_iterator vertex_iterator; - typedef edge_iterator; - typedef in_edge_iterator; - typedef out_edge_iterator; - typedef adjacency_iterator; + typedef typename vertex_it_wrapper::iterator vertex_iterator; + typedef typename edge_it_wrapper::iterator edge_iterator; + typedef typename in_edge_it_wrapper::iterator in_edge_iterator; + typedef typename out_edge_it_wrapper::iterator out_edge_iterator; + typedef typename adjacency_it_wrapper::iterator adjacency_iterator; public: - vertex_it_wrapper vertices; + vertex_it_wrapper vertices; + edge_it_wrapper edges; + in_edge_it_wrapper in_edges; + out_edge_it_wrapper out_edges; + adjacency_it_wrapper adjacent_vertices; public: @@ -169,20 +184,13 @@ class Graph { /** * Returns a property object for the given vertex. */ - VertexProperties & operator[](const vertex_descriptor & v) const; + VertexProperties & operator[](const vertex_index_type & v) const; /** * Returns a property object for the given edge. */ - EdgeProperties & operator[](const edge_descriptor & e) const; + EdgeProperties & operator[](const edge_index_type & e) const; - /** - * TODO: Return full property object for graph properties or - * use getter- and setter methods for individual fields? - * Also: Are graph properties really needed? The user could just - * save them elsewhere. - */ - /** * Returns the number of vertices in the whole graph. */ @@ -212,25 +220,25 @@ class Graph { * Adds a vertex with default properties. * NOTE: global method. */ - vertex_descriptor add_vertex(); + vertex_index_type add_vertex(); /** * Adds a vertex with the given properties. * NOTE: global method. */ - vertex_descriptor add_vertex(const VertexProperties & prop); + vertex_index_type add_vertex(const VertexProperties & prop); /** * Removes a given vertex. * NOTE: global method. */ - void remove_vertex(const vertex_descriptor & v); + void remove_vertex(const vertex_index_type & v); /** * Removes all edges (in & out) from the given vertex). * NOTE: global method. */ - void clear_vertex(vertex_descriptor & v); + void clear_vertex(vertex_index_type & v); /** * Adds an edge between two given vertices with default properties. @@ -239,29 +247,29 @@ class Graph { * \return pair, with pair::first set to the edge_descriptor of the added * edge */ - std::pair add_edge(const vertex_descriptor & v1, - const vertex_descriptor & v2); + std::pair add_edge(const vertex_index_type & v1, + const vertex_index_type & v2); /** * Adds an edge between two given vertices with the given properties * locally. * NOTE: global method. */ - std::pair add_edge(const vertex_descriptor & v1, - const vertex_descriptor & v2, const EdgeProperties & prop); + std::pair add_edge(const vertex_index_type & v1, + const vertex_index_type & v2, const EdgeProperties & prop); /** * Removes an edge between two given vertices. * NOTE: global method. */ - void remove_edge(const vertex_descriptor & v1, - const vertex_descriptor & v2); + void remove_edge(const vertex_index_type & v1, + const vertex_index_type & v2); /** - * Removes a given edge. + * Removes a given edge.a9f5c03d9de8dda4 * NOTE: global method. */ - void remove_edge(const edge_descriptor & e); + void remove_edge(const edge_index_type & e); /** * Commits local changes performed by methods classified as "global" to From befde773fac39daddd5ad605d2b67ac84fb7d169 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Tue, 7 Mar 2017 18:44:57 +0100 Subject: [PATCH 010/102] added working frame for graph implementation --- dash/include/dash/Container.h | 1 + dash/include/dash/Graph.h | 171 +++----- dash/include/dash/Pattern.h | 3 + dash/include/dash/graph/Graph-impl.h | 368 ++++++++++++++++++ dash/include/dash/graph/VertexIterator.h | 96 +++++ dash/include/dash/graph/internal/Graph.h | 31 ++ .../pattern/VertexPartitionedDynamicPattern.h | 7 +- 7 files changed, 550 insertions(+), 127 deletions(-) create mode 100644 dash/include/dash/graph/Graph-impl.h create mode 100644 dash/include/dash/graph/VertexIterator.h create mode 100644 dash/include/dash/graph/internal/Graph.h diff --git a/dash/include/dash/Container.h b/dash/include/dash/Container.h index 0a9da9881..952279db2 100644 --- a/dash/include/dash/Container.h +++ b/dash/include/dash/Container.h @@ -66,5 +66,6 @@ // Dynamic containers: #include #include +#include #endif // DASH__CONTAINER_H_ diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 49a9bf202..bbeaf5bce 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -1,6 +1,10 @@ #ifndef DASH__GRAPH_H__INCLUDED #define DASH__GRAPH_H__INCLUDED +#include +#include +#include + namespace dash { /** @@ -29,119 +33,47 @@ namespace dash { * */ -namespace graph { - -enum GraphDirection { - DirectedGraph, - BidirectionalGraph, - UndirectedGraph -}; - -} - -namespace detail { - -class vertex { - // TODO: define vertex interface -} - -template< - GraphDirection Direction, - typename VertexDescriptor> -class edge { - // TODO: define edge interface -}; - -// TODO: find template paramters -class VertexIterator { - -} - -/** - * Wrapper for the vertex iterators of the graph. - */ -struct VertexIteratorWrapper { - - typedef VertexIterator iterator; - typedef const VertexIterator const_iterator; - typedef LocalVertexIterator local_iterator; - typedef const LocalVertexIterator const_local_iterator; - - /** - * Returns global iterator to the beginning of the vertex list. - */ - iterator begin(); - - /** - * Returns global iterator to the beginning of the vertex list. - */ - const_iterator begin() const; - - /** - * Returns global iterator to the end of the vertex list. - */ - iterator end(); - - /** - * Returns global iterator to the end of the vertex list. - */ - const_iterator end() const; - - /** - * Returns local iterator to the beginning of the vertex list. - */ - local_iterator lbegin(); - - /** - * Returns local iterator to the beginning of the vertex list. - */ - const_local_iterator lbegin() const; - - /** - * Returns local iterator to the end of the vertex list. - */ - local_iterator lend(); - - /** - * Returns local iterator to the end of the vertex list. - */ - const_local_iterator lend() const; - -}; - -// TODO: add all iterator types + wrappers - -} /** * Distributed, dynamic graph container for sparse graphs. */ template< - GraphDirection Direction = dash::graph::DirectedGraph, - typename DynamicPattern = dash::graph::VertexPartitionedDynamicPattern, - typename VertexProperties = Empty, // user-defined struct - typename EdgeProperties = Empty, // user-defined struct - typename VertexContainer = std::vector, - typename EdgeContainer = std::vector, - typename VertexSizeType = std::size_t, - typename EdgeSizeType = std::size_t> + GraphDirection Direction = DirectedGraph, + //typename DynamicPattern = dash::graph::VertexPartitionedDynamicPattern, + typename DynamicPattern = void, + typename VertexProperties = internal::EmptyProperties, // user-defined struct + typename EdgeProperties = internal::EmptyProperties, // user-defined struct + typename VertexContainer = std::vector, + typename EdgeContainer = std::vector, + typename VertexIndexType = int, + typename EdgeIndexType = int> class Graph { - typedef detail::VertexIteratorWrapper vertex_it_wrapper; - typedef detail::EdgeIteratorWrapper edge_it_wrapper; - typedef detail::InEdgeIteratorWrapper in_edge_it_wrapper; - typedef detail::OutEdgeIteratorWrapper out_edge_it_wrapper; - typedef detail::AdjacencyIteratorWrapper adjacency_it_wrapper; + // TODO: add wrapper for all iterator types + typedef VertexIteratorWrapper + vertex_it_wrapper; + typedef VertexIteratorWrapper + edge_it_wrapper; + typedef VertexIteratorWrapper + in_edge_it_wrapper; + typedef VertexIteratorWrapper + out_edge_it_wrapper; + typedef VertexIteratorWrapper + adjacency_it_wrapper; public: - typedef typename detail::vertex::index_type vertex_index_type; - typedef typename detail::edge::index_type edge_index_type; - typedef VertexSizeType vertex_size_type; - typedef EdgeSizeType edge_size_type; + typedef Graph graph_type; + typedef VertexIndexType vertex_index_type; + typedef EdgeIndexType edge_index_type; + typedef typename + std::make_unsigned::type vertex_size_type; + typedef typename + std::make_unsigned::type edge_size_type; - typedef Direction direction_type; typedef DynamicPattern pattern_type; typedef VertexContainer vertex_container_type; typedef EdgeContainer edge_container_type; @@ -156,40 +88,28 @@ class Graph { public: - vertex_it_wrapper vertices; - edge_it_wrapper edges; - in_edge_it_wrapper in_edges; - out_edge_it_wrapper out_edges; - adjacency_it_wrapper adjacent_vertices; + vertex_it_wrapper vertices; + edge_it_wrapper edges; + in_edge_it_wrapper in_edges; + out_edge_it_wrapper out_edges; + adjacency_it_wrapper adjacent_vertices; public: /** - * Constructs a graph with given properties - * NOTE: GraphProperties() probably won't work, if the template argument - * is empty + * Constructs an empty graph. */ - Graph(const GraphProperties & prop = GraphProperties()); - + Graph(); + /** * Copy-constructs graph from another one. */ - Graph(const Graph & other); + Graph(const graph_type & other); /** * Copy-assigns data from another graph. */ - Graph & operator=(const Graph & other); - - /** - * Returns a property object for the given vertex. - */ - VertexProperties & operator[](const vertex_index_type & v) const; - - /** - * Returns a property object for the given edge. - */ - EdgeProperties & operator[](const edge_index_type & e) const; + graph_type & operator=(const graph_type & other); /** * Returns the number of vertices in the whole graph. @@ -276,10 +196,11 @@ class Graph { * the whole data structure. */ void barrier(); - }; } +#include + #endif // DASH__GRAPH_H__INCLUDED diff --git a/dash/include/dash/Pattern.h b/dash/include/dash/Pattern.h index 2132b49a9..6fa9dc4ca 100644 --- a/dash/include/dash/Pattern.h +++ b/dash/include/dash/Pattern.h @@ -634,4 +634,7 @@ class PatternConcept #include #include +// dynamic graph pattern types: +#include + #endif // DASH__PATTERN_H_ diff --git a/dash/include/dash/graph/Graph-impl.h b/dash/include/dash/graph/Graph-impl.h new file mode 100644 index 000000000..73499dde7 --- /dev/null +++ b/dash/include/dash/graph/Graph-impl.h @@ -0,0 +1,368 @@ +#ifndef DASH__GRAPH__INTERNAL__GRAPH_INL_H__INCLUDED +#define DASH__GRAPH__INTERNAL__GRAPH_INL_H__INCLUDED + +namespace dash { + +template +Graph::Graph() { + +} + +template +Graph +::Graph(const graph_type & other) { + +} + +template +Graph + & Graph +::operator=(const graph_type & other) { + +} + +template +const typename Graph + ::vertex_size_type Graph +::num_vertices() const { + +} + +template +const typename Graph + ::vertex_index_type Graph +::max_vertex_index() const { + +} + +template +const typename Graph + ::edge_size_type Graph +::num_edges() const { + +} + +template +const typename Graph + ::edge_index_type Graph +::max_edge_index() const { + +} + +template +bool Graph +::empty() const { + return true; +} + +template +typename Graph + ::vertex_index_type Graph +::add_vertex() { + +} + +template +typename Graph + ::vertex_index_type Graph +::add_vertex(const VertexProperties & prop) { + +} + +template +void Graph +::remove_vertex(const vertex_index_type & v) { + +} + +template +void Graph +::clear_vertex(vertex_index_type & v) { + +} + +template +std::pair + ::edge_index_type, bool> Graph +::add_edge(const vertex_index_type & v1, + const vertex_index_type & v2) { + +} + +template +std::pair + ::edge_index_type, bool> Graph +::add_edge(const vertex_index_type & v1, + const vertex_index_type & v2, const EdgeProperties & prop) { + +} + +template +void Graph +::remove_edge(const vertex_index_type & v1, + const vertex_index_type & v2) { + +} + +template +void Graph +::remove_edge(const edge_index_type & e) { + +} + +template +void Graph +::barrier() { + +} + +} + +#endif // DASH__GRAPH__INTERNAL__GRAPH_INL_H__INCLUDED diff --git a/dash/include/dash/graph/VertexIterator.h b/dash/include/dash/graph/VertexIterator.h new file mode 100644 index 000000000..9ef0e4eaa --- /dev/null +++ b/dash/include/dash/graph/VertexIterator.h @@ -0,0 +1,96 @@ +#ifndef DASH__GRAPH__VERTEX_ITERATOR_H__INCLUDED +#define DASH__GRAPH__VERTEX_ITERATOR_H__INCLUDED + +namespace dash { + +class VertexIterator { + +}; + +class LocalVertexIterator { + +}; + +/** + * Wrapper for the vertex iterators of the graph. + */ +template< + typename VertexIndexType, + typename VertexProperties> +struct VertexIteratorWrapper { + + typedef VertexIterator iterator; + typedef const VertexIterator const_iterator; + typedef LocalVertexIterator local_iterator; + typedef const LocalVertexIterator const_local_iterator; + typedef VertexIndexType vertex_index_type; + typedef VertexProperties vertex_properties_type; + + /** + * Returns a property object for the given vertex. + */ + vertex_properties_type & operator[](const vertex_index_type & v) const { + + } + + /** + * Returns global iterator to the beginning of the vertex list. + */ + iterator begin() { + + } + + /** + * Returns global iterator to the beginning of the vertex list. + */ + const_iterator begin() const { + + } + + /** + * Returns global iterator to the end of the vertex list. + */ + iterator end() { + + } + + /** + * Returns global iterator to the end of the vertex list. + */ + const_iterator end() const { + + } + + /** + * Returns local iterator to the beginning of the vertex list. + */ + local_iterator lbegin() { + + } + + /** + * Returns local iterator to the beginning of the vertex list. + */ + const_local_iterator lbegin() const { + + } + + /** + * Returns local iterator to the end of the vertex list. + */ + local_iterator lend() { + + } + + /** + * Returns local iterator to the end of the vertex list. + */ + const_local_iterator lend() const { + + } + +}; + +} + +#endif // DASH__GRAPH__VERTEX_ITERATOR_H__INCLUDED diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h new file mode 100644 index 000000000..f149a0e13 --- /dev/null +++ b/dash/include/dash/graph/internal/Graph.h @@ -0,0 +1,31 @@ +#ifndef DASH__GRAPH__INTERNAL__GRAPH_H__INCLUDED +#define DASH__GRAPH__INTERNAL__GRAPH_H__INCLUDED + +namespace dash { + +enum GraphDirection { + UndirectedGraph, + DirectedGraph + //BidirectionalGraph +}; + +namespace internal { + +class vertex { + // TODO: define vertex interface +}; + +template< + GraphDirection Direction, + typename VertexDescriptor> +class edge { + // TODO: define edge interface +}; + +class EmptyProperties { }; + +} + +} + +#endif // DASH__GRAPH__INTERNAL__GRAPH_H__INCLUDED diff --git a/dash/include/dash/graph/pattern/VertexPartitionedDynamicPattern.h b/dash/include/dash/graph/pattern/VertexPartitionedDynamicPattern.h index 895082735..c0d6b4eb4 100644 --- a/dash/include/dash/graph/pattern/VertexPartitionedDynamicPattern.h +++ b/dash/include/dash/graph/pattern/VertexPartitionedDynamicPattern.h @@ -1,5 +1,8 @@ #ifndef DASH__VERTEX_PARTITIONED_DYNAMIC_PATTERN_H__INCLUDED #define DASH__VERTEX_PARTITIONED_DYNAMIC_PATTERN_H__INCLUDED + +#include + namespace dash { namespace graph { @@ -20,7 +23,7 @@ class VertexHashDistribution { * Maps a vertex ID to the corresponding unit. * TODO: argument should be Graph::size_type */ - dart_unit_t operator()(std::size_t id); + dart_unit_t operator()(int id); }; @@ -35,7 +38,7 @@ class VertexHashDistribution { */ template< class DistributionMethod = dash::graph::detail::VertexHashDistribution, - typename IndexType = std::size_t> + typename IndexType = int> class VertexPartitionedDynamicPattern { typedef IndexType index_type; From f155179409e097f74c0bd7ee447d6ddcaec85bc4 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 9 Mar 2017 16:59:18 +0100 Subject: [PATCH 011/102] added local vertex list allocation --- dash/include/dash/GlobDynamicSequentialMem.h | 144 +++++++++++++++++++ dash/include/dash/Graph.h | 85 ++++++----- dash/include/dash/graph/Graph-impl.h | 90 +++++------- dash/include/dash/graph/internal/Graph.h | 42 +++++- 4 files changed, 268 insertions(+), 93 deletions(-) create mode 100644 dash/include/dash/GlobDynamicSequentialMem.h diff --git a/dash/include/dash/GlobDynamicSequentialMem.h b/dash/include/dash/GlobDynamicSequentialMem.h new file mode 100644 index 000000000..d0d6c7ca8 --- /dev/null +++ b/dash/include/dash/GlobDynamicSequentialMem.h @@ -0,0 +1,144 @@ +#ifndef DASH__GLOB_DYNAMIC_SEQUENTIAL_MEM_H_ +#define DASH__GLOB_DYNAMIC_SEQUENTIAL_MEM_H_ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include + +#include + + +namespace dash { + +template< + typename ContainerType> +class GlobDynamicSequentialMem +{ +private: + typedef GlobDynamicSequentialMem + self_t; + +public: + typedef ContainerType container_type; + typedef typename ContainerType::value_type value_type; + typedef typename ContainerType::iterator local_iterator; + typedef typename ContainerType::size_type size_type; + +public: + /** + * Constructor, collectively allocates the given number of elements in + * local memory of every unit in a team. + * + * \concept{DashDynamicMemorySpaceConcept} + * \concept{DashMemorySpaceConcept} + */ + GlobDynamicSequentialMem( + ContainerType & container, + size_type n_local_elem = 0, + Team & team = dash::Team::All()) + : _container(&container), + _public_container(&container), + _team(&team), + _teamid(team.dart_id()), + _nunits(team.size()), + _myid(team.myid()) + { + grow(n_local_elem); + commit(); + } + + /** + * Destructor, collectively frees underlying global memory. + */ + ~GlobDynamicSequentialMem() { } + + GlobDynamicSequentialMem() = delete; + + /** + * Copy constructor. + */ + GlobDynamicSequentialMem(const self_t & other) = default; + + /** + * Assignment operator. + */ + self_t & operator=(const self_t & rhs) = default; + + local_iterator grow(size_type num_elements) { + if(num_elements > 0) { + _container->reserve(_container->capacity() + num_elements); + _lbegin = _container->begin(); + _lend = _container->end(); + } + // TODO: return end of container or start of newly added capacity? + return _lend; + } + + void shrink(size_type num_elements) { + if(num_elements > 0) { + _container->reserve(_container->capacity() - num_elements); + } + } + + void commit() + { + } + + local_iterator lbegin() { + return _lbegin; + } + + local_iterator lend() { + return _lend; + } + + void push_back(value_type val) { + // if realloc would get triggered, copy container and keep old data so + // other units can still access the memory region + if(_container->capacity() == _container->size()) { + // TODO: new capacity? if capacity is not increased, push-back will + // trigger a realloc and we can use the increase algorithm of the + // container, but this implicates another memory copy. + container_type new_container(_container->capacity() * 2); + new_container = *_container; + _container = &new_container; + } + _container->push_back(val); + } + +private: + + container_type * _container; + container_type * _public_container; + Team * _team; + dart_team_t _teamid; + size_type _nunits = 0; + team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; + local_iterator _lbegin; + local_iterator _lend; + +}; + +} + +#endif // DASH__GLOB_DYNAMIC_SEQUENTIAL_MEM_H_ + diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index bbeaf5bce..6beedb1bc 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -4,6 +4,9 @@ #include #include #include +#include +#include +#include namespace dash { @@ -43,23 +46,31 @@ template< typename DynamicPattern = void, typename VertexProperties = internal::EmptyProperties, // user-defined struct typename EdgeProperties = internal::EmptyProperties, // user-defined struct - typename VertexContainer = std::vector, - typename EdgeContainer = std::vector, typename VertexIndexType = int, - typename EdgeIndexType = int> + typename EdgeIndexType = int, + typename EdgeContainer + = std::vector>, + typename VertexContainer + = std::vector>> class Graph { // TODO: add wrapper for all iterator types typedef VertexIteratorWrapper - vertex_it_wrapper; + vertex_it_wrapper; typedef VertexIteratorWrapper - edge_it_wrapper; + edge_it_wrapper; typedef VertexIteratorWrapper - in_edge_it_wrapper; + in_edge_it_wrapper; typedef VertexIteratorWrapper - out_edge_it_wrapper; + out_edge_it_wrapper; typedef VertexIteratorWrapper - adjacency_it_wrapper; + adjacency_it_wrapper; + + typedef internal::vertex vertex_type; + typedef internal::out_edge edge_type; + typedef GlobDynamicSequentialMem glob_mem_seq_type; public: @@ -99,17 +110,11 @@ class Graph { /** * Constructs an empty graph. */ - Graph(); - - /** - * Copy-constructs graph from another one. - */ - Graph(const graph_type & other); + Graph(vertex_size_type nvertices = 0, Team & team = dash::Team::All()); - /** - * Copy-assigns data from another graph. + /** Destructs the graph. */ - graph_type & operator=(const graph_type & other); + ~Graph(); /** * Returns the number of vertices in the whole graph. @@ -136,17 +141,12 @@ class Graph { */ bool empty() const; - /** - * Adds a vertex with default properties. - * NOTE: global method. - */ - vertex_index_type add_vertex(); - /** * Adds a vertex with the given properties. * NOTE: global method. */ - vertex_index_type add_vertex(const VertexProperties & prop); + vertex_index_type add_vertex(const VertexProperties & prop + = VertexProperties()); /** * Removes a given vertex. @@ -160,23 +160,14 @@ class Graph { */ void clear_vertex(vertex_index_type & v); - /** - * Adds an edge between two given vertices with default properties. - * NOTE: global method. - * - * \return pair, with pair::first set to the edge_descriptor of the added - * edge - */ - std::pair add_edge(const vertex_index_type & v1, - const vertex_index_type & v2); - /** * Adds an edge between two given vertices with the given properties * locally. * NOTE: global method. */ std::pair add_edge(const vertex_index_type & v1, - const vertex_index_type & v2, const EdgeProperties & prop); + const vertex_index_type & v2, + const EdgeProperties & prop = EdgeProperties()); /** * Removes an edge between two given vertices. @@ -196,6 +187,30 @@ class Graph { * the whole data structure. */ void barrier(); + + /** + * Globally allocates memory for vertex storage. + */ + bool allocate(vertex_size_type nvertices); + + /** + * Deallocates global memory of this container. + */ + void deallocate(); + +private: + + /** Stores all local vertices */ + VertexContainer _vertices; + /** Stores edge properties in undirected graphs */ + EdgeContainer _edges; + /** the team containing all units using the container */ + Team * _team = nullptr; + /** Global memory allocation and access for sequential memory regions */ + glob_mem_seq_type * _glob_mem_seq = nullptr; + /** Unit ID of the current unit */ + team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; + }; } diff --git a/dash/include/dash/graph/Graph-impl.h b/dash/include/dash/graph/Graph-impl.h index 73499dde7..639d7a693 100644 --- a/dash/include/dash/graph/Graph-impl.h +++ b/dash/include/dash/graph/Graph-impl.h @@ -3,25 +3,6 @@ namespace dash { -template -Graph::Graph() { - -} - template -::Graph(const graph_type & other) { - +::Graph(vertex_size_type nvertices, + Team & team) + : _team(&team), + _myid(team.myid()) +{ + allocate(nvertices); } template -Graph - & Graph -::operator=(const graph_type & other) { - +::~Graph() { + deallocate(); } template -::add_vertex() { - +::add_vertex(const VertexProperties & prop) { + vertex_type v(prop); + _glob_mem_seq->push_back(v); } template -typename Graph - ::vertex_index_type Graph -::add_vertex(const VertexProperties & prop) { +::remove_vertex(const vertex_index_type & v) { } @@ -232,7 +214,7 @@ void Graph -::remove_vertex(const vertex_index_type & v) { +::clear_vertex(vertex_index_type & v) { } @@ -244,7 +226,9 @@ template -void Graph + ::edge_index_type, bool> Graph -::clear_vertex(vertex_index_type & v) { +::add_edge(const vertex_index_type & v1, + const vertex_index_type & v2, const EdgeProperties & prop) { } @@ -264,9 +249,7 @@ template -std::pair - ::edge_index_type, bool> Graph -::add_edge(const vertex_index_type & v1, +::remove_edge(const vertex_index_type & v1, const vertex_index_type & v2) { } @@ -287,9 +270,7 @@ template -std::pair - ::edge_index_type, bool> Graph -::add_edge(const vertex_index_type & v1, - const vertex_index_type & v2, const EdgeProperties & prop) { +::remove_edge(const edge_index_type & e) { } @@ -318,8 +298,7 @@ void Graph -::remove_edge(const vertex_index_type & v1, - const vertex_index_type & v2) { +::barrier() { } @@ -331,7 +310,7 @@ template -void Graph -::remove_edge(const edge_index_type & e) { - +::allocate(vertex_size_type nvertices) { + auto lcap = dash::math::div_ceil(nvertices, _team->size()); + _glob_mem_seq = new glob_mem_seq_type(_vertices, lcap, *_team); + // Register deallocator of this list instance at the team + // instance that has been used to initialize it: + _team->register_deallocator(this, std::bind(&Graph::deallocate, this)); } template -::barrier() { - +::deallocate() { + if(_glob_mem_seq != nullptr) { + delete _glob_mem_seq; + } + // Remove this function from team deallocator list to avoid + // double-free: + _team->unregister_deallocator(this, std::bind(&Graph::deallocate, this)); } } diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index f149a0e13..c743e81ce 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -11,18 +11,46 @@ enum GraphDirection { namespace internal { -class vertex { - // TODO: define vertex interface +template< + typename EdgeContainer, + typename VertexProperties> +struct vertex { + + /** + * Creates a vertex with given properties. + */ + vertex(VertexProperties properties = VertexProperties()) + : _properties(properties) + { } + + /** Outgoing edges from this vertex */ + EdgeContainer _out_edges; + /** Properties of this vertex */ + VertexProperties _properties; + }; template< - GraphDirection Direction, - typename VertexDescriptor> -class edge { - // TODO: define edge interface + typename VertexIndexType, + typename EdgeProperties> +struct out_edge { + + /** + * Creates an edge from its parent vertex to target. + */ + out_edge(VertexIndexType target, EdgeProperties properties = EdgeProperties()) + : _target(target), + _properties(properties) + { } + + /** Target vertex the edge is pointing to */ + VertexIndexType _target; + /** Properties of this edge */ + EdgeProperties _properties; + }; -class EmptyProperties { }; +struct EmptyProperties { }; } From 162d6c66b7c88429abbade1148994b244fc7f392 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 9 Mar 2017 19:18:52 +0100 Subject: [PATCH 012/102] added local vertex iterator --- dash/include/dash/GlobDynamicSequentialMem.h | 11 ++-- dash/include/dash/Graph.h | 58 ++++++++++---------- dash/include/dash/graph/Graph-impl.h | 2 +- dash/include/dash/graph/VertexIterator.h | 36 +++++++----- 4 files changed, 60 insertions(+), 47 deletions(-) diff --git a/dash/include/dash/GlobDynamicSequentialMem.h b/dash/include/dash/GlobDynamicSequentialMem.h index d0d6c7ca8..6366113b2 100644 --- a/dash/include/dash/GlobDynamicSequentialMem.h +++ b/dash/include/dash/GlobDynamicSequentialMem.h @@ -52,11 +52,10 @@ class GlobDynamicSequentialMem * \concept{DashMemorySpaceConcept} */ GlobDynamicSequentialMem( - ContainerType & container, size_type n_local_elem = 0, Team & team = dash::Team::All()) - : _container(&container), - _public_container(&container), + : _container(new container_type()), + _public_container(_container), _team(&team), _teamid(team.dart_id()), _nunits(team.size()), @@ -120,7 +119,11 @@ class GlobDynamicSequentialMem // container, but this implicates another memory copy. container_type new_container(_container->capacity() * 2); new_container = *_container; - _container = &new_container; + if(_public_container != _container) { + delete _public_container; + _public_container = _container; + } + _container = &new_container; } _container->push_back(val); } diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 6beedb1bc..8c1195647 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -54,30 +54,32 @@ template< = std::vector>> class Graph { +public: + + typedef Graph graph_type; + +private: + // TODO: add wrapper for all iterator types - typedef VertexIteratorWrapper - vertex_it_wrapper; - typedef VertexIteratorWrapper - edge_it_wrapper; - typedef VertexIteratorWrapper - in_edge_it_wrapper; - typedef VertexIteratorWrapper - out_edge_it_wrapper; - typedef VertexIteratorWrapper - adjacency_it_wrapper; + typedef VertexIteratorWrapper vertex_it_wrapper; + typedef VertexIteratorWrapper edge_it_wrapper; + typedef VertexIteratorWrapper in_edge_it_wrapper; + typedef VertexIteratorWrapper out_edge_it_wrapper; + typedef VertexIteratorWrapper adjacency_it_wrapper; + + friend vertex_it_wrapper; typedef internal::vertex vertex_type; + VertexProperties> vertex_type; typedef internal::out_edge edge_type; - typedef GlobDynamicSequentialMem glob_mem_seq_type; + EdgeProperties> edge_type; + typedef GlobDynamicSequentialMem glob_mem_seq_type; public: - typedef Graph graph_type; typedef VertexIndexType vertex_index_type; typedef EdgeIndexType edge_index_type; typedef typename @@ -91,6 +93,8 @@ class Graph { typedef VertexProperties vertex_properties_type; typedef EdgeProperties edge_properties_type; + typedef typename glob_mem_seq_type::local_iterator local_vertex_iterator; + typedef typename vertex_it_wrapper::iterator vertex_iterator; typedef typename edge_it_wrapper::iterator edge_iterator; typedef typename in_edge_it_wrapper::iterator in_edge_iterator; @@ -99,11 +103,11 @@ class Graph { public: - vertex_it_wrapper vertices; - edge_it_wrapper edges; - in_edge_it_wrapper in_edges; - out_edge_it_wrapper out_edges; - adjacency_it_wrapper adjacent_vertices; + vertex_it_wrapper vertices = vertex_it_wrapper(this); + edge_it_wrapper edges = edge_it_wrapper(this); + in_edge_it_wrapper in_edges = in_edge_it_wrapper(this); + out_edge_it_wrapper out_edges = out_edge_it_wrapper(this); + adjacency_it_wrapper adjacent_vertices = adjacency_it_wrapper(this); public: @@ -200,16 +204,12 @@ class Graph { private: - /** Stores all local vertices */ - VertexContainer _vertices; - /** Stores edge properties in undirected graphs */ - EdgeContainer _edges; /** the team containing all units using the container */ - Team * _team = nullptr; + Team * _team = nullptr; /** Global memory allocation and access for sequential memory regions */ - glob_mem_seq_type * _glob_mem_seq = nullptr; + glob_mem_seq_type * _glob_mem_seq = nullptr; /** Unit ID of the current unit */ - team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; + team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; }; diff --git a/dash/include/dash/graph/Graph-impl.h b/dash/include/dash/graph/Graph-impl.h index 639d7a693..b13d3653d 100644 --- a/dash/include/dash/graph/Graph-impl.h +++ b/dash/include/dash/graph/Graph-impl.h @@ -320,7 +320,7 @@ bool Graph ::allocate(vertex_size_type nvertices) { auto lcap = dash::math::div_ceil(nvertices, _team->size()); - _glob_mem_seq = new glob_mem_seq_type(_vertices, lcap, *_team); + _glob_mem_seq = new glob_mem_seq_type(lcap, *_team); // Register deallocator of this list instance at the team // instance that has been used to initialize it: _team->register_deallocator(this, std::bind(&Graph::deallocate, this)); diff --git a/dash/include/dash/graph/VertexIterator.h b/dash/include/dash/graph/VertexIterator.h index 9ef0e4eaa..4d8a0ee83 100644 --- a/dash/include/dash/graph/VertexIterator.h +++ b/dash/include/dash/graph/VertexIterator.h @@ -14,17 +14,23 @@ class LocalVertexIterator { /** * Wrapper for the vertex iterators of the graph. */ -template< - typename VertexIndexType, - typename VertexProperties> +template struct VertexIteratorWrapper { - typedef VertexIterator iterator; - typedef const VertexIterator const_iterator; - typedef LocalVertexIterator local_iterator; - typedef const LocalVertexIterator const_local_iterator; - typedef VertexIndexType vertex_index_type; - typedef VertexProperties vertex_properties_type; + typedef Graph graph_type; + typedef VertexIterator iterator; + typedef const VertexIterator const_iterator; + typedef typename Graph::local_vertex_iterator local_iterator; + typedef const local_iterator const_local_iterator; + typedef typename Graph::vertex_index_type vertex_index_type; + typedef typename Graph::vertex_properties_type vertex_properties_type; + + /** + * Constructs the wrapper. + */ + VertexIteratorWrapper(graph_type * graph) + : _graph(graph) + { } /** * Returns a property object for the given vertex. @@ -65,30 +71,34 @@ struct VertexIteratorWrapper { * Returns local iterator to the beginning of the vertex list. */ local_iterator lbegin() { - + return _graph->_glob_mem_seq->lbegin(); } /** * Returns local iterator to the beginning of the vertex list. */ const_local_iterator lbegin() const { - + return _graph->_glob_mem_seq->lbegin(); } /** * Returns local iterator to the end of the vertex list. */ local_iterator lend() { - + return _graph->_glob_mem_seq->lend(); } /** * Returns local iterator to the end of the vertex list. */ const_local_iterator lend() const { - + return _graph->_glob_mem_seq->lend(); } +private: + + graph_type * _graph; + }; } From 8f3dfa8a2a97f68c478965ded16e745bda870aab Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Fri, 10 Mar 2017 10:58:53 +0100 Subject: [PATCH 013/102] fixed local vertex iterators --- dash/include/dash/GlobDynamicSequentialMem.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dash/include/dash/GlobDynamicSequentialMem.h b/dash/include/dash/GlobDynamicSequentialMem.h index 6366113b2..a56589c95 100644 --- a/dash/include/dash/GlobDynamicSequentialMem.h +++ b/dash/include/dash/GlobDynamicSequentialMem.h @@ -124,8 +124,10 @@ class GlobDynamicSequentialMem _public_container = _container; } _container = &new_container; + _lbegin = _container->begin(); } _container->push_back(val); + _lend = _container->end(); } private: From 245556ff1efd1ba3783c265064aea5f71badbe72 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Fri, 10 Mar 2017 11:06:23 +0100 Subject: [PATCH 014/102] fixed seq globmem shrink --- dash/include/dash/GlobDynamicSequentialMem.h | 1 + 1 file changed, 1 insertion(+) diff --git a/dash/include/dash/GlobDynamicSequentialMem.h b/dash/include/dash/GlobDynamicSequentialMem.h index a56589c95..172ea0f31 100644 --- a/dash/include/dash/GlobDynamicSequentialMem.h +++ b/dash/include/dash/GlobDynamicSequentialMem.h @@ -95,6 +95,7 @@ class GlobDynamicSequentialMem void shrink(size_type num_elements) { if(num_elements > 0) { _container->reserve(_container->capacity() - num_elements); + _lend = _container->end(); } } From a2ba2a2ec2f46436517bf04755e57864290db27f Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Fri, 10 Mar 2017 15:18:02 +0100 Subject: [PATCH 015/102] fixed increment of LocalBucketIter --- dash/include/dash/allocator/LocalBucketIter.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dash/include/dash/allocator/LocalBucketIter.h b/dash/include/dash/allocator/LocalBucketIter.h index ca70832a2..ef48b4b57 100644 --- a/dash/include/dash/allocator/LocalBucketIter.h +++ b/dash/include/dash/allocator/LocalBucketIter.h @@ -395,8 +395,9 @@ class LocalBucketIter } else { // find bucket containing element at given offset: for (; _bucket_it != _bucket_last; ++_bucket_it) { - if (offset >= _bucket_it->size) { - offset -= _bucket_it->size; + if (offset + _bucket_phase >= _bucket_it->size) { + offset -= _bucket_it->size - _bucket_phase; + _bucket_phase = 0; } else if (offset < _bucket_it->size) { _bucket_phase = offset; break; From dc592f024b1016decf52dddfe53d4a137fc66d12 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 13 Mar 2017 13:52:10 +0100 Subject: [PATCH 016/102] changed seq globmem functionality --- dash/include/dash/GlobDynamicSequentialMem.h | 116 +++++++++++++------ 1 file changed, 80 insertions(+), 36 deletions(-) diff --git a/dash/include/dash/GlobDynamicSequentialMem.h b/dash/include/dash/GlobDynamicSequentialMem.h index 172ea0f31..486dd911b 100644 --- a/dash/include/dash/GlobDynamicSequentialMem.h +++ b/dash/include/dash/GlobDynamicSequentialMem.h @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -40,8 +41,12 @@ class GlobDynamicSequentialMem public: typedef ContainerType container_type; typedef typename ContainerType::value_type value_type; - typedef typename ContainerType::iterator local_iterator; + typedef typename ContainerType::difference_type index_type; + typedef LocalBucketIter local_iterator; typedef typename ContainerType::size_type size_type; + typedef typename local_iterator::bucket_type bucket_type; + // TODO: use std::array instead of list -> change LocalBucketIter + typedef typename std::list bucket_list; public: /** @@ -55,13 +60,28 @@ class GlobDynamicSequentialMem size_type n_local_elem = 0, Team & team = dash::Team::All()) : _container(new container_type()), - _public_container(_container), + _unattached_container(new container_type()), + _buckets(), _team(&team), _teamid(team.dart_id()), _nunits(team.size()), _myid(team.myid()) { - grow(n_local_elem); + _container->reserve(n_local_elem); + bucket_type cont_bucket { + 0, + _container->data(), + DART_GPTR_NULL, + false + }; + bucket_type unattached_cont_bucket { + 0, + _unattached_container->data(), + DART_GPTR_NULL, + false + }; + _buckets.push_back(cont_bucket); + _buckets.push_back(unattached_cont_bucket); commit(); } @@ -82,23 +102,6 @@ class GlobDynamicSequentialMem */ self_t & operator=(const self_t & rhs) = default; - local_iterator grow(size_type num_elements) { - if(num_elements > 0) { - _container->reserve(_container->capacity() + num_elements); - _lbegin = _container->begin(); - _lend = _container->end(); - } - // TODO: return end of container or start of newly added capacity? - return _lend; - } - - void shrink(size_type num_elements) { - if(num_elements > 0) { - _container->reserve(_container->capacity() - num_elements); - _lend = _container->end(); - } - } - void commit() { } @@ -112,29 +115,70 @@ class GlobDynamicSequentialMem } void push_back(value_type val) { - // if realloc would get triggered, copy container and keep old data so - // other units can still access the memory region + // bucket of _container + auto it = _buckets.begin(); + // use _unattached container, if _container is full + // we don't want a realloc of _container because this changes the memory + // location, which invalidates global pointers of other units if(_container->capacity() == _container->size()) { - // TODO: new capacity? if capacity is not increased, push-back will - // trigger a realloc and we can use the increase algorithm of the - // container, but this implicates another memory copy. - container_type new_container(_container->capacity() * 2); - new_container = *_container; - if(_public_container != _container) { - delete _public_container; - _public_container = _container; - } - _container = &new_container; - _lbegin = _container->begin(); + // bucket of _unattached_container + ++it; + _unattached_container->push_back(val); + // adding data might change the memory location + it->lptr = _unattached_container->data(); + } else { + _container->push_back(val); } - _container->push_back(val); - _lend = _container->end(); + ++(it->size); + update_lbegin(); + update_lend(); + } + +private: + /** + * Native pointer of the initial address of the local memory of + * a unit. + * + */ + void update_lbegin() noexcept + { + DASH_LOG_TRACE("GlobDynamicMem.update_lbegin()"); + local_iterator unit_lbegin( + // iteration space + _buckets.begin(), _buckets.end(), + // position in iteration space + 0, + // bucket at position in iteration space, + // offset in bucket + _buckets.begin(), 0); + DASH_LOG_TRACE("GlobDynamicMem.update_lbegin >", unit_lbegin); + _lbegin = unit_lbegin; + } + + /** + * Update internal native pointer of the final address of the local memory#include + * of a unit. + */ + void update_lend() noexcept + { + DASH_LOG_TRACE("GlobDynamicMem.update_lend()"); + local_iterator unit_lend( + // iteration space + _buckets.begin(), _buckets.end(), + // position in iteration space + _container->size() + _unattached_container->size(), + // bucket at position in iteration space, + // offset in bucket + _buckets.end(), 0); + DASH_LOG_TRACE("GlobDynamicMem.update_lend >", unit_lend); + _lend = unit_lend; } private: container_type * _container; - container_type * _public_container; + container_type * _unattached_container = nullptr; + bucket_list _buckets; Team * _team; dart_team_t _teamid; size_type _nunits = 0; From 10594f97fa2b44f097d987534ef94f0c429811ef Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 13 Mar 2017 14:01:09 +0100 Subject: [PATCH 017/102] renamed GlobDynamicSequentialMem --- ...micSequentialMem.h => GlobDynamicContiguousMem.h} | 12 ++++++------ dash/include/dash/Graph.h | 8 ++++---- dash/include/dash/graph/Graph-impl.h | 8 ++++---- dash/include/dash/graph/VertexIterator.h | 8 ++++---- 4 files changed, 18 insertions(+), 18 deletions(-) rename dash/include/dash/{GlobDynamicSequentialMem.h => GlobDynamicContiguousMem.h} (95%) diff --git a/dash/include/dash/GlobDynamicSequentialMem.h b/dash/include/dash/GlobDynamicContiguousMem.h similarity index 95% rename from dash/include/dash/GlobDynamicSequentialMem.h rename to dash/include/dash/GlobDynamicContiguousMem.h index 486dd911b..d2e43d4ff 100644 --- a/dash/include/dash/GlobDynamicSequentialMem.h +++ b/dash/include/dash/GlobDynamicContiguousMem.h @@ -32,10 +32,10 @@ namespace dash { template< typename ContainerType> -class GlobDynamicSequentialMem +class GlobDynamicContiguousMem { private: - typedef GlobDynamicSequentialMem + typedef GlobDynamicContiguousMem self_t; public: @@ -56,7 +56,7 @@ class GlobDynamicSequentialMem * \concept{DashDynamicMemorySpaceConcept} * \concept{DashMemorySpaceConcept} */ - GlobDynamicSequentialMem( + GlobDynamicContiguousMem( size_type n_local_elem = 0, Team & team = dash::Team::All()) : _container(new container_type()), @@ -88,14 +88,14 @@ class GlobDynamicSequentialMem /** * Destructor, collectively frees underlying global memory. */ - ~GlobDynamicSequentialMem() { } + ~GlobDynamicContiguousMem() { } - GlobDynamicSequentialMem() = delete; + GlobDynamicContiguousMem() = delete; /** * Copy constructor. */ - GlobDynamicSequentialMem(const self_t & other) = default; + GlobDynamicContiguousMem(const self_t & other) = default; /** * Assignment operator. diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 8c1195647..2537e5531 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include @@ -76,7 +76,7 @@ class Graph { VertexProperties> vertex_type; typedef internal::out_edge edge_type; - typedef GlobDynamicSequentialMem glob_mem_seq_type; + typedef GlobDynamicContiguousMem glob_mem_con_type; public: @@ -93,7 +93,7 @@ class Graph { typedef VertexProperties vertex_properties_type; typedef EdgeProperties edge_properties_type; - typedef typename glob_mem_seq_type::local_iterator local_vertex_iterator; + typedef typename glob_mem_con_type::local_iterator local_vertex_iterator; typedef typename vertex_it_wrapper::iterator vertex_iterator; typedef typename edge_it_wrapper::iterator edge_iterator; @@ -207,7 +207,7 @@ class Graph { /** the team containing all units using the container */ Team * _team = nullptr; /** Global memory allocation and access for sequential memory regions */ - glob_mem_seq_type * _glob_mem_seq = nullptr; + glob_mem_con_type * _glob_mem_con = nullptr; /** Unit ID of the current unit */ team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; diff --git a/dash/include/dash/graph/Graph-impl.h b/dash/include/dash/graph/Graph-impl.h index b13d3653d..bfecc2375 100644 --- a/dash/include/dash/graph/Graph-impl.h +++ b/dash/include/dash/graph/Graph-impl.h @@ -175,7 +175,7 @@ typename Graph ::add_vertex(const VertexProperties & prop) { vertex_type v(prop); - _glob_mem_seq->push_back(v); + _glob_mem_con->push_back(v); } template ::allocate(vertex_size_type nvertices) { auto lcap = dash::math::div_ceil(nvertices, _team->size()); - _glob_mem_seq = new glob_mem_seq_type(lcap, *_team); + _glob_mem_con = new glob_mem_con_type(lcap, *_team); // Register deallocator of this list instance at the team // instance that has been used to initialize it: _team->register_deallocator(this, std::bind(&Graph::deallocate, this)); @@ -343,8 +343,8 @@ void Graph ::deallocate() { - if(_glob_mem_seq != nullptr) { - delete _glob_mem_seq; + if(_glob_mem_con != nullptr) { + delete _glob_mem_con; } // Remove this function from team deallocator list to avoid // double-free: diff --git a/dash/include/dash/graph/VertexIterator.h b/dash/include/dash/graph/VertexIterator.h index 4d8a0ee83..3efedbb43 100644 --- a/dash/include/dash/graph/VertexIterator.h +++ b/dash/include/dash/graph/VertexIterator.h @@ -71,28 +71,28 @@ struct VertexIteratorWrapper { * Returns local iterator to the beginning of the vertex list. */ local_iterator lbegin() { - return _graph->_glob_mem_seq->lbegin(); + return _graph->_glob_mem_con->lbegin(); } /** * Returns local iterator to the beginning of the vertex list. */ const_local_iterator lbegin() const { - return _graph->_glob_mem_seq->lbegin(); + return _graph->_glob_mem_con->lbegin(); } /** * Returns local iterator to the end of the vertex list. */ local_iterator lend() { - return _graph->_glob_mem_seq->lend(); + return _graph->_glob_mem_con->lend(); } /** * Returns local iterator to the end of the vertex list. */ const_local_iterator lend() const { - return _graph->_glob_mem_seq->lend(); + return _graph->_glob_mem_con->lend(); } private: From 091aac9f610094b9364ab927224461c19f64e723 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 13 Mar 2017 19:12:50 +0100 Subject: [PATCH 018/102] added (non-working) collective allocation --- dash/include/dash/GlobDynamicContiguousMem.h | 53 ++++++++++++++++++-- dash/include/dash/Graph.h | 7 +++ dash/include/dash/graph/Graph-impl.h | 2 +- 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/dash/include/dash/GlobDynamicContiguousMem.h b/dash/include/dash/GlobDynamicContiguousMem.h index d2e43d4ff..15a7dc7c6 100644 --- a/dash/include/dash/GlobDynamicContiguousMem.h +++ b/dash/include/dash/GlobDynamicContiguousMem.h @@ -82,7 +82,6 @@ class GlobDynamicContiguousMem }; _buckets.push_back(cont_bucket); _buckets.push_back(unattached_cont_bucket); - commit(); } /** @@ -104,6 +103,30 @@ class GlobDynamicContiguousMem void commit() { + // merge public and local containers together + _container->insert(_container->end(), + _unattached_container->begin(), + _unattached_container->end()); + _unattached_container->clear(); + // update memory location & size of _container + auto it = _buckets.begin(); + it->lptr = _container->data(); + it->size = _container->size(); + // update memory location & size of _unattached_container + ++it; + it->lptr = _unattached_container->data(); + it->size = 0; + + update_lbegin(); + update_lend(); + + // attach new container to global memory space + dart_gptr_t gptr = DART_GPTR_NULL; + dart_storage_t ds = dart_storage(_container->size()); + dart_team_memregister( + _team->dart_id(), ds.nelem, ds.dtype, _container->data(), &gptr); + it = _buckets.begin(); + it->gptr = gptr; } local_iterator lbegin() { @@ -134,6 +157,30 @@ class GlobDynamicContiguousMem update_lend(); } + /** + * Global pointer referencing an element position in a unit's bucket. + */ + dart_gptr_t dart_gptr_at(team_unit_t unit, index_type bucket_phase) { + // we only have 1 global pointer for _container + auto bucket = *_buckets.begin(); + auto dart_gptr = bucket.gptr; + + if (DART_GPTR_ISNULL(dart_gptr)) { + dart_gptr = DART_GPTR_NULL; + } else { + // Move dart_gptr to unit and local offset: + DASH_ASSERT_RETURNS( + dart_gptr_setunit(&dart_gptr, unit), + DART_OK); + DASH_ASSERT_RETURNS( + dart_gptr_incaddr( + &dart_gptr, + bucket_phase * sizeof(value_type)), + DART_OK); + } + return dart_gptr; + } + private: /** * Native pointer of the initial address of the local memory of @@ -142,7 +189,6 @@ class GlobDynamicContiguousMem */ void update_lbegin() noexcept { - DASH_LOG_TRACE("GlobDynamicMem.update_lbegin()"); local_iterator unit_lbegin( // iteration space _buckets.begin(), _buckets.end(), @@ -151,7 +197,6 @@ class GlobDynamicContiguousMem // bucket at position in iteration space, // offset in bucket _buckets.begin(), 0); - DASH_LOG_TRACE("GlobDynamicMem.update_lbegin >", unit_lbegin); _lbegin = unit_lbegin; } @@ -161,7 +206,6 @@ class GlobDynamicContiguousMem */ void update_lend() noexcept { - DASH_LOG_TRACE("GlobDynamicMem.update_lend()"); local_iterator unit_lend( // iteration space _buckets.begin(), _buckets.end(), @@ -170,7 +214,6 @@ class GlobDynamicContiguousMem // bucket at position in iteration space, // offset in bucket _buckets.end(), 0); - DASH_LOG_TRACE("GlobDynamicMem.update_lend >", unit_lend); _lend = unit_lend; } diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 2537e5531..b3c1291ba 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -93,6 +93,8 @@ class Graph { typedef VertexProperties vertex_properties_type; typedef EdgeProperties edge_properties_type; + typedef GlobRef reference; + typedef typename glob_mem_con_type::local_iterator local_vertex_iterator; typedef typename vertex_it_wrapper::iterator vertex_iterator; @@ -202,6 +204,11 @@ class Graph { */ void deallocate(); + vertex_type test() { + auto gptr = _glob_mem_con->dart_gptr_at(team_unit_t(0), 0); + return reference(gptr); + } + private: /** the team containing all units using the container */ diff --git a/dash/include/dash/graph/Graph-impl.h b/dash/include/dash/graph/Graph-impl.h index bfecc2375..c92805ac7 100644 --- a/dash/include/dash/graph/Graph-impl.h +++ b/dash/include/dash/graph/Graph-impl.h @@ -299,7 +299,7 @@ void Graph ::barrier() { - + _glob_mem_con->commit(); } template Date: Tue, 14 Mar 2017 14:54:23 +0100 Subject: [PATCH 019/102] Minor, added notes --- dash/include/dash/GlobDynamicContiguousMem.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dash/include/dash/GlobDynamicContiguousMem.h b/dash/include/dash/GlobDynamicContiguousMem.h index 15a7dc7c6..1c2e24857 100644 --- a/dash/include/dash/GlobDynamicContiguousMem.h +++ b/dash/include/dash/GlobDynamicContiguousMem.h @@ -30,6 +30,18 @@ namespace dash { +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +// NOTE: Must be compatible (= same interface) to GlobDynamicMem so it +// can be replaced in e.g. +// +// dash::Grap +// dash::Grap +// +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +template< + typename ContainerType> +class GlobDynamicContiguousMem +{ template< typename ContainerType> class GlobDynamicContiguousMem @@ -60,6 +72,7 @@ class GlobDynamicContiguousMem size_type n_local_elem = 0, Team & team = dash::Team::All()) : _container(new container_type()), + // TODO: use std::make_shared() _unattached_container(new container_type()), _buckets(), _team(&team), From e823a2959fab98ac1ba1e81c4ee21e808f675687 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Fri, 17 Mar 2017 15:15:32 +0100 Subject: [PATCH 020/102] added global vertex iterator --- dash/include/dash/GlobDynamicContiguousMem.h | 78 ++++++++++++++++---- dash/include/dash/Graph.h | 11 +-- dash/include/dash/graph/VertexIterator.h | 14 ++-- 3 files changed, 76 insertions(+), 27 deletions(-) diff --git a/dash/include/dash/GlobDynamicContiguousMem.h b/dash/include/dash/GlobDynamicContiguousMem.h index 1c2e24857..295a25b89 100644 --- a/dash/include/dash/GlobDynamicContiguousMem.h +++ b/dash/include/dash/GlobDynamicContiguousMem.h @@ -42,23 +42,30 @@ template< typename ContainerType> class GlobDynamicContiguousMem { -template< - typename ContainerType> -class GlobDynamicContiguousMem -{ private: - typedef GlobDynamicContiguousMem - self_t; + typedef GlobDynamicContiguousMem self_t; + public: - typedef ContainerType container_type; - typedef typename ContainerType::value_type value_type; - typedef typename ContainerType::difference_type index_type; - typedef LocalBucketIter local_iterator; - typedef typename ContainerType::size_type size_type; - typedef typename local_iterator::bucket_type bucket_type; + typedef ContainerType container_type; + typedef typename ContainerType::value_type value_type; + typedef typename ContainerType::difference_type index_type; + typedef LocalBucketIter local_iterator; + typedef GlobBucketIter global_iterator; + typedef typename ContainerType::size_type size_type; + typedef typename local_iterator::bucket_type bucket_type; // TODO: use std::array instead of list -> change LocalBucketIter - typedef typename std::list bucket_list; + typedef typename std::list bucket_list; + typedef local_iterator local_pointer; + typedef local_iterator const_local_pointer; + +private: + + typedef std::vector > + bucket_cumul_sizes_map; + + template + friend class dash::GlobBucketIter; public: /** @@ -78,7 +85,8 @@ class GlobDynamicContiguousMem _team(&team), _teamid(team.dart_id()), _nunits(team.size()), - _myid(team.myid()) + _myid(team.myid()), + _bucket_cumul_sizes(team.size()) { _container->reserve(n_local_elem); bucket_type cont_bucket { @@ -95,6 +103,10 @@ class GlobDynamicContiguousMem }; _buckets.push_back(cont_bucket); _buckets.push_back(unattached_cont_bucket); + for(auto it = _bucket_cumul_sizes.begin(); + it != _bucket_cumul_sizes.end(); ++it) { + it->resize(1); + } } /** @@ -140,6 +152,28 @@ class GlobDynamicContiguousMem _team->dart_id(), ds.nelem, ds.dtype, _container->data(), &gptr); it = _buckets.begin(); it->gptr = gptr; + + // distribute bucket sizes between all units + size_type current_size = _container->size(); + std::vector bucket_sizes(_team->size()); + dart_allgather(¤t_size, bucket_sizes.data(), + sizeof(size_type), DART_TYPE_BYTE, _team->dart_id()); + _size = 0; + for(int i = 0; i < _team->size(); ++i) { + _bucket_cumul_sizes[i][0] = bucket_sizes[i]; + _size += bucket_sizes[i]; + } + + _begin = global_iterator(this, 0); + _end = _begin + _size; + } + + global_iterator begin() { + return _begin; + } + + global_iterator end() { + return _end; } local_iterator lbegin() { @@ -168,12 +202,14 @@ class GlobDynamicContiguousMem ++(it->size); update_lbegin(); update_lend(); + _bucket_cumul_sizes[_myid][0] += 1; } /** * Global pointer referencing an element position in a unit's bucket. */ - dart_gptr_t dart_gptr_at(team_unit_t unit, index_type bucket_phase) { + dart_gptr_t dart_gptr_at(team_unit_t unit, + index_type bucket_index, index_type bucket_phase) { // we only have 1 global pointer for _container auto bucket = *_buckets.begin(); auto dart_gptr = bucket.gptr; @@ -194,6 +230,14 @@ class GlobDynamicContiguousMem return dart_gptr; } + size_type size() { + return _size; + } + + Team & team() { + return (_team != nullptr) ? *_team : dash::Team::Null(); + } + private: /** * Native pointer of the initial address of the local memory of @@ -239,8 +283,12 @@ class GlobDynamicContiguousMem dart_team_t _teamid; size_type _nunits = 0; team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; + global_iterator _begin; + global_iterator _end; local_iterator _lbegin; local_iterator _lend; + bucket_cumul_sizes_map _bucket_cumul_sizes; + size_type _size = 0; }; diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index b3c1291ba..99b98998e 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -72,14 +72,14 @@ class Graph { friend vertex_it_wrapper; - typedef internal::vertex vertex_type; - typedef internal::out_edge edge_type; typedef GlobDynamicContiguousMem glob_mem_con_type; public: + typedef internal::vertex vertex_type; + typedef internal::out_edge edge_type; typedef VertexIndexType vertex_index_type; typedef EdgeIndexType edge_index_type; typedef typename @@ -95,6 +95,7 @@ class Graph { typedef GlobRef reference; + typedef typename glob_mem_con_type::global_iterator global_vertex_iterator; typedef typename glob_mem_con_type::local_iterator local_vertex_iterator; typedef typename vertex_it_wrapper::iterator vertex_iterator; @@ -205,7 +206,7 @@ class Graph { void deallocate(); vertex_type test() { - auto gptr = _glob_mem_con->dart_gptr_at(team_unit_t(0), 0); + auto gptr = _glob_mem_con->dart_gptr_at(team_unit_t(1), 0, 0); return reference(gptr); } diff --git a/dash/include/dash/graph/VertexIterator.h b/dash/include/dash/graph/VertexIterator.h index 3efedbb43..73aff3ee5 100644 --- a/dash/include/dash/graph/VertexIterator.h +++ b/dash/include/dash/graph/VertexIterator.h @@ -18,8 +18,8 @@ template struct VertexIteratorWrapper { typedef Graph graph_type; - typedef VertexIterator iterator; - typedef const VertexIterator const_iterator; + typedef typename Graph::global_vertex_iterator iterator; + typedef const iterator const_iterator; typedef typename Graph::local_vertex_iterator local_iterator; typedef const local_iterator const_local_iterator; typedef typename Graph::vertex_index_type vertex_index_type; @@ -28,7 +28,7 @@ struct VertexIteratorWrapper { /** * Constructs the wrapper. */ - VertexIteratorWrapper(graph_type * graph) + VertexIteratorWrapper(graph_type * graph) : _graph(graph) { } @@ -43,28 +43,28 @@ struct VertexIteratorWrapper { * Returns global iterator to the beginning of the vertex list. */ iterator begin() { - + return _graph->_glob_mem_con->begin(); } /** * Returns global iterator to the beginning of the vertex list. */ const_iterator begin() const { - + return _graph->_glob_mem_con->begin(); } /** * Returns global iterator to the end of the vertex list. */ iterator end() { - + return _graph->_glob_mem_con->end(); } /** * Returns global iterator to the end of the vertex list. */ const_iterator end() const { - + return _graph->_glob_mem_con->end(); } /** From a5336542a055e7c781be6ec08fc6eeee1092f198 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Fri, 17 Mar 2017 15:36:29 +0100 Subject: [PATCH 021/102] merged Graph.h and Graph-impl.h --- dash/include/dash/Graph.h | 76 ++++-- dash/include/dash/graph/Graph-impl.h | 356 --------------------------- 2 files changed, 58 insertions(+), 374 deletions(-) delete mode 100644 dash/include/dash/graph/Graph-impl.h diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 99b98998e..7a3bb1a81 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -117,55 +117,79 @@ class Graph { /** * Constructs an empty graph. */ - Graph(vertex_size_type nvertices = 0, Team & team = dash::Team::All()); + Graph(vertex_size_type nvertices = 0, Team & team = dash::Team::All()) + : _team(&team), + _myid(team.myid()) + { + allocate(nvertices); + } /** Destructs the graph. */ - ~Graph(); + ~Graph() { + deallocate(); + } /** * Returns the number of vertices in the whole graph. */ - const vertex_size_type num_vertices() const; + const vertex_size_type num_vertices() const { + + } /** * Returns the index of the vertex with the highest index in the whole graph. */ - const vertex_index_type max_vertex_index() const; + const vertex_index_type max_vertex_index() const { + + } /** * Returns the number of edges in the whole graph. */ - const edge_size_type num_edges() const; + const edge_size_type num_edges() const { + + } /** * Returns the index of the edge with the highest index in the whole graph. */ - const edge_index_type max_edge_index() const; + const edge_index_type max_edge_index() const { + + } /** * Returns, whether the graph is empty. */ - bool empty() const; + bool empty() const { + return true; + } /** * Adds a vertex with the given properties. * NOTE: global method. */ vertex_index_type add_vertex(const VertexProperties & prop - = VertexProperties()); + = VertexProperties()) { + vertex_type v(prop); + _glob_mem_con->push_back(v); + } /** * Removes a given vertex. * NOTE: global method. */ - void remove_vertex(const vertex_index_type & v); + void remove_vertex(const vertex_index_type & v) { + + } /** * Removes all edges (in & out) from the given vertex). * NOTE: global method. */ - void clear_vertex(vertex_index_type & v); + void clear_vertex(vertex_index_type & v) { + + } /** * Adds an edge between two given vertices with the given properties @@ -174,36 +198,54 @@ class Graph { */ std::pair add_edge(const vertex_index_type & v1, const vertex_index_type & v2, - const EdgeProperties & prop = EdgeProperties()); + const EdgeProperties & prop = EdgeProperties()) { + } /** * Removes an edge between two given vertices. * NOTE: global method. */ void remove_edge(const vertex_index_type & v1, - const vertex_index_type & v2); + const vertex_index_type & v2) { + + } /** * Removes a given edge.a9f5c03d9de8dda4 * NOTE: global method. */ - void remove_edge(const edge_index_type & e); + void remove_edge(const edge_index_type & e) { + + } /** * Commits local changes performed by methods classified as "global" to * the whole data structure. */ - void barrier(); + void barrier() { + _glob_mem_con->commit(); + } /** * Globally allocates memory for vertex storage. */ - bool allocate(vertex_size_type nvertices); + bool allocate(vertex_size_type nvertices) { + auto lcap = dash::math::div_ceil(nvertices, _team->size()); + _glob_mem_con = new glob_mem_con_type(lcap, *_team); + // Register deallocator at the respective team instance + _team->register_deallocator(this, std::bind(&Graph::deallocate, this)); + } /** * Deallocates global memory of this container. */ - void deallocate(); + void deallocate() { + if(_glob_mem_con != nullptr) { + delete _glob_mem_con; + } + // Remove deallocator from the respective team instance + _team->unregister_deallocator(this, std::bind(&Graph::deallocate, this)); + } vertex_type test() { auto gptr = _glob_mem_con->dart_gptr_at(team_unit_t(1), 0, 0); @@ -223,7 +265,5 @@ class Graph { } -#include - #endif // DASH__GRAPH_H__INCLUDED diff --git a/dash/include/dash/graph/Graph-impl.h b/dash/include/dash/graph/Graph-impl.h deleted file mode 100644 index c92805ac7..000000000 --- a/dash/include/dash/graph/Graph-impl.h +++ /dev/null @@ -1,356 +0,0 @@ -#ifndef DASH__GRAPH__INTERNAL__GRAPH_INL_H__INCLUDED -#define DASH__GRAPH__INTERNAL__GRAPH_INL_H__INCLUDED - -namespace dash { - -template -Graph -::Graph(vertex_size_type nvertices, - Team & team) - : _team(&team), - _myid(team.myid()) -{ - allocate(nvertices); -} - -template -Graph -::~Graph() { - deallocate(); -} - -template -const typename Graph - ::vertex_size_type Graph -::num_vertices() const { - -} - -template -const typename Graph - ::vertex_index_type Graph -::max_vertex_index() const { - -} - -template -const typename Graph - ::edge_size_type Graph -::num_edges() const { - -} - -template -const typename Graph - ::edge_index_type Graph -::max_edge_index() const { - -} - -template -bool Graph -::empty() const { - return true; -} - -template -typename Graph - ::vertex_index_type Graph -::add_vertex(const VertexProperties & prop) { - vertex_type v(prop); - _glob_mem_con->push_back(v); -} - -template -void Graph -::remove_vertex(const vertex_index_type & v) { - -} - -template -void Graph -::clear_vertex(vertex_index_type & v) { - -} - -template -std::pair - ::edge_index_type, bool> Graph -::add_edge(const vertex_index_type & v1, - const vertex_index_type & v2, const EdgeProperties & prop) { - -} - -template -void Graph -::remove_edge(const vertex_index_type & v1, - const vertex_index_type & v2) { - -} - -template -void Graph -::remove_edge(const edge_index_type & e) { - -} - -template -void Graph -::barrier() { - _glob_mem_con->commit(); -} - -template -bool Graph -::allocate(vertex_size_type nvertices) { - auto lcap = dash::math::div_ceil(nvertices, _team->size()); - _glob_mem_con = new glob_mem_con_type(lcap, *_team); - // Register deallocator of this list instance at the team - // instance that has been used to initialize it: - _team->register_deallocator(this, std::bind(&Graph::deallocate, this)); -} - -template -void Graph -::deallocate() { - if(_glob_mem_con != nullptr) { - delete _glob_mem_con; - } - // Remove this function from team deallocator list to avoid - // double-free: - _team->unregister_deallocator(this, std::bind(&Graph::deallocate, this)); -} - -} - -#endif // DASH__GRAPH__INTERNAL__GRAPH_INL_H__INCLUDED From da3fef7feeabcc5732e27d5fb37aa4b4ad1b8845 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 20 Mar 2017 10:34:53 +0100 Subject: [PATCH 022/102] added local vertex index support --- dash/include/dash/Graph.h | 5 +++++ dash/include/dash/graph/internal/Graph.h | 3 +++ 2 files changed, 8 insertions(+) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 7a3bb1a81..7947ab27a 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -172,7 +172,10 @@ class Graph { vertex_index_type add_vertex(const VertexProperties & prop = VertexProperties()) { vertex_type v(prop); + v._local_id = ++_local_vertex_max_index; _glob_mem_con->push_back(v); + // TODO: return global index + return _local_vertex_max_index; } /** @@ -260,6 +263,8 @@ class Graph { glob_mem_con_type * _glob_mem_con = nullptr; /** Unit ID of the current unit */ team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; + /** Index of last added vertex */ + vertex_index_type _local_vertex_max_index = -1; }; diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index c743e81ce..0edc66f1b 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -12,6 +12,7 @@ enum GraphDirection { namespace internal { template< + typename IndexType, typename EdgeContainer, typename VertexProperties> struct vertex { @@ -27,6 +28,8 @@ struct vertex { EdgeContainer _out_edges; /** Properties of this vertex */ VertexProperties _properties; + /** Index of the vertex in local index space */ + IndexType _local_id; }; From be6abb759004575480f5b6c3f95b992fa72a8fc7 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 20 Mar 2017 17:29:13 +0100 Subject: [PATCH 023/102] refactored GlobDynamicContiguousMem to support multiple containers --- dash/include/dash/GlobDynamicContiguousMem.h | 242 +++++++++++++------ dash/include/dash/Graph.h | 15 +- 2 files changed, 184 insertions(+), 73 deletions(-) diff --git a/dash/include/dash/GlobDynamicContiguousMem.h b/dash/include/dash/GlobDynamicContiguousMem.h index 295a25b89..63a18cb50 100644 --- a/dash/include/dash/GlobDynamicContiguousMem.h +++ b/dash/include/dash/GlobDynamicContiguousMem.h @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -30,6 +31,90 @@ namespace dash { +namespace internal { + +template +struct container_data { + + typedef ContainerType container_type; + typedef typename ContainerType::value_type value_type; + typedef typename ContainerType::difference_type index_type; + typedef typename ContainerType::size_type size_type; + typedef LocalBucketIter local_iterator; + typedef typename local_iterator::bucket_type bucket_type; + typedef typename std::list bucket_list; + + container_data(size_type n_local_elem) + : container(new container_type()), + unattached_container(new container_type()) + { + container->reserve(n_local_elem); + bucket_type cont_bucket { + 0, + container->data(), + DART_GPTR_NULL, + false + }; + bucket_type unattached_cont_bucket { + 0, + unattached_container->data(), + DART_GPTR_NULL, + false + }; + buckets.push_back(cont_bucket); + buckets.push_back(unattached_cont_bucket); + + update_lbegin(); + update_lend(); + } + + /** + * Native pointer of the initial address of the local memory of + * a unit. + * + */ + void update_lbegin() noexcept + { + local_iterator unit_lbegin( + // iteration space + buckets.begin(), buckets.end(), + // position in iteration space + 0, + // bucket at position in iteration space, + // offset in bucket + buckets.begin(), 0); + lbegin = unit_lbegin; + } + + /** + * Update internal native pointer of the final address of the local memory#include + * of a unit. + */ + void update_lend() noexcept + { + local_iterator unit_lend( + // iteration space + buckets.begin(), buckets.end(), + // position in iteration space + container->size() + unattached_container->size(), + // bucket at position in iteration space, + // offset in bucket + buckets.end(), 0); + lend = unit_lend; + } + + ContainerType * container; + ContainerType * unattached_container; + bucket_list buckets; + bucket_type * container_bucket; + bucket_type * unattached_container_bucket; + local_iterator lbegin; + local_iterator lend; + +}; + +} + // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! // NOTE: Must be compatible (= same interface) to GlobDynamicMem so it // can be replaced in e.g. @@ -48,13 +133,15 @@ class GlobDynamicContiguousMem public: typedef ContainerType container_type; + typedef internal::container_data data_type; + typedef std::list container_list_type; + typedef typename container_list_type::iterator container_list_iter; typedef typename ContainerType::value_type value_type; typedef typename ContainerType::difference_type index_type; typedef LocalBucketIter local_iterator; typedef GlobBucketIter global_iterator; typedef typename ContainerType::size_type size_type; typedef typename local_iterator::bucket_type bucket_type; - // TODO: use std::array instead of list -> change LocalBucketIter typedef typename std::list bucket_list; typedef local_iterator local_pointer; typedef local_iterator const_local_pointer; @@ -69,46 +156,37 @@ class GlobDynamicContiguousMem public: /** - * Constructor, collectively allocates the given number of elements in - * local memory of every unit in a team. - * - * \concept{DashDynamicMemorySpaceConcept} - * \concept{DashMemorySpaceConcept} + * Constructor */ - GlobDynamicContiguousMem( - size_type n_local_elem = 0, - Team & team = dash::Team::All()) - : _container(new container_type()), - // TODO: use std::make_shared() - _unattached_container(new container_type()), - _buckets(), + GlobDynamicContiguousMem(Team & team = dash::Team::All()) + : _buckets(), _team(&team), _teamid(team.dart_id()), _nunits(team.size()), _myid(team.myid()), _bucket_cumul_sizes(team.size()) { - _container->reserve(n_local_elem); - bucket_type cont_bucket { - 0, - _container->data(), - DART_GPTR_NULL, - false - }; - bucket_type unattached_cont_bucket { - 0, - _unattached_container->data(), - DART_GPTR_NULL, - false - }; - _buckets.push_back(cont_bucket); - _buckets.push_back(unattached_cont_bucket); for(auto it = _bucket_cumul_sizes.begin(); it != _bucket_cumul_sizes.end(); ++it) { it->resize(1); } } + container_list_iter add_container(size_type n_elements) { + auto c_data = data_type(n_elements); + // TODO: use pointers instead of copies -> no maintenance of 2 object + // copies + auto it = _buckets.insert(_buckets.end(), + c_data.buckets.begin(), c_data.buckets.end()); + c_data.container_bucket = &(*it); + ++it; + c_data.unattached_container_bucket = &(*it); + // insert won't invalidate iterators for std::list, so we can use them + // to access the container + + return _container_list.insert(_container_list.end(), c_data); + } + /** * Destructor, collectively frees underlying global memory. */ @@ -126,43 +204,59 @@ class GlobDynamicContiguousMem */ self_t & operator=(const self_t & rhs) = default; - void commit() - { - // merge public and local containers together - _container->insert(_container->end(), - _unattached_container->begin(), - _unattached_container->end()); - _unattached_container->clear(); - // update memory location & size of _container - auto it = _buckets.begin(); - it->lptr = _container->data(); - it->size = _container->size(); - // update memory location & size of _unattached_container - ++it; - it->lptr = _unattached_container->data(); - it->size = 0; - + void commit() { + for(auto c_data : _container_list) { + // merge public and local containers + c_data.container->insert(c_data.container->end(), + c_data.unattached_container->begin(), + c_data.unattached_container->end()); + c_data.unattached_container->clear(); + // update memory location & size of _container + auto it = c_data.buckets.begin(); + it->lptr = c_data.container->data(); + it->size = c_data.container->size(); + c_data.container_bucket->lptr = c_data.container->data(); + c_data.container_bucket->size = c_data.container->size(); + // update memory location & size of _unattached_container + ++it; + it->lptr = c_data.unattached_container->data(); + it->size = 0; + c_data.unattached_container_bucket->lptr + = c_data.unattached_container->data(); + c_data.unattached_container_bucket->size = 0; + + c_data.update_lbegin(); + c_data.update_lend(); + + // attach new container to global memory space + dart_gptr_t gptr = DART_GPTR_NULL; + dart_storage_t ds = dart_storage(c_data.container->size()); + dart_team_memregister( + _team->dart_id(), + ds.nelem, + ds.dtype, + c_data.container->data(), + &gptr + ); + // no need to update gptr of local bucekt list in c_data + c_data.container_bucket->gptr = gptr; + + //TODO: Adjust bucket size accumulation, so that multiple containers + // are supported + // distribute bucket sizes between all units + std::vector bucket_sizes(_team->size()); + dart_allgather(&_local_size, bucket_sizes.data(), + sizeof(size_type), DART_TYPE_BYTE, _team->dart_id()); + _size = 0; + for(int i = 0; i < _team->size(); ++i) { + _bucket_cumul_sizes[i][0] = bucket_sizes[i]; + _size += bucket_sizes[i]; + } + } update_lbegin(); update_lend(); - // attach new container to global memory space - dart_gptr_t gptr = DART_GPTR_NULL; - dart_storage_t ds = dart_storage(_container->size()); - dart_team_memregister( - _team->dart_id(), ds.nelem, ds.dtype, _container->data(), &gptr); - it = _buckets.begin(); - it->gptr = gptr; - - // distribute bucket sizes between all units - size_type current_size = _container->size(); - std::vector bucket_sizes(_team->size()); - dart_allgather(¤t_size, bucket_sizes.data(), - sizeof(size_type), DART_TYPE_BYTE, _team->dart_id()); - _size = 0; - for(int i = 0; i < _team->size(); ++i) { - _bucket_cumul_sizes[i][0] = bucket_sizes[i]; - _size += bucket_sizes[i]; - } + _begin = global_iterator(this, 0); _end = _begin + _size; @@ -184,22 +278,30 @@ class GlobDynamicContiguousMem return _lend; } - void push_back(value_type val) { + void push_back(container_list_iter pos, value_type val) { + auto c_data = *pos; // bucket of _container - auto it = _buckets.begin(); + auto it = c_data.buckets.begin(); // use _unattached container, if _container is full // we don't want a realloc of _container because this changes the memory // location, which invalidates global pointers of other units - if(_container->capacity() == _container->size()) { + if(c_data.container->capacity() == c_data.container->size()) { // bucket of _unattached_container ++it; - _unattached_container->push_back(val); + c_data.unattached_container->push_back(val); // adding data might change the memory location - it->lptr = _unattached_container->data(); + it->lptr = c_data.unattached_container->data(); + c_data.unattached_container_bucket->lptr + = c_data.unattached_container->data(); + ++(c_data.unattached_container_bucket->size); } else { - _container->push_back(val); + c_data.container->push_back(val); + ++(c_data.container_bucket->size); } ++(it->size); + ++_local_size; + c_data.update_lbegin(); + c_data.update_lend(); update_lbegin(); update_lend(); _bucket_cumul_sizes[_myid][0] += 1; @@ -267,7 +369,7 @@ class GlobDynamicContiguousMem // iteration space _buckets.begin(), _buckets.end(), // position in iteration space - _container->size() + _unattached_container->size(), + _local_size, // bucket at position in iteration space, // offset in bucket _buckets.end(), 0); @@ -276,6 +378,7 @@ class GlobDynamicContiguousMem private: + container_list_type _container_list; container_type * _container; container_type * _unattached_container = nullptr; bucket_list _buckets; @@ -289,6 +392,7 @@ class GlobDynamicContiguousMem local_iterator _lend; bucket_cumul_sizes_map _bucket_cumul_sizes; size_type _size = 0; + size_type _local_size = 0; }; diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 7947ab27a..acf7d458b 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -51,7 +51,8 @@ template< typename EdgeContainer = std::vector>, typename VertexContainer - = std::vector>> + = std::vector>> class Graph { public: @@ -73,10 +74,13 @@ class Graph { friend vertex_it_wrapper; typedef GlobDynamicContiguousMem glob_mem_con_type; + typedef typename + glob_mem_con_type::container_list_iter vertex_cont_ref_type; public: - typedef internal::vertex vertex_type; typedef internal::out_edge edge_type; @@ -173,7 +177,7 @@ class Graph { = VertexProperties()) { vertex_type v(prop); v._local_id = ++_local_vertex_max_index; - _glob_mem_con->push_back(v); + _glob_mem_con->push_back(_vertex_container_ref, v); // TODO: return global index return _local_vertex_max_index; } @@ -234,7 +238,8 @@ class Graph { */ bool allocate(vertex_size_type nvertices) { auto lcap = dash::math::div_ceil(nvertices, _team->size()); - _glob_mem_con = new glob_mem_con_type(lcap, *_team); + _glob_mem_con = new glob_mem_con_type(*_team); + _vertex_container_ref = _glob_mem_con->add_container(lcap); // Register deallocator at the respective team instance _team->register_deallocator(this, std::bind(&Graph::deallocate, this)); } @@ -265,6 +270,8 @@ class Graph { team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; /** Index of last added vertex */ vertex_index_type _local_vertex_max_index = -1; + /** Iterator to the vertex container in GlobMem object */ + vertex_cont_ref_type _vertex_container_ref; }; From 93a38e637a1847ed87511b955841eab675795a54 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Tue, 21 Mar 2017 16:02:39 +0100 Subject: [PATCH 024/102] added local edge iteration capabilities --- dash/include/dash/GlobDynamicContiguousMem.h | 14 ++- dash/include/dash/Graph.h | 108 +++++++++++++------ dash/include/dash/graph/EdgeIterator.h | 98 +++++++++++++++++ dash/include/dash/graph/VertexIterator.h | 16 +-- dash/include/dash/graph/internal/Graph.h | 53 ++++++--- 5 files changed, 227 insertions(+), 62 deletions(-) create mode 100644 dash/include/dash/graph/EdgeIterator.h diff --git a/dash/include/dash/GlobDynamicContiguousMem.h b/dash/include/dash/GlobDynamicContiguousMem.h index 63a18cb50..7e4e85f48 100644 --- a/dash/include/dash/GlobDynamicContiguousMem.h +++ b/dash/include/dash/GlobDynamicContiguousMem.h @@ -183,10 +183,18 @@ class GlobDynamicContiguousMem c_data.unattached_container_bucket = &(*it); // insert won't invalidate iterators for std::list, so we can use them // to access the container - return _container_list.insert(_container_list.end(), c_data); } + value_type & get(container_list_iter cont, index_type pos) { + auto c_data = *cont; + if(c_data.container->size() > pos) { + return c_data.container->operator[](pos); + } + pos -= c_data.container->size(); + return c_data.unattached_container->operator[](pos); + } + /** * Destructor, collectively frees underlying global memory. */ @@ -278,8 +286,8 @@ class GlobDynamicContiguousMem return _lend; } - void push_back(container_list_iter pos, value_type val) { - auto c_data = *pos; + void push_back(container_list_iter cont, value_type val) { + auto c_data = *cont; // bucket of _container auto it = c_data.buckets.begin(); // use _unattached container, if _container is full diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index acf7d458b..b10e3b661 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -48,11 +49,9 @@ template< typename EdgeProperties = internal::EmptyProperties, // user-defined struct typename VertexIndexType = int, typename EdgeIndexType = int, - typename EdgeContainer - = std::vector>, - typename VertexContainer - = std::vector>> + template typename EdgeContainer = std::vector, + template typename VertexContainer = std::vector +> class Graph { public: @@ -61,29 +60,37 @@ class Graph { VertexProperties, EdgeProperties, VertexIndexType, EdgeIndexType, EdgeContainer, VertexContainer> graph_type; + typedef internal::vertex vertex_type; + typedef internal::out_edge edge_type; + typedef VertexContainer> vertex_container_type; + typedef EdgeContainer> edge_container_type; private: // TODO: add wrapper for all iterator types typedef VertexIteratorWrapper vertex_it_wrapper; - typedef VertexIteratorWrapper edge_it_wrapper; + typedef EdgeIteratorWrapper edge_it_wrapper; typedef VertexIteratorWrapper in_edge_it_wrapper; typedef VertexIteratorWrapper out_edge_it_wrapper; typedef VertexIteratorWrapper adjacency_it_wrapper; friend vertex_it_wrapper; + friend edge_it_wrapper; + + typedef GlobDynamicContiguousMem< + vertex_container_type> glob_mem_vert_type; + typedef GlobDynamicContiguousMem< + edge_container_type> glob_mem_edge_type; - typedef GlobDynamicContiguousMem glob_mem_con_type; - typedef typename - glob_mem_con_type::container_list_iter vertex_cont_ref_type; public: - typedef internal::vertex vertex_type; - typedef internal::out_edge edge_type; + typedef typename + glob_mem_vert_type::container_list_iter vertex_cont_ref_type; + typedef typename + glob_mem_edge_type::container_list_iter edge_cont_ref_type; typedef VertexIndexType vertex_index_type; typedef EdgeIndexType edge_index_type; typedef typename @@ -92,16 +99,20 @@ class Graph { std::make_unsigned::type edge_size_type; typedef DynamicPattern pattern_type; - typedef VertexContainer vertex_container_type; - typedef EdgeContainer edge_container_type; typedef VertexProperties vertex_properties_type; typedef EdgeProperties edge_properties_type; typedef GlobRef reference; - typedef typename glob_mem_con_type::global_iterator global_vertex_iterator; - typedef typename glob_mem_con_type::local_iterator local_vertex_iterator; - + typedef typename + glob_mem_vert_type::global_iterator global_vertex_iterator; + typedef typename + glob_mem_vert_type::local_iterator local_vertex_iterator; + typedef typename + glob_mem_edge_type::global_iterator global_edge_iterator; + typedef typename + glob_mem_edge_type::local_iterator local_edge_iterator; + typedef typename vertex_it_wrapper::iterator vertex_iterator; typedef typename edge_it_wrapper::iterator edge_iterator; typedef typename in_edge_it_wrapper::iterator in_edge_iterator; @@ -121,11 +132,15 @@ class Graph { /** * Constructs an empty graph. */ - Graph(vertex_size_type nvertices = 0, Team & team = dash::Team::All()) + Graph( + vertex_size_type n_vertices = 0, + edge_size_type n_vertex_edges = 0, + Team & team = dash::Team::All() + ) : _team(&team), _myid(team.myid()) { - allocate(nvertices); + allocate(n_vertices, n_vertex_edges); } /** Destructs the graph. @@ -175,9 +190,10 @@ class Graph { */ vertex_index_type add_vertex(const VertexProperties & prop = VertexProperties()) { - vertex_type v(prop); - v._local_id = ++_local_vertex_max_index; - _glob_mem_con->push_back(_vertex_container_ref, v); + ++_local_vertex_max_index; + auto ref = _glob_mem_edge->add_container(_alloc_edges_per_vertex); + vertex_type v(_local_vertex_max_index, ref, prop); + _glob_mem_vertex->push_back(_vertex_container_ref, v); // TODO: return global index return _local_vertex_max_index; } @@ -206,7 +222,20 @@ class Graph { std::pair add_edge(const vertex_index_type & v1, const vertex_index_type & v2, const EdgeProperties & prop = EdgeProperties()) { - + //TODO: Check, whether the edge already exists + //TODO:_Handle removed vertices + // (vertex index is no longer == vertex container index) + auto vertex1 = _glob_mem_vertex->get(_vertex_container_ref, v1); + auto vertex2 = _glob_mem_vertex->get(_vertex_container_ref, v2); + //TODO: if feasible, use pointer to prop, so it gets saved only once + ++_local_edge_max_index; + edge_type edge1 = edge_type(_local_edge_max_index, v2, prop); + edge_type edge2 = edge_type(_local_edge_max_index, v1, prop); + //TODO: find out, how to save edges for different graph types + // (directed, undirected) + _glob_mem_edge->push_back(vertex1._edge_ref, edge1); + _glob_mem_edge->push_back(vertex2._edge_ref, edge2); + return std::make_pair(_local_edge_max_index, true); } /** * Removes an edge between two given vertices. @@ -230,16 +259,21 @@ class Graph { * the whole data structure. */ void barrier() { - _glob_mem_con->commit(); + _glob_mem_vertex->commit(); + _glob_mem_edge->commit(); } /** * Globally allocates memory for vertex storage. */ - bool allocate(vertex_size_type nvertices) { - auto lcap = dash::math::div_ceil(nvertices, _team->size()); - _glob_mem_con = new glob_mem_con_type(*_team); - _vertex_container_ref = _glob_mem_con->add_container(lcap); + bool allocate(vertex_size_type n_vertices, edge_size_type n_vertex_edges) { + auto vertex_lcap = dash::math::div_ceil(n_vertices, _team->size()); + _glob_mem_vertex = new glob_mem_vert_type(*_team); + _vertex_container_ref = _glob_mem_vertex->add_container(vertex_lcap); + // no edge list allocation yet, this will happen once the vertices are + // created. Each edge list will have n_vertex_edges elements reserved. + _alloc_edges_per_vertex = n_vertex_edges; + _glob_mem_edge = new glob_mem_edge_type(*_team); // Register deallocator at the respective team instance _team->register_deallocator(this, std::bind(&Graph::deallocate, this)); } @@ -248,15 +282,15 @@ class Graph { * Deallocates global memory of this container. */ void deallocate() { - if(_glob_mem_con != nullptr) { - delete _glob_mem_con; + if(_glob_mem_vertex != nullptr) { + delete _glob_mem_vertex; } // Remove deallocator from the respective team instance _team->unregister_deallocator(this, std::bind(&Graph::deallocate, this)); } vertex_type test() { - auto gptr = _glob_mem_con->dart_gptr_at(team_unit_t(1), 0, 0); + auto gptr = _glob_mem_vertex->dart_gptr_at(team_unit_t(1), 0, 0); return reference(gptr); } @@ -265,14 +299,20 @@ class Graph { /** the team containing all units using the container */ Team * _team = nullptr; /** Global memory allocation and access for sequential memory regions */ - glob_mem_con_type * _glob_mem_con = nullptr; + glob_mem_vert_type * _glob_mem_vertex = nullptr; + + glob_mem_edge_type * _glob_mem_edge = nullptr; /** Unit ID of the current unit */ team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; /** Index of last added vertex */ vertex_index_type _local_vertex_max_index = -1; + + edge_index_type _local_edge_max_index = -1; /** Iterator to the vertex container in GlobMem object */ vertex_cont_ref_type _vertex_container_ref; + edge_size_type _alloc_edges_per_vertex; + }; } diff --git a/dash/include/dash/graph/EdgeIterator.h b/dash/include/dash/graph/EdgeIterator.h new file mode 100644 index 000000000..5e2ab43b0 --- /dev/null +++ b/dash/include/dash/graph/EdgeIterator.h @@ -0,0 +1,98 @@ +#ifndef DASH__GRAPH__EDGE_ITERATOR_H__INCLUDED +#define DASH__GRAPH__EDGE_ITERATOR_H__INCLUDED + +namespace dash { + +/** + * Wrapper for the edge iterators of the graph. + */ +template +struct EdgeIteratorWrapper { + + typedef Graph graph_type; + typedef typename Graph::global_edge_iterator iterator; + typedef const iterator const_iterator; + typedef typename Graph::local_edge_iterator local_iterator; + typedef const local_iterator const_local_iterator; + typedef typename Graph::edge_index_type edge_index_type; + typedef typename Graph::edge_properties_type edge_properties_type; + + /** + * Constructs the wrapper. + */ + EdgeIteratorWrapper(graph_type * graph) + : _graph(graph) + { } + + /** + * Returns a property object for the given edge. + */ + edge_properties_type & operator[](const edge_index_type & v) const { + + } + + /** + * Returns global iterator to the beginning of the edge list. + */ + iterator begin() { + return _graph->_glob_mem_edge->begin(); + } + + /** + * Returns global iterator to the beginning of the edge list. + */ + const_iterator begin() const { + return _graph->_glob_mem_edge->begin(); + } + + /** + * Returns global iterator to the end of the edge list. + */ + iterator end() { + return _graph->_glob_mem_edge->end(); + } + + /** + * Returns global iterator to the end of the edge list. + */ + const_iterator end() const { + return _graph->_glob_mem_edge->end(); + } + + /** + * Returns local iterator to the beginning of the edge list. + */ + local_iterator lbegin() { + return _graph->_glob_mem_edge->lbegin(); + } + + /** + * Returns local iterator to the beginning of the edge list. + */ + const_local_iterator lbegin() const { + return _graph->_glob_mem_edge->lbegin(); + } + + /** + * Returns local iterator to the end of the edge list. + */ + local_iterator lend() { + return _graph->_glob_mem_edge->lend(); + } + + /** + * Returns local iterator to the end of the edge list. + */ + const_local_iterator lend() const { + return _graph->_glob_mem_edge->lend(); + } + +private: + + graph_type * _graph; + +}; + +} + +#endif // DASH__GRAPH__EDGE_ITERATOR_H__INCLUDED diff --git a/dash/include/dash/graph/VertexIterator.h b/dash/include/dash/graph/VertexIterator.h index 73aff3ee5..9404c5493 100644 --- a/dash/include/dash/graph/VertexIterator.h +++ b/dash/include/dash/graph/VertexIterator.h @@ -43,56 +43,56 @@ struct VertexIteratorWrapper { * Returns global iterator to the beginning of the vertex list. */ iterator begin() { - return _graph->_glob_mem_con->begin(); + return _graph->_glob_mem_vertex->begin(); } /** * Returns global iterator to the beginning of the vertex list. */ const_iterator begin() const { - return _graph->_glob_mem_con->begin(); + return _graph->_glob_mem_vertex->begin(); } /** * Returns global iterator to the end of the vertex list. */ iterator end() { - return _graph->_glob_mem_con->end(); + return _graph->_glob_mem_vertex->end(); } /** * Returns global iterator to the end of the vertex list. */ const_iterator end() const { - return _graph->_glob_mem_con->end(); + return _graph->_glob_mem_vertex->end(); } /** * Returns local iterator to the beginning of the vertex list. */ local_iterator lbegin() { - return _graph->_glob_mem_con->lbegin(); + return _graph->_glob_mem_vertex->lbegin(); } /** * Returns local iterator to the beginning of the vertex list. */ const_local_iterator lbegin() const { - return _graph->_glob_mem_con->lbegin(); + return _graph->_glob_mem_vertex->lbegin(); } /** * Returns local iterator to the end of the vertex list. */ local_iterator lend() { - return _graph->_glob_mem_con->lend(); + return _graph->_glob_mem_vertex->lend(); } /** * Returns local iterator to the end of the vertex list. */ const_local_iterator lend() const { - return _graph->_glob_mem_con->lend(); + return _graph->_glob_mem_vertex->lend(); } private: diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index 0edc66f1b..53ba05a88 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -11,45 +11,64 @@ enum GraphDirection { namespace internal { -template< - typename IndexType, - typename EdgeContainer, - typename VertexProperties> +template struct vertex { + + typedef typename GraphType::edge_cont_ref_type edge_container_ref; + typedef typename GraphType::vertex_index_type index_type; + typedef typename GraphType::vertex_properties_type properties_type; /** * Creates a vertex with given properties. */ - vertex(VertexProperties properties = VertexProperties()) - : _properties(properties) + vertex( + index_type & index, + edge_container_ref & edge_ref, + properties_type properties = properties_type() + ) + : _local_id(index), + _edge_ref(edge_ref), + _properties(properties) { } - /** Outgoing edges from this vertex */ - EdgeContainer _out_edges; /** Properties of this vertex */ - VertexProperties _properties; + properties_type _properties; /** Index of the vertex in local index space */ - IndexType _local_id; + index_type _local_id; + /* + * Iterator to the position of the respective edge container in glob_mem + * object + */ + edge_container_ref _edge_ref; }; -template< - typename VertexIndexType, - typename EdgeProperties> +template struct out_edge { + + typedef typename GraphType::vertex_index_type vertex_index_type; + typedef typename GraphType::edge_index_type index_type; + typedef typename GraphType::edge_properties_type properties_type; /** * Creates an edge from its parent vertex to target. */ - out_edge(VertexIndexType target, EdgeProperties properties = EdgeProperties()) - : _target(target), + out_edge( + index_type index, + vertex_index_type target, + properties_type properties = properties_type() + ) + : _local_id(index), + _target(target), _properties(properties) { } /** Target vertex the edge is pointing to */ - VertexIndexType _target; + vertex_index_type _target; /** Properties of this edge */ - EdgeProperties _properties; + properties_type _properties; + /** Index of the edge in local index space */ + index_type _local_id; }; From 3f2e0e92dfac073e5124c8a27b39e0cbefe48af1 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Wed, 22 Mar 2017 17:17:47 +0100 Subject: [PATCH 025/102] fixed global iterators for more than one globmem container --- dash/include/dash/GlobDynamicContiguousMem.h | 184 ++++++++++++------- dash/include/dash/graph/internal/Graph.h | 10 + 2 files changed, 130 insertions(+), 64 deletions(-) diff --git a/dash/include/dash/GlobDynamicContiguousMem.h b/dash/include/dash/GlobDynamicContiguousMem.h index 7e4e85f48..41b8ca2c9 100644 --- a/dash/include/dash/GlobDynamicContiguousMem.h +++ b/dash/include/dash/GlobDynamicContiguousMem.h @@ -33,20 +33,23 @@ namespace dash { namespace internal { -template +template struct container_data { - typedef ContainerType container_type; - typedef typename ContainerType::value_type value_type; - typedef typename ContainerType::difference_type index_type; - typedef typename ContainerType::size_type size_type; + typedef typename GlobMemType::container_type container_type; + typedef typename GlobMemType::value_type value_type; + typedef typename GlobMemType::index_type index_type; + typedef typename GlobMemType::size_type size_type; typedef LocalBucketIter local_iterator; typedef typename local_iterator::bucket_type bucket_type; typedef typename std::list bucket_list; + typedef typename + GlobMemType::bucket_cumul_sizes_map::difference_type bucket_sizes_index; - container_data(size_type n_local_elem) + container_data(size_type n_local_elem, bucket_sizes_index b_index) : container(new container_type()), - unattached_container(new container_type()) + unattached_container(new container_type()), + bucket_index(b_index) { container->reserve(n_local_elem); bucket_type cont_bucket { @@ -103,13 +106,14 @@ struct container_data { lend = unit_lend; } - ContainerType * container; - ContainerType * unattached_container; - bucket_list buckets; - bucket_type * container_bucket; - bucket_type * unattached_container_bucket; - local_iterator lbegin; - local_iterator lend; + container_type * container; + container_type * unattached_container; + bucket_list buckets; + bucket_type * container_bucket; + bucket_type * unattached_container_bucket; + local_iterator lbegin; + local_iterator lend; + bucket_sizes_index bucket_index; }; @@ -132,24 +136,20 @@ class GlobDynamicContiguousMem public: - typedef ContainerType container_type; - typedef internal::container_data data_type; - typedef std::list container_list_type; - typedef typename container_list_type::iterator container_list_iter; - typedef typename ContainerType::value_type value_type; - typedef typename ContainerType::difference_type index_type; - typedef LocalBucketIter local_iterator; - typedef GlobBucketIter global_iterator; - typedef typename ContainerType::size_type size_type; - typedef typename local_iterator::bucket_type bucket_type; - typedef typename std::list bucket_list; - typedef local_iterator local_pointer; - typedef local_iterator const_local_pointer; - -private: - - typedef std::vector > - bucket_cumul_sizes_map; + typedef ContainerType container_type; + typedef internal::container_data data_type; + typedef std::list container_list_type; + typedef typename container_list_type::iterator container_list_iter; + typedef typename ContainerType::value_type value_type; + typedef typename ContainerType::difference_type index_type; + typedef LocalBucketIter local_iterator; + typedef GlobBucketIter global_iterator; + typedef typename ContainerType::size_type size_type; + typedef typename local_iterator::bucket_type bucket_type; + typedef typename std::list bucket_list; + typedef local_iterator local_pointer; + typedef local_iterator const_local_pointer; + typedef std::vector> bucket_cumul_sizes_map; template friend class dash::GlobBucketIter; @@ -165,15 +165,12 @@ class GlobDynamicContiguousMem _nunits(team.size()), _myid(team.myid()), _bucket_cumul_sizes(team.size()) - { - for(auto it = _bucket_cumul_sizes.begin(); - it != _bucket_cumul_sizes.end(); ++it) { - it->resize(1); - } - } + { } container_list_iter add_container(size_type n_elements) { - auto c_data = data_type(n_elements); + increment_bucket_sizes(); + auto bucket_index = _bucket_cumul_sizes[_myid].size(); + auto c_data = data_type(n_elements, bucket_index); // TODO: use pointers instead of copies -> no maintenance of 2 object // copies auto it = _buckets.insert(_buckets.end(), @@ -213,6 +210,10 @@ class GlobDynamicContiguousMem self_t & operator=(const self_t & rhs) = default; void commit() { + int bucket_num = 0; + int bucket_cumul = 0; + + // TODO: put multiple containers into one bucket for(auto c_data : _container_list) { // merge public and local containers c_data.container->insert(c_data.container->end(), @@ -246,25 +247,49 @@ class GlobDynamicContiguousMem c_data.container->data(), &gptr ); - // no need to update gptr of local bucekt list in c_data + // no need to update gptr of local bucket list in c_data c_data.container_bucket->gptr = gptr; + c_data.unattached_container_bucket->gptr = gptr; + + // update cumulated bucket sizes + bucket_cumul += c_data.container->size(); + _bucket_cumul_sizes[_myid][bucket_num] = bucket_cumul; + ++bucket_num; + _bucket_cumul_sizes[_myid][bucket_num] = bucket_cumul; + ++bucket_num; + } - //TODO: Adjust bucket size accumulation, so that multiple containers - // are supported - // distribute bucket sizes between all units - std::vector bucket_sizes(_team->size()); - dart_allgather(&_local_size, bucket_sizes.data(), - sizeof(size_type), DART_TYPE_BYTE, _team->dart_id()); + // distribute bucket sizes between all units + // TODO: use one allgather for all buckets + // TODO: make it work for unevenls distributed amount of buckets + auto bucket_count = _bucket_cumul_sizes[_myid].size(); + for(auto c_data : _container_list) { + std::vector bucket_sizes(bucket_count * _team->size()); + std::vector local_buckets(_bucket_cumul_sizes[_myid]); + dart_allgather(local_buckets.data(), bucket_sizes.data(), + sizeof(size_type) * local_buckets.size(), DART_TYPE_BYTE, _team->dart_id()); _size = 0; + auto begin = bucket_sizes.begin(); for(int i = 0; i < _team->size(); ++i) { - _bucket_cumul_sizes[i][0] = bucket_sizes[i]; - _size += bucket_sizes[i]; + auto end = begin + bucket_count - 1; + std::copy(begin, end + 1, _bucket_cumul_sizes[i].begin()); + begin = end + 1; + _size += *end; } } + update_lbegin(); update_lend(); - + int b_num = 0; + std::cout << "-----------------" << std::endl; + for(auto el : _bucket_cumul_sizes) { + for(auto el2 : el) { + std::cout << "bucket " << b_num <<" of " << _myid << ": " << el2 << std::endl; + } + b_num++; + } + std::cout << "-----------------" << std::endl; _begin = global_iterator(this, 0); _end = _begin + _size; @@ -312,19 +337,49 @@ class GlobDynamicContiguousMem c_data.update_lend(); update_lbegin(); update_lend(); - _bucket_cumul_sizes[_myid][0] += 1; + } + + + + size_type size() { + return _size; + } + + Team & team() { + return (_team != nullptr) ? *_team : dash::Team::Null(); } /** * Global pointer referencing an element position in a unit's bucket. */ - dart_gptr_t dart_gptr_at(team_unit_t unit, - index_type bucket_index, index_type bucket_phase) { - // we only have 1 global pointer for _container - auto bucket = *_buckets.begin(); - auto dart_gptr = bucket.gptr; - + dart_gptr_t dart_gptr_at( + /// Unit id mapped to address in global memory space. + team_unit_t unit, + /// Index of bucket containing the referenced address. + index_type bucket_index, + /// Offset of the referenced address in the bucket's memory space. + index_type bucket_phase) const + { + DASH_LOG_DEBUG("GlobDynamicMem.dart_gptr_at(u,bi,bp)", + unit, bucket_index, bucket_phase); + if (_nunits == 0) { + DASH_THROW(dash::exception::RuntimeError, "No units in team"); + } + // Get the referenced bucket's dart_gptr: + auto bucket_it = _buckets.begin(); + std::advance(bucket_it, bucket_index); + auto dart_gptr = bucket_it->gptr; + DASH_LOG_TRACE_VAR("GlobDynamicMem.dart_gptr_at", bucket_it->attached); + DASH_LOG_TRACE_VAR("GlobDynamicMem.dart_gptr_at", bucket_it->gptr); + if (unit == _myid) { + DASH_LOG_TRACE_VAR("GlobDynamicMem.dart_gptr_at", bucket_it->lptr); + DASH_LOG_TRACE_VAR("GlobDynamicMem.dart_gptr_at", bucket_it->size); + DASH_ASSERT_LT(bucket_phase, bucket_it->size, + "bucket phase out of bounds"); + } if (DART_GPTR_ISNULL(dart_gptr)) { + DASH_LOG_TRACE("GlobDynamicMem.dart_gptr_at", + "bucket.gptr is DART_GPTR_NULL"); dart_gptr = DART_GPTR_NULL; } else { // Move dart_gptr to unit and local offset: @@ -337,17 +392,10 @@ class GlobDynamicContiguousMem bucket_phase * sizeof(value_type)), DART_OK); } + DASH_LOG_DEBUG("GlobDynamicMem.dart_gptr_at >", dart_gptr); return dart_gptr; } - size_type size() { - return _size; - } - - Team & team() { - return (_team != nullptr) ? *_team : dash::Team::Null(); - } - private: /** * Native pointer of the initial address of the local memory of @@ -384,6 +432,14 @@ class GlobDynamicContiguousMem _lend = unit_lend; } + void increment_bucket_sizes() { + for(auto it = _bucket_cumul_sizes.begin(); + it != _bucket_cumul_sizes.end(); ++it) { + // gets initiliazed with 0 automatically + it->resize(it->size() + 2); + } + } + private: container_list_type _container_list; diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index 53ba05a88..fad22c6d8 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -17,6 +17,11 @@ struct vertex { typedef typename GraphType::edge_cont_ref_type edge_container_ref; typedef typename GraphType::vertex_index_type index_type; typedef typename GraphType::vertex_properties_type properties_type; + + /** + * Default constructor needed for GlobRef / GlobSharedRef dereference op + */ + vertex() = default; /** * Creates a vertex with given properties. @@ -50,6 +55,11 @@ struct out_edge { typedef typename GraphType::edge_index_type index_type; typedef typename GraphType::edge_properties_type properties_type; + /** + * Default constructor needed for GlobRef / GlobSharedRef dereference op + */ + out_edge() = default; + /** * Creates an edge from its parent vertex to target. */ From f8a109da3885886817f6856934e9c35c6ed2b202 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Wed, 29 Mar 2017 13:27:41 +0200 Subject: [PATCH 026/102] chnaged index types --- dash/include/dash/Graph.h | 46 +++++++++++++++--------- dash/include/dash/graph/internal/Graph.h | 34 ++++++++++++++++-- 2 files changed, 62 insertions(+), 18 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index b10e3b661..98497b305 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -83,7 +83,7 @@ class Graph { vertex_container_type> glob_mem_vert_type; typedef GlobDynamicContiguousMem< edge_container_type> glob_mem_edge_type; - + typedef std::list edge_list_type; public: @@ -91,8 +91,10 @@ class Graph { glob_mem_vert_type::container_list_iter vertex_cont_ref_type; typedef typename glob_mem_edge_type::container_list_iter edge_cont_ref_type; - typedef VertexIndexType vertex_index_type; - typedef EdgeIndexType edge_index_type; + typedef VertexIndexType vertex_offset_type; + typedef EdgeIndexType edge_offset_type; + typedef internal::VertexIndex vertex_index_type; + typedef internal::EdgeIndex edge_index_type; typedef typename std::make_unsigned::type vertex_size_type; typedef typename @@ -195,7 +197,7 @@ class Graph { vertex_type v(_local_vertex_max_index, ref, prop); _glob_mem_vertex->push_back(_vertex_container_ref, v); // TODO: return global index - return _local_vertex_max_index; + return vertex_index_type(_myid, _local_vertex_max_index); } /** @@ -222,20 +224,30 @@ class Graph { std::pair add_edge(const vertex_index_type & v1, const vertex_index_type & v2, const EdgeProperties & prop = EdgeProperties()) { + ++_local_edge_max_index; + //TODO: Handle errors, if vertices do not exist + auto edge1 = edge_type(_local_edge_max_index, v2, prop); + if(v1.unit == _myid) { + auto vertex1 = _glob_mem_vertex->get(_vertex_container_ref, v1.offset); + _glob_mem_edge->push_back(vertex1._edge_ref, edge1); + } else { + _remote_edges.push_back(edge1); + } + auto edge2 = edge_type(_local_edge_max_index, v1, prop); + if(v2.unit == _myid) { + auto vertex2 = _glob_mem_vertex->get(_vertex_container_ref, v2.offset); + _glob_mem_edge->push_back(vertex2._edge_ref, edge2); + } else { + _remote_edges.push_back(edge2); + } //TODO: Check, whether the edge already exists - //TODO:_Handle removed vertices - // (vertex index is no longer == vertex container index) - auto vertex1 = _glob_mem_vertex->get(_vertex_container_ref, v1); - auto vertex2 = _glob_mem_vertex->get(_vertex_container_ref, v2); //TODO: if feasible, use pointer to prop, so it gets saved only once - ++_local_edge_max_index; - edge_type edge1 = edge_type(_local_edge_max_index, v2, prop); - edge_type edge2 = edge_type(_local_edge_max_index, v1, prop); //TODO: find out, how to save edges for different graph types // (directed, undirected) - _glob_mem_edge->push_back(vertex1._edge_ref, edge1); - _glob_mem_edge->push_back(vertex2._edge_ref, edge2); - return std::make_pair(_local_edge_max_index, true); + return std::make_pair( + edge_index_type(_myid, _local_edge_max_index), + true + ); } /** * Removes an edge between two given vertices. @@ -305,13 +317,15 @@ class Graph { /** Unit ID of the current unit */ team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; /** Index of last added vertex */ - vertex_index_type _local_vertex_max_index = -1; + vertex_offset_type _local_vertex_max_index = -1; - edge_index_type _local_edge_max_index = -1; + edge_offset_type _local_edge_max_index = -1; /** Iterator to the vertex container in GlobMem object */ vertex_cont_ref_type _vertex_container_ref; edge_size_type _alloc_edges_per_vertex; + /** edges that have to be added to vertices on another unit */ + edge_list_type _remote_edges; }; diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index fad22c6d8..a360f6523 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -11,11 +11,41 @@ enum GraphDirection { namespace internal { +template +struct VertexIndex { + + VertexIndex() = default; + + VertexIndex(team_unit_t u, IndexType o) + : unit(u), + offset(o) + { } + + team_unit_t unit; + IndexType offset; + +}; + +template +struct EdgeIndex { + + EdgeIndex() = default; + + EdgeIndex(team_unit_t u, IndexType o) + : unit(u), + offset(o) + { } + + team_unit_t unit; + IndexType offset; + +}; + template struct vertex { typedef typename GraphType::edge_cont_ref_type edge_container_ref; - typedef typename GraphType::vertex_index_type index_type; + typedef typename GraphType::vertex_offset_type index_type; typedef typename GraphType::vertex_properties_type properties_type; /** @@ -52,7 +82,7 @@ template struct out_edge { typedef typename GraphType::vertex_index_type vertex_index_type; - typedef typename GraphType::edge_index_type index_type; + typedef typename GraphType::edge_offset_type index_type; typedef typename GraphType::edge_properties_type properties_type; /** From d2a743caf9bb0e6e59592420d5d0df49eaaac1a8 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Wed, 29 Mar 2017 16:04:59 +0200 Subject: [PATCH 027/102] added alltoall dart function --- .../include/dash/dart/if/dart_communication.h | 54 +++++++ dart-impl/mpi/src/dart_communication.c | 134 ++++++++++++++++++ 2 files changed, 188 insertions(+) diff --git a/dart-if/include/dash/dart/if/dart_communication.h b/dart-if/include/dash/dart/if/dart_communication.h index 6fcdecef7..da23eb8e5 100644 --- a/dart-if/include/dash/dart/if/dart_communication.h +++ b/dart-if/include/dash/dart/if/dart_communication.h @@ -165,6 +165,60 @@ dart_ret_t dart_allgatherv( const size_t * recvdispls, dart_team_t teamid) DART_NOTHROW; +/** + * DART Equivalent to MPI alltoall. + * + * \param sendbuf The buffer containing the data to be sent by each unit. + * \param recvbuf The buffer to hold the received data. + * \param nelem Number of values sent by each process and received from + * each unit. + * \param dtype The data type of values in \c sendbuf and \c recvbuf. + * \param team The team to participate in the allgather. + * + * \return \c DART_OK on success, any other of \ref dart_ret_t otherwise. + * + * \threadsafe_data{team} + * \ingroup DartCommunication + */ +dart_ret_t dart_alltoall( + const void * sendbuf, + void * recvbuf, + size_t nelem, + dart_datatype_t dtype, + dart_team_t team) DART_NOTHROW; + +/** + * DART Equivalent to MPI alltoallv. + * + * \param sendbuf The buffer containing the data to be sent by each unit. + * \param nsendcounts Array containing the number of values to be sent by + * each unit. + * \param senddispls Array containing the displacements of data to be sent + * by each unit. + * \param dtype The data type of values in \c sendbuf and \c recvbuf. + * \param recvbuf The buffer to hold the received data. + * \param nrecvelem Array containing the number of values to receive from + * each unit. + * \param recvdispls Array containing the displacements of data received + * from each unit in \c recvbuf. + * \param teamid The team to participate in the allgatherv. + * + * \return \c DART_OK on success, any other of \ref dart_ret_t otherwise. + * + * \threadsafe_data{team} + * \ingroup DartCommunication + */ +dart_ret_t dart_alltoallv( + const void * sendbuf, + const size_t * nsendcounts, + const size_t * senddispls, + dart_datatype_t dtype, + void * recvbuf, + const size_t * nrecvelem, + const size_t * recvdispls, + dart_team_t teamid) DART_NOTHROW; + + /** * DART Equivalent to MPI allreduce. * diff --git a/dart-impl/mpi/src/dart_communication.c b/dart-impl/mpi/src/dart_communication.c index a9e97aa9a..a6d59fa75 100644 --- a/dart-impl/mpi/src/dart_communication.c +++ b/dart-impl/mpi/src/dart_communication.c @@ -1726,6 +1726,140 @@ dart_ret_t dart_allgatherv( return DART_OK; } +dart_ret_t dart_alltoall( + const void * sendbuf, + void * recvbuf, + size_t nelem, + dart_datatype_t dtype, + dart_team_t teamid) +{ + MPI_Datatype mpi_dtype = dart_mpi_datatype(dtype); + MPI_Comm comm; + DART_LOG_TRACE("dart_alltoall() team:%d nelem:%"PRIu64"", + teamid, nelem); + + if (teamid == DART_UNDEFINED_TEAM_ID) { + DART_LOG_ERROR("dart_alltoall ! failed: team may not be DART_UNDEFINED_TEAM_ID"); + return DART_ERR_INVAL; + } + + /* + * MPI uses offset type int, do not copy more than INT_MAX elements: + */ + if (nelem > INT_MAX) { + DART_LOG_ERROR("dart_alltoall ! failed: nelem > INT_MAX"); + return DART_ERR_INVAL; + } + + dart_team_data_t *team_data = dart_adapt_teamlist_get(teamid); + if (team_data == NULL) { + DART_LOG_ERROR("dart_alltoall ! team:%d " + "dart_adapt_teamlist_convert failed", teamid); + return DART_ERR_INVAL; + } + if (sendbuf == recvbuf || NULL == sendbuf) { + sendbuf = MPI_IN_PLACE; + } + comm = team_data->comm; + if (MPI_Alltoall( + sendbuf, + nelem, + mpi_dtype, + recvbuf, + nelem, + mpi_dtype, + comm) != MPI_SUCCESS) { + DART_LOG_ERROR("dart_alltoall ! team:%d nelem:%"PRIu64" failed", + teamid, nelem); + return DART_ERR_INVAL; + } + DART_LOG_TRACE("dart_alltoall > team:%d nelem:%"PRIu64"", + teamid, nelem); + return DART_OK; +} + +dart_ret_t dart_alltoallv( + const void * sendbuf, + const size_t * nsendcounts, + const size_t * senddispls, + dart_datatype_t dtype, + void * recvbuf, + const size_t * nrecvcounts, + const size_t * recvdispls, + dart_team_t teamid) +{ + MPI_Datatype mpi_dtype = dart_mpi_datatype(dtype); + MPI_Comm comm; + int comm_size; + DART_LOG_TRACE("dart_alltoallv() team:%d nsendelem:%"PRIu64"", + teamid, nsendelem); + + if (teamid == DART_UNDEFINED_TEAM_ID) { + DART_LOG_ERROR("dart_alltoallv ! failed: team may not be DART_UNDEFINED_TEAM_ID"); + return DART_ERR_INVAL; + } + + dart_team_data_t *team_data = dart_adapt_teamlist_get(teamid); + if (team_data == NULL) { + DART_LOG_ERROR("dart_alltoallv ! team:%d " + "dart_adapt_teamlist_convert failed", teamid); + return DART_ERR_INVAL; + } + if (sendbuf == recvbuf || NULL == sendbuf) { + sendbuf = MPI_IN_PLACE; + } + comm = team_data->comm; + + // convert nrecvcounts and recvdispls + MPI_Comm_size(comm, &comm_size); + int *insendcounts = malloc(sizeof(int) * comm_size); + int *isenddispls = malloc(sizeof(int) * comm_size); + int *inrecvcounts = malloc(sizeof(int) * comm_size); + int *irecvdispls = malloc(sizeof(int) * comm_size); + for (int i = 0; i < comm_size; i++) { + if (nsendcounts[i] > INT_MAX || senddispls[i] > INT_MAX) { + DART_LOG_ERROR("dart_alltoallv ! failed: nsendcounts[%i] > INT_MAX || senddispls[%i] > INT_MAX", i, i); + free(insendcounts); + free(isenddispls); + return DART_ERR_INVAL; + } + if (nrecvcounts[i] > INT_MAX || recvdispls[i] > INT_MAX) { + DART_LOG_ERROR("dart_alltoallv ! failed: nrecvcounts[%i] > INT_MAX || recvdispls[%i] > INT_MAX", i, i); + free(inrecvcounts); + free(irecvdispls); + return DART_ERR_INVAL; + } + insendcounts[i] = nsendcounts[i]; + isenddispls[i] = senddispls[i]; + inrecvcounts[i] = nrecvcounts[i]; + irecvdispls[i] = recvdispls[i]; + } + + if (MPI_Alltoallv( + sendbuf, + insendcounts, + isenddispls, + mpi_dtype, + recvbuf, + inrecvcounts, + irecvdispls, + mpi_dtype, + comm) != MPI_SUCCESS) { + DART_LOG_ERROR("dart_alltoallv ! team:%d failed", teamid); + free(insendcounts); + free(isenddispls); + free(inrecvcounts); + free(irecvdispls); + return DART_ERR_INVAL; + } + free(insendcounts); + free(isenddispls); + free(inrecvcounts); + free(irecvdispls); + DART_LOG_TRACE("dart_alltoallv > team:%d", teamid); + return DART_OK; +} + dart_ret_t dart_allreduce( const void * sendbuf, void * recvbuf, From 535d9d02ee90bac5999eadfe4f00c2b0201385dd Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Wed, 29 Mar 2017 16:05:52 +0200 Subject: [PATCH 028/102] added edge exchange functionality --- dash/include/dash/Graph.h | 63 +++++++++++++++++++++--- dash/include/dash/graph/internal/Graph.h | 5 ++ 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 98497b305..17e081e0a 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -83,7 +83,7 @@ class Graph { vertex_container_type> glob_mem_vert_type; typedef GlobDynamicContiguousMem< edge_container_type> glob_mem_edge_type; - typedef std::list edge_list_type; + typedef std::vector> edge_list_type; public: @@ -140,7 +140,8 @@ class Graph { Team & team = dash::Team::All() ) : _team(&team), - _myid(team.myid()) + _myid(team.myid()), + _remote_edges(team.size()) { allocate(n_vertices, n_vertex_edges); } @@ -224,21 +225,23 @@ class Graph { std::pair add_edge(const vertex_index_type & v1, const vertex_index_type & v2, const EdgeProperties & prop = EdgeProperties()) { - ++_local_edge_max_index; //TODO: Handle errors, if vertices do not exist - auto edge1 = edge_type(_local_edge_max_index, v2, prop); + auto edge1 = edge_type(0, v1, v2, prop); if(v1.unit == _myid) { auto vertex1 = _glob_mem_vertex->get(_vertex_container_ref, v1.offset); + edge1._local_id = ++_local_edge_max_index; _glob_mem_edge->push_back(vertex1._edge_ref, edge1); } else { - _remote_edges.push_back(edge1); + //TODO: check, if unit ID is valid + _remote_edges[v1.unit].push_back(edge1); } - auto edge2 = edge_type(_local_edge_max_index, v1, prop); + auto edge2 = edge_type(0, v2, v1, prop); if(v2.unit == _myid) { auto vertex2 = _glob_mem_vertex->get(_vertex_container_ref, v2.offset); + edge2._local_id = ++_local_edge_max_index; _glob_mem_edge->push_back(vertex2._edge_ref, edge2); } else { - _remote_edges.push_back(edge2); + _remote_edges[v2.unit].push_back(edge2); } //TODO: Check, whether the edge already exists //TODO: if feasible, use pointer to prop, so it gets saved only once @@ -271,6 +274,52 @@ class Graph { * the whole data structure. */ void barrier() { + // move all edges that have to be added by other units in a contiguous + // memory region + std::vector remote_edges; + std::vector remote_edges_count(_team->size()); + std::vector remote_edges_displs(_team->size()); + for(int i = 0; i < _remote_edges.size(); ++i) { + for(auto remote_edge : _remote_edges[i]) { + remote_edges.push_back(remote_edge); + remote_edges_count[i] += sizeof(edge_type); + } + for(int j = i + 1; j < remote_edges_displs.size(); ++j) { + remote_edges_displs[j] += remote_edges_count[i]; + } + } + _remote_edges.clear(); + // exchange amount of edges to be transferred with other units + std::vector edge_count(_team->size()); + dart_alltoall(remote_edges_count.data(), edge_count.data(), + sizeof(std::size_t), DART_TYPE_BYTE, _team->dart_id()); + int total_count = 0; + std::vector edge_displs(_team->size()); + for(int i = 0; i < edge_count.size(); ++i) { + total_count += edge_count[i]; + for(int j = i + 1; j < edge_displs.size(); ++j) { + edge_displs[j] += edge_count[i]; + } + } + // exchange edges + std::vector edges(total_count / sizeof(edge_type)); + dart_alltoallv(remote_edges.data(), + remote_edges_count.data(), + remote_edges_displs.data(), + DART_TYPE_BYTE, + edges.data(), + edge_count.data(), + edge_displs.data(), + _team->dart_id() + ); + // add missing edges to local memory space + for(auto edge : edges) { + auto vertex = _glob_mem_vertex->get(_vertex_container_ref, + edge._source.offset); + edge._local_id = ++_local_edge_max_index; + _glob_mem_edge->push_back(vertex._edge_ref, edge); + } + // commit changes in local memory space globally _glob_mem_vertex->commit(); _glob_mem_edge->commit(); } diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index a360f6523..6d28fa259 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -95,14 +95,19 @@ struct out_edge { */ out_edge( index_type index, + vertex_index_type source, vertex_index_type target, properties_type properties = properties_type() ) : _local_id(index), + _source(source), _target(target), _properties(properties) { } + //TODO: Examine, if saving source can be avoided + /** Source vertex the edge is pointing from */ + vertex_index_type _source; /** Target vertex the edge is pointing to */ vertex_index_type _target; /** Properties of this edge */ From ec6ef3bb211bd6724cc9ff09b676c6134c7f4841 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Wed, 29 Mar 2017 19:07:27 +0200 Subject: [PATCH 029/102] adjusted to changes from merge (NOT working with current state of DART) --- dart-impl/mpi/src/dart_communication.c | 4 +- dash/include/dash/GlobDynamicContiguousMem.h | 49 +- dash/include/dash/Graph.h | 9 +- dash/include/dash/graph/EdgeIterator.h | 8 +- dash/include/dash/graph/GlobGraphIter.h | 732 +++++++++++++++++++ 5 files changed, 781 insertions(+), 21 deletions(-) create mode 100644 dash/include/dash/graph/GlobGraphIter.h diff --git a/dart-impl/mpi/src/dart_communication.c b/dart-impl/mpi/src/dart_communication.c index db9ad6529..a66ebaf87 100644 --- a/dart-impl/mpi/src/dart_communication.c +++ b/dart-impl/mpi/src/dart_communication.c @@ -1835,7 +1835,7 @@ dart_ret_t dart_alltoall( dart_datatype_t dtype, dart_team_t teamid) { - MPI_Datatype mpi_dtype = dart_mpi_datatype(dtype); + MPI_Datatype mpi_dtype = dart__mpi__datatype(dtype); MPI_Comm comm; DART_LOG_TRACE("dart_alltoall() team:%d nelem:%"PRIu64"", teamid, nelem); @@ -1890,7 +1890,7 @@ dart_ret_t dart_alltoallv( const size_t * recvdispls, dart_team_t teamid) { - MPI_Datatype mpi_dtype = dart_mpi_datatype(dtype); + MPI_Datatype mpi_dtype = dart__mpi__datatype(dtype); MPI_Comm comm; int comm_size; DART_LOG_TRACE("dart_alltoallv() team:%d nsendelem:%"PRIu64"", diff --git a/dash/include/dash/GlobDynamicContiguousMem.h b/dash/include/dash/GlobDynamicContiguousMem.h index 41b8ca2c9..0ebf31a45 100644 --- a/dash/include/dash/GlobDynamicContiguousMem.h +++ b/dash/include/dash/GlobDynamicContiguousMem.h @@ -14,8 +14,9 @@ #include #include -#include -#include +#include +#include +#include #include @@ -40,7 +41,7 @@ struct container_data { typedef typename GlobMemType::value_type value_type; typedef typename GlobMemType::index_type index_type; typedef typename GlobMemType::size_type size_type; - typedef LocalBucketIter local_iterator; + typedef typename GlobMemType::local_iterator local_iterator; typedef typename local_iterator::bucket_type bucket_type; typedef typename std::list bucket_list; typedef typename @@ -142,17 +143,19 @@ class GlobDynamicContiguousMem typedef typename container_list_type::iterator container_list_iter; typedef typename ContainerType::value_type value_type; typedef typename ContainerType::difference_type index_type; - typedef LocalBucketIter local_iterator; - typedef GlobBucketIter global_iterator; + typedef GlobHeapLocalPtr local_iterator; + typedef GlobPtr global_iterator; typedef typename ContainerType::size_type size_type; typedef typename local_iterator::bucket_type bucket_type; typedef typename std::list bucket_list; typedef local_iterator local_pointer; typedef local_iterator const_local_pointer; typedef std::vector> bucket_cumul_sizes_map; - + + //TODO: Use local_size function instead of _bucket_cumul_sizes in iterator + // to remove this dependency template - friend class dash::GlobBucketIter; + friend class dash::GlobGraphIter; public: /** @@ -282,6 +285,7 @@ class GlobDynamicContiguousMem update_lend(); int b_num = 0; + /* std::cout << "-----------------" << std::endl; for(auto el : _bucket_cumul_sizes) { for(auto el2 : el) { @@ -290,9 +294,8 @@ class GlobDynamicContiguousMem b_num++; } std::cout << "-----------------" << std::endl; - - _begin = global_iterator(this, 0); - _end = _begin + _size; + */ + // TODO: set global iterators here } global_iterator begin() { @@ -345,7 +348,7 @@ class GlobDynamicContiguousMem return _size; } - Team & team() { + Team & team() const { return (_team != nullptr) ? *_team : dash::Team::Null(); } @@ -396,6 +399,30 @@ class GlobDynamicContiguousMem return dart_gptr; } + /** + * Number of elements in local memory space of given unit. + * + * \return Local capacity as published by the specified unit in last + * commit. + */ + inline size_type local_size(team_unit_t unit) const + { + DASH_LOG_TRACE("GlobHeapMem.local_size(u)", "unit:", unit); + DASH_ASSERT_RANGE(0, unit, _nunits-1, "unit id out of range"); + DASH_LOG_TRACE_VAR("GlobHeapMem.local_size", + _bucket_cumul_sizes[unit]); + size_type unit_local_size; + if (unit == _myid) { + // Value of _local_sizes[u] is the local size as visible by the unit, + // i.e. including size of unattached buckets. + unit_local_size = _local_size; + } else { + unit_local_size = _bucket_cumul_sizes[unit].back(); + } + DASH_LOG_TRACE("GlobHeapMem.local_size >", unit_local_size); + return unit_local_size; + } + private: /** * Native pointer of the initial address of the local memory of diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 17e081e0a..570bcbed0 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace dash { @@ -106,12 +107,12 @@ class Graph { typedef GlobRef reference; - typedef typename - glob_mem_vert_type::global_iterator global_vertex_iterator; + typedef + GlobGraphIter global_vertex_iterator; typedef typename glob_mem_vert_type::local_iterator local_vertex_iterator; - typedef typename - glob_mem_edge_type::global_iterator global_edge_iterator; + typedef + GlobGraphIter global_edge_iterator; typedef typename glob_mem_edge_type::local_iterator local_edge_iterator; diff --git a/dash/include/dash/graph/EdgeIterator.h b/dash/include/dash/graph/EdgeIterator.h index 5e2ab43b0..535880681 100644 --- a/dash/include/dash/graph/EdgeIterator.h +++ b/dash/include/dash/graph/EdgeIterator.h @@ -35,28 +35,28 @@ struct EdgeIteratorWrapper { * Returns global iterator to the beginning of the edge list. */ iterator begin() { - return _graph->_glob_mem_edge->begin(); + return iterator(_graph->_glob_mem_edge, 0); } /** * Returns global iterator to the beginning of the edge list. */ const_iterator begin() const { - return _graph->_glob_mem_edge->begin(); + return iterator(_graph->_glob_mem_edge, 0); } /** * Returns global iterator to the end of the edge list. */ iterator end() { - return _graph->_glob_mem_edge->end(); + return iterator(_graph->_glob_mem_edge, _graph->_glob_mem_edge->size()); } /** * Returns global iterator to the end of the edge list. */ const_iterator end() const { - return _graph->_glob_mem_edge->end(); + return iterator(_graph->_glob_mem_edge, _graph->_glob_mem_edge->size()); } /** diff --git a/dash/include/dash/graph/GlobGraphIter.h b/dash/include/dash/graph/GlobGraphIter.h new file mode 100644 index 000000000..7619d9915 --- /dev/null +++ b/dash/include/dash/graph/GlobGraphIter.h @@ -0,0 +1,732 @@ +#ifndef DASH__GRAPH__GLOB_GRAPH_ITER_H__INCLUDED +#define DASH__GRAPH__GLOB_GRAPH_ITER_H__INCLUDED + +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + + +namespace dash { + +// Forward-declaration +template< + typename ElementType, + class AllocatorType > +class GlobDynamicMem; + +/** + * Iterator on global buckets. Represents global pointer type. + */ +template< + typename ElementType, + class GlobMemType, + class PointerType = dash::GlobPtr, + class ReferenceType = dash::GlobSharedRef > +class GlobGraphIter +: public std::iterator< + std::random_access_iterator_tag, + ElementType, + typename GlobMemType::index_type, + PointerType, + ReferenceType > +{ + template< + typename ElementType_, + class GlobMemType_, + class Pointer_, + class Reference_> + friend std::ostream & dash::operator<<( + std::ostream & os, + const dash::GlobGraphIter< + ElementType_, GlobMemType_, Pointer_, Reference_> & it); + +private: + typedef GlobGraphIter< + ElementType, + GlobMemType, + PointerType, + ReferenceType> + self_t; + +public: + typedef typename GlobMemType::index_type index_type; + typedef typename std::make_unsigned::type size_type; + + typedef ElementType value_type; + typedef ReferenceType reference; + typedef ReferenceType const_reference; + typedef PointerType pointer; + typedef PointerType const_pointer; + + typedef ElementType * raw_pointer; + + typedef typename + std::conditional< + std::is_const::value, + const GlobMemType, + GlobMemType + >::type + globmem_type; + + typedef typename + std::conditional< + std::is_const::value, + typename GlobMemType::const_local_pointer, + typename GlobMemType::local_pointer + >::type + local_pointer; + + typedef struct { + team_unit_t unit; + index_type index; + } local_index; + +private: + typedef std::vector > + bucket_cumul_sizes_map; + +public: + /** + * Default constructor. + */ + GlobGraphIter() + : _globmem(nullptr), + _bucket_cumul_sizes(nullptr), + _idx(0), + _max_idx(0), + _myid(dash::Team::GlobalUnitID()), + _idx_unit_id(DART_UNDEFINED_UNIT_ID), + _idx_local_idx(-1), + _idx_bucket_idx(-1), + _idx_bucket_phase(-1) + { + DASH_LOG_TRACE_VAR("GlobGraphIter()", _idx); + DASH_LOG_TRACE_VAR("GlobGraphIter()", _max_idx); + } + + /** + * Constructor, creates a global iterator on global memory from global + * offset in logical storage order. + */ + GlobGraphIter( + globmem_type * gmem, + index_type position = 0) + : _globmem(gmem), + _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), + _lbegin(_globmem->lbegin()), + _idx(position), + _max_idx(gmem->size() - 1), + _myid(gmem->team().myid()), + _idx_unit_id(0), + _idx_local_idx(0), + _idx_bucket_idx(0), + _idx_bucket_phase(0) + { + DASH_LOG_TRACE("GlobGraphIter(gmem,idx)", "gidx:", position); + for (auto unit_bucket_cumul_sizes : *_bucket_cumul_sizes) { + DASH_LOG_TRACE_VAR("GlobGraphIter(gmem,idx)", + unit_bucket_cumul_sizes); + size_type bucket_cumul_size_prev = 0; + for (auto bucket_cumul_size : unit_bucket_cumul_sizes) { + DASH_LOG_TRACE_VAR("GlobGraphIter(gmem,idx)", bucket_cumul_size); + if (position < bucket_cumul_size) { + DASH_LOG_TRACE_VAR("GlobGraphIter(gmem,idx)", position); + _idx_bucket_phase -= bucket_cumul_size; + position = 0; + _idx_local_idx = position; + _idx_bucket_phase = position - bucket_cumul_size_prev; + break; + } + bucket_cumul_size_prev = bucket_cumul_size; + ++_idx_bucket_idx; + } + if (position == 0) { + break; + } + // Advance to next unit, adjust position relative to next unit's + // local index space: + position -= unit_bucket_cumul_sizes.back(); + ++_idx_unit_id; + } + DASH_LOG_TRACE("GlobGraphIter(gmem,idx)", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bucket:", _idx_bucket_idx, + "phase:", _idx_bucket_phase); + } + + /** + * Constructor, creates a global iterator on global memory from unit and + * local offset in logical storage order. + */ + GlobGraphIter( + globmem_type * gmem, + team_unit_t unit, + index_type local_index) + : _globmem(gmem), + _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), + _lbegin(_globmem->lbegin()), + _idx(0), + _max_idx(gmem->size() - 1), + _myid(gmem->team().myid()), + _idx_unit_id(unit), + _idx_local_idx(0), + _idx_bucket_idx(0), + _idx_bucket_phase(0) + { + DASH_LOG_TRACE("GlobGraphIter(gmem,unit,lidx)", + "unit:", unit, + "lidx:", local_index); + DASH_ASSERT_LT(unit, _bucket_cumul_sizes->size(), "invalid unit id"); + + for (size_type unit = 0; unit < _idx_unit_id; ++unit) { + auto prec_unit_local_size = (*_bucket_cumul_sizes)[unit].back(); + _idx += prec_unit_local_size; + } + increment(local_index); + DASH_LOG_TRACE("GlobGraphIter(gmem,unit,lidx) >", + "gidx:", _idx, + "maxidx:", _max_idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bucket:", _idx_bucket_idx, + "phase:", _idx_bucket_phase); + } + + /** + * Copy constructor. + */ + template + GlobGraphIter( + const GlobGraphIter & other) + : _globmem(other._globmem), + _bucket_cumul_sizes(other._bucket_cumul_sizes), + _lbegin(other._lbegin), + _idx(other._idx), + _max_idx(other._max_idx), + _idx_unit_id(other._idx_unit_id), + _idx_local_idx(other._idx_local_idx), + _idx_bucket_idx(other._idx_bucket_idx), + _idx_bucket_phase(other._idx_bucket_phase) + { } + + /** + * Assignment operator. + */ + template + self_t & operator=( + const GlobGraphIter & other) + { + _globmem = other._globmem; + _bucket_cumul_sizes = other._bucket_cumul_sizes; + _lbegin = other._lbegin; + _idx = other._idx; + _max_idx = other._max_idx; + _idx_unit_id = other._idx_unit_id; + _idx_local_idx = other._idx_local_idx; + _idx_bucket_idx = other._idx_bucket_idx; + _idx_bucket_phase = other._idx_bucket_phase; + } + + /** + * Type conversion operator to \c GlobPtr. + * + * \return A global reference to the element at the iterator's position + */ + operator pointer() const + { + return pointer(dart_gptr()); + } + + /** + * Explicit conversion to \c dart_gptr_t. + * + * \return A DART global pointer to the element at the iterator's + * position + */ + dart_gptr_t dart_gptr() const + { + DASH_LOG_TRACE_VAR("GlobGraphIter.dart_gptr()", _idx); + // Create global pointer from unit, bucket and phase: + dart_gptr_t dart_gptr = _globmem->dart_gptr_at( + _idx_unit_id, + _idx_bucket_idx, + _idx_bucket_phase); + DASH_LOG_TRACE_VAR("GlobGraphIter.dart_gptr >", dart_gptr); + return dart_gptr; + } + + /** + * Dereference operator. + * + * \return A global reference to the element at the iterator's position. + */ + reference operator*() const + { + auto lptr = local(); + if (lptr != nullptr) { + return reference(static_cast(lptr)); + } else { + return reference(dart_gptr()); + } + } + + /** + * Subscript operator, returns global reference to element at given + * global index. + */ + reference operator[]( + /// The global position of the element + index_type g_index) const + { + DASH_LOG_TRACE_VAR("GlobGraphIter.[]()", g_index); + auto git = *this; + git += g_index; + auto lptr = git.local(); + if (lptr != nullptr) { + return reference(lptr); + } else { + auto gref = *git; + DASH_LOG_TRACE_VAR("GlobGraphIter.[] >", gref); + return gref; + } + } + + /** + * Checks whether the element referenced by this global iterator is in + * the calling unit's local memory. + */ + inline bool is_local() const + { + return (_myid == _idx_unit_id); + } + + /** + * Conversion to local bucket iterator. + */ + local_pointer local() const + { + if (_myid != _idx_unit_id) { + // Iterator position does not point to local element + return nullptr; + } + return (_lbegin + _idx_local_idx); + } + + /** + * Unit and local offset at the iterator's position. + */ + inline local_index lpos() const + { + local_index local_pos; + local_pos.unit = _idx_unit_id; + local_pos.index = _idx_local_idx; + return local_pos; + } + + /** + * Map iterator to global index domain. + */ + inline self_t global() const + { + return *this; + } + + /** + * Position of the iterator in global index space. + */ + inline index_type pos() const + { + return _idx; + } + + /** + * Position of the iterator in global index range. + */ + inline index_type gpos() const + { + return _idx; + } + + /** + * The instance of \c GlobMem used by this iterator to resolve addresses + * in global memory. + */ + inline const globmem_type & globmem() const + { + return *_globmem; + } + + /** + * The instance of \c GlobMem used by this iterator to resolve addresses + * in global memory. + */ + inline globmem_type & globmem() + { + return *_globmem; + } + + /** + * Prefix increment operator. + */ + inline self_t & operator++() + { + increment(1); + return *this; + } + + /** + * Prefix decrement operator. + */ + inline self_t & operator--() + { + decrement(1); + return *this; + } + + /** + * Postfix increment operator. + */ + inline self_t operator++(int) + { + auto result = *this; + increment(1); + return result; + } + + /** + * Postfix decrement operator. + */ + inline self_t operator--(int) + { + auto result = *this; + decrement(1); + return result; + } + + inline self_t & operator+=(index_type offset) + { + increment(offset); + return *this; + } + + inline self_t & operator-=(index_type offset) + { + increment(offset); + return *this; + } + + inline self_t operator+(index_type offset) const + { + auto res = *this; + res.increment(offset); + return res; + } + + inline self_t operator-(index_type offset) const + { + auto res = *this; + res.decrement(offset); + return res; + } + + inline index_type operator+( + const self_t & other) const + { + return _idx + other._idx; + } + + inline index_type operator-( + const self_t & other) const + { + return _idx - other._idx; + } + + template + inline bool operator<(const GlobGraphIter & other) const + { + return (_idx < other._idx); + } + + template + inline bool operator<=(const GlobGraphIter & other) const + { + return (_idx <= other._idx); + } + + template + inline bool operator>(const GlobGraphIter & other) const + { + return (_idx > other._idx); + } + + template + inline bool operator>=(const GlobGraphIter & other) const + { + return (_idx >= other._idx); + } + + template + inline bool operator==(const GlobGraphIter & other) const + { + return _idx == other._idx; + } + + template + inline bool operator!=(const GlobGraphIter & other) const + { + return _idx != other._idx; + } + +private: + /** + * Advance pointer by specified position offset. + */ + void increment(int offset) + { + DASH_LOG_TRACE("GlobGraphIter.increment()", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase, + "offset:", offset); + _idx += offset; + auto current_bucket_size = + (*_bucket_cumul_sizes)[_idx_unit_id][_idx_bucket_idx]; + if (_idx_local_idx + offset < current_bucket_size) { + DASH_LOG_TRACE("GlobGraphIter.increment", "position current bucket"); + // element is in bucket currently referenced by this iterator: + _idx_bucket_phase += offset; + _idx_local_idx += offset; + } else { + DASH_LOG_TRACE("GlobGraphIter.increment", + "position in succeeding bucket"); + // iterate units: + auto unit_id_max = _bucket_cumul_sizes->size() - 1; + for (; _idx_unit_id <= unit_id_max; ++_idx_unit_id) { + if (offset == 0) { + break; + } + auto unit_bkt_sizes = (*_bucket_cumul_sizes)[_idx_unit_id]; + auto unit_bkt_sizes_total = unit_bkt_sizes.back(); + auto unit_num_bkts = unit_bkt_sizes.size(); + DASH_LOG_TRACE("GlobGraphIter.increment", + "unit:", _idx_unit_id, + "remaining offset:", offset, + "total local bucket size:", unit_bkt_sizes_total); + if (_idx_local_idx + offset >= unit_bkt_sizes_total) { + // offset refers to next unit: + DASH_LOG_TRACE("GlobGraphIter.increment", + "position in remote range"); + // subtract remaining own local size from remaining offset: + offset -= (unit_bkt_sizes_total - _idx_local_idx); + if (_idx_unit_id == unit_id_max) { + // end iterator, offset exceeds iteration space: + _idx_bucket_idx = unit_num_bkts - 1; + auto last_bkt_size = unit_bkt_sizes.back(); + if (unit_num_bkts > 1) { + last_bkt_size -= unit_bkt_sizes[_idx_bucket_idx-1]; + } + _idx_bucket_phase = last_bkt_size + offset; + _idx_local_idx += unit_bkt_sizes_total + offset; + break; + } + _idx_local_idx = 0; + _idx_bucket_idx = 0; + _idx_bucket_phase = 0; + } else { + // offset refers to current unit: + DASH_LOG_TRACE("GlobGraphIter.increment", + "position in local range", + "current bucket phase:", _idx_bucket_phase, + "cumul. bucket sizes:", unit_bkt_sizes); + _idx_local_idx += offset; + // iterate the unit's bucket sizes: + for (; _idx_bucket_idx < unit_num_bkts; ++_idx_bucket_idx) { + auto cumul_bucket_size = unit_bkt_sizes[_idx_bucket_idx]; + if (_idx_local_idx < cumul_bucket_size) { + auto cumul_prev = _idx_bucket_idx > 0 + ? unit_bkt_sizes[_idx_bucket_idx-1] + : 0; + // offset refers to current bucket: + _idx_bucket_phase = _idx_local_idx - cumul_prev; + offset = 0; + break; + } + } + if (offset == 0) { + break; + } + } + } + } + DASH_LOG_TRACE("GlobGraphIter.increment >", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase); + } + + /** + * Decrement pointer by specified position offset. + */ + void decrement(int offset) + { + DASH_LOG_TRACE("GlobGraphIter.decrement()", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase, + "offset:", -offset); + if (offset > _idx) { + DASH_THROW(dash::exception::OutOfRange, + "offset " << offset << " is out of range"); + } + _idx -= offset; + if (offset <= _idx_bucket_phase) { + // element is in bucket currently referenced by this iterator: + _idx_bucket_phase -= offset; + _idx_local_idx -= offset; + } else { + // iterate units: + auto first_unit = _idx_unit_id; + for (; _idx_unit_id >= 0; --_idx_unit_id) { + auto unit_bkt_sizes = (*_bucket_cumul_sizes)[_idx_unit_id]; + auto unit_bkt_sizes_total = unit_bkt_sizes.back(); + auto unit_num_bkts = unit_bkt_sizes.size(); + if (_idx_unit_id != first_unit) { + --offset; + _idx_bucket_idx = unit_num_bkts - 1; + _idx_local_idx = unit_bkt_sizes_total - 1; + auto last_bkt_size = unit_bkt_sizes.back(); + if (unit_num_bkts > 1) { + last_bkt_size -= unit_bkt_sizes[_idx_bucket_idx-1]; + } + _idx_bucket_phase = last_bkt_size - 1; + } + if (offset <= _idx_local_idx) { + // offset refers to current unit: + // iterate the unit's bucket sizes: + for (; _idx_bucket_idx >= 0; --_idx_bucket_idx) { + auto bucket_size = unit_bkt_sizes[_idx_bucket_idx]; + if (offset <= _idx_bucket_phase) { + // offset refers to current bucket: + _idx_local_idx -= offset; + _idx_bucket_phase -= offset; + offset = 0; + break; + } else { + // offset refers to preceeding bucket: + _idx_local_idx -= (_idx_bucket_phase + 1); + offset -= (_idx_bucket_phase + 1); + _idx_bucket_phase = unit_bkt_sizes[_idx_bucket_idx-1] - 1; + } + } + } else { + // offset refers to preceeding unit: + offset -= _idx_local_idx; + } + if (offset == 0) { + break; + } + } + } + DASH_LOG_TRACE("GlobGraphIter.decrement >", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase); + } + +private: + /// Global memory used to dereference iterated values. + globmem_type * _globmem = nullptr; + /// Mapping unit id to buckets in the unit's attached local storage. + const bucket_cumul_sizes_map * _bucket_cumul_sizes = nullptr; + /// Pointer to first element in local data space. + local_pointer _lbegin; + /// Current position of the iterator in global canonical index space. + index_type _idx = 0; + /// Maximum position allowed for this iterator. + index_type _max_idx = 0; + /// Unit id of the active unit. + team_unit_t _myid; + /// Unit id at the iterator's current position. + team_unit_t _idx_unit_id; + /// Logical offset in local index space at the iterator's current position. + index_type _idx_local_idx = -1; + /// Local bucket index at the iterator's current position. + index_type _idx_bucket_idx = -1; + /// Element offset in bucket at the iterator's current position. + index_type _idx_bucket_phase = -1; + +}; // class GlobGraphIter + +/** + * Resolve the number of elements between two global bucket iterators. + * + * \complexity O(1) + * + * \ingroup Algorithms + */ +template< + typename ElementType, + class GlobMemType, + class Pointer, + class Reference> +auto distance( + /// Global iterator to the first position in the global sequence + const dash::GlobGraphIter< + ElementType, GlobMemType, Pointer, Reference> & first, + /// Global iterator to the final position in the global sequence + const dash::GlobGraphIter< + ElementType, GlobMemType, Pointer, Reference> & last) +-> typename GlobMemType::index_type +{ + return last - first; +} + +template< + typename ElementType, + class GlobMemType, + class Pointer, + class Reference> +std::ostream & operator<<( + std::ostream & os, + const dash::GlobGraphIter< + ElementType, GlobMemType, Pointer, Reference> & it) +{ + std::ostringstream ss; + ss << "dash::GlobGraphIter<" + << typeid(ElementType).name() << ">(" + << "gidx:" << it._idx << ", (" + << "unit:" << it._idx_unit_id << ", " + << "lidx:" << it._idx_local_idx << "), (" + << "bidx:" << it._idx_bucket_idx << ", " + << "bphase:" << it._idx_bucket_phase << ")" + << ")"; + return operator<<(os, ss.str()); +} + +} // namespace dash + +#endif // DASH__GRAPH__GLOB_GRAPH_ITER_H__INCLUDED From fd4ec7656ede2e2b9d565b2c851aa87378f8d70c Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 30 Mar 2017 20:49:36 +0200 Subject: [PATCH 030/102] specialized GlobPtr for GlobDynamicContiguousMem --- dash/include/dash/GlobDynamicContiguousMem.h | 25 +- dash/include/dash/Graph.h | 7 +- dash/include/dash/graph/EdgeIterator.h | 12 +- .../dash/memory/GlobHeapContiguousPtr.h | 710 ++++++++++++++++++ 4 files changed, 733 insertions(+), 21 deletions(-) create mode 100644 dash/include/dash/memory/GlobHeapContiguousPtr.h diff --git a/dash/include/dash/GlobDynamicContiguousMem.h b/dash/include/dash/GlobDynamicContiguousMem.h index 0ebf31a45..640f31690 100644 --- a/dash/include/dash/GlobDynamicContiguousMem.h +++ b/dash/include/dash/GlobDynamicContiguousMem.h @@ -17,6 +17,7 @@ #include #include #include +#include #include @@ -128,12 +129,11 @@ struct container_data { // dash::Grap // // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -template< - typename ContainerType> +template class GlobDynamicContiguousMem { private: - typedef GlobDynamicContiguousMem self_t; + typedef GlobDynamicContiguousMem self_t; public: @@ -152,10 +152,8 @@ class GlobDynamicContiguousMem typedef local_iterator const_local_pointer; typedef std::vector> bucket_cumul_sizes_map; - //TODO: Use local_size function instead of _bucket_cumul_sizes in iterator - // to remove this dependency - template - friend class dash::GlobGraphIter; + template + friend class dash::GlobPtr; public: /** @@ -295,22 +293,23 @@ class GlobDynamicContiguousMem } std::cout << "-----------------" << std::endl; */ - // TODO: set global iterators here + _begin = global_iterator(this, 0); + _end = global_iterator(this, _size); } - global_iterator begin() { + global_iterator begin() const { return _begin; } - global_iterator end() { + global_iterator end() const { return _end; } - local_iterator lbegin() { + local_iterator lbegin() const { return _lbegin; } - local_iterator lend() { + local_iterator lend() const { return _lend; } @@ -344,7 +343,7 @@ class GlobDynamicContiguousMem - size_type size() { + size_type size() const { return _size; } diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 570bcbed0..a37932fcc 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -107,14 +107,13 @@ class Graph { typedef GlobRef reference; - typedef - GlobGraphIter global_vertex_iterator; typedef typename glob_mem_vert_type::local_iterator local_vertex_iterator; - typedef - GlobGraphIter global_edge_iterator; typedef typename glob_mem_edge_type::local_iterator local_edge_iterator; + + typedef typename glob_mem_vert_type::global_iterator global_vertex_iterator; + typedef typename glob_mem_edge_type::global_iterator global_edge_iterator; typedef typename vertex_it_wrapper::iterator vertex_iterator; typedef typename edge_it_wrapper::iterator edge_iterator; diff --git a/dash/include/dash/graph/EdgeIterator.h b/dash/include/dash/graph/EdgeIterator.h index 535880681..7fcf8c71e 100644 --- a/dash/include/dash/graph/EdgeIterator.h +++ b/dash/include/dash/graph/EdgeIterator.h @@ -35,28 +35,32 @@ struct EdgeIteratorWrapper { * Returns global iterator to the beginning of the edge list. */ iterator begin() { - return iterator(_graph->_glob_mem_edge, 0); + return _graph->_glob_mem_edge->begin(); + //return iterator(_graph->_glob_mem_edge, 0); } /** * Returns global iterator to the beginning of the edge list. */ const_iterator begin() const { - return iterator(_graph->_glob_mem_edge, 0); + return _graph->_glob_mem_edge->begin(); + //return iterator(_graph->_glob_mem_edge, 0); } /** * Returns global iterator to the end of the edge list. */ iterator end() { - return iterator(_graph->_glob_mem_edge, _graph->_glob_mem_edge->size()); + return _graph->_glob_mem_edge->end(); + //return iterator(_graph->_glob_mem_edge, _graph->_glob_mem_edge->size()); } /** * Returns global iterator to the end of the edge list. */ const_iterator end() const { - return iterator(_graph->_glob_mem_edge, _graph->_glob_mem_edge->size()); + return _graph->_glob_mem_edge->end(); + //return iterator(_graph->_glob_mem_edge, _graph->_glob_mem_edge->size()); } /** diff --git a/dash/include/dash/memory/GlobHeapContiguousPtr.h b/dash/include/dash/memory/GlobHeapContiguousPtr.h new file mode 100644 index 000000000..83a89a827 --- /dev/null +++ b/dash/include/dash/memory/GlobHeapContiguousPtr.h @@ -0,0 +1,710 @@ +#ifndef DASH__MEMORY__GLOB_HEAP_CONTIGUOUS_PTR_H__INCLUDED +#define DASH__MEMORY__GLOB_HEAP_CONTIGUOUS_PTR_H__INCLUDED + +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + + +namespace dash { + +// Forward-declaration +template +class GlobDynamicContiguousMem; + +/** + * Iterator on global buckets. Represents global pointer type. + */ +template +class GlobPtr< + ElementType, + GlobDynamicContiguousMem + > +{ + typedef GlobDynamicContiguousMem GlobHeapMemType; + typedef GlobPtr self_t; + + template< + typename ElementType_, + class ContainerType_ > + friend std::ostream & operator<<( + std::ostream & os, + const dash::GlobPtr< + ElementType_, + GlobDynamicContiguousMem + > & gptr); + +public: + typedef typename GlobHeapMemType::index_type index_type; + typedef typename std::make_unsigned::type size_type; + + typedef ElementType value_type; + + typedef GlobSharedRef< value_type, self_t> reference; + typedef GlobSharedRef const_reference; + + typedef value_type * raw_pointer; + + typedef GlobHeapMemType globmem_type; + typedef typename GlobHeapMemType::local_pointer local_pointer; + + typedef struct { + team_unit_t unit; + index_type index; + } local_index; + + /* + * TODO: Implement rebind to allow for the allocation of different types + * than ElementType + * + template + struct rebind { + typedef GlobPtr< + U, + GlobDynamicContiguousMem< + typename AllocatorType::template rebind< + typename std::remove_const::type + >::other + > + > other; + }; + */ + +private: + typedef std::vector > + bucket_cumul_sizes_map; + +private: + /// Global memory used to dereference iterated values. + const globmem_type * _globmem = nullptr; + /// Mapping unit id to buckets in the unit's attached local storage. + const bucket_cumul_sizes_map * _bucket_cumul_sizes = nullptr; + /// Pointer to first element in local data space. + local_pointer _lbegin; + /// Current position of the pointer in global canonical index space. + index_type _idx = 0; + /// Maximum position allowed for this pointer. + index_type _max_idx = 0; + /// Unit id of the active unit. + team_unit_t _myid; + /// Unit id at the pointer's current position. + team_unit_t _idx_unit_id; + /// Logical offset in local index space at the pointer's current position. + index_type _idx_local_idx = -1; + /// Local bucket index at the pointer's current position. + index_type _idx_bucket_idx = -1; + /// Element offset in bucket at the pointer's current position. + index_type _idx_bucket_phase = -1; + +public: + /** + * Default constructor. + */ + GlobPtr() + : _globmem(nullptr), + _bucket_cumul_sizes(nullptr), + _idx(0), + _max_idx(0), + _myid(dash::Team::GlobalUnitID()), + _idx_unit_id(DART_UNDEFINED_UNIT_ID), + _idx_local_idx(-1), + _idx_bucket_idx(-1), + _idx_bucket_phase(-1) + { + DASH_LOG_TRACE_VAR("GlobPtr()", _idx); + DASH_LOG_TRACE_VAR("GlobPtr()", _max_idx); + } + + /** + * Constructor, creates a global pointer on global memory from global + * offset in logical storage order. + */ + template + GlobPtr( + const MemSpaceT * gmem, + index_type position = 0) + : _globmem(reinterpret_cast(gmem)), + _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), + _lbegin(_globmem->lbegin()), + _idx(position), + _max_idx(gmem->size() - 1), + _myid(gmem->team().myid()), + _idx_unit_id(0), + _idx_local_idx(0), + _idx_bucket_idx(0), + _idx_bucket_phase(0) + { + DASH_LOG_TRACE("GlobPtr(gmem,idx)", "gidx:", position); + for (auto unit_bucket_cumul_sizes : *_bucket_cumul_sizes) { + DASH_LOG_TRACE_VAR("GlobPtr(gmem,idx)", + unit_bucket_cumul_sizes); + size_type bucket_cumul_size_prev = 0; + for (auto bucket_cumul_size : unit_bucket_cumul_sizes) { + DASH_LOG_TRACE_VAR("GlobPtr(gmem,idx)", bucket_cumul_size); + if (position < bucket_cumul_size) { + DASH_LOG_TRACE_VAR("GlobPtr(gmem,idx)", position); + _idx_bucket_phase -= bucket_cumul_size; + position = 0; + _idx_local_idx = position; + _idx_bucket_phase = position - bucket_cumul_size_prev; + break; + } + bucket_cumul_size_prev = bucket_cumul_size; + ++_idx_bucket_idx; + } + if (position == 0) { + break; + } + // Advance to next unit, adjust position relative to next unit's + // local index space: + position -= unit_bucket_cumul_sizes.back(); + ++_idx_unit_id; + } + DASH_LOG_TRACE("GlobPtr(gmem,idx)", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bucket:", _idx_bucket_idx, + "phase:", _idx_bucket_phase); + } + + /** + * Constructor, creates a global pointer on global memory from unit and + * local offset in logical storage order. + */ + template + GlobPtr( + const MemSpaceT * gmem, + team_unit_t unit, + index_type local_index) + : _globmem(reinterpret_cast(gmem)), + _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), + _lbegin(_globmem->lbegin()), + _idx(0), + _max_idx(gmem->size() - 1), + _myid(gmem->team().myid()), + _idx_unit_id(unit), + _idx_local_idx(0), + _idx_bucket_idx(0), + _idx_bucket_phase(0) + { + DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx)", + "unit:", unit, + "lidx:", local_index); + DASH_ASSERT_LT(unit, _bucket_cumul_sizes->size(), "invalid unit id"); + + for (size_type unit = 0; unit < _idx_unit_id; ++unit) { + auto prec_unit_local_size = (*_bucket_cumul_sizes)[unit].back(); + _idx += prec_unit_local_size; + } + increment(local_index); + DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx) >", + "gidx:", _idx, + "maxidx:", _max_idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bucket:", _idx_bucket_idx, + "phase:", _idx_bucket_phase); + } + + /** + * Copy constructor. + */ + template + GlobPtr( + const GlobPtr & other) + : _globmem(other._globmem), + _bucket_cumul_sizes(other._bucket_cumul_sizes), + _lbegin(other._lbegin), + _idx(other._idx), + _max_idx(other._max_idx), + _idx_unit_id(other._idx_unit_id), + _idx_local_idx(other._idx_local_idx), + _idx_bucket_idx(other._idx_bucket_idx), + _idx_bucket_phase(other._idx_bucket_phase) + { } + + /** + * Assignment operator. + */ + template + self_t & operator=( + const GlobPtr & other) + { + _globmem = other._globmem; + _bucket_cumul_sizes = other._bucket_cumul_sizes; + _lbegin = other._lbegin; + _idx = other._idx; + _max_idx = other._max_idx; + _idx_unit_id = other._idx_unit_id; + _idx_local_idx = other._idx_local_idx; + _idx_bucket_idx = other._idx_bucket_idx; + _idx_bucket_phase = other._idx_bucket_phase; + } + + /** + * Explicit conversion to \c dart_gptr_t. + * + * \return A DART global pointer to the element at the pointer's + * position + */ + dart_gptr_t dart_gptr() const + { + DASH_LOG_TRACE_VAR("GlobPtr.dart_gptr()", _idx); + // Create global pointer from unit, bucket and phase: + dart_gptr_t dart_gptr = _globmem->dart_gptr_at( + _idx_unit_id, + _idx_bucket_idx, + _idx_bucket_phase); + DASH_LOG_TRACE_VAR("GlobPtr.dart_gptr >", dart_gptr); + return dart_gptr; + } + + /** + * Dereference operator. + * + * \return A global reference to the element at the pointer's position. + */ + reference operator*() const + { + auto lptr = local(); + if (lptr != nullptr) { + return reference(static_cast(lptr)); + } else { + return reference(dart_gptr()); + } + } + + /** + * Subscript operator, returns global reference to element at given + * global index. + */ + reference operator[]( + /// The global position of the element + index_type g_index) const + { + DASH_LOG_TRACE_VAR("GlobPtr.[]()", g_index); + auto git = *this; + git += g_index; + auto lptr = git.local(); + if (lptr != nullptr) { + return reference(lptr); + } else { + auto gref = *git; + DASH_LOG_TRACE_VAR("GlobPtr.[] >", gref); + return gref; + } + } + + /** + * Checks whether the element referenced by this global pointer is in + * the calling unit's local memory. + */ + inline bool is_local() const + { + return (_myid == _idx_unit_id); + } + + /** + * Conversion to local bucket pointer. + */ + local_pointer local() const + { + if (_myid != _idx_unit_id) { + // Iterator position does not point to local element + return nullptr; + } + return (_lbegin + _idx_local_idx); + } + + /** + * Unit and local offset at the pointer's position. + */ + inline local_index lpos() const + { + local_index local_pos; + local_pos.unit = _idx_unit_id; + local_pos.index = _idx_local_idx; + return local_pos; + } + + /** + * Map pointer to global index domain. + */ + inline self_t global() const + { + return *this; + } + + /** + * Position of the pointer in global index space. + */ + inline index_type pos() const + { + return _idx; + } + + /** + * Position of the pointer in global index range. + */ + inline index_type gpos() const + { + return _idx; + } + + /** + * The instance of \c GlobStaticMem used by this pointer to resolve + * addresses in global memory. + */ + inline const globmem_type & globmem() const + { + return *_globmem; + } + + /** + * The instance of \c GlobStaticMem used by this pointer to resolve + * addresses in global memory. + */ + inline globmem_type & globmem() + { + return *_globmem; + } + + /** + * Prefix increment operator. + */ + inline self_t & operator++() + { + increment(1); + return *this; + } + + /** + * Prefix decrement operator. + */ + inline self_t & operator--() + { + decrement(1); + return *this; + } + + /** + * Postfix increment operator. + */ + inline self_t operator++(int) + { + auto result = *this; + increment(1); + return result; + } + + /** + * Postfix decrement operator. + */ + inline self_t operator--(int) + { + auto result = *this; + decrement(1); + return result; + } + + inline self_t & operator+=(index_type offset) + { + increment(offset); + return *this; + } + + inline self_t & operator-=(index_type offset) + { + increment(offset); + return *this; + } + + inline self_t operator+(index_type offset) const + { + auto res = *this; + res.increment(offset); + return res; + } + + inline self_t operator-(index_type offset) const + { + auto res = *this; + res.decrement(offset); + return res; + } + + inline index_type operator+( + const self_t & other) const + { + return _idx + other._idx; + } + + inline index_type operator-( + const self_t & other) const + { + return _idx - other._idx; + } + + template + inline bool operator<(const GlobPtr & other) const + { + return (_idx < other._idx); + } + + template + inline bool operator<=(const GlobPtr & other) const + { + return (_idx <= other._idx); + } + + template + inline bool operator>(const GlobPtr & other) const + { + return (_idx > other._idx); + } + + template + inline bool operator>=(const GlobPtr & other) const + { + return (_idx >= other._idx); + } + + template + inline bool operator==(const GlobPtr & other) const + { + return _idx == other._idx; + } + + template + inline bool operator!=(const GlobPtr & other) const + { + return _idx != other._idx; + } + +private: + /** + * Advance pointer by specified position offset. + */ + void increment(int offset) + { + DASH_LOG_TRACE("GlobPtr.increment()", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase, + "offset:", offset); + _idx += offset; + auto current_bucket_size = + (*_bucket_cumul_sizes)[_idx_unit_id][_idx_bucket_idx]; + if (_idx_local_idx + offset < current_bucket_size) { + DASH_LOG_TRACE("GlobPtr.increment", "position current bucket"); + // element is in bucket currently referenced by this pointer: + _idx_bucket_phase += offset; + _idx_local_idx += offset; + } else { + DASH_LOG_TRACE("GlobPtr.increment", + "position in succeeding bucket"); + // iterate units: + auto unit_id_max = _bucket_cumul_sizes->size() - 1; + for (; _idx_unit_id <= unit_id_max; ++_idx_unit_id) { + if (offset == 0) { + break; + } + auto unit_bkt_sizes = (*_bucket_cumul_sizes)[_idx_unit_id]; + auto unit_bkt_sizes_total = unit_bkt_sizes.back(); + auto unit_num_bkts = unit_bkt_sizes.size(); + DASH_LOG_TRACE("GlobPtr.increment", + "unit:", _idx_unit_id, + "remaining offset:", offset, + "total local bucket size:", unit_bkt_sizes_total); + if (_idx_local_idx + offset >= unit_bkt_sizes_total) { + // offset refers to next unit: + DASH_LOG_TRACE("GlobPtr.increment", + "position in remote range"); + // subtract remaining own local size from remaining offset: + offset -= (unit_bkt_sizes_total - _idx_local_idx); + if (_idx_unit_id == unit_id_max) { + // end pointer, offset exceeds iteration space: + _idx_bucket_idx = unit_num_bkts - 1; + auto last_bkt_size = unit_bkt_sizes.back(); + if (unit_num_bkts > 1) { + last_bkt_size -= unit_bkt_sizes[_idx_bucket_idx-1]; + } + _idx_bucket_phase = last_bkt_size + offset; + _idx_local_idx += unit_bkt_sizes_total + offset; + break; + } + _idx_local_idx = 0; + _idx_bucket_idx = 0; + _idx_bucket_phase = 0; + } else { + // offset refers to current unit: + DASH_LOG_TRACE("GlobPtr.increment", + "position in local range", + "current bucket phase:", _idx_bucket_phase, + "cumul. bucket sizes:", unit_bkt_sizes); + _idx_local_idx += offset; + // iterate the unit's bucket sizes: + for (; _idx_bucket_idx < unit_num_bkts; ++_idx_bucket_idx) { + auto cumul_bucket_size = unit_bkt_sizes[_idx_bucket_idx]; + if (_idx_local_idx < cumul_bucket_size) { + auto cumul_prev = _idx_bucket_idx > 0 + ? unit_bkt_sizes[_idx_bucket_idx-1] + : 0; + // offset refers to current bucket: + _idx_bucket_phase = _idx_local_idx - cumul_prev; + offset = 0; + break; + } + } + if (offset == 0) { + break; + } + } + } + } + DASH_LOG_TRACE("GlobPtr.increment >", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase); + } + + /** + * Decrement pointer by specified position offset. + */ + void decrement(int offset) + { + DASH_LOG_TRACE("GlobPtr.decrement()", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase, + "offset:", -offset); + if (offset > _idx) { + DASH_THROW(dash::exception::OutOfRange, + "offset " << offset << " is out of range"); + } + _idx -= offset; + if (offset <= _idx_bucket_phase) { + // element is in bucket currently referenced by this pointer: + _idx_bucket_phase -= offset; + _idx_local_idx -= offset; + } else { + // iterate units: + auto first_unit = _idx_unit_id; + for (; _idx_unit_id >= 0; --_idx_unit_id) { + auto unit_bkt_sizes = (*_bucket_cumul_sizes)[_idx_unit_id]; + auto unit_bkt_sizes_total = unit_bkt_sizes.back(); + auto unit_num_bkts = unit_bkt_sizes.size(); + if (_idx_unit_id != first_unit) { + --offset; + _idx_bucket_idx = unit_num_bkts - 1; + _idx_local_idx = unit_bkt_sizes_total - 1; + auto last_bkt_size = unit_bkt_sizes.back(); + if (unit_num_bkts > 1) { + last_bkt_size -= unit_bkt_sizes[_idx_bucket_idx-1]; + } + _idx_bucket_phase = last_bkt_size - 1; + } + if (offset <= _idx_local_idx) { + // offset refers to current unit: + // iterate the unit's bucket sizes: + for (; _idx_bucket_idx >= 0; --_idx_bucket_idx) { + auto bucket_size = unit_bkt_sizes[_idx_bucket_idx]; + if (offset <= _idx_bucket_phase) { + // offset refers to current bucket: + _idx_local_idx -= offset; + _idx_bucket_phase -= offset; + offset = 0; + break; + } else { + // offset refers to preceeding bucket: + _idx_local_idx -= (_idx_bucket_phase + 1); + offset -= (_idx_bucket_phase + 1); + _idx_bucket_phase = unit_bkt_sizes[_idx_bucket_idx-1] - 1; + } + } + } else { + // offset refers to preceeding unit: + offset -= _idx_local_idx; + } + if (offset == 0) { + break; + } + } + } + DASH_LOG_TRACE("GlobPtr.decrement >", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase); + } + +}; // class GlobPtr + +/** + * Resolve the number of elements between two global bucket pointers. + * + * \complexity O(1) + * + * \ingroup Algorithms + */ +template< + typename ElementType, + class ContainerType > +auto distance( + /// Global pointer to the first position in the global sequence + const dash::GlobPtr< + ElementType, GlobDynamicContiguousMem + > & first, + /// Global pointer to the final position in the global sequence + const dash::GlobPtr< + ElementType, GlobDynamicContiguousMem + > & last) +-> typename GlobDynamicContiguousMem::index_type +{ + return last - first; +} + +template< + typename ElementType, + class ContainerType > +std::ostream & operator<<( + std::ostream & os, + const dash::GlobPtr< + ElementType, + GlobDynamicContiguousMem + > & gptr) +{ + std::ostringstream ss; + ss << "dash::GlobPtr<" << typeid(ElementType).name() << ">(" + << "gidx:" << gptr._idx << ", (" + << "unit:" << gptr._idx_unit_id << ", " + << "lidx:" << gptr._idx_local_idx << "), (" + << "bidx:" << gptr._idx_bucket_idx << ", " + << "bphase:" << gptr._idx_bucket_phase << ")" + << ")"; + return operator<<(os, ss.str()); +} + +} // namespace dash + +#endif // DASH__MEMORY__GLOB_HEAP_CONTIGUOUS_PTR_H__INCLUDED From 3723ca8264a2937054e36eea96c1f85466a9bc96 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 30 Mar 2017 20:50:12 +0200 Subject: [PATCH 031/102] removed GlobGraphIter --- dash/include/dash/graph/GlobGraphIter.h | 732 ------------------------ 1 file changed, 732 deletions(-) delete mode 100644 dash/include/dash/graph/GlobGraphIter.h diff --git a/dash/include/dash/graph/GlobGraphIter.h b/dash/include/dash/graph/GlobGraphIter.h deleted file mode 100644 index 7619d9915..000000000 --- a/dash/include/dash/graph/GlobGraphIter.h +++ /dev/null @@ -1,732 +0,0 @@ -#ifndef DASH__GRAPH__GLOB_GRAPH_ITER_H__INCLUDED -#define DASH__GRAPH__GLOB_GRAPH_ITER_H__INCLUDED - -#include - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include - - -namespace dash { - -// Forward-declaration -template< - typename ElementType, - class AllocatorType > -class GlobDynamicMem; - -/** - * Iterator on global buckets. Represents global pointer type. - */ -template< - typename ElementType, - class GlobMemType, - class PointerType = dash::GlobPtr, - class ReferenceType = dash::GlobSharedRef > -class GlobGraphIter -: public std::iterator< - std::random_access_iterator_tag, - ElementType, - typename GlobMemType::index_type, - PointerType, - ReferenceType > -{ - template< - typename ElementType_, - class GlobMemType_, - class Pointer_, - class Reference_> - friend std::ostream & dash::operator<<( - std::ostream & os, - const dash::GlobGraphIter< - ElementType_, GlobMemType_, Pointer_, Reference_> & it); - -private: - typedef GlobGraphIter< - ElementType, - GlobMemType, - PointerType, - ReferenceType> - self_t; - -public: - typedef typename GlobMemType::index_type index_type; - typedef typename std::make_unsigned::type size_type; - - typedef ElementType value_type; - typedef ReferenceType reference; - typedef ReferenceType const_reference; - typedef PointerType pointer; - typedef PointerType const_pointer; - - typedef ElementType * raw_pointer; - - typedef typename - std::conditional< - std::is_const::value, - const GlobMemType, - GlobMemType - >::type - globmem_type; - - typedef typename - std::conditional< - std::is_const::value, - typename GlobMemType::const_local_pointer, - typename GlobMemType::local_pointer - >::type - local_pointer; - - typedef struct { - team_unit_t unit; - index_type index; - } local_index; - -private: - typedef std::vector > - bucket_cumul_sizes_map; - -public: - /** - * Default constructor. - */ - GlobGraphIter() - : _globmem(nullptr), - _bucket_cumul_sizes(nullptr), - _idx(0), - _max_idx(0), - _myid(dash::Team::GlobalUnitID()), - _idx_unit_id(DART_UNDEFINED_UNIT_ID), - _idx_local_idx(-1), - _idx_bucket_idx(-1), - _idx_bucket_phase(-1) - { - DASH_LOG_TRACE_VAR("GlobGraphIter()", _idx); - DASH_LOG_TRACE_VAR("GlobGraphIter()", _max_idx); - } - - /** - * Constructor, creates a global iterator on global memory from global - * offset in logical storage order. - */ - GlobGraphIter( - globmem_type * gmem, - index_type position = 0) - : _globmem(gmem), - _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), - _lbegin(_globmem->lbegin()), - _idx(position), - _max_idx(gmem->size() - 1), - _myid(gmem->team().myid()), - _idx_unit_id(0), - _idx_local_idx(0), - _idx_bucket_idx(0), - _idx_bucket_phase(0) - { - DASH_LOG_TRACE("GlobGraphIter(gmem,idx)", "gidx:", position); - for (auto unit_bucket_cumul_sizes : *_bucket_cumul_sizes) { - DASH_LOG_TRACE_VAR("GlobGraphIter(gmem,idx)", - unit_bucket_cumul_sizes); - size_type bucket_cumul_size_prev = 0; - for (auto bucket_cumul_size : unit_bucket_cumul_sizes) { - DASH_LOG_TRACE_VAR("GlobGraphIter(gmem,idx)", bucket_cumul_size); - if (position < bucket_cumul_size) { - DASH_LOG_TRACE_VAR("GlobGraphIter(gmem,idx)", position); - _idx_bucket_phase -= bucket_cumul_size; - position = 0; - _idx_local_idx = position; - _idx_bucket_phase = position - bucket_cumul_size_prev; - break; - } - bucket_cumul_size_prev = bucket_cumul_size; - ++_idx_bucket_idx; - } - if (position == 0) { - break; - } - // Advance to next unit, adjust position relative to next unit's - // local index space: - position -= unit_bucket_cumul_sizes.back(); - ++_idx_unit_id; - } - DASH_LOG_TRACE("GlobGraphIter(gmem,idx)", - "gidx:", _idx, - "unit:", _idx_unit_id, - "lidx:", _idx_local_idx, - "bucket:", _idx_bucket_idx, - "phase:", _idx_bucket_phase); - } - - /** - * Constructor, creates a global iterator on global memory from unit and - * local offset in logical storage order. - */ - GlobGraphIter( - globmem_type * gmem, - team_unit_t unit, - index_type local_index) - : _globmem(gmem), - _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), - _lbegin(_globmem->lbegin()), - _idx(0), - _max_idx(gmem->size() - 1), - _myid(gmem->team().myid()), - _idx_unit_id(unit), - _idx_local_idx(0), - _idx_bucket_idx(0), - _idx_bucket_phase(0) - { - DASH_LOG_TRACE("GlobGraphIter(gmem,unit,lidx)", - "unit:", unit, - "lidx:", local_index); - DASH_ASSERT_LT(unit, _bucket_cumul_sizes->size(), "invalid unit id"); - - for (size_type unit = 0; unit < _idx_unit_id; ++unit) { - auto prec_unit_local_size = (*_bucket_cumul_sizes)[unit].back(); - _idx += prec_unit_local_size; - } - increment(local_index); - DASH_LOG_TRACE("GlobGraphIter(gmem,unit,lidx) >", - "gidx:", _idx, - "maxidx:", _max_idx, - "unit:", _idx_unit_id, - "lidx:", _idx_local_idx, - "bucket:", _idx_bucket_idx, - "phase:", _idx_bucket_phase); - } - - /** - * Copy constructor. - */ - template - GlobGraphIter( - const GlobGraphIter & other) - : _globmem(other._globmem), - _bucket_cumul_sizes(other._bucket_cumul_sizes), - _lbegin(other._lbegin), - _idx(other._idx), - _max_idx(other._max_idx), - _idx_unit_id(other._idx_unit_id), - _idx_local_idx(other._idx_local_idx), - _idx_bucket_idx(other._idx_bucket_idx), - _idx_bucket_phase(other._idx_bucket_phase) - { } - - /** - * Assignment operator. - */ - template - self_t & operator=( - const GlobGraphIter & other) - { - _globmem = other._globmem; - _bucket_cumul_sizes = other._bucket_cumul_sizes; - _lbegin = other._lbegin; - _idx = other._idx; - _max_idx = other._max_idx; - _idx_unit_id = other._idx_unit_id; - _idx_local_idx = other._idx_local_idx; - _idx_bucket_idx = other._idx_bucket_idx; - _idx_bucket_phase = other._idx_bucket_phase; - } - - /** - * Type conversion operator to \c GlobPtr. - * - * \return A global reference to the element at the iterator's position - */ - operator pointer() const - { - return pointer(dart_gptr()); - } - - /** - * Explicit conversion to \c dart_gptr_t. - * - * \return A DART global pointer to the element at the iterator's - * position - */ - dart_gptr_t dart_gptr() const - { - DASH_LOG_TRACE_VAR("GlobGraphIter.dart_gptr()", _idx); - // Create global pointer from unit, bucket and phase: - dart_gptr_t dart_gptr = _globmem->dart_gptr_at( - _idx_unit_id, - _idx_bucket_idx, - _idx_bucket_phase); - DASH_LOG_TRACE_VAR("GlobGraphIter.dart_gptr >", dart_gptr); - return dart_gptr; - } - - /** - * Dereference operator. - * - * \return A global reference to the element at the iterator's position. - */ - reference operator*() const - { - auto lptr = local(); - if (lptr != nullptr) { - return reference(static_cast(lptr)); - } else { - return reference(dart_gptr()); - } - } - - /** - * Subscript operator, returns global reference to element at given - * global index. - */ - reference operator[]( - /// The global position of the element - index_type g_index) const - { - DASH_LOG_TRACE_VAR("GlobGraphIter.[]()", g_index); - auto git = *this; - git += g_index; - auto lptr = git.local(); - if (lptr != nullptr) { - return reference(lptr); - } else { - auto gref = *git; - DASH_LOG_TRACE_VAR("GlobGraphIter.[] >", gref); - return gref; - } - } - - /** - * Checks whether the element referenced by this global iterator is in - * the calling unit's local memory. - */ - inline bool is_local() const - { - return (_myid == _idx_unit_id); - } - - /** - * Conversion to local bucket iterator. - */ - local_pointer local() const - { - if (_myid != _idx_unit_id) { - // Iterator position does not point to local element - return nullptr; - } - return (_lbegin + _idx_local_idx); - } - - /** - * Unit and local offset at the iterator's position. - */ - inline local_index lpos() const - { - local_index local_pos; - local_pos.unit = _idx_unit_id; - local_pos.index = _idx_local_idx; - return local_pos; - } - - /** - * Map iterator to global index domain. - */ - inline self_t global() const - { - return *this; - } - - /** - * Position of the iterator in global index space. - */ - inline index_type pos() const - { - return _idx; - } - - /** - * Position of the iterator in global index range. - */ - inline index_type gpos() const - { - return _idx; - } - - /** - * The instance of \c GlobMem used by this iterator to resolve addresses - * in global memory. - */ - inline const globmem_type & globmem() const - { - return *_globmem; - } - - /** - * The instance of \c GlobMem used by this iterator to resolve addresses - * in global memory. - */ - inline globmem_type & globmem() - { - return *_globmem; - } - - /** - * Prefix increment operator. - */ - inline self_t & operator++() - { - increment(1); - return *this; - } - - /** - * Prefix decrement operator. - */ - inline self_t & operator--() - { - decrement(1); - return *this; - } - - /** - * Postfix increment operator. - */ - inline self_t operator++(int) - { - auto result = *this; - increment(1); - return result; - } - - /** - * Postfix decrement operator. - */ - inline self_t operator--(int) - { - auto result = *this; - decrement(1); - return result; - } - - inline self_t & operator+=(index_type offset) - { - increment(offset); - return *this; - } - - inline self_t & operator-=(index_type offset) - { - increment(offset); - return *this; - } - - inline self_t operator+(index_type offset) const - { - auto res = *this; - res.increment(offset); - return res; - } - - inline self_t operator-(index_type offset) const - { - auto res = *this; - res.decrement(offset); - return res; - } - - inline index_type operator+( - const self_t & other) const - { - return _idx + other._idx; - } - - inline index_type operator-( - const self_t & other) const - { - return _idx - other._idx; - } - - template - inline bool operator<(const GlobGraphIter & other) const - { - return (_idx < other._idx); - } - - template - inline bool operator<=(const GlobGraphIter & other) const - { - return (_idx <= other._idx); - } - - template - inline bool operator>(const GlobGraphIter & other) const - { - return (_idx > other._idx); - } - - template - inline bool operator>=(const GlobGraphIter & other) const - { - return (_idx >= other._idx); - } - - template - inline bool operator==(const GlobGraphIter & other) const - { - return _idx == other._idx; - } - - template - inline bool operator!=(const GlobGraphIter & other) const - { - return _idx != other._idx; - } - -private: - /** - * Advance pointer by specified position offset. - */ - void increment(int offset) - { - DASH_LOG_TRACE("GlobGraphIter.increment()", - "gidx:", _idx, - "unit:", _idx_unit_id, - "lidx:", _idx_local_idx, - "bidx:", _idx_bucket_idx, - "bphase:", _idx_bucket_phase, - "offset:", offset); - _idx += offset; - auto current_bucket_size = - (*_bucket_cumul_sizes)[_idx_unit_id][_idx_bucket_idx]; - if (_idx_local_idx + offset < current_bucket_size) { - DASH_LOG_TRACE("GlobGraphIter.increment", "position current bucket"); - // element is in bucket currently referenced by this iterator: - _idx_bucket_phase += offset; - _idx_local_idx += offset; - } else { - DASH_LOG_TRACE("GlobGraphIter.increment", - "position in succeeding bucket"); - // iterate units: - auto unit_id_max = _bucket_cumul_sizes->size() - 1; - for (; _idx_unit_id <= unit_id_max; ++_idx_unit_id) { - if (offset == 0) { - break; - } - auto unit_bkt_sizes = (*_bucket_cumul_sizes)[_idx_unit_id]; - auto unit_bkt_sizes_total = unit_bkt_sizes.back(); - auto unit_num_bkts = unit_bkt_sizes.size(); - DASH_LOG_TRACE("GlobGraphIter.increment", - "unit:", _idx_unit_id, - "remaining offset:", offset, - "total local bucket size:", unit_bkt_sizes_total); - if (_idx_local_idx + offset >= unit_bkt_sizes_total) { - // offset refers to next unit: - DASH_LOG_TRACE("GlobGraphIter.increment", - "position in remote range"); - // subtract remaining own local size from remaining offset: - offset -= (unit_bkt_sizes_total - _idx_local_idx); - if (_idx_unit_id == unit_id_max) { - // end iterator, offset exceeds iteration space: - _idx_bucket_idx = unit_num_bkts - 1; - auto last_bkt_size = unit_bkt_sizes.back(); - if (unit_num_bkts > 1) { - last_bkt_size -= unit_bkt_sizes[_idx_bucket_idx-1]; - } - _idx_bucket_phase = last_bkt_size + offset; - _idx_local_idx += unit_bkt_sizes_total + offset; - break; - } - _idx_local_idx = 0; - _idx_bucket_idx = 0; - _idx_bucket_phase = 0; - } else { - // offset refers to current unit: - DASH_LOG_TRACE("GlobGraphIter.increment", - "position in local range", - "current bucket phase:", _idx_bucket_phase, - "cumul. bucket sizes:", unit_bkt_sizes); - _idx_local_idx += offset; - // iterate the unit's bucket sizes: - for (; _idx_bucket_idx < unit_num_bkts; ++_idx_bucket_idx) { - auto cumul_bucket_size = unit_bkt_sizes[_idx_bucket_idx]; - if (_idx_local_idx < cumul_bucket_size) { - auto cumul_prev = _idx_bucket_idx > 0 - ? unit_bkt_sizes[_idx_bucket_idx-1] - : 0; - // offset refers to current bucket: - _idx_bucket_phase = _idx_local_idx - cumul_prev; - offset = 0; - break; - } - } - if (offset == 0) { - break; - } - } - } - } - DASH_LOG_TRACE("GlobGraphIter.increment >", - "gidx:", _idx, - "unit:", _idx_unit_id, - "lidx:", _idx_local_idx, - "bidx:", _idx_bucket_idx, - "bphase:", _idx_bucket_phase); - } - - /** - * Decrement pointer by specified position offset. - */ - void decrement(int offset) - { - DASH_LOG_TRACE("GlobGraphIter.decrement()", - "gidx:", _idx, - "unit:", _idx_unit_id, - "lidx:", _idx_local_idx, - "bidx:", _idx_bucket_idx, - "bphase:", _idx_bucket_phase, - "offset:", -offset); - if (offset > _idx) { - DASH_THROW(dash::exception::OutOfRange, - "offset " << offset << " is out of range"); - } - _idx -= offset; - if (offset <= _idx_bucket_phase) { - // element is in bucket currently referenced by this iterator: - _idx_bucket_phase -= offset; - _idx_local_idx -= offset; - } else { - // iterate units: - auto first_unit = _idx_unit_id; - for (; _idx_unit_id >= 0; --_idx_unit_id) { - auto unit_bkt_sizes = (*_bucket_cumul_sizes)[_idx_unit_id]; - auto unit_bkt_sizes_total = unit_bkt_sizes.back(); - auto unit_num_bkts = unit_bkt_sizes.size(); - if (_idx_unit_id != first_unit) { - --offset; - _idx_bucket_idx = unit_num_bkts - 1; - _idx_local_idx = unit_bkt_sizes_total - 1; - auto last_bkt_size = unit_bkt_sizes.back(); - if (unit_num_bkts > 1) { - last_bkt_size -= unit_bkt_sizes[_idx_bucket_idx-1]; - } - _idx_bucket_phase = last_bkt_size - 1; - } - if (offset <= _idx_local_idx) { - // offset refers to current unit: - // iterate the unit's bucket sizes: - for (; _idx_bucket_idx >= 0; --_idx_bucket_idx) { - auto bucket_size = unit_bkt_sizes[_idx_bucket_idx]; - if (offset <= _idx_bucket_phase) { - // offset refers to current bucket: - _idx_local_idx -= offset; - _idx_bucket_phase -= offset; - offset = 0; - break; - } else { - // offset refers to preceeding bucket: - _idx_local_idx -= (_idx_bucket_phase + 1); - offset -= (_idx_bucket_phase + 1); - _idx_bucket_phase = unit_bkt_sizes[_idx_bucket_idx-1] - 1; - } - } - } else { - // offset refers to preceeding unit: - offset -= _idx_local_idx; - } - if (offset == 0) { - break; - } - } - } - DASH_LOG_TRACE("GlobGraphIter.decrement >", - "gidx:", _idx, - "unit:", _idx_unit_id, - "lidx:", _idx_local_idx, - "bidx:", _idx_bucket_idx, - "bphase:", _idx_bucket_phase); - } - -private: - /// Global memory used to dereference iterated values. - globmem_type * _globmem = nullptr; - /// Mapping unit id to buckets in the unit's attached local storage. - const bucket_cumul_sizes_map * _bucket_cumul_sizes = nullptr; - /// Pointer to first element in local data space. - local_pointer _lbegin; - /// Current position of the iterator in global canonical index space. - index_type _idx = 0; - /// Maximum position allowed for this iterator. - index_type _max_idx = 0; - /// Unit id of the active unit. - team_unit_t _myid; - /// Unit id at the iterator's current position. - team_unit_t _idx_unit_id; - /// Logical offset in local index space at the iterator's current position. - index_type _idx_local_idx = -1; - /// Local bucket index at the iterator's current position. - index_type _idx_bucket_idx = -1; - /// Element offset in bucket at the iterator's current position. - index_type _idx_bucket_phase = -1; - -}; // class GlobGraphIter - -/** - * Resolve the number of elements between two global bucket iterators. - * - * \complexity O(1) - * - * \ingroup Algorithms - */ -template< - typename ElementType, - class GlobMemType, - class Pointer, - class Reference> -auto distance( - /// Global iterator to the first position in the global sequence - const dash::GlobGraphIter< - ElementType, GlobMemType, Pointer, Reference> & first, - /// Global iterator to the final position in the global sequence - const dash::GlobGraphIter< - ElementType, GlobMemType, Pointer, Reference> & last) --> typename GlobMemType::index_type -{ - return last - first; -} - -template< - typename ElementType, - class GlobMemType, - class Pointer, - class Reference> -std::ostream & operator<<( - std::ostream & os, - const dash::GlobGraphIter< - ElementType, GlobMemType, Pointer, Reference> & it) -{ - std::ostringstream ss; - ss << "dash::GlobGraphIter<" - << typeid(ElementType).name() << ">(" - << "gidx:" << it._idx << ", (" - << "unit:" << it._idx_unit_id << ", " - << "lidx:" << it._idx_local_idx << "), (" - << "bidx:" << it._idx_bucket_idx << ", " - << "bphase:" << it._idx_bucket_phase << ")" - << ")"; - return operator<<(os, ss.str()); -} - -} // namespace dash - -#endif // DASH__GRAPH__GLOB_GRAPH_ITER_H__INCLUDED From a15872b4ea6d53eb84707c5e5ebea385733f9032 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Fri, 31 Mar 2017 12:21:58 +0200 Subject: [PATCH 032/102] simplified global iteration --- dash/include/dash/GlobDynamicContiguousMem.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dash/include/dash/GlobDynamicContiguousMem.h b/dash/include/dash/GlobDynamicContiguousMem.h index 640f31690..c6c5ac7dc 100644 --- a/dash/include/dash/GlobDynamicContiguousMem.h +++ b/dash/include/dash/GlobDynamicContiguousMem.h @@ -148,6 +148,7 @@ class GlobDynamicContiguousMem typedef typename ContainerType::size_type size_type; typedef typename local_iterator::bucket_type bucket_type; typedef typename std::list bucket_list; + typedef typename std::list bucket_ptr_list; typedef local_iterator local_pointer; typedef local_iterator const_local_pointer; typedef std::vector> bucket_cumul_sizes_map; @@ -214,7 +215,10 @@ class GlobDynamicContiguousMem int bucket_num = 0; int bucket_cumul = 0; + // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + // Important for performance: // TODO: put multiple containers into one bucket + // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! for(auto c_data : _container_list) { // merge public and local containers c_data.container->insert(c_data.container->end(), @@ -252,6 +256,8 @@ class GlobDynamicContiguousMem c_data.container_bucket->gptr = gptr; c_data.unattached_container_bucket->gptr = gptr; + //TODO: If possible, avoid adding unattached_container to global + // iteration space with size 0 // update cumulated bucket sizes bucket_cumul += c_data.container->size(); _bucket_cumul_sizes[_myid][bucket_num] = bucket_cumul; @@ -262,7 +268,7 @@ class GlobDynamicContiguousMem // distribute bucket sizes between all units // TODO: use one allgather for all buckets - // TODO: make it work for unevenls distributed amount of buckets + // TODO: make it work for unevenly distributed amount of buckets auto bucket_count = _bucket_cumul_sizes[_myid].size(); for(auto c_data : _container_list) { std::vector bucket_sizes(bucket_count * _team->size()); @@ -472,6 +478,7 @@ class GlobDynamicContiguousMem container_type * _container; container_type * _unattached_container = nullptr; bucket_list _buckets; + bucket_ptr_list _global_buckets; Team * _team; dart_team_t _teamid; size_type _nunits = 0; From bdfca656944c9be4758ba1f9b0dc03bdc55f179a Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Fri, 31 Mar 2017 18:43:31 +0200 Subject: [PATCH 033/102] added support for allocating different amounts of containers on different units --- dash/include/dash/GlobDynamicContiguousMem.h | 103 ++++++++++++------- 1 file changed, 64 insertions(+), 39 deletions(-) diff --git a/dash/include/dash/GlobDynamicContiguousMem.h b/dash/include/dash/GlobDynamicContiguousMem.h index c6c5ac7dc..d54cdd398 100644 --- a/dash/include/dash/GlobDynamicContiguousMem.h +++ b/dash/include/dash/GlobDynamicContiguousMem.h @@ -162,6 +162,7 @@ class GlobDynamicContiguousMem */ GlobDynamicContiguousMem(Team & team = dash::Team::All()) : _buckets(), + _global_buckets(), _team(&team), _teamid(team.dart_id()), _nunits(team.size()), @@ -178,6 +179,8 @@ class GlobDynamicContiguousMem auto it = _buckets.insert(_buckets.end(), c_data.buckets.begin(), c_data.buckets.end()); c_data.container_bucket = &(*it); + // for global iteration, only _container's bucket is needed + _global_buckets.push_back(&(*it)); ++it; c_data.unattached_container_bucket = &(*it); // insert won't invalidate iterators for std::list, so we can use them @@ -212,59 +215,81 @@ class GlobDynamicContiguousMem self_t & operator=(const self_t & rhs) = default; void commit() { + // Gather information about the max amount of containers a single unit + // currently holds + std::vector container_count(_team->size()); + int my_container_count = _container_list.size(); + dart_allgather(&my_container_count, container_count.data(), + sizeof(size_type), DART_TYPE_BYTE, _team->dart_id()); + auto max_containers = std::max_element(container_count.begin(), + container_count.end()); + int bucket_num = 0; int bucket_cumul = 0; - // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! // Important for performance: // TODO: put multiple containers into one bucket - // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - for(auto c_data : _container_list) { - // merge public and local containers - c_data.container->insert(c_data.container->end(), - c_data.unattached_container->begin(), - c_data.unattached_container->end()); - c_data.unattached_container->clear(); - // update memory location & size of _container - auto it = c_data.buckets.begin(); - it->lptr = c_data.container->data(); - it->size = c_data.container->size(); - c_data.container_bucket->lptr = c_data.container->data(); - c_data.container_bucket->size = c_data.container->size(); - // update memory location & size of _unattached_container - ++it; - it->lptr = c_data.unattached_container->data(); - it->size = 0; - c_data.unattached_container_bucket->lptr - = c_data.unattached_container->data(); - c_data.unattached_container_bucket->size = 0; - - c_data.update_lbegin(); - c_data.update_lend(); + // TODO: update only containers with unattached_container.size() > 0 + // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + // attach buckets for the maximum amount of containers, even if this unit + // holds less containers + { + data_type * c_data; + auto c_data_it = _container_list.begin(); + auto current_size = _container_list.size(); + for(int i = 0; i < *max_containers; ++i) { + if(i < current_size) { + c_data = &(*c_data_it); + // merge public and local containers + c_data->container->insert(c_data->container->end(), + c_data->unattached_container->begin(), + c_data->unattached_container->end()); + c_data->unattached_container->clear(); + // update memory location & size of _container + auto it = c_data->buckets.begin(); + it->lptr = c_data->container->data(); + it->size = c_data->container->size(); + c_data->container_bucket->lptr = c_data->container->data(); + c_data->container_bucket->size = c_data->container->size(); + // update memory location & size of _unattached_container + ++it; + it->lptr = c_data->unattached_container->data(); + it->size = 0; + c_data->unattached_container_bucket->lptr + = c_data->unattached_container->data(); + c_data->unattached_container_bucket->size = 0; + + c_data->update_lbegin(); + c_data->update_lend(); + ++c_data_it; + } else { + // if other units add more containers, create an empty container to + // store a dart_gptr for the collective allocation + c_data = &(*(add_container(0))); + } // attach new container to global memory space dart_gptr_t gptr = DART_GPTR_NULL; - dart_storage_t ds = dart_storage(c_data.container->size()); + dart_storage_t ds = dart_storage(c_data->container->size()); dart_team_memregister( _team->dart_id(), ds.nelem, ds.dtype, - c_data.container->data(), + c_data->container->data(), &gptr ); // no need to update gptr of local bucket list in c_data - c_data.container_bucket->gptr = gptr; - c_data.unattached_container_bucket->gptr = gptr; + c_data->container_bucket->gptr = gptr; //TODO: If possible, avoid adding unattached_container to global // iteration space with size 0 // update cumulated bucket sizes - bucket_cumul += c_data.container->size(); - _bucket_cumul_sizes[_myid][bucket_num] = bucket_cumul; - ++bucket_num; + bucket_cumul += c_data->container->size(); _bucket_cumul_sizes[_myid][bucket_num] = bucket_cumul; ++bucket_num; } + } // distribute bucket sizes between all units // TODO: use one allgather for all buckets @@ -374,15 +399,15 @@ class GlobDynamicContiguousMem DASH_THROW(dash::exception::RuntimeError, "No units in team"); } // Get the referenced bucket's dart_gptr: - auto bucket_it = _buckets.begin(); + auto bucket_it = _global_buckets.begin(); std::advance(bucket_it, bucket_index); - auto dart_gptr = bucket_it->gptr; - DASH_LOG_TRACE_VAR("GlobDynamicMem.dart_gptr_at", bucket_it->attached); - DASH_LOG_TRACE_VAR("GlobDynamicMem.dart_gptr_at", bucket_it->gptr); + auto dart_gptr = (*bucket_it)->gptr; + DASH_LOG_TRACE_VAR("GlobDynamicMem.dart_gptr_at", (*bucket_it)->attached); + DASH_LOG_TRACE_VAR("GlobDynamicMem.dart_gptr_at", (*bucket_it)->gptr); if (unit == _myid) { - DASH_LOG_TRACE_VAR("GlobDynamicMem.dart_gptr_at", bucket_it->lptr); - DASH_LOG_TRACE_VAR("GlobDynamicMem.dart_gptr_at", bucket_it->size); - DASH_ASSERT_LT(bucket_phase, bucket_it->size, + DASH_LOG_TRACE_VAR("GlobDynamicMem.dart_gptr_at", (*bucket_it)->lptr); + DASH_LOG_TRACE_VAR("GlobDynamicMem.dart_gptr_at", (*bucket_it)->size); + DASH_ASSERT_LT(bucket_phase, (*bucket_it)->size, "bucket phase out of bounds"); } if (DART_GPTR_ISNULL(dart_gptr)) { @@ -468,7 +493,7 @@ class GlobDynamicContiguousMem for(auto it = _bucket_cumul_sizes.begin(); it != _bucket_cumul_sizes.end(); ++it) { // gets initiliazed with 0 automatically - it->resize(it->size() + 2); + it->resize(it->size() + 1); } } From c90483518e499c830458ba523deb054a6ab0efab Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 3 Apr 2017 12:40:03 +0200 Subject: [PATCH 034/102] changed edge index --- dash/include/dash/GlobDynamicContiguousMem.h | 26 ++++++++++++++------ dash/include/dash/Graph.h | 24 ++++++++++++------ dash/include/dash/graph/internal/Graph.h | 4 ++- 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/dash/include/dash/GlobDynamicContiguousMem.h b/dash/include/dash/GlobDynamicContiguousMem.h index d54cdd398..6e4ac5258 100644 --- a/dash/include/dash/GlobDynamicContiguousMem.h +++ b/dash/include/dash/GlobDynamicContiguousMem.h @@ -140,7 +140,8 @@ class GlobDynamicContiguousMem typedef ContainerType container_type; typedef internal::container_data data_type; typedef std::list container_list_type; - typedef typename container_list_type::iterator container_list_iter; + typedef typename + container_list_type::difference_type container_list_index; typedef typename ContainerType::value_type value_type; typedef typename ContainerType::difference_type index_type; typedef GlobHeapLocalPtr local_iterator; @@ -170,7 +171,7 @@ class GlobDynamicContiguousMem _bucket_cumul_sizes(team.size()) { } - container_list_iter add_container(size_type n_elements) { + container_list_index add_container(size_type n_elements) { increment_bucket_sizes(); auto bucket_index = _bucket_cumul_sizes[_myid].size(); auto c_data = data_type(n_elements, bucket_index); @@ -185,11 +186,14 @@ class GlobDynamicContiguousMem c_data.unattached_container_bucket = &(*it); // insert won't invalidate iterators for std::list, so we can use them // to access the container - return _container_list.insert(_container_list.end(), c_data); + _container_list.push_back(c_data); + return _container_list.size() - 1; } - value_type & get(container_list_iter cont, index_type pos) { - auto c_data = *cont; + value_type & get(container_list_index cont, index_type pos) { + auto it = _container_list.begin(); + std::advance(it, cont); + auto c_data = *it; if(c_data.container->size() > pos) { return c_data.container->operator[](pos); } @@ -266,7 +270,10 @@ class GlobDynamicContiguousMem } else { // if other units add more containers, create an empty container to // store a dart_gptr for the collective allocation - c_data = &(*(add_container(0))); + auto cont_it = _container_list.begin(); + auto cont_index = add_container(0); + std::advance(cont_it, cont_index); + c_data = &(*cont_it); } // attach new container to global memory space @@ -344,8 +351,10 @@ class GlobDynamicContiguousMem return _lend; } - void push_back(container_list_iter cont, value_type val) { - auto c_data = *cont; + size_type push_back(container_list_index cont, value_type val) { + auto cont_it = _container_list.begin(); + std::advance(cont_it, cont); + auto c_data = *cont_it; // bucket of _container auto it = c_data.buckets.begin(); // use _unattached container, if _container is full @@ -370,6 +379,7 @@ class GlobDynamicContiguousMem c_data.update_lend(); update_lbegin(); update_lend(); + return static_cast(it->size); } diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index a37932fcc..2067515b4 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -89,9 +89,9 @@ class Graph { public: typedef typename - glob_mem_vert_type::container_list_iter vertex_cont_ref_type; + glob_mem_vert_type::container_list_index vertex_cont_ref_type; typedef typename - glob_mem_edge_type::container_list_iter edge_cont_ref_type; + glob_mem_edge_type::container_list_index edge_cont_ref_type; typedef VertexIndexType vertex_offset_type; typedef EdgeIndexType edge_offset_type; typedef internal::VertexIndex vertex_index_type; @@ -225,12 +225,16 @@ class Graph { std::pair add_edge(const vertex_index_type & v1, const vertex_index_type & v2, const EdgeProperties & prop = EdgeProperties()) { + edge_index_type local_index(_myid, 0, -1); //TODO: Handle errors, if vertices do not exist auto edge1 = edge_type(0, v1, v2, prop); if(v1.unit == _myid) { auto vertex1 = _glob_mem_vertex->get(_vertex_container_ref, v1.offset); edge1._local_id = ++_local_edge_max_index; - _glob_mem_edge->push_back(vertex1._edge_ref, edge1); + auto local_offset = _glob_mem_edge->push_back(vertex1._edge_ref, edge1); + local_index.unit = v1.unit; + local_index.container = vertex1._edge_ref; + local_index.offset = local_offset; } else { //TODO: check, if unit ID is valid _remote_edges[v1.unit].push_back(edge1); @@ -239,7 +243,12 @@ class Graph { if(v2.unit == _myid) { auto vertex2 = _glob_mem_vertex->get(_vertex_container_ref, v2.offset); edge2._local_id = ++_local_edge_max_index; - _glob_mem_edge->push_back(vertex2._edge_ref, edge2); + auto local_offset = _glob_mem_edge->push_back(vertex2._edge_ref, edge2); + if(local_index.offset == -1) { + local_index.unit = v2.unit; + local_index.container = vertex2._edge_ref; + local_index.offset = local_offset; + } } else { _remote_edges[v2.unit].push_back(edge2); } @@ -247,10 +256,9 @@ class Graph { //TODO: if feasible, use pointer to prop, so it gets saved only once //TODO: find out, how to save edges for different graph types // (directed, undirected) - return std::make_pair( - edge_index_type(_myid, _local_edge_max_index), - true - ); + //TODO: Find a way to return index, if both vertices reside on different + // unit. + return std::make_pair(local_index, true); } /** * Removes an edge between two given vertices. diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index 6d28fa259..f97491c2f 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -31,12 +31,14 @@ struct EdgeIndex { EdgeIndex() = default; - EdgeIndex(team_unit_t u, IndexType o) + EdgeIndex(team_unit_t u, IndexType c, IndexType o) : unit(u), + container(c), offset(o) { } team_unit_t unit; + IndexType container; IndexType offset; }; From 5182926056bc8f29fe84f35a456ecbdcf7a133e0 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 3 Apr 2017 16:54:50 +0200 Subject: [PATCH 035/102] added in- and out-edge iteration functionality --- dash/include/dash/GlobDynamicContiguousMem.h | 119 ++++++++-------- dash/include/dash/Graph.h | 141 ++++++++++++------- dash/include/dash/graph/EdgeIterator.h | 24 ++-- dash/include/dash/graph/InEdgeIterator.h | 102 ++++++++++++++ dash/include/dash/graph/OutEdgeIterator.h | 102 ++++++++++++++ dash/include/dash/graph/internal/Graph.h | 23 +-- 6 files changed, 381 insertions(+), 130 deletions(-) create mode 100644 dash/include/dash/graph/InEdgeIterator.h create mode 100644 dash/include/dash/graph/OutEdgeIterator.h diff --git a/dash/include/dash/GlobDynamicContiguousMem.h b/dash/include/dash/GlobDynamicContiguousMem.h index 6e4ac5258..2864da61e 100644 --- a/dash/include/dash/GlobDynamicContiguousMem.h +++ b/dash/include/dash/GlobDynamicContiguousMem.h @@ -239,63 +239,63 @@ class GlobDynamicContiguousMem // attach buckets for the maximum amount of containers, even if this unit // holds less containers { - data_type * c_data; - auto c_data_it = _container_list.begin(); - auto current_size = _container_list.size(); - for(int i = 0; i < *max_containers; ++i) { - if(i < current_size) { - c_data = &(*c_data_it); - // merge public and local containers - c_data->container->insert(c_data->container->end(), - c_data->unattached_container->begin(), - c_data->unattached_container->end()); - c_data->unattached_container->clear(); - // update memory location & size of _container - auto it = c_data->buckets.begin(); - it->lptr = c_data->container->data(); - it->size = c_data->container->size(); - c_data->container_bucket->lptr = c_data->container->data(); - c_data->container_bucket->size = c_data->container->size(); - // update memory location & size of _unattached_container - ++it; - it->lptr = c_data->unattached_container->data(); - it->size = 0; - c_data->unattached_container_bucket->lptr - = c_data->unattached_container->data(); - c_data->unattached_container_bucket->size = 0; - - c_data->update_lbegin(); - c_data->update_lend(); - ++c_data_it; - } else { - // if other units add more containers, create an empty container to - // store a dart_gptr for the collective allocation - auto cont_it = _container_list.begin(); - auto cont_index = add_container(0); - std::advance(cont_it, cont_index); - c_data = &(*cont_it); + data_type * c_data; + auto c_data_it = _container_list.begin(); + auto current_size = _container_list.size(); + for(int i = 0; i < *max_containers; ++i) { + if(i < current_size) { + c_data = &(*c_data_it); + // merge public and local containers + c_data->container->insert(c_data->container->end(), + c_data->unattached_container->begin(), + c_data->unattached_container->end()); + c_data->unattached_container->clear(); + // update memory location & size of _container + auto it = c_data->buckets.begin(); + it->lptr = c_data->container->data(); + it->size = c_data->container->size(); + c_data->container_bucket->lptr = c_data->container->data(); + c_data->container_bucket->size = c_data->container->size(); + // update memory location & size of _unattached_container + ++it; + it->lptr = c_data->unattached_container->data(); + it->size = 0; + c_data->unattached_container_bucket->lptr + = c_data->unattached_container->data(); + c_data->unattached_container_bucket->size = 0; + + c_data->update_lbegin(); + c_data->update_lend(); + ++c_data_it; + } else { + // if other units add more containers, create an empty container to + // store a dart_gptr for the collective allocation + auto cont_it = _container_list.begin(); + auto cont_index = add_container(0); + std::advance(cont_it, cont_index); + c_data = &(*cont_it); + } + + // attach new container to global memory space + dart_gptr_t gptr = DART_GPTR_NULL; + dart_storage_t ds = dart_storage(c_data->container->size()); + dart_team_memregister( + _team->dart_id(), + ds.nelem, + ds.dtype, + c_data->container->data(), + &gptr + ); + // no need to update gptr of local bucket list in c_data + c_data->container_bucket->gptr = gptr; + + //TODO: If possible, avoid adding unattached_container to global + // iteration space with size 0 + // update cumulated bucket sizes + bucket_cumul += c_data->container->size(); + _bucket_cumul_sizes[_myid][bucket_num] = bucket_cumul; + ++bucket_num; } - - // attach new container to global memory space - dart_gptr_t gptr = DART_GPTR_NULL; - dart_storage_t ds = dart_storage(c_data->container->size()); - dart_team_memregister( - _team->dart_id(), - ds.nelem, - ds.dtype, - c_data->container->data(), - &gptr - ); - // no need to update gptr of local bucket list in c_data - c_data->container_bucket->gptr = gptr; - - //TODO: If possible, avoid adding unattached_container to global - // iteration space with size 0 - // update cumulated bucket sizes - bucket_cumul += c_data->container->size(); - _bucket_cumul_sizes[_myid][bucket_num] = bucket_cumul; - ++bucket_num; - } } // distribute bucket sizes between all units @@ -382,7 +382,12 @@ class GlobDynamicContiguousMem return static_cast(it->size); } - + size_type container_size(container_list_index index) const { + auto cont_it = _container_list.begin(); + std::advance(cont_it, index); + auto c_data = *cont_it; + return c_data.container->size() + c_data.unattached_container->size(); + } size_type size() const { return _size; diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 2067515b4..1092912a7 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include @@ -61,8 +63,8 @@ class Graph { VertexProperties, EdgeProperties, VertexIndexType, EdgeIndexType, EdgeContainer, VertexContainer> graph_type; - typedef internal::vertex vertex_type; - typedef internal::out_edge edge_type; + typedef internal::Vertex vertex_type; + typedef internal::Edge edge_type; typedef VertexContainer> vertex_container_type; typedef EdgeContainer vertex_it_wrapper; typedef EdgeIteratorWrapper edge_it_wrapper; - typedef VertexIteratorWrapper in_edge_it_wrapper; - typedef VertexIteratorWrapper out_edge_it_wrapper; - typedef VertexIteratorWrapper adjacency_it_wrapper; + typedef InEdgeIteratorWrapper in_edge_it_wrapper; + typedef OutEdgeIteratorWrapper out_edge_it_wrapper; friend vertex_it_wrapper; friend edge_it_wrapper; + friend in_edge_it_wrapper; + friend out_edge_it_wrapper; typedef GlobDynamicContiguousMem< vertex_container_type> glob_mem_vert_type; @@ -112,14 +115,15 @@ class Graph { typedef typename glob_mem_edge_type::local_iterator local_edge_iterator; - typedef typename glob_mem_vert_type::global_iterator global_vertex_iterator; - typedef typename glob_mem_edge_type::global_iterator global_edge_iterator; + typedef typename + glob_mem_vert_type::global_iterator global_vertex_iterator; + typedef typename + glob_mem_edge_type::global_iterator global_edge_iterator; typedef typename vertex_it_wrapper::iterator vertex_iterator; typedef typename edge_it_wrapper::iterator edge_iterator; typedef typename in_edge_it_wrapper::iterator in_edge_iterator; typedef typename out_edge_it_wrapper::iterator out_edge_iterator; - typedef typename adjacency_it_wrapper::iterator adjacency_iterator; public: @@ -127,7 +131,6 @@ class Graph { edge_it_wrapper edges = edge_it_wrapper(this); in_edge_it_wrapper in_edges = in_edge_it_wrapper(this); out_edge_it_wrapper out_edges = out_edge_it_wrapper(this); - adjacency_it_wrapper adjacent_vertices = adjacency_it_wrapper(this); public: @@ -193,12 +196,13 @@ class Graph { */ vertex_index_type add_vertex(const VertexProperties & prop = VertexProperties()) { - ++_local_vertex_max_index; - auto ref = _glob_mem_edge->add_container(_alloc_edges_per_vertex); - vertex_type v(_local_vertex_max_index, ref, prop); + auto in_ref = _glob_mem_in_edge->add_container(_alloc_edges_per_vertex); + auto out_ref = _glob_mem_out_edge->add_container(_alloc_edges_per_vertex); + auto max_index = _glob_mem_vertex->container_size(_vertex_container_ref); + vertex_type v(max_index, in_ref, out_ref, prop); _glob_mem_vertex->push_back(_vertex_container_ref, v); // TODO: return global index - return vertex_index_type(_myid, _local_vertex_max_index); + return vertex_index_type(_myid, max_index); } /** @@ -222,44 +226,35 @@ class Graph { * locally. * NOTE: global method. */ - std::pair add_edge(const vertex_index_type & v1, - const vertex_index_type & v2, - const EdgeProperties & prop = EdgeProperties()) { - edge_index_type local_index(_myid, 0, -1); - //TODO: Handle errors, if vertices do not exist - auto edge1 = edge_type(0, v1, v2, prop); - if(v1.unit == _myid) { - auto vertex1 = _glob_mem_vertex->get(_vertex_container_ref, v1.offset); - edge1._local_id = ++_local_edge_max_index; - auto local_offset = _glob_mem_edge->push_back(vertex1._edge_ref, edge1); - local_index.unit = v1.unit; - local_index.container = vertex1._edge_ref; - local_index.offset = local_offset; + std::pair add_edge( + const vertex_index_type & source, + const vertex_index_type & target, + const EdgeProperties & prop = EdgeProperties() + ) { + bool remote_edge_set = false; + edge_index_type local_index; + if(source.unit == _myid) { + local_index = add_local_edge(source, target, prop, _glob_mem_out_edge); } else { - //TODO: check, if unit ID is valid - _remote_edges[v1.unit].push_back(edge1); + auto edge = edge_type(0, source, target, prop); + _remote_edges[source.unit].push_back(edge); + remote_edge_set = true; } - auto edge2 = edge_type(0, v2, v1, prop); - if(v2.unit == _myid) { - auto vertex2 = _glob_mem_vertex->get(_vertex_container_ref, v2.offset); - edge2._local_id = ++_local_edge_max_index; - auto local_offset = _glob_mem_edge->push_back(vertex2._edge_ref, edge2); + if(target.unit == _myid) { + // _glob_mem_in_edge == _glob_mem_out_edge for undirected graph types + edge_index_type local_index_tmp = add_local_edge(target, + source, prop, _glob_mem_in_edge); if(local_index.offset == -1) { - local_index.unit = v2.unit; - local_index.container = vertex2._edge_ref; - local_index.offset = local_offset; + local_index = local_index_tmp; } - } else { - _remote_edges[v2.unit].push_back(edge2); + // do not double-send edges + } else if(!remote_edge_set) { + auto edge = edge_type(0, source, target, prop); + _remote_edges[target.unit].push_back(edge); } - //TODO: Check, whether the edge already exists - //TODO: if feasible, use pointer to prop, so it gets saved only once - //TODO: find out, how to save edges for different graph types - // (directed, undirected) - //TODO: Find a way to return index, if both vertices reside on different - // unit. return std::make_pair(local_index, true); } + /** * Removes an edge between two given vertices. * NOTE: global method. @@ -322,14 +317,20 @@ class Graph { ); // add missing edges to local memory space for(auto edge : edges) { - auto vertex = _glob_mem_vertex->get(_vertex_container_ref, - edge._source.offset); - edge._local_id = ++_local_edge_max_index; - _glob_mem_edge->push_back(vertex._edge_ref, edge); + if(edge._source.unit == _myid) { + add_local_edge(edge._source, edge._target, edge._properties, + _glob_mem_out_edge); + } + if(edge._target.unit == _myid) { + // _glob_mem_in_edge == _glob_mem_out_edge for undirected graph types + add_local_edge(edge._target, edge._source, edge._properties, + _glob_mem_in_edge); + } } // commit changes in local memory space globally _glob_mem_vertex->commit(); - _glob_mem_edge->commit(); + _glob_mem_out_edge->commit(); + _glob_mem_in_edge->commit(); } /** @@ -342,7 +343,14 @@ class Graph { // no edge list allocation yet, this will happen once the vertices are // created. Each edge list will have n_vertex_edges elements reserved. _alloc_edges_per_vertex = n_vertex_edges; - _glob_mem_edge = new glob_mem_edge_type(*_team); + _glob_mem_out_edge = new glob_mem_edge_type(*_team); + if(Direction == DirectedGraph) { + _glob_mem_in_edge = new glob_mem_edge_type(*_team); + } else { + // there is no distinction between in- and out-edges in an undirected + // graph + _glob_mem_in_edge = _glob_mem_out_edge; + } // Register deallocator at the respective team instance _team->register_deallocator(this, std::bind(&Graph::deallocate, this)); } @@ -363,6 +371,35 @@ class Graph { return reference(gptr); } +private: + + edge_index_type add_local_edge( + const vertex_index_type & source, + const vertex_index_type & target, + const EdgeProperties & prop, + glob_mem_edge_type * glob_mem_edge + ) { + edge_index_type local_index(_myid, 0, -1); + auto edge = edge_type(0, source, target, prop); + // if vertex is local, place edge in edge list of the given vertex. + // mark the edge for the next commit otherwise + if(source.unit == _myid) { + auto vertex = _glob_mem_vertex->get(_vertex_container_ref, + source.offset); + edge._local_id = glob_mem_edge->container_size( + vertex._out_edge_ref); + auto local_offset = glob_mem_edge->push_back( + vertex._out_edge_ref, edge); + local_index.unit = source.unit; + local_index.container = vertex._out_edge_ref; + local_index.offset = local_offset; + } else { + //TODO: check, if unit ID is valid + _remote_edges[source.unit].push_back(edge); + } + return local_index; + } + private: /** the team containing all units using the container */ @@ -370,7 +407,9 @@ class Graph { /** Global memory allocation and access for sequential memory regions */ glob_mem_vert_type * _glob_mem_vertex = nullptr; - glob_mem_edge_type * _glob_mem_edge = nullptr; + glob_mem_edge_type * _glob_mem_in_edge = nullptr; + + glob_mem_edge_type * _glob_mem_out_edge = nullptr; /** Unit ID of the current unit */ team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; /** Index of last added vertex */ diff --git a/dash/include/dash/graph/EdgeIterator.h b/dash/include/dash/graph/EdgeIterator.h index 7fcf8c71e..eafcb14c9 100644 --- a/dash/include/dash/graph/EdgeIterator.h +++ b/dash/include/dash/graph/EdgeIterator.h @@ -35,60 +35,60 @@ struct EdgeIteratorWrapper { * Returns global iterator to the beginning of the edge list. */ iterator begin() { - return _graph->_glob_mem_edge->begin(); - //return iterator(_graph->_glob_mem_edge, 0); + return _graph->_glob_mem_out_edge->begin(); + //return iterator(_graph->_glob_mem_out_edge, 0); } /** * Returns global iterator to the beginning of the edge list. */ const_iterator begin() const { - return _graph->_glob_mem_edge->begin(); - //return iterator(_graph->_glob_mem_edge, 0); + return _graph->_glob_mem_out_edge->begin(); + //return iterator(_graph->_glob_mem_out_edge, 0); } /** * Returns global iterator to the end of the edge list. */ iterator end() { - return _graph->_glob_mem_edge->end(); - //return iterator(_graph->_glob_mem_edge, _graph->_glob_mem_edge->size()); + return _graph->_glob_mem_out_edge->end(); + //return iterator(_graph->_glob_mem_out_edge, _graph->_glob_mem_out_edge->size()); } /** * Returns global iterator to the end of the edge list. */ const_iterator end() const { - return _graph->_glob_mem_edge->end(); - //return iterator(_graph->_glob_mem_edge, _graph->_glob_mem_edge->size()); + return _graph->_glob_mem_out_edge->end(); + //return iterator(_graph->_glob_mem_out_edge, _graph->_glob_mem_out_edge->size()); } /** * Returns local iterator to the beginning of the edge list. */ local_iterator lbegin() { - return _graph->_glob_mem_edge->lbegin(); + return _graph->_glob_mem_out_edge->lbegin(); } /** * Returns local iterator to the beginning of the edge list. */ const_local_iterator lbegin() const { - return _graph->_glob_mem_edge->lbegin(); + return _graph->_glob_mem_out_edge->lbegin(); } /** * Returns local iterator to the end of the edge list. */ local_iterator lend() { - return _graph->_glob_mem_edge->lend(); + return _graph->_glob_mem_out_edge->lend(); } /** * Returns local iterator to the end of the edge list. */ const_local_iterator lend() const { - return _graph->_glob_mem_edge->lend(); + return _graph->_glob_mem_out_edge->lend(); } private: diff --git a/dash/include/dash/graph/InEdgeIterator.h b/dash/include/dash/graph/InEdgeIterator.h new file mode 100644 index 000000000..9d28e1cf0 --- /dev/null +++ b/dash/include/dash/graph/InEdgeIterator.h @@ -0,0 +1,102 @@ +#ifndef DASH__GRAPH__IN_EDGE_ITERATOR_H__INCLUDED +#define DASH__GRAPH__IN_EDGE_ITERATOR_H__INCLUDED + +namespace dash { + +/** + * Wrapper for the edge iterators of the graph. + */ +template +struct InEdgeIteratorWrapper { + + typedef Graph graph_type; + typedef typename Graph::global_edge_iterator iterator; + typedef const iterator const_iterator; + typedef typename Graph::local_edge_iterator local_iterator; + typedef const local_iterator const_local_iterator; + typedef typename Graph::edge_index_type edge_index_type; + typedef typename Graph::edge_properties_type edge_properties_type; + + /** + * Constructs the wrapper. + */ + InEdgeIteratorWrapper(graph_type * graph) + : _graph(graph) + { } + + /** + * Returns a property object for the given edge. + */ + edge_properties_type & operator[](const edge_index_type & v) const { + + } + + /** + * Returns global iterator to the beginning of the edge list. + */ + iterator begin() { + return _graph->_glob_mem_in_edge->begin(); + //return iterator(_graph->_glob_mem_in_edge, 0); + } + + /** + * Returns global iterator to the beginning of the edge list. + */ + const_iterator begin() const { + return _graph->_glob_mem_in_edge->begin(); + //return iterator(_graph->_glob_mem_in_edge, 0); + } + + /** + * Returns global iterator to the end of the edge list. + */ + iterator end() { + return _graph->_glob_mem_in_edge->end(); + //return iterator(_graph->_glob_mem_in_edge, _graph->_glob_mem_in_edge->size()); + } + + /** + * Returns global iterator to the end of the edge list. + */ + const_iterator end() const { + return _graph->_glob_mem_in_edge->end(); + //return iterator(_graph->_glob_mem_in_edge, _graph->_glob_mem_in_edge->size()); + } + + /** + * Returns local iterator to the beginning of the edge list. + */ + local_iterator lbegin() { + return _graph->_glob_mem_in_edge->lbegin(); + } + + /** + * Returns local iterator to the beginning of the edge list. + */ + const_local_iterator lbegin() const { + return _graph->_glob_mem_in_edge->lbegin(); + } + + /** + * Returns local iterator to the end of the edge list. + */ + local_iterator lend() { + return _graph->_glob_mem_in_edge->lend(); + } + + /** + * Returns local iterator to the end of the edge list. + */ + const_local_iterator lend() const { + return _graph->_glob_mem_in_edge->lend(); + } + +private: + + graph_type * _graph; + +}; + +} + +#endif // DASH__GRAPH__IN_EDGE_ITERATOR_H__INCLUDED diff --git a/dash/include/dash/graph/OutEdgeIterator.h b/dash/include/dash/graph/OutEdgeIterator.h new file mode 100644 index 000000000..381b61abd --- /dev/null +++ b/dash/include/dash/graph/OutEdgeIterator.h @@ -0,0 +1,102 @@ +#ifndef DASH__GRAPH__OUT_EDGE_ITERATOR_H__INCLUDED +#define DASH__GRAPH__OUT_EDGE_ITERATOR_H__INCLUDED + +namespace dash { + +/** + * Wrapper for the edge iterators of the graph. + */ +template +struct OutEdgeIteratorWrapper { + + typedef Graph graph_type; + typedef typename Graph::global_edge_iterator iterator; + typedef const iterator const_iterator; + typedef typename Graph::local_edge_iterator local_iterator; + typedef const local_iterator const_local_iterator; + typedef typename Graph::edge_index_type edge_index_type; + typedef typename Graph::edge_properties_type edge_properties_type; + + /** + * Constructs the wrapper. + */ + OutEdgeIteratorWrapper(graph_type * graph) + : _graph(graph) + { } + + /** + * Returns a property object for the given edge. + */ + edge_properties_type & operator[](const edge_index_type & v) const { + + } + + /** + * Returns global iterator to the beginning of the edge list. + */ + iterator begin() { + return _graph->_glob_mem_out_edge->begin(); + //return iterator(_graph->_glob_mem_out_edge, 0); + } + + /** + * Returns global iterator to the beginning of the edge list. + */ + const_iterator begin() const { + return _graph->_glob_mem_out_edge->begin(); + //return iterator(_graph->_glob_mem_out_edge, 0); + } + + /** + * Returns global iterator to the end of the edge list. + */ + iterator end() { + return _graph->_glob_mem_out_edge->end(); + //return iterator(_graph->_glob_mem_out_edge, _graph->_glob_mem_out_edge->size()); + } + + /** + * Returns global iterator to the end of the edge list. + */ + const_iterator end() const { + return _graph->_glob_mem_out_edge->end(); + //return iterator(_graph->_glob_mem_out_edge, _graph->_glob_mem_out_edge->size()); + } + + /** + * Returns local iterator to the beginning of the edge list. + */ + local_iterator lbegin() { + return _graph->_glob_mem_out_edge->lbegin(); + } + + /** + * Returns local iterator to the beginning of the edge list. + */ + const_local_iterator lbegin() const { + return _graph->_glob_mem_out_edge->lbegin(); + } + + /** + * Returns local iterator to the end of the edge list. + */ + local_iterator lend() { + return _graph->_glob_mem_out_edge->lend(); + } + + /** + * Returns local iterator to the end of the edge list. + */ + const_local_iterator lend() const { + return _graph->_glob_mem_out_edge->lend(); + } + +private: + + graph_type * _graph; + +}; + +} + +#endif // DASH__GRAPH__OUT_EDGE_ITERATOR_H__INCLUDED diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index f97491c2f..924795ed2 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -44,7 +44,7 @@ struct EdgeIndex { }; template -struct vertex { +struct Vertex { typedef typename GraphType::edge_cont_ref_type edge_container_ref; typedef typename GraphType::vertex_offset_type index_type; @@ -53,18 +53,20 @@ struct vertex { /** * Default constructor needed for GlobRef / GlobSharedRef dereference op */ - vertex() = default; + Vertex() = default; /** * Creates a vertex with given properties. */ - vertex( - index_type & index, - edge_container_ref & edge_ref, + Vertex( + index_type index, + edge_container_ref & in_edge_ref, + edge_container_ref & out_edge_ref, properties_type properties = properties_type() ) : _local_id(index), - _edge_ref(edge_ref), + _in_edge_ref(in_edge_ref), + _out_edge_ref(out_edge_ref), _properties(properties) { } @@ -76,12 +78,13 @@ struct vertex { * Iterator to the position of the respective edge container in glob_mem * object */ - edge_container_ref _edge_ref; + edge_container_ref _in_edge_ref; + edge_container_ref _out_edge_ref; }; template -struct out_edge { +struct Edge { typedef typename GraphType::vertex_index_type vertex_index_type; typedef typename GraphType::edge_offset_type index_type; @@ -90,12 +93,12 @@ struct out_edge { /** * Default constructor needed for GlobRef / GlobSharedRef dereference op */ - out_edge() = default; + Edge() = default; /** * Creates an edge from its parent vertex to target. */ - out_edge( + Edge( index_type index, vertex_index_type source, vertex_index_type target, From bdf065656efa28eea398928f4eb6bfbd7172ed93 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Sat, 8 Apr 2017 13:55:16 +0200 Subject: [PATCH 036/102] added memory space unifier --- dash/include/dash/GlobDynamicCombinedMem.h | 244 ++++++ dash/include/dash/GlobDynamicContiguousMem.h | 3 + dash/include/dash/Graph.h | 59 +- dash/include/dash/graph/EdgeIterator.h | 30 +- .../include/dash/memory/GlobHeapCombinedPtr.h | 712 ++++++++++++++++++ 5 files changed, 1010 insertions(+), 38 deletions(-) create mode 100644 dash/include/dash/GlobDynamicCombinedMem.h create mode 100644 dash/include/dash/memory/GlobHeapCombinedPtr.h diff --git a/dash/include/dash/GlobDynamicCombinedMem.h b/dash/include/dash/GlobDynamicCombinedMem.h new file mode 100644 index 000000000..380f60219 --- /dev/null +++ b/dash/include/dash/GlobDynamicCombinedMem.h @@ -0,0 +1,244 @@ +#ifndef DASH__GLOB_DYNAMIC_COMBINED_MEM_H +#define DASH__GLOB_DYNAMIC_COMBINED_MEM_H + +#include +#include + +namespace dash { + +template +class GlobDynamicCombinedMem { + +public: + + typedef GlobDynamicCombinedMem self_t; + typedef GlobMemType glob_mem_type; + typedef typename glob_mem_type::index_type index_type; + typedef typename glob_mem_type::size_type size_type; + typedef typename glob_mem_type::value_type value_type; + typedef std::vector> bucket_cumul_sizes_map; + typedef std::list glob_mem_list; + typedef GlobPtr global_iterator; + typedef typename glob_mem_type::local_iterator local_iterator; + typedef typename local_iterator::bucket_type bucket_type; + typedef typename std::list bucket_list; + typedef local_iterator local_pointer; + typedef local_iterator const_local_pointer; + + template + friend class GlobPtr; + +public: + + GlobDynamicCombinedMem(Team & team = dash::Team::All()) + : _bucket_cumul_sizes(team.size()), + _team(&team), + _buckets() + { } + + void add_globmem(glob_mem_type & glob_mem) { + // only GlobMem objects with the same Team can be used + if(*_team == glob_mem.team()) { + _glob_mem_list.push_back(&glob_mem); + //update_bucket_sizes(); + } + } + + void commit() { + update_bucket_sizes(); + // TODO: The bucket list should normally update on every element insertion, + // to always be consistent with the current local memory space + update_bucket_list(); + update_size(); + + _begin = global_iterator(this, 0); + _end = global_iterator(this, _size); + } + + dart_gptr_t dart_gptr_at(team_unit_t unit, + index_type bucket_index, index_type bucket_phase) const { + auto gmem_index = bucket_index % _glob_mem_list.size(); + auto gmem_it = _glob_mem_list.begin(); + std::advance(gmem_it, gmem_index); + auto gmem = *gmem_it; + // translate combined bucket index to index for particular GlobMem object + int gmem_bucket_index = bucket_index / _glob_mem_list.size(); + return gmem->dart_gptr_at(unit, gmem_bucket_index, bucket_phase); + } + + global_iterator begin() const { + return _begin; + } + + global_iterator end() const { + return _end; + } + + local_iterator lbegin() const { + return _lbegin; + } + + local_iterator lend() const { + return _lend; + } + + size_type size() const { + return _size; + } + + Team & team() const { + return (_team != nullptr) ? *_team : dash::Team::Null(); + } + +private: + + /** + * Combines bucket sizes of all currently added GlobMem objects. + * Resulting order for gmem_0 & gmem_1: + * [unit_0] : [gmem_0 b_0][gmem_1 b_0] ... [gmem_0 b_n][gmem_1 b_n] + * : : : : : + * [unit_n] : [gmem_0 b_0][gmem_1 b_0] ... [gmem_0 b_n][gmem_1 b_n] + */ + void update_bucket_sizes() { + // the index of a bucket directly translates to its corresponding GlobMem + // object: + // example for 2 GlobMem objects: + // bucket 0 -> GlobMem 0 + // bucket 1 -> GlobMem 1 + // bucket 2 -> GlobMem 0 + // ... + if(_glob_mem_list.size() > 0) { + for(int i = 0; i < _bucket_cumul_sizes.size(); ++i) { + int max = 0; + for(auto gmem : _glob_mem_list) { + if(gmem->_bucket_cumul_sizes[i].size() > max) { + max = gmem->_bucket_cumul_sizes[i].size(); + } + } + _bucket_cumul_sizes[i].clear(); + _bucket_cumul_sizes[i].resize(max * _glob_mem_list.size()); + } + int offset = 0; + for(auto gmem : _glob_mem_list) { + auto & gmem_buckets = gmem->_bucket_cumul_sizes; + int pos = offset; + for(int i = 0; i < gmem_buckets.size(); ++i) { + size_type last = 0; + for(int j = 0; j + offset < _bucket_cumul_sizes[i].size(); ++j) { + int gmem_index = j / _glob_mem_list.size(); + // if number of buckets shorter than the maximum, repeat the last + // bucket size for full accumulation + if(gmem_index < gmem_buckets[i].size()) { + last = gmem_buckets[i][gmem_index]; + } + _bucket_cumul_sizes[i][j + offset] += last; + } + } + ++offset; + } + } + } + + void update_bucket_list() { + _buckets.clear(); + std::vector> iters; + int max = 0; + if(_glob_mem_list.size() > 0) { + for(auto gmem : _glob_mem_list) { + iters.push_back(std::make_pair(gmem->_buckets.begin(), + gmem->_buckets.end())); + if(max < gmem->_buckets.size()) { + max = gmem->_buckets.size(); + } + } + // TODO: Currently onlys working for GlobMemContiguousMem because of + // the double buckets + for(int i = 0; i < max; i += 2) { + for(auto & it : iters) { + if(it.first != it.second) { + _buckets.push_back(*(it.first)); + ++(it.first); + } + if(it.first != it.second) { + _buckets.push_back(*(it.first)); + ++(it.first); + } + } + } + } + update_local_size(); + update_lbegin(); + update_lend(); + } + +private: + + void update_size() { + _size = 0; + for(auto gmem : _glob_mem_list) { + _size += gmem->_size; + } + } + + void update_local_size() { + _local_size = 0; + for(auto gmem : _glob_mem_list) { + _local_size += gmem->_local_size; + } + } + + /** + * Native pointer of the initial address of the local memory of + * a unit. + * + */ + void update_lbegin() noexcept + { + local_iterator unit_lbegin( + // iteration space + _buckets.begin(), _buckets.end(), + // position in iteration space + 0, + // bucket at position in iteration space, + // offset in bucket + _buckets.begin(), 0); + _lbegin = unit_lbegin; + } + + /** + * Update internal native pointer of the final address of the local memory#include + * of a unit. + */ + void update_lend() noexcept + { + local_iterator unit_lend( + // iteration space + _buckets.begin(), _buckets.end(), + // position in iteration space + _local_size, + // bucket at position in iteration space, + // offset in bucket + _buckets.end(), 0); + _lend = unit_lend; + } + +private: + + bucket_list _buckets; + bucket_cumul_sizes_map _bucket_cumul_sizes; + glob_mem_list _glob_mem_list; + Team * _team = nullptr; + size_type _size = 0; + global_iterator _begin; + global_iterator _end; + local_iterator _lbegin; + local_iterator _lend; + size_type _local_size = 0; + +}; + +} + +#endif //DASH__GLOB_DYNAMIC_COMBINED_MEM_H + diff --git a/dash/include/dash/GlobDynamicContiguousMem.h b/dash/include/dash/GlobDynamicContiguousMem.h index 2864da61e..52a8a4cb9 100644 --- a/dash/include/dash/GlobDynamicContiguousMem.h +++ b/dash/include/dash/GlobDynamicContiguousMem.h @@ -18,6 +18,7 @@ #include #include #include +#include #include @@ -157,6 +158,8 @@ class GlobDynamicContiguousMem template friend class dash::GlobPtr; + friend class GlobDynamicCombinedMem; + public: /** * Constructor diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 1092912a7..157fe0a60 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -84,46 +85,50 @@ class Graph { friend out_edge_it_wrapper; typedef GlobDynamicContiguousMem< - vertex_container_type> glob_mem_vert_type; + vertex_container_type> glob_mem_vert_type; typedef GlobDynamicContiguousMem< - edge_container_type> glob_mem_edge_type; - typedef std::vector> edge_list_type; + edge_container_type> glob_mem_edge_type; + typedef + GlobDynamicCombinedMem glob_mem_edge_comb_type; + typedef std::vector> edge_list_type; public: typedef typename - glob_mem_vert_type::container_list_index vertex_cont_ref_type; + glob_mem_vert_type::container_list_index vertex_cont_ref_type; typedef typename - glob_mem_edge_type::container_list_index edge_cont_ref_type; - typedef VertexIndexType vertex_offset_type; - typedef EdgeIndexType edge_offset_type; - typedef internal::VertexIndex vertex_index_type; - typedef internal::EdgeIndex edge_index_type; + glob_mem_edge_type::container_list_index edge_cont_ref_type; + typedef VertexIndexType vertex_offset_type; + typedef EdgeIndexType edge_offset_type; + typedef internal::VertexIndex vertex_index_type; + typedef internal::EdgeIndex edge_index_type; typedef typename - std::make_unsigned::type vertex_size_type; + std::make_unsigned::type vertex_size_type; typedef typename - std::make_unsigned::type edge_size_type; + std::make_unsigned::type edge_size_type; - typedef DynamicPattern pattern_type; - typedef VertexProperties vertex_properties_type; - typedef EdgeProperties edge_properties_type; + typedef DynamicPattern pattern_type; + typedef VertexProperties vertex_properties_type; + typedef EdgeProperties edge_properties_type; - typedef GlobRef reference; + typedef GlobRef reference; typedef typename - glob_mem_vert_type::local_iterator local_vertex_iterator; + glob_mem_vert_type::local_iterator local_vertex_iterator; typedef typename - glob_mem_edge_type::local_iterator local_edge_iterator; + glob_mem_edge_type::local_iterator local_edge_iterator; typedef typename - glob_mem_vert_type::global_iterator global_vertex_iterator; + glob_mem_vert_type::global_iterator global_vertex_iterator; typedef typename - glob_mem_edge_type::global_iterator global_edge_iterator; + glob_mem_edge_type::global_iterator global_edge_iterator; + typedef typename + glob_mem_edge_comb_type::global_iterator global_edge_comb_iterator; - typedef typename vertex_it_wrapper::iterator vertex_iterator; - typedef typename edge_it_wrapper::iterator edge_iterator; - typedef typename in_edge_it_wrapper::iterator in_edge_iterator; - typedef typename out_edge_it_wrapper::iterator out_edge_iterator; + typedef typename vertex_it_wrapper::iterator vertex_iterator; + typedef typename edge_it_wrapper::iterator edge_iterator; + typedef typename in_edge_it_wrapper::iterator in_edge_iterator; + typedef typename out_edge_it_wrapper::iterator out_edge_iterator; public: @@ -331,6 +336,7 @@ class Graph { _glob_mem_vertex->commit(); _glob_mem_out_edge->commit(); _glob_mem_in_edge->commit(); + _glob_mem_edge->commit(); } /** @@ -351,6 +357,9 @@ class Graph { // graph _glob_mem_in_edge = _glob_mem_out_edge; } + _glob_mem_edge = new glob_mem_edge_comb_type(*_team); + _glob_mem_edge->add_globmem(*_glob_mem_out_edge); + _glob_mem_edge->add_globmem(*_glob_mem_in_edge); // Register deallocator at the respective team instance _team->register_deallocator(this, std::bind(&Graph::deallocate, this)); } @@ -359,6 +368,8 @@ class Graph { * Deallocates global memory of this container. */ void deallocate() { + // TODO: Delete all objects created on the heap, and look for objects with + // shared ownership if(_glob_mem_vertex != nullptr) { delete _glob_mem_vertex; } @@ -410,6 +421,8 @@ class Graph { glob_mem_edge_type * _glob_mem_in_edge = nullptr; glob_mem_edge_type * _glob_mem_out_edge = nullptr; + + glob_mem_edge_comb_type * _glob_mem_edge = nullptr; /** Unit ID of the current unit */ team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; /** Index of last added vertex */ diff --git a/dash/include/dash/graph/EdgeIterator.h b/dash/include/dash/graph/EdgeIterator.h index eafcb14c9..2e4037af8 100644 --- a/dash/include/dash/graph/EdgeIterator.h +++ b/dash/include/dash/graph/EdgeIterator.h @@ -9,13 +9,13 @@ namespace dash { template struct EdgeIteratorWrapper { - typedef Graph graph_type; - typedef typename Graph::global_edge_iterator iterator; - typedef const iterator const_iterator; - typedef typename Graph::local_edge_iterator local_iterator; - typedef const local_iterator const_local_iterator; - typedef typename Graph::edge_index_type edge_index_type; - typedef typename Graph::edge_properties_type edge_properties_type; + typedef Graph graph_type; + typedef typename Graph::global_edge_comb_iterator iterator; + typedef const iterator const_iterator; + typedef typename Graph::local_edge_iterator local_iterator; + typedef const local_iterator const_local_iterator; + typedef typename Graph::edge_index_type edge_index_type; + typedef typename Graph::edge_properties_type edge_properties_type; /** * Constructs the wrapper. @@ -35,7 +35,7 @@ struct EdgeIteratorWrapper { * Returns global iterator to the beginning of the edge list. */ iterator begin() { - return _graph->_glob_mem_out_edge->begin(); + return _graph->_glob_mem_edge->begin(); //return iterator(_graph->_glob_mem_out_edge, 0); } @@ -43,7 +43,7 @@ struct EdgeIteratorWrapper { * Returns global iterator to the beginning of the edge list. */ const_iterator begin() const { - return _graph->_glob_mem_out_edge->begin(); + return _graph->_glob_mem_edge->begin(); //return iterator(_graph->_glob_mem_out_edge, 0); } @@ -51,7 +51,7 @@ struct EdgeIteratorWrapper { * Returns global iterator to the end of the edge list. */ iterator end() { - return _graph->_glob_mem_out_edge->end(); + return _graph->_glob_mem_edge->end(); //return iterator(_graph->_glob_mem_out_edge, _graph->_glob_mem_out_edge->size()); } @@ -59,7 +59,7 @@ struct EdgeIteratorWrapper { * Returns global iterator to the end of the edge list. */ const_iterator end() const { - return _graph->_glob_mem_out_edge->end(); + return _graph->_glob_mem_edge->end(); //return iterator(_graph->_glob_mem_out_edge, _graph->_glob_mem_out_edge->size()); } @@ -67,28 +67,28 @@ struct EdgeIteratorWrapper { * Returns local iterator to the beginning of the edge list. */ local_iterator lbegin() { - return _graph->_glob_mem_out_edge->lbegin(); + return _graph->_glob_mem_edge->lbegin(); } /** * Returns local iterator to the beginning of the edge list. */ const_local_iterator lbegin() const { - return _graph->_glob_mem_out_edge->lbegin(); + return _graph->_glob_mem_edge->lbegin(); } /** * Returns local iterator to the end of the edge list. */ local_iterator lend() { - return _graph->_glob_mem_out_edge->lend(); + return _graph->_glob_mem_edge->lend(); } /** * Returns local iterator to the end of the edge list. */ const_local_iterator lend() const { - return _graph->_glob_mem_out_edge->lend(); + return _graph->_glob_mem_edge->lend(); } private: diff --git a/dash/include/dash/memory/GlobHeapCombinedPtr.h b/dash/include/dash/memory/GlobHeapCombinedPtr.h new file mode 100644 index 000000000..cd3d1a3c6 --- /dev/null +++ b/dash/include/dash/memory/GlobHeapCombinedPtr.h @@ -0,0 +1,712 @@ +#ifndef DASH__MEMORY__GLOB_HEAP_COMBINED_PTR_H__INCLUDED +#define DASH__MEMORY__GLOB_HEAP_COMBINED_PTR_H__INCLUDED + +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + + +namespace dash { + +// Forward-declaration +template +class GlobDynamicCombinedMem; + +/** + * Iterator on global buckets. Represents global pointer type. + */ +template +class GlobPtr< + ElementType, + GlobDynamicCombinedMem + > +{ + typedef GlobDynamicCombinedMem GlobHeapMemType; + typedef GlobPtr self_t; + + template< + typename ElementType_, + class ContainerType_ > + friend std::ostream & operator<<( + std::ostream & os, + const dash::GlobPtr< + ElementType_, + GlobDynamicCombinedMem + > & gptr); + + friend GlobPtr; + +public: + typedef typename GlobHeapMemType::index_type index_type; + typedef typename std::make_unsigned::type size_type; + + typedef ElementType value_type; + + typedef GlobSharedRef< value_type, self_t> reference; + typedef GlobSharedRef const_reference; + + typedef value_type * raw_pointer; + + typedef GlobHeapMemType globmem_type; + typedef typename GlobHeapMemType::local_pointer local_pointer; + + typedef struct { + team_unit_t unit; + index_type index; + } local_index; + + /* + * TODO: Implement rebind to allow for the allocation of different types + * than ElementType + * + template + struct rebind { + typedef GlobPtr< + U, + GlobDynamicContiguousMem< + typename AllocatorType::template rebind< + typename std::remove_const::type + >::other + > + > other; + }; + */ + +private: + typedef std::vector > + bucket_cumul_sizes_map; + +private: + /// Global memory used to dereference iterated values. + const globmem_type * _globmem = nullptr; + /// Mapping unit id to buckets in the unit's attached local storage. + const bucket_cumul_sizes_map * _bucket_cumul_sizes = nullptr; + /// Pointer to first element in local data space. + local_pointer _lbegin; + /// Current position of the pointer in global canonical index space. + index_type _idx = 0; + /// Maximum position allowed for this pointer. + index_type _max_idx = 0; + /// Unit id of the active unit. + team_unit_t _myid; + /// Unit id at the pointer's current position. + team_unit_t _idx_unit_id; + /// Logical offset in local index space at the pointer's current position. + index_type _idx_local_idx = -1; + /// Local bucket index at the pointer's current position. + index_type _idx_bucket_idx = -1; + /// Element offset in bucket at the pointer's current position. + index_type _idx_bucket_phase = -1; + +public: + /** + * Default constructor. + */ + GlobPtr() + : _globmem(nullptr), + _bucket_cumul_sizes(nullptr), + _idx(0), + _max_idx(0), + _myid(dash::Team::GlobalUnitID()), + _idx_unit_id(DART_UNDEFINED_UNIT_ID), + _idx_local_idx(-1), + _idx_bucket_idx(-1), + _idx_bucket_phase(-1) + { + DASH_LOG_TRACE_VAR("GlobPtr()", _idx); + DASH_LOG_TRACE_VAR("GlobPtr()", _max_idx); + } + + /** + * Constructor, creates a global pointer on global memory from global + * offset in logical storage order. + */ + template + GlobPtr( + const MemSpaceT * gmem, + index_type position = 0) + : _globmem(reinterpret_cast(gmem)), + _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), + _lbegin(_globmem->lbegin()), + _idx(position), + _max_idx(gmem->size() - 1), + _myid(gmem->team().myid()), + _idx_unit_id(0), + _idx_local_idx(0), + _idx_bucket_idx(0), + _idx_bucket_phase(0) + { + DASH_LOG_TRACE("GlobPtr(gmem,idx)", "gidx:", position); + for (auto unit_bucket_cumul_sizes : *_bucket_cumul_sizes) { + DASH_LOG_TRACE_VAR("GlobPtr(gmem,idx)", + unit_bucket_cumul_sizes); + size_type bucket_cumul_size_prev = 0; + for (auto bucket_cumul_size : unit_bucket_cumul_sizes) { + DASH_LOG_TRACE_VAR("GlobPtr(gmem,idx)", bucket_cumul_size); + if (position < bucket_cumul_size) { + DASH_LOG_TRACE_VAR("GlobPtr(gmem,idx)", position); + _idx_bucket_phase -= bucket_cumul_size; + position = 0; + _idx_local_idx = position; + _idx_bucket_phase = position - bucket_cumul_size_prev; + break; + } + bucket_cumul_size_prev = bucket_cumul_size; + ++_idx_bucket_idx; + } + if (position == 0) { + break; + } + // Advance to next unit, adjust position relative to next unit's + // local index space: + position -= unit_bucket_cumul_sizes.back(); + ++_idx_unit_id; + } + DASH_LOG_TRACE("GlobPtr(gmem,idx)", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bucket:", _idx_bucket_idx, + "phase:", _idx_bucket_phase); + } + + /** + * Constructor, creates a global pointer on global memory from unit and + * local offset in logical storage order. + */ + template + GlobPtr( + const MemSpaceT * gmem, + team_unit_t unit, + index_type local_index) + : _globmem(reinterpret_cast(gmem)), + _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), + _lbegin(_globmem->lbegin()), + _idx(0), + _max_idx(gmem->size() - 1), + _myid(gmem->team().myid()), + _idx_unit_id(unit), + _idx_local_idx(0), + _idx_bucket_idx(0), + _idx_bucket_phase(0) + { + DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx)", + "unit:", unit, + "lidx:", local_index); + DASH_ASSERT_LT(unit, _bucket_cumul_sizes->size(), "invalid unit id"); + + for (size_type unit = 0; unit < _idx_unit_id; ++unit) { + auto prec_unit_local_size = (*_bucket_cumul_sizes)[unit].back(); + _idx += prec_unit_local_size; + } + increment(local_index); + DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx) >", + "gidx:", _idx, + "maxidx:", _max_idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bucket:", _idx_bucket_idx, + "phase:", _idx_bucket_phase); + } + + /** + * Copy constructor. + */ + template + GlobPtr( + const GlobPtr & other) + : _globmem(other._globmem), + _bucket_cumul_sizes(other._bucket_cumul_sizes), + _lbegin(other._lbegin), + _idx(other._idx), + _max_idx(other._max_idx), + _idx_unit_id(other._idx_unit_id), + _idx_local_idx(other._idx_local_idx), + _idx_bucket_idx(other._idx_bucket_idx), + _idx_bucket_phase(other._idx_bucket_phase) + { } + + /** + * Assignment operator. + */ + template + self_t & operator=( + const GlobPtr & other) + { + _globmem = other._globmem; + _bucket_cumul_sizes = other._bucket_cumul_sizes; + _lbegin = other._lbegin; + _idx = other._idx; + _max_idx = other._max_idx; + _idx_unit_id = other._idx_unit_id; + _idx_local_idx = other._idx_local_idx; + _idx_bucket_idx = other._idx_bucket_idx; + _idx_bucket_phase = other._idx_bucket_phase; + } + + /** + * Explicit conversion to \c dart_gptr_t. + * + * \return A DART global pointer to the element at the pointer's + * position + */ + dart_gptr_t dart_gptr() const + { + DASH_LOG_TRACE_VAR("GlobPtr.dart_gptr()", _idx); + // Create global pointer from unit, bucket and phase: + dart_gptr_t dart_gptr = _globmem->dart_gptr_at( + _idx_unit_id, + _idx_bucket_idx, + _idx_bucket_phase); + DASH_LOG_TRACE_VAR("GlobPtr.dart_gptr >", dart_gptr); + return dart_gptr; + } + + /** + * Dereference operator. + * + * \return A global reference to the element at the pointer's position. + */ + reference operator*() const + { + auto lptr = local(); + if (lptr != nullptr) { + return reference(static_cast(lptr)); + } else { + return reference(dart_gptr()); + } + } + + /** + * Subscript operator, returns global reference to element at given + * global index. + */ + reference operator[]( + /// The global position of the element + index_type g_index) const + { + DASH_LOG_TRACE_VAR("GlobPtr.[]()", g_index); + auto git = *this; + git += g_index; + auto lptr = git.local(); + if (lptr != nullptr) { + return reference(lptr); + } else { + auto gref = *git; + DASH_LOG_TRACE_VAR("GlobPtr.[] >", gref); + return gref; + } + } + + /** + * Checks whether the element referenced by this global pointer is in + * the calling unit's local memory. + */ + inline bool is_local() const + { + return (_myid == _idx_unit_id); + } + + /** + * Conversion to local bucket pointer. + */ + local_pointer local() const + { + if (_myid != _idx_unit_id) { + // Iterator position does not point to local element + return nullptr; + } + return (_lbegin + _idx_local_idx); + } + + /** + * Unit and local offset at the pointer's position. + */ + inline local_index lpos() const + { + local_index local_pos; + local_pos.unit = _idx_unit_id; + local_pos.index = _idx_local_idx; + return local_pos; + } + + /** + * Map pointer to global index domain. + */ + inline self_t global() const + { + return *this; + } + + /** + * Position of the pointer in global index space. + */ + inline index_type pos() const + { + return _idx; + } + + /** + * Position of the pointer in global index range. + */ + inline index_type gpos() const + { + return _idx; + } + + /** + * The instance of \c GlobStaticMem used by this pointer to resolve + * addresses in global memory. + */ + inline const globmem_type & globmem() const + { + return *_globmem; + } + + /** + * The instance of \c GlobStaticMem used by this pointer to resolve + * addresses in global memory. + */ + inline globmem_type & globmem() + { + return *_globmem; + } + + /** + * Prefix increment operator. + */ + inline self_t & operator++() + { + increment(1); + return *this; + } + + /** + * Prefix decrement operator. + */ + inline self_t & operator--() + { + decrement(1); + return *this; + } + + /** + * Postfix increment operator. + */ + inline self_t operator++(int) + { + auto result = *this; + increment(1); + return result; + } + + /** + * Postfix decrement operator. + */ + inline self_t operator--(int) + { + auto result = *this; + decrement(1); + return result; + } + + inline self_t & operator+=(index_type offset) + { + increment(offset); + return *this; + } + + inline self_t & operator-=(index_type offset) + { + increment(offset); + return *this; + } + + inline self_t operator+(index_type offset) const + { + auto res = *this; + res.increment(offset); + return res; + } + + inline self_t operator-(index_type offset) const + { + auto res = *this; + res.decrement(offset); + return res; + } + + inline index_type operator+( + const self_t & other) const + { + return _idx + other._idx; + } + + inline index_type operator-( + const self_t & other) const + { + return _idx - other._idx; + } + + template + inline bool operator<(const GlobPtr & other) const + { + return (_idx < other._idx); + } + + template + inline bool operator<=(const GlobPtr & other) const + { + return (_idx <= other._idx); + } + + template + inline bool operator>(const GlobPtr & other) const + { + return (_idx > other._idx); + } + + template + inline bool operator>=(const GlobPtr & other) const + { + return (_idx >= other._idx); + } + + template + inline bool operator==(const GlobPtr & other) const + { + return _idx == other._idx; + } + + template + inline bool operator!=(const GlobPtr & other) const + { + return _idx != other._idx; + } + +private: + /** + * Advance pointer by specified position offset. + */ + void increment(int offset) + { + DASH_LOG_TRACE("GlobPtr.increment()", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase, + "offset:", offset); + _idx += offset; + auto current_bucket_size = + (*_bucket_cumul_sizes)[_idx_unit_id][_idx_bucket_idx]; + if (_idx_local_idx + offset < current_bucket_size) { + DASH_LOG_TRACE("GlobPtr.increment", "position current bucket"); + // element is in bucket currently referenced by this pointer: + _idx_bucket_phase += offset; + _idx_local_idx += offset; + } else { + DASH_LOG_TRACE("GlobPtr.increment", + "position in succeeding bucket"); + // iterate units: + auto unit_id_max = _bucket_cumul_sizes->size() - 1; + for (; _idx_unit_id <= unit_id_max; ++_idx_unit_id) { + if (offset == 0) { + break; + } + auto unit_bkt_sizes = (*_bucket_cumul_sizes)[_idx_unit_id]; + auto unit_bkt_sizes_total = unit_bkt_sizes.back(); + auto unit_num_bkts = unit_bkt_sizes.size(); + DASH_LOG_TRACE("GlobPtr.increment", + "unit:", _idx_unit_id, + "remaining offset:", offset, + "total local bucket size:", unit_bkt_sizes_total); + if (_idx_local_idx + offset >= unit_bkt_sizes_total) { + // offset refers to next unit: + DASH_LOG_TRACE("GlobPtr.increment", + "position in remote range"); + // subtract remaining own local size from remaining offset: + offset -= (unit_bkt_sizes_total - _idx_local_idx); + if (_idx_unit_id == unit_id_max) { + // end pointer, offset exceeds iteration space: + _idx_bucket_idx = unit_num_bkts - 1; + auto last_bkt_size = unit_bkt_sizes.back(); + if (unit_num_bkts > 1) { + last_bkt_size -= unit_bkt_sizes[_idx_bucket_idx-1]; + } + _idx_bucket_phase = last_bkt_size + offset; + _idx_local_idx += unit_bkt_sizes_total + offset; + break; + } + _idx_local_idx = 0; + _idx_bucket_idx = 0; + _idx_bucket_phase = 0; + } else { + // offset refers to current unit: + DASH_LOG_TRACE("GlobPtr.increment", + "position in local range", + "current bucket phase:", _idx_bucket_phase, + "cumul. bucket sizes:", unit_bkt_sizes); + _idx_local_idx += offset; + // iterate the unit's bucket sizes: + for (; _idx_bucket_idx < unit_num_bkts; ++_idx_bucket_idx) { + auto cumul_bucket_size = unit_bkt_sizes[_idx_bucket_idx]; + if (_idx_local_idx < cumul_bucket_size) { + auto cumul_prev = _idx_bucket_idx > 0 + ? unit_bkt_sizes[_idx_bucket_idx-1] + : 0; + // offset refers to current bucket: + _idx_bucket_phase = _idx_local_idx - cumul_prev; + offset = 0; + break; + } + } + if (offset == 0) { + break; + } + } + } + } + DASH_LOG_TRACE("GlobPtr.increment >", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase); + } + + /** + * Decrement pointer by specified position offset. + */ + void decrement(int offset) + { + DASH_LOG_TRACE("GlobPtr.decrement()", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase, + "offset:", -offset); + if (offset > _idx) { + DASH_THROW(dash::exception::OutOfRange, + "offset " << offset << " is out of range"); + } + _idx -= offset; + if (offset <= _idx_bucket_phase) { + // element is in bucket currently referenced by this pointer: + _idx_bucket_phase -= offset; + _idx_local_idx -= offset; + } else { + // iterate units: + auto first_unit = _idx_unit_id; + for (; _idx_unit_id >= 0; --_idx_unit_id) { + auto unit_bkt_sizes = (*_bucket_cumul_sizes)[_idx_unit_id]; + auto unit_bkt_sizes_total = unit_bkt_sizes.back(); + auto unit_num_bkts = unit_bkt_sizes.size(); + if (_idx_unit_id != first_unit) { + --offset; + _idx_bucket_idx = unit_num_bkts - 1; + _idx_local_idx = unit_bkt_sizes_total - 1; + auto last_bkt_size = unit_bkt_sizes.back(); + if (unit_num_bkts > 1) { + last_bkt_size -= unit_bkt_sizes[_idx_bucket_idx-1]; + } + _idx_bucket_phase = last_bkt_size - 1; + } + if (offset <= _idx_local_idx) { + // offset refers to current unit: + // iterate the unit's bucket sizes: + for (; _idx_bucket_idx >= 0; --_idx_bucket_idx) { + auto bucket_size = unit_bkt_sizes[_idx_bucket_idx]; + if (offset <= _idx_bucket_phase) { + // offset refers to current bucket: + _idx_local_idx -= offset; + _idx_bucket_phase -= offset; + offset = 0; + break; + } else { + // offset refers to preceeding bucket: + _idx_local_idx -= (_idx_bucket_phase + 1); + offset -= (_idx_bucket_phase + 1); + _idx_bucket_phase = unit_bkt_sizes[_idx_bucket_idx-1] - 1; + } + } + } else { + // offset refers to preceeding unit: + offset -= _idx_local_idx; + } + if (offset == 0) { + break; + } + } + } + DASH_LOG_TRACE("GlobPtr.decrement >", + "gidx:", _idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bidx:", _idx_bucket_idx, + "bphase:", _idx_bucket_phase); + } + +}; // class GlobPtr + +/** + * Resolve the number of elements between two global bucket pointers. + * + * \complexity O(1) + * + * \ingroup Algorithms + */ +template< + typename ElementType, + class GlobMemType > +auto distance( + /// Global pointer to the first position in the global sequence + const dash::GlobPtr< + ElementType, GlobDynamicCombinedMem + > & first, + /// Global pointer to the final position in the global sequence + const dash::GlobPtr< + ElementType, GlobDynamicCombinedMem + > & last) +-> typename GlobDynamicCombinedMem::index_type +{ + return last - first; +} + +template< + typename ElementType, + class GlobMemType > +std::ostream & operator<<( + std::ostream & os, + const dash::GlobPtr< + ElementType, + GlobDynamicCombinedMem + > & gptr) +{ + std::ostringstream ss; + ss << "dash::GlobPtr<" << typeid(ElementType).name() << ">(" + << "gidx:" << gptr._idx << ", (" + << "unit:" << gptr._idx_unit_id << ", " + << "lidx:" << gptr._idx_local_idx << "), (" + << "bidx:" << gptr._idx_bucket_idx << ", " + << "bphase:" << gptr._idx_bucket_phase << ")" + << ")"; + return operator<<(os, ss.str()); +} + +} // namespace dash + +#endif // DASH__MEMORY__GLOB_HEAP_COMBINED_PTR_H__INCLUDED From 85c3ad8a7645dbda231619c56623b4eba4b577e8 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Wed, 19 Apr 2017 13:20:26 +0200 Subject: [PATCH 037/102] refactored Edge and Vertex --- dash/include/dash/Graph.h | 4 +-- dash/include/dash/graph/internal/Graph.h | 38 +++++++++++++++++------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 157fe0a60..f723ec4f7 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -323,12 +323,12 @@ class Graph { // add missing edges to local memory space for(auto edge : edges) { if(edge._source.unit == _myid) { - add_local_edge(edge._source, edge._target, edge._properties, + add_local_edge(edge._source, edge._target, edge.properties, _glob_mem_out_edge); } if(edge._target.unit == _myid) { // _glob_mem_in_edge == _glob_mem_out_edge for undirected graph types - add_local_edge(edge._target, edge._source, edge._properties, + add_local_edge(edge._target, edge._source, edge.properties, _glob_mem_in_edge); } } diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index 924795ed2..46c4bc49a 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -44,12 +44,16 @@ struct EdgeIndex { }; template -struct Vertex { +class Vertex { typedef typename GraphType::edge_cont_ref_type edge_container_ref; typedef typename GraphType::vertex_offset_type index_type; typedef typename GraphType::vertex_properties_type properties_type; + friend GraphType; + +public: + /** * Default constructor needed for GlobRef / GlobSharedRef dereference op */ @@ -67,29 +71,36 @@ struct Vertex { : _local_id(index), _in_edge_ref(in_edge_ref), _out_edge_ref(out_edge_ref), - _properties(properties) + properties(properties) { } +public: + /** Properties of this vertex */ - properties_type _properties; + properties_type properties; + +private: + /** Index of the vertex in local index space */ index_type _local_id; - /* - * Iterator to the position of the respective edge container in glob_mem - * object - */ + /** index of the in-edge list belonging to this vertex */ edge_container_ref _in_edge_ref; + /** index of the out-edge list belonging to this vertex */ edge_container_ref _out_edge_ref; }; template -struct Edge { +class Edge { typedef typename GraphType::vertex_index_type vertex_index_type; typedef typename GraphType::edge_offset_type index_type; typedef typename GraphType::edge_properties_type properties_type; + + friend GraphType; +public: + /** * Default constructor needed for GlobRef / GlobSharedRef dereference op */ @@ -107,16 +118,21 @@ struct Edge { : _local_id(index), _source(source), _target(target), - _properties(properties) + properties(properties) { } +public: + + /** Properties of this edge */ + properties_type properties; + +private: + //TODO: Examine, if saving source can be avoided /** Source vertex the edge is pointing from */ vertex_index_type _source; /** Target vertex the edge is pointing to */ vertex_index_type _target; - /** Properties of this edge */ - properties_type _properties; /** Index of the edge in local index space */ index_type _local_id; From 985c8a3981dfde40ba20490c531bf9a1e931c551 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Wed, 19 Apr 2017 19:40:17 +0200 Subject: [PATCH 038/102] added adjacency iterators --- dash/include/dash/GlobDynamicCombinedMem.h | 9 ++++ dash/include/dash/GlobDynamicContiguousMem.h | 14 +++-- dash/include/dash/Graph.h | 48 +++++++++-------- dash/include/dash/graph/EdgeIterator.h | 54 +++++++++++++++++++ dash/include/dash/graph/InEdgeIterator.h | 54 +++++++++++++++++++ dash/include/dash/graph/OutEdgeIterator.h | 54 +++++++++++++++++++ dash/include/dash/graph/internal/Graph.h | 36 +++++++++---- .../include/dash/memory/GlobHeapCombinedPtr.h | 43 +++++++++++++++ .../dash/memory/GlobHeapContiguousPtr.h | 43 +++++++++++++++ 9 files changed, 320 insertions(+), 35 deletions(-) diff --git a/dash/include/dash/GlobDynamicCombinedMem.h b/dash/include/dash/GlobDynamicCombinedMem.h index 380f60219..8cfa4c4ec 100644 --- a/dash/include/dash/GlobDynamicCombinedMem.h +++ b/dash/include/dash/GlobDynamicCombinedMem.h @@ -90,6 +90,15 @@ class GlobDynamicCombinedMem { return (_team != nullptr) ? *_team : dash::Team::Null(); } + size_type container_size(team_unit_t unit, size_type index) const { + size_type bucket_size = _bucket_cumul_sizes[unit][index + + _glob_mem_list.size() - 1]; + if(index > 0) { + bucket_size -= _bucket_cumul_sizes[unit][index - 1]; + } + return bucket_size; + } + private: /** diff --git a/dash/include/dash/GlobDynamicContiguousMem.h b/dash/include/dash/GlobDynamicContiguousMem.h index 52a8a4cb9..32c902c33 100644 --- a/dash/include/dash/GlobDynamicContiguousMem.h +++ b/dash/include/dash/GlobDynamicContiguousMem.h @@ -299,6 +299,7 @@ class GlobDynamicContiguousMem _bucket_cumul_sizes[_myid][bucket_num] = bucket_cumul; ++bucket_num; } + _team->barrier(); } // distribute bucket sizes between all units @@ -354,7 +355,7 @@ class GlobDynamicContiguousMem return _lend; } - size_type push_back(container_list_index cont, value_type val) { + void push_back(container_list_index cont, value_type val) { auto cont_it = _container_list.begin(); std::advance(cont_it, cont); auto c_data = *cont_it; @@ -382,16 +383,23 @@ class GlobDynamicContiguousMem c_data.update_lend(); update_lbegin(); update_lend(); - return static_cast(it->size); } - size_type container_size(container_list_index index) const { + size_type container_local_size(container_list_index index) const { auto cont_it = _container_list.begin(); std::advance(cont_it, index); auto c_data = *cont_it; return c_data.container->size() + c_data.unattached_container->size(); } + size_type container_size(team_unit_t unit, size_type index) const { + if(index <= 0) { + return _bucket_cumul_sizes[unit][index]; + } + return _bucket_cumul_sizes[unit][index] - + _bucket_cumul_sizes[unit][index - 1]; + } + size_type size() const { return _size; } diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index f723ec4f7..2e471be17 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -203,8 +203,10 @@ class Graph { = VertexProperties()) { auto in_ref = _glob_mem_in_edge->add_container(_alloc_edges_per_vertex); auto out_ref = _glob_mem_out_edge->add_container(_alloc_edges_per_vertex); - auto max_index = _glob_mem_vertex->container_size(_vertex_container_ref); - vertex_type v(max_index, in_ref, out_ref, prop); + auto max_index = _glob_mem_vertex->container_local_size( + _vertex_container_ref); + vertex_index_type v_index(_myid, max_index); + vertex_type v(v_index, in_ref, out_ref, prop); _glob_mem_vertex->push_back(_vertex_container_ref, v); // TODO: return global index return vertex_index_type(_myid, max_index); @@ -236,14 +238,12 @@ class Graph { const vertex_index_type & target, const EdgeProperties & prop = EdgeProperties() ) { - bool remote_edge_set = false; edge_index_type local_index; if(source.unit == _myid) { local_index = add_local_edge(source, target, prop, _glob_mem_out_edge); } else { - auto edge = edge_type(0, source, target, prop); + edge_type edge(source, target, prop); _remote_edges[source.unit].push_back(edge); - remote_edge_set = true; } if(target.unit == _myid) { // _glob_mem_in_edge == _glob_mem_out_edge for undirected graph types @@ -253,10 +253,14 @@ class Graph { local_index = local_index_tmp; } // do not double-send edges - } else if(!remote_edge_set) { - auto edge = edge_type(0, source, target, prop); + } else if(source.unit != target.unit) { + edge_type edge(source, target, prop); _remote_edges[target.unit].push_back(edge); } + //TODO: handle cases, were both vertices reside on a different unit + + // currently, double edges are allowed for all cases, so we always return + // true return std::make_pair(local_index, true); } @@ -323,6 +327,7 @@ class Graph { // add missing edges to local memory space for(auto edge : edges) { if(edge._source.unit == _myid) { + add_local_edge(edge._source, edge._target, edge.properties, _glob_mem_out_edge); } @@ -391,23 +396,20 @@ class Graph { glob_mem_edge_type * glob_mem_edge ) { edge_index_type local_index(_myid, 0, -1); - auto edge = edge_type(0, source, target, prop); - // if vertex is local, place edge in edge list of the given vertex. - // mark the edge for the next commit otherwise - if(source.unit == _myid) { - auto vertex = _glob_mem_vertex->get(_vertex_container_ref, - source.offset); - edge._local_id = glob_mem_edge->container_size( - vertex._out_edge_ref); - auto local_offset = glob_mem_edge->push_back( - vertex._out_edge_ref, edge); - local_index.unit = source.unit; - local_index.container = vertex._out_edge_ref; - local_index.offset = local_offset; - } else { - //TODO: check, if unit ID is valid - _remote_edges[source.unit].push_back(edge); + auto edge = edge_type(source, target, prop); + auto vertex = _glob_mem_vertex->get(_vertex_container_ref, + source.offset); + auto edge_ref = vertex._out_edge_ref; + if(glob_mem_edge == _glob_mem_in_edge) { + edge_ref = vertex._in_edge_ref; } + local_index = edge_index_type( + _myid, + edge_ref, + glob_mem_edge->container_local_size(edge_ref) + ); + edge._index = local_index; + glob_mem_edge->push_back(edge_ref, edge); return local_index; } diff --git a/dash/include/dash/graph/EdgeIterator.h b/dash/include/dash/graph/EdgeIterator.h index 2e4037af8..7add8087b 100644 --- a/dash/include/dash/graph/EdgeIterator.h +++ b/dash/include/dash/graph/EdgeIterator.h @@ -16,6 +16,8 @@ struct EdgeIteratorWrapper { typedef const local_iterator const_local_iterator; typedef typename Graph::edge_index_type edge_index_type; typedef typename Graph::edge_properties_type edge_properties_type; + typedef typename Graph::vertex_type vertex_type; + typedef typename Graph::vertex_index_type vertex_index_type; /** * Constructs the wrapper. @@ -91,6 +93,58 @@ struct EdgeIteratorWrapper { return _graph->_glob_mem_edge->lend(); } + /** + * Returns global iterator to the beginning of the edge list of the given + * vertex + */ + iterator vbegin(const vertex_type & v) { + return iterator( + _graph->_glob_mem_edge, + v._index.unit, + v._in_edge_ref + v._out_edge_ref, + 0 + ); + } + + /** + * Returns global iterator to the end of the edge list of the given vertex + */ + iterator vend(const vertex_type & v) { + auto edge_ref = v._in_edge_ref + v._out_edge_ref; + return iterator( + _graph->_glob_mem_edge, + v._index.unit, + edge_ref, + _graph->_glob_mem_edge->container_size(v._index.unit, edge_ref) + ); + } + + /** + * Returns global iterator to the beginning of the edge list of the given + * vertex + */ + const_iterator vbegin(const vertex_type & v) const { + return iterator( + _graph->_glob_mem_edge, + v._index.unit, + v._in_edge_ref + v._out_edge_ref, + 0 + ); + } + + /** + * Returns global iterator to the end of the edge list of the given vertex + */ + const_iterator vend(const vertex_type & v) const { + auto edge_ref = v._in_edge_ref + v._out_edge_ref; + return iterator( + _graph->_glob_mem_edge, + v._index.unit, + edge_ref, + _graph->_glob_mem_edge->container_size(v._index.unit, edge_ref) + ); + } + private: graph_type * _graph; diff --git a/dash/include/dash/graph/InEdgeIterator.h b/dash/include/dash/graph/InEdgeIterator.h index 9d28e1cf0..77490567a 100644 --- a/dash/include/dash/graph/InEdgeIterator.h +++ b/dash/include/dash/graph/InEdgeIterator.h @@ -16,6 +16,8 @@ struct InEdgeIteratorWrapper { typedef const local_iterator const_local_iterator; typedef typename Graph::edge_index_type edge_index_type; typedef typename Graph::edge_properties_type edge_properties_type; + typedef typename Graph::vertex_type vertex_type; + typedef typename Graph::vertex_index_type vertex_index_type; /** * Constructs the wrapper. @@ -91,6 +93,58 @@ struct InEdgeIteratorWrapper { return _graph->_glob_mem_in_edge->lend(); } + /** + * Returns global iterator to the beginning of the edge list of the given + * vertex + */ + iterator vbegin(const vertex_type & v) { + return iterator( + _graph->_glob_mem_in_edge, + v._index.unit, + v._in_edge_ref, + 0 + ); + } + + /** + * Returns global iterator to the end of the edge list of the given vertex + */ + iterator vend(const vertex_type & v) { + return iterator( + _graph->_glob_mem_in_edge, + v._index.unit, + v._in_edge_ref, + _graph->_glob_mem_in_edge->container_size(v._index.unit, + v._in_edge_ref) + ); + } + + /** + * Returns global iterator to the beginning of the edge list of the given + * vertex + */ + const_iterator vbegin(const vertex_type & v) const { + return iterator( + _graph->_glob_mem_in_edge, + v._index.unit, + v._in_edge_ref, + 0 + ); + } + + /** + * Returns global iterator to the end of the edge list of the given vertex + */ + const_iterator vend(const vertex_type & v) const { + return iterator( + _graph->_glob_mem_in_edge, + v._index.unit, + v._in_edge_ref, + _graph->_glob_mem_in_edge->container_size(v._index.unit, + v._in_edge_ref) + ); + } + private: graph_type * _graph; diff --git a/dash/include/dash/graph/OutEdgeIterator.h b/dash/include/dash/graph/OutEdgeIterator.h index 381b61abd..0d0523da6 100644 --- a/dash/include/dash/graph/OutEdgeIterator.h +++ b/dash/include/dash/graph/OutEdgeIterator.h @@ -16,6 +16,8 @@ struct OutEdgeIteratorWrapper { typedef const local_iterator const_local_iterator; typedef typename Graph::edge_index_type edge_index_type; typedef typename Graph::edge_properties_type edge_properties_type; + typedef typename Graph::vertex_type vertex_type; + typedef typename Graph::vertex_index_type vertex_index_type; /** * Constructs the wrapper. @@ -91,6 +93,58 @@ struct OutEdgeIteratorWrapper { return _graph->_glob_mem_out_edge->lend(); } + /** + * Returns global iterator to the beginning of the edge list of the given + * vertex + */ + iterator vbegin(const vertex_type & v) { + return iterator( + _graph->_glob_mem_out_edge, + v._index.unit, + v._in_edge_ref, + 0 + ); + } + + /** + * Returns global iterator to the end of the edge list of the given vertex + */ + iterator vend(const vertex_type & v) { + return iterator( + _graph->_glob_mem_out_edge, + v._index.unit, + v._in_edge_ref, + _graph->_glob_mem_out_edge->container_size(v._index.unit, + v._in_edge_ref) + ); + } + + /** + * Returns global iterator to the beginning of the edge list of the given + * vertex + */ + const_iterator vbegin(const vertex_type & v) const { + return iterator( + _graph->_glob_mem_out_edge, + v._index.unit, + v._out_edge_ref, + 0 + ); + } + + /** + * Returns global iterator to the end of the edge list of the given vertex + */ + const_iterator vend(const vertex_type & v) const { + return iterator( + _graph->_glob_mem_out_edge, + v._index.unit, + v._out_edge_ref, + _graph->_glob_mem_out_edge->container_size(v._index.unit, + v._out_edge_ref) + ); + } + private: graph_type * _graph; diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index 46c4bc49a..ec675ad27 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -47,10 +47,13 @@ template class Vertex { typedef typename GraphType::edge_cont_ref_type edge_container_ref; - typedef typename GraphType::vertex_offset_type index_type; + typedef typename GraphType::vertex_index_type index_type; typedef typename GraphType::vertex_properties_type properties_type; friend GraphType; + friend InEdgeIteratorWrapper; + friend OutEdgeIteratorWrapper; + friend EdgeIteratorWrapper; public: @@ -63,12 +66,12 @@ class Vertex { * Creates a vertex with given properties. */ Vertex( - index_type index, + index_type & index, edge_container_ref & in_edge_ref, edge_container_ref & out_edge_ref, properties_type properties = properties_type() ) - : _local_id(index), + : _index(index), _in_edge_ref(in_edge_ref), _out_edge_ref(out_edge_ref), properties(properties) @@ -82,7 +85,7 @@ class Vertex { private: /** Index of the vertex in local index space */ - index_type _local_id; + index_type _index; /** index of the in-edge list belonging to this vertex */ edge_container_ref _in_edge_ref; /** index of the out-edge list belonging to this vertex */ @@ -94,7 +97,7 @@ template class Edge { typedef typename GraphType::vertex_index_type vertex_index_type; - typedef typename GraphType::edge_offset_type index_type; + typedef typename GraphType::edge_index_type index_type; typedef typename GraphType::edge_properties_type properties_type; friend GraphType; @@ -107,15 +110,30 @@ class Edge { Edge() = default; /** - * Creates an edge from its parent vertex to target. + * Creates an edge with index */ Edge( - index_type index, + index_type & index, vertex_index_type source, vertex_index_type target, properties_type properties = properties_type() ) - : _local_id(index), + : _index(index), + _source(source), + _target(target), + properties(properties) + { } + + /** + * Creates an edge without index. Needed for edges that belong to other + * units. + */ + Edge( + vertex_index_type source, + vertex_index_type target, + properties_type properties = properties_type() + ) + : _index(), _source(source), _target(target), properties(properties) @@ -134,7 +152,7 @@ class Edge { /** Target vertex the edge is pointing to */ vertex_index_type _target; /** Index of the edge in local index space */ - index_type _local_id; + index_type _index; }; diff --git a/dash/include/dash/memory/GlobHeapCombinedPtr.h b/dash/include/dash/memory/GlobHeapCombinedPtr.h index cd3d1a3c6..67e35aaef 100644 --- a/dash/include/dash/memory/GlobHeapCombinedPtr.h +++ b/dash/include/dash/memory/GlobHeapCombinedPtr.h @@ -221,6 +221,49 @@ class GlobPtr< "phase:", _idx_bucket_phase); } + /** + * Constructor, creates a global pointer on global memory from unit, bucket + * index and local offset in logical storage order. + */ + template + GlobPtr( + const MemSpaceT * gmem, + team_unit_t unit, + index_type bucket_index, + index_type local_index) + : _globmem(reinterpret_cast(gmem)), + _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), + _lbegin(_globmem->lbegin()), + _idx(0), + _max_idx(gmem->size() - 1), + _myid(gmem->team().myid()), + _idx_unit_id(unit), + _idx_local_idx(0), + _idx_bucket_idx(0), + _idx_bucket_phase(0) + { + DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx)", + "unit:", unit, + "lidx:", local_index); + DASH_ASSERT_LT(unit, _bucket_cumul_sizes->size(), "invalid unit id"); + + for (size_type unit = 0; unit < _idx_unit_id; ++unit) { + auto prec_unit_local_size = (*_bucket_cumul_sizes)[unit].back(); + _idx += prec_unit_local_size; + } + if(bucket_index > 0) { + local_index += (*_bucket_cumul_sizes)[_idx_unit_id.id][bucket_index - 1]; + } + increment(local_index); + DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx) >", + "gidx:", _idx, + "maxidx:", _max_idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bucket:", _idx_bucket_idx, + "phase:", _idx_bucket_phase); + } + /** * Copy constructor. */ diff --git a/dash/include/dash/memory/GlobHeapContiguousPtr.h b/dash/include/dash/memory/GlobHeapContiguousPtr.h index 83a89a827..8833121a4 100644 --- a/dash/include/dash/memory/GlobHeapContiguousPtr.h +++ b/dash/include/dash/memory/GlobHeapContiguousPtr.h @@ -219,6 +219,49 @@ class GlobPtr< "phase:", _idx_bucket_phase); } + /** + * Constructor, creates a global pointer on global memory from unit, bucket + * index and local offset in logical storage order. + */ + template + GlobPtr( + const MemSpaceT * gmem, + team_unit_t unit, + index_type bucket_index, + index_type local_index) + : _globmem(reinterpret_cast(gmem)), + _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), + _lbegin(_globmem->lbegin()), + _idx(0), + _max_idx(gmem->size() - 1), + _myid(gmem->team().myid()), + _idx_unit_id(unit), + _idx_local_idx(0), + _idx_bucket_idx(0), + _idx_bucket_phase(0) + { + DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx)", + "unit:", unit, + "lidx:", local_index); + DASH_ASSERT_LT(unit, _bucket_cumul_sizes->size(), "invalid unit id"); + + for (size_type unit = 0; unit < _idx_unit_id; ++unit) { + auto prec_unit_local_size = (*_bucket_cumul_sizes)[unit].back(); + _idx += prec_unit_local_size; + } + if(bucket_index > 0) { + local_index += (*_bucket_cumul_sizes)[_idx_unit_id.id][bucket_index - 1]; + } + increment(local_index); + DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx) >", + "gidx:", _idx, + "maxidx:", _max_idx, + "unit:", _idx_unit_id, + "lidx:", _idx_local_idx, + "bucket:", _idx_bucket_idx, + "phase:", _idx_bucket_phase); + } + /** * Copy constructor. */ From b52de743a4f37cf2d18b80bdd8a4a13f1c7d641b Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 20 Apr 2017 15:45:32 +0200 Subject: [PATCH 039/102] added various refactorings and comments --- dash/include/dash/Graph.h | 235 +++++++---- dash/include/dash/graph/internal/Graph.h | 68 +++- .../GlobHeapCombinedMem.h} | 84 +++- .../include/dash/memory/GlobHeapCombinedPtr.h | 18 +- .../GlobHeapContiguousMem.h} | 370 ++++++++---------- .../dash/memory/GlobHeapContiguousPtr.h | 18 +- 6 files changed, 468 insertions(+), 325 deletions(-) rename dash/include/dash/{GlobDynamicCombinedMem.h => memory/GlobHeapCombinedMem.h} (80%) rename dash/include/dash/{GlobDynamicContiguousMem.h => memory/GlobHeapContiguousMem.h} (63%) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 2e471be17..23ea6b675 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -7,8 +7,8 @@ #include #include #include -#include -#include +#include +#include #include #include #include @@ -49,8 +49,8 @@ template< GraphDirection Direction = DirectedGraph, //typename DynamicPattern = dash::graph::VertexPartitionedDynamicPattern, typename DynamicPattern = void, - typename VertexProperties = internal::EmptyProperties, // user-defined struct - typename EdgeProperties = internal::EmptyProperties, // user-defined struct + typename VertexProperties = EmptyProperties, // user-defined struct + typename EdgeProperties = EmptyProperties, // user-defined struct typename VertexIndexType = int, typename EdgeIndexType = int, template typename EdgeContainer = std::vector, @@ -63,9 +63,9 @@ class Graph { typedef Graph graph_type; - typedef internal::Vertex vertex_type; - typedef internal::Edge edge_type; + EdgeContainer, VertexContainer> self_t; + typedef Vertex vertex_type; + typedef Edge edge_type; typedef VertexContainer> vertex_container_type; typedef EdgeContainer vertex_it_wrapper; - typedef EdgeIteratorWrapper edge_it_wrapper; - typedef InEdgeIteratorWrapper in_edge_it_wrapper; - typedef OutEdgeIteratorWrapper out_edge_it_wrapper; + typedef VertexIteratorWrapper vertex_it_wrapper; + typedef EdgeIteratorWrapper edge_it_wrapper; + typedef InEdgeIteratorWrapper in_edge_it_wrapper; + typedef OutEdgeIteratorWrapper out_edge_it_wrapper; friend vertex_it_wrapper; friend edge_it_wrapper; friend in_edge_it_wrapper; friend out_edge_it_wrapper; - typedef GlobDynamicContiguousMem< - vertex_container_type> glob_mem_vert_type; - typedef GlobDynamicContiguousMem< - edge_container_type> glob_mem_edge_type; + typedef GlobHeapContiguousMem< + vertex_container_type> glob_mem_vert_type; + typedef GlobHeapContiguousMem< + edge_container_type> glob_mem_edge_type; typedef - GlobDynamicCombinedMem glob_mem_edge_comb_type; - typedef std::vector> edge_list_type; + GlobHeapCombinedMem glob_mem_edge_comb_type; + typedef std::vector> edge_list_type; public: typedef typename - glob_mem_vert_type::container_list_index vertex_cont_ref_type; + glob_mem_vert_type::container_list_index vertex_cont_ref_type; typedef typename - glob_mem_edge_type::container_list_index edge_cont_ref_type; - typedef VertexIndexType vertex_offset_type; - typedef EdgeIndexType edge_offset_type; - typedef internal::VertexIndex vertex_index_type; - typedef internal::EdgeIndex edge_index_type; + glob_mem_edge_type::container_list_index edge_cont_ref_type; + typedef VertexIndexType vertex_offset_type; + typedef EdgeIndexType edge_offset_type; + typedef VertexIndex vertex_index_type; + typedef EdgeIndex edge_index_type; typedef typename - std::make_unsigned::type vertex_size_type; + std::make_unsigned::type vertex_size_type; typedef typename - std::make_unsigned::type edge_size_type; + std::make_unsigned::type edge_size_type; - typedef DynamicPattern pattern_type; - typedef VertexProperties vertex_properties_type; - typedef EdgeProperties edge_properties_type; + typedef DynamicPattern pattern_type; + typedef VertexProperties vertex_properties_type; + typedef EdgeProperties edge_properties_type; - typedef GlobRef reference; + typedef GlobRef reference; typedef typename - glob_mem_vert_type::local_iterator local_vertex_iterator; + glob_mem_vert_type::local_iterator local_vertex_iterator; typedef typename - glob_mem_edge_type::local_iterator local_edge_iterator; + glob_mem_edge_type::local_iterator local_edge_iterator; typedef typename - glob_mem_vert_type::global_iterator global_vertex_iterator; + glob_mem_vert_type::global_iterator global_vertex_iterator; typedef typename - glob_mem_edge_type::global_iterator global_edge_iterator; + glob_mem_edge_type::global_iterator global_edge_iterator; typedef typename - glob_mem_edge_comb_type::global_iterator global_edge_comb_iterator; + glob_mem_edge_comb_type::global_iterator global_edge_comb_iterator; - typedef typename vertex_it_wrapper::iterator vertex_iterator; - typedef typename edge_it_wrapper::iterator edge_iterator; - typedef typename in_edge_it_wrapper::iterator in_edge_iterator; - typedef typename out_edge_it_wrapper::iterator out_edge_iterator; + typedef typename vertex_it_wrapper::iterator vertex_iterator; + typedef typename edge_it_wrapper::iterator edge_iterator; + typedef typename in_edge_it_wrapper::iterator in_edge_iterator; + typedef typename out_edge_it_wrapper::iterator out_edge_iterator; public: @@ -139,12 +139,14 @@ class Graph { public: + //TODO: add constructor that can construct a graph with edge iterators + /** * Constructs an empty graph. */ Graph( - vertex_size_type n_vertices = 0, - edge_size_type n_vertex_edges = 0, + const vertex_size_type n_vertices = 0, + const edge_size_type n_vertex_edges = 0, Team & team = dash::Team::All() ) : _team(&team), @@ -161,46 +163,77 @@ class Graph { } /** - * Returns the number of vertices in the whole graph. + * Copy-constructor. Explicitly deleted. + */ + Graph(const self_t &) = delete; + + /** + * Move-constructor. Explicitly deleted. + */ + Graph(self_t &&) = delete; + + /** + * Copy-assignment operator. Explicitly deleted. + */ + self_t & operator=(const self_t &) = delete; + + /** + * Move-assignment operator. Explicitly deleted. */ - const vertex_size_type num_vertices() const { + self_t & operator=(self_t &&) = delete; + /** + * Returns the number of vertices in the whole graph. + * + * \return The amount of vertices in the whole graph. + */ + vertex_size_type num_vertices() const { + return _glob_mem_vertex->size(); } /** * Returns the index of the vertex with the highest index in the whole graph. + * + * \return Offset of the vertex with the highest index in local address + * space. */ - const vertex_index_type max_vertex_index() const { - + vertex_index_type max_vertex_index() const { + return _local_vertex_max_index; } /** * Returns the number of edges in the whole graph. + * + * \return The amount of edges in the whole graph. */ - const edge_size_type num_edges() const { - + edge_size_type num_edges() const { + return _glob_mem_edge->size(); } /** * Returns the index of the edge with the highest index in the whole graph. + * + * \return Offset of the edge with the highest index in local address space. */ - const edge_index_type max_edge_index() const { - + edge_offset_type max_edge_index() const { + return _local_edge_max_index; } /** * Returns, whether the graph is empty. + * + * \return True, if the graph holds 0 vertices. Fals otherwise. */ bool empty() const { - return true; + return _glob_mem_vertex->size() == 0 ? true : false; } /** - * Adds a vertex with the given properties. - * NOTE: global method. + * Adds a vertex with the given properties locally. + * + * \return Index of the newly created vertex. */ - vertex_index_type add_vertex(const VertexProperties & prop - = VertexProperties()) { + vertex_index_type add_vertex(const VertexProperties & prop) { auto in_ref = _glob_mem_in_edge->add_container(_alloc_edges_per_vertex); auto out_ref = _glob_mem_out_edge->add_container(_alloc_edges_per_vertex); auto max_index = _glob_mem_vertex->container_local_size( @@ -212,9 +245,18 @@ class Graph { return vertex_index_type(_myid, max_index); } + /** + * Adds a vertex locally. + * + * \return Index of the newly created vertex. + */ + vertex_index_type add_vertex() { + VertexProperties prop; + add_vertex(prop); + } + /** * Removes a given vertex. - * NOTE: global method. */ void remove_vertex(const vertex_index_type & v) { @@ -222,21 +264,25 @@ class Graph { /** * Removes all edges (in & out) from the given vertex). - * NOTE: global method. */ - void clear_vertex(vertex_index_type & v) { + void clear_vertex(const vertex_index_type & v) { } /** * Adds an edge between two given vertices with the given properties * locally. - * NOTE: global method. + * Edges that belong to vertices held on a different unit are marked for + * transfer. These edges will be transferred after calling \c barrier(). + * + * \return Pair, with pair::first set to the index of the newly created edge + * and pair::second set to a boolean indicating whether the edge has + * actually been added. */ std::pair add_edge( const vertex_index_type & source, const vertex_index_type & target, - const EdgeProperties & prop = EdgeProperties() + const EdgeProperties & prop ) { edge_index_type local_index; if(source.unit == _myid) { @@ -265,8 +311,24 @@ class Graph { } /** - * Removes an edge between two given vertices. - * NOTE: global method. + * Adds an edge between two given vertices locally. + * Edges that belong to vertices held on a different unit are marked for + * transfer. These edges will be transferred after calling \c barrier(). + * + * \return Pair, with pair::first set to the index of the newly created edge + * and pair::second set to a boolean indicating whether the edge has + * actually been added. + */ + std::pair add_edge( + const vertex_index_type & source, + const vertex_index_type & target + ) { + EdgeProperties prop; + add_edge(source, target, prop); + } + + /** + * Removes the edges between two given vertices. */ void remove_edge(const vertex_index_type & v1, const vertex_index_type & v2) { @@ -274,16 +336,15 @@ class Graph { } /** - * Removes a given edge.a9f5c03d9de8dda4 - * NOTE: global method. + * Removes a given edge. */ void remove_edge(const edge_index_type & e) { } /** - * Commits local changes performed by methods classified as "global" to - * the whole data structure. + * Commits local changes of the graph to global memory space since the last + * call of this method. */ void barrier() { // move all edges that have to be added by other units in a contiguous @@ -335,6 +396,8 @@ class Graph { // _glob_mem_in_edge == _glob_mem_out_edge for undirected graph types add_local_edge(edge._target, edge._source, edge.properties, _glob_mem_in_edge); + //TODO: for directed graphs, should target and source really be mutated + // in the in-edge list? } } // commit changes in local memory space globally @@ -345,7 +408,7 @@ class Graph { } /** - * Globally allocates memory for vertex storage. + * Globally allocates memory for vertex and edge storage. */ bool allocate(vertex_size_type n_vertices, edge_size_type n_vertex_edges) { auto vertex_lcap = dash::math::div_ceil(n_vertices, _team->size()); @@ -367,35 +430,43 @@ class Graph { _glob_mem_edge->add_globmem(*_glob_mem_in_edge); // Register deallocator at the respective team instance _team->register_deallocator(this, std::bind(&Graph::deallocate, this)); + return true; } /** - * Deallocates global memory of this container. + * Deallocates global memory. */ void deallocate() { - // TODO: Delete all objects created on the heap, and look for objects with - // shared ownership if(_glob_mem_vertex != nullptr) { delete _glob_mem_vertex; } + if(_glob_mem_in_edge != nullptr) { + delete _glob_mem_in_edge; + } + if(_glob_mem_out_edge != nullptr) { + delete _glob_mem_out_edge; + } + if(_glob_mem_edge != nullptr) { + delete _glob_mem_edge; + } // Remove deallocator from the respective team instance _team->unregister_deallocator(this, std::bind(&Graph::deallocate, this)); } - vertex_type test() { - auto gptr = _glob_mem_vertex->dart_gptr_at(team_unit_t(1), 0, 0); - return reference(gptr); - } - private: + /** + * Inserts an edge locally. The source vertex must belong to this unit. + * + * \return Index of the newly created edge. + */ edge_index_type add_local_edge( const vertex_index_type & source, const vertex_index_type & target, const EdgeProperties & prop, glob_mem_edge_type * glob_mem_edge ) { - edge_index_type local_index(_myid, 0, -1); + edge_index_type local_index; auto edge = edge_type(source, target, prop); auto vertex = _glob_mem_vertex->get(_vertex_container_ref, source.offset); @@ -417,25 +488,25 @@ class Graph { /** the team containing all units using the container */ Team * _team = nullptr; - /** Global memory allocation and access for sequential memory regions */ + /** Global memory allocation and access to vertices */ glob_mem_vert_type * _glob_mem_vertex = nullptr; - + /** Global memory allocation and access to inbound edges */ glob_mem_edge_type * _glob_mem_in_edge = nullptr; - + /** Global memory allocation and access to outbound edges */ glob_mem_edge_type * _glob_mem_out_edge = nullptr; - + /** Access to inbound and outbound edges */ glob_mem_edge_comb_type * _glob_mem_edge = nullptr; /** Unit ID of the current unit */ team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; /** Index of last added vertex */ vertex_offset_type _local_vertex_max_index = -1; - + /** Index of last added edge */ edge_offset_type _local_edge_max_index = -1; - /** Iterator to the vertex container in GlobMem object */ + /** Index of the vertex container in _glob_mem_vertex */ vertex_cont_ref_type _vertex_container_ref; - + /** Amount of edge elements to be pre-allocated for every vertex */ edge_size_type _alloc_edges_per_vertex; - /** edges that have to be added to vertices on another unit */ + /** Edges that have to be added to vertices on another unit in next commit */ edge_list_type _remote_edges; }; diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index ec675ad27..d982aadaa 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -3,46 +3,74 @@ namespace dash { +/** + * Enum declaring the different graph types. + */ enum GraphDirection { UndirectedGraph, DirectedGraph //BidirectionalGraph }; -namespace internal { - +/** + * Index type for vertices. + */ template struct VertexIndex { + /** + * Default constructor. + */ VertexIndex() = default; + /** + * Index Constructor. + */ VertexIndex(team_unit_t u, IndexType o) : unit(u), offset(o) { } + /** The unit holding the referenced vertex */ team_unit_t unit; + /** The offset of the vertex in local memory space of unit */ IndexType offset; }; +/** + * Index type for edges. + */ template struct EdgeIndex { + /** + * Default constructor. + */ EdgeIndex() = default; + /** + * Index Constructor. + */ EdgeIndex(team_unit_t u, IndexType c, IndexType o) : unit(u), container(c), offset(o) { } + /** The unit holding the referenced edge */ team_unit_t unit; + /** The container in globale memory space of unit containing the edge */ IndexType container; + /** The offset in the referenced container */ IndexType offset; }; +/** + * Vertex type holding properties and references to its corresponding edge + * lists. + */ template class Vertex { @@ -58,7 +86,7 @@ class Vertex { public: /** - * Default constructor needed for GlobRef / GlobSharedRef dereference op + * Default constructor needed for GlobRef / GlobSharedRef dereference op. */ Vertex() = default; @@ -66,10 +94,10 @@ class Vertex { * Creates a vertex with given properties. */ Vertex( - index_type & index, - edge_container_ref & in_edge_ref, - edge_container_ref & out_edge_ref, - properties_type properties = properties_type() + const index_type & index, + const edge_container_ref & in_edge_ref, + const edge_container_ref & out_edge_ref, + const properties_type & properties ) : _index(index), _in_edge_ref(in_edge_ref), @@ -93,6 +121,9 @@ class Vertex { }; +/** + * Edge type holding properties and references to the vertices it belongs to. + */ template class Edge { @@ -105,18 +136,18 @@ class Edge { public: /** - * Default constructor needed for GlobRef / GlobSharedRef dereference op + * Default constructor needed for GlobRef / GlobSharedRef dereference op. */ Edge() = default; /** - * Creates an edge with index + * Creates an edge with index. */ Edge( - index_type & index, - vertex_index_type source, - vertex_index_type target, - properties_type properties = properties_type() + const index_type & index, + const vertex_index_type & source, + const vertex_index_type & target, + const properties_type & properties ) : _index(index), _source(source), @@ -129,9 +160,9 @@ class Edge { * units. */ Edge( - vertex_index_type source, - vertex_index_type target, - properties_type properties = properties_type() + const vertex_index_type & source, + const vertex_index_type & target, + const properties_type & properties ) : _index(), _source(source), @@ -156,10 +187,11 @@ class Edge { }; +/** + * Default property type holding no data. + */ struct EmptyProperties { }; } -} - #endif // DASH__GRAPH__INTERNAL__GRAPH_H__INCLUDED diff --git a/dash/include/dash/GlobDynamicCombinedMem.h b/dash/include/dash/memory/GlobHeapCombinedMem.h similarity index 80% rename from dash/include/dash/GlobDynamicCombinedMem.h rename to dash/include/dash/memory/GlobHeapCombinedMem.h index 8cfa4c4ec..85e28f5a8 100644 --- a/dash/include/dash/GlobDynamicCombinedMem.h +++ b/dash/include/dash/memory/GlobHeapCombinedMem.h @@ -1,5 +1,5 @@ -#ifndef DASH__GLOB_DYNAMIC_COMBINED_MEM_H -#define DASH__GLOB_DYNAMIC_COMBINED_MEM_H +#ifndef DASH__MEMORY__GLOB_HEAP_COMBINED_MEM_H +#define DASH__MEMORY__GLOB_HEAP_COMBINED_MEM_H #include #include @@ -7,11 +7,11 @@ namespace dash { template -class GlobDynamicCombinedMem { +class GlobHeapCombinedMem { public: - typedef GlobDynamicCombinedMem self_t; + typedef GlobHeapCombinedMem self_t; typedef GlobMemType glob_mem_type; typedef typename glob_mem_type::index_type index_type; typedef typename glob_mem_type::size_type size_type; @@ -30,12 +30,43 @@ class GlobDynamicCombinedMem { public: - GlobDynamicCombinedMem(Team & team = dash::Team::All()) + /** + * Constructor. + */ + GlobHeapCombinedMem(Team & team = dash::Team::All()) : _bucket_cumul_sizes(team.size()), _team(&team), _buckets() { } + /** + * Default constructor. Explicitly deleted. + */ + GlobHeapCombinedMem() = delete; + + /** + * Copy constructor. + */ + GlobHeapCombinedMem(const self_t & other) = default; + + /** + * Move constructor. + */ + GlobHeapCombinedMem(self_t && other) = default; + + /** + * Copy-assignment operator. + */ + self_t & operator=(const self_t & other) = default; + + /** + * Move-assignment operator. + */ + self_t & operator=(self_t && other) = default; + + /** + * Adds a GlobMem object to the unifier. + */ void add_globmem(glob_mem_type & glob_mem) { // only GlobMem objects with the same Team can be used if(*_team == glob_mem.team()) { @@ -44,6 +75,11 @@ class GlobDynamicCombinedMem { } } + /** + * Updates memory spaces of underlying GlobMem objects. + * + * Non-collective operation. + */ void commit() { update_bucket_sizes(); // TODO: The bucket list should normally update on every element insertion, @@ -66,30 +102,51 @@ class GlobDynamicCombinedMem { return gmem->dart_gptr_at(unit, gmem_bucket_index, bucket_phase); } + /** + * Iterator to the beginning of the memory space. + */ global_iterator begin() const { return _begin; } + /** + * Iterator past the end of the memory space. + */ global_iterator end() const { return _end; } + /** + * Iterator to the beginning of the mamory space's local portion. + */ local_iterator lbegin() const { return _lbegin; } + /** + * Iterator to the end of the memory space's local portion. + */ local_iterator lend() const { return _lend; } + /** + * Returns the amount of elements currently available in global memory space. + */ size_type size() const { return _size; } + /** + * Returns the team containing all units associated with this memory space. + */ Team & team() const { return (_team != nullptr) ? *_team : dash::Team::Null(); } + /** + * Returns the size of a the memory space belonging to a certain bucket. + */ size_type container_size(team_unit_t unit, size_type index) const { size_type bucket_size = _bucket_cumul_sizes[unit][index + _glob_mem_list.size() - 1]; @@ -148,6 +205,9 @@ class GlobDynamicCombinedMem { } } + /** + * Updates the bucket list. + */ void update_bucket_list() { _buckets.clear(); std::vector + * Update internal native pointer of the final address of the local memory * of a unit. */ void update_lend() noexcept @@ -249,5 +317,5 @@ class GlobDynamicCombinedMem { } -#endif //DASH__GLOB_DYNAMIC_COMBINED_MEM_H +#endif //DASH__MEMORY__GLOB_HEAP_COMBINED_MEM_H diff --git a/dash/include/dash/memory/GlobHeapCombinedPtr.h b/dash/include/dash/memory/GlobHeapCombinedPtr.h index 67e35aaef..f88c397e1 100644 --- a/dash/include/dash/memory/GlobHeapCombinedPtr.h +++ b/dash/include/dash/memory/GlobHeapCombinedPtr.h @@ -22,7 +22,7 @@ namespace dash { // Forward-declaration template -class GlobDynamicCombinedMem; +class GlobHeapCombinedMem; /** * Iterator on global buckets. Represents global pointer type. @@ -30,10 +30,10 @@ class GlobDynamicCombinedMem; template class GlobPtr< ElementType, - GlobDynamicCombinedMem + GlobHeapCombinedMem > { - typedef GlobDynamicCombinedMem GlobHeapMemType; + typedef GlobHeapCombinedMem GlobHeapMemType; typedef GlobPtr self_t; template< @@ -43,7 +43,7 @@ class GlobPtr< std::ostream & os, const dash::GlobPtr< ElementType_, - GlobDynamicCombinedMem + GlobHeapCombinedMem > & gptr); friend GlobPtr; @@ -75,7 +75,7 @@ class GlobPtr< struct rebind { typedef GlobPtr< U, - GlobDynamicContiguousMem< + GlobHeapContiguousMem< typename AllocatorType::template rebind< typename std::remove_const::type >::other @@ -718,13 +718,13 @@ template< auto distance( /// Global pointer to the first position in the global sequence const dash::GlobPtr< - ElementType, GlobDynamicCombinedMem + ElementType, GlobHeapCombinedMem > & first, /// Global pointer to the final position in the global sequence const dash::GlobPtr< - ElementType, GlobDynamicCombinedMem + ElementType, GlobHeapCombinedMem > & last) --> typename GlobDynamicCombinedMem::index_type +-> typename GlobHeapCombinedMem::index_type { return last - first; } @@ -736,7 +736,7 @@ std::ostream & operator<<( std::ostream & os, const dash::GlobPtr< ElementType, - GlobDynamicCombinedMem + GlobHeapCombinedMem > & gptr) { std::ostringstream ss; diff --git a/dash/include/dash/GlobDynamicContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h similarity index 63% rename from dash/include/dash/GlobDynamicContiguousMem.h rename to dash/include/dash/memory/GlobHeapContiguousMem.h index 32c902c33..59e4ee0c6 100644 --- a/dash/include/dash/GlobDynamicContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -1,41 +1,31 @@ -#ifndef DASH__GLOB_DYNAMIC_SEQUENTIAL_MEM_H_ -#define DASH__GLOB_DYNAMIC_SEQUENTIAL_MEM_H_ +#ifndef DASH__MEMORY__GLOB_HEAP_CONTIGUOUS_MEM_H_ +#define DASH__MEMORY__GLOB_HEAP_CONTIGUOUS_MEM_H_ #include #include #include #include -#include #include -#include -#include - -#include -#include #include #include #include #include -#include +#include #include #include #include -#include -#include -#include -#include - -#include - namespace dash { namespace internal { +/** + * Helper class encapsulating single bucket container. + */ template struct container_data { @@ -49,93 +39,45 @@ struct container_data { typedef typename GlobMemType::bucket_cumul_sizes_map::difference_type bucket_sizes_index; - container_data(size_type n_local_elem, bucket_sizes_index b_index) + /** + * Constructor. Creates containers for globally available elements and + * locally available (unattached) elements. + */ + container_data(size_type n_local_elem) : container(new container_type()), - unattached_container(new container_type()), - bucket_index(b_index) + unattached_container(new container_type()) { container->reserve(n_local_elem); - bucket_type cont_bucket { - 0, - container->data(), - DART_GPTR_NULL, - false - }; - bucket_type unattached_cont_bucket { - 0, - unattached_container->data(), - DART_GPTR_NULL, - false - }; - buckets.push_back(cont_bucket); - buckets.push_back(unattached_cont_bucket); - - update_lbegin(); - update_lend(); - } - - /** - * Native pointer of the initial address of the local memory of - * a unit. - * - */ - void update_lbegin() noexcept - { - local_iterator unit_lbegin( - // iteration space - buckets.begin(), buckets.end(), - // position in iteration space - 0, - // bucket at position in iteration space, - // offset in bucket - buckets.begin(), 0); - lbegin = unit_lbegin; } /** - * Update internal native pointer of the final address of the local memory#include - * of a unit. + * Default constructor. Explicitly deleted. */ - void update_lend() noexcept - { - local_iterator unit_lend( - // iteration space - buckets.begin(), buckets.end(), - // position in iteration space - container->size() + unattached_container->size(), - // bucket at position in iteration space, - // offset in bucket - buckets.end(), 0); - lend = unit_lend; - } + container_data() = delete; - container_type * container; - container_type * unattached_container; - bucket_list buckets; - bucket_type * container_bucket; - bucket_type * unattached_container_bucket; - local_iterator lbegin; - local_iterator lend; - bucket_sizes_index bucket_index; + /** container for globally available elements */ + std::shared_ptr container; + /** container for locally available (unattached) elements */ + std::shared_ptr unattached_container; + /** pointer to corresponding bucket in GlobHeapContiguousMem object */ + bucket_type * container_bucket; + /** pointer to corresponding bucket in GlobHeapContiguousMem object */ + bucket_type * unattached_container_bucket; }; } -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -// NOTE: Must be compatible (= same interface) to GlobDynamicMem so it -// can be replaced in e.g. -// -// dash::Grap -// dash::Grap -// -// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +//TODO: adapt to GlobMem concept +/** + * Global memory space for multiple, dynaimcally allocated contiguous memory + * regions. + */ template -class GlobDynamicContiguousMem -{ -private: - typedef GlobDynamicContiguousMem self_t; +class GlobHeapContiguousMem { +private: + typedef GlobHeapContiguousMem self_t; public: typedef ContainerType container_type; @@ -157,44 +99,61 @@ class GlobDynamicContiguousMem template friend class dash::GlobPtr; - - friend class GlobDynamicCombinedMem; + friend class GlobHeapCombinedMem; public: + /** - * Constructor + * Constructor. */ - GlobDynamicContiguousMem(Team & team = dash::Team::All()) - : _buckets(), - _global_buckets(), - _team(&team), - _teamid(team.dart_id()), - _nunits(team.size()), - _myid(team.myid()), - _bucket_cumul_sizes(team.size()) + GlobHeapContiguousMem(Team & team = dash::Team::All()) + : _container_list(new container_list_type()), + _buckets(), + _global_buckets(), + _team(&team), + _teamid(team.dart_id()), + _nunits(team.size()), + _myid(team.myid()), + _bucket_cumul_sizes(team.size()) { } + /** + * Adds a new bucket into memory space. + */ container_list_index add_container(size_type n_elements) { increment_bucket_sizes(); - auto bucket_index = _bucket_cumul_sizes[_myid].size(); - auto c_data = data_type(n_elements, bucket_index); - // TODO: use pointers instead of copies -> no maintenance of 2 object - // copies - auto it = _buckets.insert(_buckets.end(), - c_data.buckets.begin(), c_data.buckets.end()); - c_data.container_bucket = &(*it); + auto c_data = data_type(n_elements); + + // create bucket data and add to bucket list + bucket_type cont_bucket { + 0, + c_data.container->data(), + DART_GPTR_NULL, + false + }; + bucket_type unattached_cont_bucket { + 0, + c_data.unattached_container->data(), + DART_GPTR_NULL, + false + }; + _buckets.push_back(cont_bucket); + // add a pointer to the corresponding bucket data + c_data.container_bucket = &(_buckets.back()); // for global iteration, only _container's bucket is needed - _global_buckets.push_back(&(*it)); - ++it; - c_data.unattached_container_bucket = &(*it); - // insert won't invalidate iterators for std::list, so we can use them - // to access the container - _container_list.push_back(c_data); - return _container_list.size() - 1; + _global_buckets.push_back(&(_buckets.back())); + _buckets.push_back(unattached_cont_bucket); + c_data.unattached_container_bucket = &(_buckets.back()); + + _container_list->push_back(c_data); + return _container_list->size() - 1; } + /** + * Returns a reference to the requested element. + */ value_type & get(container_list_index cont, index_type pos) { - auto it = _container_list.begin(); + auto it = _container_list->begin(); std::advance(it, cont); auto c_data = *it; if(c_data.container->size() > pos) { @@ -207,25 +166,43 @@ class GlobDynamicContiguousMem /** * Destructor, collectively frees underlying global memory. */ - ~GlobDynamicContiguousMem() { } + ~GlobHeapContiguousMem() { } - GlobDynamicContiguousMem() = delete; + /** + * Default constructor. Explicitly deleted. + */ + GlobHeapContiguousMem() = delete; /** * Copy constructor. */ - GlobDynamicContiguousMem(const self_t & other) = default; + GlobHeapContiguousMem(const self_t & other) = default; /** - * Assignment operator. + * Move constructor. */ - self_t & operator=(const self_t & rhs) = default; + GlobHeapContiguousMem(self_t && other) = default; + /** + * Copy-assignment operator. + */ + self_t & operator=(const self_t & other) = default; + + /** + * Move-ssignment operator. + */ + self_t & operator=(self_t && other) = default; + + /** + * Commit local changes in memory to global memory space. + * + * Collective operation. + */ void commit() { // Gather information about the max amount of containers a single unit // currently holds std::vector container_count(_team->size()); - int my_container_count = _container_list.size(); + int my_container_count = _container_list->size(); dart_allgather(&my_container_count, container_count.data(), sizeof(size_type), DART_TYPE_BYTE, _team->dart_id()); auto max_containers = std::max_element(container_count.begin(), @@ -243,8 +220,8 @@ class GlobDynamicContiguousMem // holds less containers { data_type * c_data; - auto c_data_it = _container_list.begin(); - auto current_size = _container_list.size(); + auto c_data_it = _container_list->begin(); + auto current_size = _container_list->size(); for(int i = 0; i < *max_containers; ++i) { if(i < current_size) { c_data = &(*c_data_it); @@ -254,26 +231,17 @@ class GlobDynamicContiguousMem c_data->unattached_container->end()); c_data->unattached_container->clear(); // update memory location & size of _container - auto it = c_data->buckets.begin(); - it->lptr = c_data->container->data(); - it->size = c_data->container->size(); c_data->container_bucket->lptr = c_data->container->data(); c_data->container_bucket->size = c_data->container->size(); // update memory location & size of _unattached_container - ++it; - it->lptr = c_data->unattached_container->data(); - it->size = 0; c_data->unattached_container_bucket->lptr = c_data->unattached_container->data(); c_data->unattached_container_bucket->size = 0; - - c_data->update_lbegin(); - c_data->update_lend(); ++c_data_it; } else { // if other units add more containers, create an empty container to // store a dart_gptr for the collective allocation - auto cont_it = _container_list.begin(); + auto cont_it = _container_list->begin(); auto cont_index = add_container(0); std::advance(cont_it, cont_index); c_data = &(*cont_it); @@ -306,7 +274,7 @@ class GlobDynamicContiguousMem // TODO: use one allgather for all buckets // TODO: make it work for unevenly distributed amount of buckets auto bucket_count = _bucket_cumul_sizes[_myid].size(); - for(auto c_data : _container_list) { + for(auto c_data : *_container_list) { std::vector bucket_sizes(bucket_count * _team->size()); std::vector local_buckets(_bucket_cumul_sizes[_myid]); dart_allgather(local_buckets.data(), bucket_sizes.data(), @@ -323,53 +291,52 @@ class GlobDynamicContiguousMem update_lbegin(); update_lend(); - int b_num = 0; - /* - std::cout << "-----------------" << std::endl; - for(auto el : _bucket_cumul_sizes) { - for(auto el2 : el) { - std::cout << "bucket " << b_num <<" of " << _myid << ": " << el2 << std::endl; - } - b_num++; - } - std::cout << "-----------------" << std::endl; - */ + _begin = global_iterator(this, 0); _end = global_iterator(this, _size); } + /** + * Iterator to the beginning of the memory space. + */ global_iterator begin() const { return _begin; } + /** + * Iterator to the end of the memory space. + */ global_iterator end() const { return _end; } + /** + * Iterator to the beginning of the memory space's local portion. + */ local_iterator lbegin() const { return _lbegin; } + /** + * Iterator to the end of the memory space's local portion. + */ local_iterator lend() const { return _lend; } - void push_back(container_list_index cont, value_type val) { - auto cont_it = _container_list.begin(); + /** + * Insert value at the end of the given bucket. + */ + void push_back(container_list_index cont, value_type & val) { + auto cont_it = _container_list->begin(); std::advance(cont_it, cont); auto c_data = *cont_it; - // bucket of _container - auto it = c_data.buckets.begin(); // use _unattached container, if _container is full // we don't want a realloc of _container because this changes the memory // location, which invalidates global pointers of other units if(c_data.container->capacity() == c_data.container->size()) { - // bucket of _unattached_container - ++it; c_data.unattached_container->push_back(val); - // adding data might change the memory location - it->lptr = c_data.unattached_container->data(); c_data.unattached_container_bucket->lptr = c_data.unattached_container->data(); ++(c_data.unattached_container_bucket->size); @@ -377,21 +344,25 @@ class GlobDynamicContiguousMem c_data.container->push_back(val); ++(c_data.container_bucket->size); } - ++(it->size); ++_local_size; - c_data.update_lbegin(); - c_data.update_lend(); + update_lbegin(); update_lend(); } + /** + * Returns the local size of a given bucket. + */ size_type container_local_size(container_list_index index) const { - auto cont_it = _container_list.begin(); + auto cont_it = _container_list->begin(); std::advance(cont_it, index); auto c_data = *cont_it; return c_data.container->size() + c_data.unattached_container->size(); } + /** + * Returns the global size of a given bucket. + */ size_type container_size(team_unit_t unit, size_type index) const { if(index <= 0) { return _bucket_cumul_sizes[unit][index]; @@ -400,14 +371,21 @@ class GlobDynamicContiguousMem _bucket_cumul_sizes[unit][index - 1]; } + /** + * Returns the amount of elements in global memory space. + */ size_type size() const { return _size; } + /** + * Returns the team containing all units associated with this memory space. + */ Team & team() const { return (_team != nullptr) ? *_team : dash::Team::Null(); } + // NOTE: method copied from GlobHeapMem.h /** * Global pointer referencing an element position in a unit's bucket. */ @@ -455,32 +433,10 @@ class GlobDynamicContiguousMem return dart_gptr; } - /** - * Number of elements in local memory space of given unit. - * - * \return Local capacity as published by the specified unit in last - * commit. - */ - inline size_type local_size(team_unit_t unit) const - { - DASH_LOG_TRACE("GlobHeapMem.local_size(u)", "unit:", unit); - DASH_ASSERT_RANGE(0, unit, _nunits-1, "unit id out of range"); - DASH_LOG_TRACE_VAR("GlobHeapMem.local_size", - _bucket_cumul_sizes[unit]); - size_type unit_local_size; - if (unit == _myid) { - // Value of _local_sizes[u] is the local size as visible by the unit, - // i.e. including size of unattached buckets. - unit_local_size = _local_size; - } else { - unit_local_size = _bucket_cumul_sizes[unit].back(); - } - DASH_LOG_TRACE("GlobHeapMem.local_size >", unit_local_size); - return unit_local_size; - } - private: - /** + + // NOTE: method copied from GlobHeapMem.h + /** * Native pointer of the initial address of the local memory of * a unit. * @@ -498,8 +454,9 @@ class GlobDynamicContiguousMem _lbegin = unit_lbegin; } + // NOTE: method copied from GlobHeapMem.h /** - * Update internal native pointer of the final address of the local memory#include + * Update internal native pointer of the final address of the local memory * of a unit. */ void update_lend() noexcept @@ -515,6 +472,9 @@ class GlobDynamicContiguousMem _lend = unit_lend; } + /** + * Increments the bucket count of each unit by one. + */ void increment_bucket_sizes() { for(auto it = _bucket_cumul_sizes.begin(); it != _bucket_cumul_sizes.end(); ++it) { @@ -525,26 +485,38 @@ class GlobDynamicContiguousMem private: - container_list_type _container_list; - container_type * _container; - container_type * _unattached_container = nullptr; - bucket_list _buckets; - bucket_ptr_list _global_buckets; - Team * _team; - dart_team_t _teamid; - size_type _nunits = 0; - team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; - global_iterator _begin; - global_iterator _end; - local_iterator _lbegin; - local_iterator _lend; - bucket_cumul_sizes_map _bucket_cumul_sizes; - size_type _size = 0; - size_type _local_size = 0; + /** List of all attached container */ + std::shared_ptr _container_list; + /** List of buckets for GlobHeapLocalPtr */ + bucket_list _buckets; + /** list of buckets available for global iteration */ + bucket_ptr_list _global_buckets; + /** Team associated with this memory space */ + Team * _team; + /** ID of the team */ + dart_team_t _teamid; + /** Number of units in the team */ + size_type _nunits = 0; + /** ID of this unit */ + team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; + /** Iterator to the beginning ot the memory space */ + global_iterator _begin; + /** Iterator to the end ot the memory space */ + global_iterator _end; + /** Iterator to the beginning of this memory space's local portion */ + local_iterator _lbegin; + /** Iterator to the end of this memory space's local portion */ + local_iterator _lend; + /** Accumulated sizes of the buckets of each unit. See GlobHeapMem */ + bucket_cumul_sizes_map _bucket_cumul_sizes; + /** Global size of the memory space */ + size_type _size = 0; + /** Local size of the memory space */ + size_type _local_size = 0; }; } -#endif // DASH__GLOB_DYNAMIC_SEQUENTIAL_MEM_H_ +#endif // DASH_MEMORY__GLOB_HEAP_CONTIGUOUS_MEM_H_ diff --git a/dash/include/dash/memory/GlobHeapContiguousPtr.h b/dash/include/dash/memory/GlobHeapContiguousPtr.h index 8833121a4..a1e7af3aa 100644 --- a/dash/include/dash/memory/GlobHeapContiguousPtr.h +++ b/dash/include/dash/memory/GlobHeapContiguousPtr.h @@ -22,7 +22,7 @@ namespace dash { // Forward-declaration template -class GlobDynamicContiguousMem; +class GlobHeapContiguousMem; /** * Iterator on global buckets. Represents global pointer type. @@ -30,10 +30,10 @@ class GlobDynamicContiguousMem; template class GlobPtr< ElementType, - GlobDynamicContiguousMem + GlobHeapContiguousMem > { - typedef GlobDynamicContiguousMem GlobHeapMemType; + typedef GlobHeapContiguousMem GlobHeapMemType; typedef GlobPtr self_t; template< @@ -43,7 +43,7 @@ class GlobPtr< std::ostream & os, const dash::GlobPtr< ElementType_, - GlobDynamicContiguousMem + GlobHeapContiguousMem > & gptr); public: @@ -73,7 +73,7 @@ class GlobPtr< struct rebind { typedef GlobPtr< U, - GlobDynamicContiguousMem< + GlobHeapContiguousMem< typename AllocatorType::template rebind< typename std::remove_const::type >::other @@ -716,13 +716,13 @@ template< auto distance( /// Global pointer to the first position in the global sequence const dash::GlobPtr< - ElementType, GlobDynamicContiguousMem + ElementType, GlobHeapContiguousMem > & first, /// Global pointer to the final position in the global sequence const dash::GlobPtr< - ElementType, GlobDynamicContiguousMem + ElementType, GlobHeapContiguousMem > & last) --> typename GlobDynamicContiguousMem::index_type +-> typename GlobHeapContiguousMem::index_type { return last - first; } @@ -734,7 +734,7 @@ std::ostream & operator<<( std::ostream & os, const dash::GlobPtr< ElementType, - GlobDynamicContiguousMem + GlobHeapContiguousMem > & gptr) { std::ostringstream ss; From d28e5f0ec783a20b7fd82dbdb852a42f53166c8e Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 20 Apr 2017 15:56:17 +0200 Subject: [PATCH 040/102] added missing memory detach --- .../dash/memory/GlobHeapContiguousMem.h | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h index 59e4ee0c6..360cfc244 100644 --- a/dash/include/dash/memory/GlobHeapContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -203,8 +203,11 @@ class GlobHeapContiguousMem { // currently holds std::vector container_count(_team->size()); int my_container_count = _container_list->size(); - dart_allgather(&my_container_count, container_count.data(), - sizeof(size_type), DART_TYPE_BYTE, _team->dart_id()); + DASH_ASSERT_RETURNS( + dart_allgather(&my_container_count, container_count.data(), + sizeof(size_type), DART_TYPE_BYTE, _team->dart_id()), + DART_OK + ); auto max_containers = std::max_element(container_count.begin(), container_count.end()); @@ -247,15 +250,26 @@ class GlobHeapContiguousMem { c_data = &(*cont_it); } - // attach new container to global memory space + // detach old container location from global memory space, if it has + // been attached before + if(c_data->container_bucket->gptr != DART_GPTR_NULL) { + DASH_ASSERT_RETURNS( + dart_team_memderegister(c_data->container_bucket->gptr), + DART_OK + ); + } + + // attach new container location to global memory space dart_gptr_t gptr = DART_GPTR_NULL; dart_storage_t ds = dart_storage(c_data->container->size()); - dart_team_memregister( + DASH_ASSERT_RETURNS( + dart_team_memregister( _team->dart_id(), ds.nelem, ds.dtype, c_data->container->data(), - &gptr + &gptr), + DART_OK ); // no need to update gptr of local bucket list in c_data c_data->container_bucket->gptr = gptr; @@ -277,8 +291,11 @@ class GlobHeapContiguousMem { for(auto c_data : *_container_list) { std::vector bucket_sizes(bucket_count * _team->size()); std::vector local_buckets(_bucket_cumul_sizes[_myid]); - dart_allgather(local_buckets.data(), bucket_sizes.data(), - sizeof(size_type) * local_buckets.size(), DART_TYPE_BYTE, _team->dart_id()); + DASH_ASSERT_RETURNS( + dart_allgather(local_buckets.data(), bucket_sizes.data(), + sizeof(size_type) * local_buckets.size(), DART_TYPE_BYTE, _team->dart_id()), + DART_OK + ); _size = 0; auto begin = bucket_sizes.begin(); for(int i = 0; i < _team->size(); ++i) { From e27087802eb3ae5b13cd29efac4e633bc5e06437 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Fri, 28 Apr 2017 18:33:11 +0200 Subject: [PATCH 041/102] minor bug fixes --- dash/include/dash/Graph.h | 44 +++++++++++++------ dash/include/dash/graph/OutEdgeIterator.h | 6 +-- dash/include/dash/graph/VertexIterator.h | 18 +++----- dash/include/dash/graph/internal/Graph.h | 14 ++++++ .../dash/memory/GlobHeapContiguousMem.h | 4 +- 5 files changed, 55 insertions(+), 31 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 23ea6b675..76f0c8726 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -234,8 +234,11 @@ class Graph { * \return Index of the newly created vertex. */ vertex_index_type add_vertex(const VertexProperties & prop) { - auto in_ref = _glob_mem_in_edge->add_container(_alloc_edges_per_vertex); auto out_ref = _glob_mem_out_edge->add_container(_alloc_edges_per_vertex); + auto in_ref = out_ref; + if(_glob_mem_in_edge != _glob_mem_out_edge) { + in_ref = _glob_mem_in_edge->add_container(_alloc_edges_per_vertex); + } auto max_index = _glob_mem_vertex->container_local_size( _vertex_container_ref); vertex_index_type v_index(_myid, max_index); @@ -353,7 +356,7 @@ class Graph { std::vector remote_edges_count(_team->size()); std::vector remote_edges_displs(_team->size()); for(int i = 0; i < _remote_edges.size(); ++i) { - for(auto remote_edge : _remote_edges[i]) { + for(auto & remote_edge : _remote_edges[i]) { remote_edges.push_back(remote_edge); remote_edges_count[i] += sizeof(edge_type); } @@ -364,8 +367,11 @@ class Graph { _remote_edges.clear(); // exchange amount of edges to be transferred with other units std::vector edge_count(_team->size()); - dart_alltoall(remote_edges_count.data(), edge_count.data(), - sizeof(std::size_t), DART_TYPE_BYTE, _team->dart_id()); + DASH_ASSERT_RETURNS( + dart_alltoall(remote_edges_count.data(), edge_count.data(), + sizeof(std::size_t), DART_TYPE_BYTE, _team->dart_id()), + DART_OK + ); int total_count = 0; std::vector edge_displs(_team->size()); for(int i = 0; i < edge_count.size(); ++i) { @@ -376,19 +382,21 @@ class Graph { } // exchange edges std::vector edges(total_count / sizeof(edge_type)); - dart_alltoallv(remote_edges.data(), - remote_edges_count.data(), - remote_edges_displs.data(), - DART_TYPE_BYTE, - edges.data(), - edge_count.data(), - edge_displs.data(), - _team->dart_id() + DASH_ASSERT_RETURNS( + dart_alltoallv(remote_edges.data(), + remote_edges_count.data(), + remote_edges_displs.data(), + DART_TYPE_BYTE, + edges.data(), + edge_count.data(), + edge_displs.data(), + _team->dart_id() + ), + DART_OK ); // add missing edges to local memory space for(auto edge : edges) { if(edge._source.unit == _myid) { - add_local_edge(edge._source, edge._target, edge.properties, _glob_mem_out_edge); } @@ -443,7 +451,8 @@ class Graph { if(_glob_mem_in_edge != nullptr) { delete _glob_mem_in_edge; } - if(_glob_mem_out_edge != nullptr) { + if(_glob_mem_out_edge != nullptr + && _glob_mem_out_edge != _glob_mem_in_edge) { delete _glob_mem_out_edge; } if(_glob_mem_edge != nullptr) { @@ -453,6 +462,13 @@ class Graph { _team->unregister_deallocator(this, std::bind(&Graph::deallocate, this)); } + /** + * Returns the team containing all units associated with this container + */ + Team & team() const { + return *_team; + } + private: /** diff --git a/dash/include/dash/graph/OutEdgeIterator.h b/dash/include/dash/graph/OutEdgeIterator.h index 0d0523da6..86c5f2546 100644 --- a/dash/include/dash/graph/OutEdgeIterator.h +++ b/dash/include/dash/graph/OutEdgeIterator.h @@ -101,7 +101,7 @@ struct OutEdgeIteratorWrapper { return iterator( _graph->_glob_mem_out_edge, v._index.unit, - v._in_edge_ref, + v._out_edge_ref, 0 ); } @@ -113,9 +113,9 @@ struct OutEdgeIteratorWrapper { return iterator( _graph->_glob_mem_out_edge, v._index.unit, - v._in_edge_ref, + v._out_edge_ref, _graph->_glob_mem_out_edge->container_size(v._index.unit, - v._in_edge_ref) + v._out_edge_ref) ); } diff --git a/dash/include/dash/graph/VertexIterator.h b/dash/include/dash/graph/VertexIterator.h index 9404c5493..9a6b7c84c 100644 --- a/dash/include/dash/graph/VertexIterator.h +++ b/dash/include/dash/graph/VertexIterator.h @@ -3,14 +3,6 @@ namespace dash { -class VertexIterator { - -}; - -class LocalVertexIterator { - -}; - /** * Wrapper for the vertex iterators of the graph. */ @@ -23,7 +15,7 @@ struct VertexIteratorWrapper { typedef typename Graph::local_vertex_iterator local_iterator; typedef const local_iterator const_local_iterator; typedef typename Graph::vertex_index_type vertex_index_type; - typedef typename Graph::vertex_properties_type vertex_properties_type; + typedef typename Graph::vertex_type vertex_type; /** * Constructs the wrapper. @@ -35,8 +27,12 @@ struct VertexIteratorWrapper { /** * Returns a property object for the given vertex. */ - vertex_properties_type & operator[](const vertex_index_type & v) const { - + vertex_type & operator[](const vertex_index_type & v) const { + if(_graph->_myid == v.unit) { + return _graph->_glob_mem_vertex->get(_graph->_vertex_container_ref, + v.offset); + } + // TODO: handle error here } /** diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index d982aadaa..361a1545c 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -175,6 +175,20 @@ class Edge { /** Properties of this edge */ properties_type properties; + /** + * Returns the source vertex of the edge + */ + vertex_index_type source() const { + return _source; + } + + /** + * Returns the target vertex of the edge + */ + vertex_index_type target() const { + return _target; + } + private: //TODO: Examine, if saving source can be avoided diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h index 360cfc244..509ac61b0 100644 --- a/dash/include/dash/memory/GlobHeapContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -274,8 +274,6 @@ class GlobHeapContiguousMem { // no need to update gptr of local bucket list in c_data c_data->container_bucket->gptr = gptr; - //TODO: If possible, avoid adding unattached_container to global - // iteration space with size 0 // update cumulated bucket sizes bucket_cumul += c_data->container->size(); _bucket_cumul_sizes[_myid][bucket_num] = bucket_cumul; @@ -399,7 +397,7 @@ class GlobHeapContiguousMem { * Returns the team containing all units associated with this memory space. */ Team & team() const { - return (_team != nullptr) ? *_team : dash::Team::Null(); + return *_team; } // NOTE: method copied from GlobHeapMem.h From 34deff0f38880cbe52cf9033e47ce84866faefd5 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Fri, 28 Apr 2017 18:34:18 +0200 Subject: [PATCH 042/102] fixed include error --- dash/include/dash/memory/GlobHeapContiguousMem.h | 1 - 1 file changed, 1 deletion(-) diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h index 509ac61b0..8d104a603 100644 --- a/dash/include/dash/memory/GlobHeapContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -10,7 +10,6 @@ #include #include -#include #include #include From 8ca57965ff002b52e16540a573f1d03465c05da6 Mon Sep 17 00:00:00 2001 From: Tobias Fuchs Date: Fri, 28 Apr 2017 18:21:16 +0000 Subject: [PATCH 043/102] Added Stefan's graph example --- dash/examples/ex.02.graph/Makefile | 5 + dash/examples/ex.02.graph/main.cc | 145 +++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 dash/examples/ex.02.graph/Makefile create mode 100644 dash/examples/ex.02.graph/main.cc diff --git a/dash/examples/ex.02.graph/Makefile b/dash/examples/ex.02.graph/Makefile new file mode 100644 index 000000000..42043aca7 --- /dev/null +++ b/dash/examples/ex.02.graph/Makefile @@ -0,0 +1,5 @@ +# +# In-place makefile for use side-by-side with the +# CMake build system +# +include ../Makefile_cpp diff --git a/dash/examples/ex.02.graph/main.cc b/dash/examples/ex.02.graph/main.cc new file mode 100644 index 000000000..fc78c3c15 --- /dev/null +++ b/dash/examples/ex.02.graph/main.cc @@ -0,0 +1,145 @@ +#include +#include + +struct vprop { + int id; + int level; +}; + +struct eprop { + int id; +}; + +typedef dash::Graph graph_t; +typedef graph_t::vertex_index_type vertex_index_t; +typedef graph_t::vertex_type vertex_t; +typedef graph_t::edge_type edge_t; + +void create_graph(graph_t &g) { + // add 4 v to every unit + for (int i = (dash::myid() * 4) + 1; i <= (dash::myid() * 4) + 4; ++i) { + vprop prop{i, -1}; + g.add_vertex(prop); + } + + std::vector v; + v.emplace_back(dash::team_unit_t(0), 0); + v.emplace_back(dash::team_unit_t(0), 1); + v.emplace_back(dash::team_unit_t(0), 2); + v.emplace_back(dash::team_unit_t(0), 3); + v.emplace_back(dash::team_unit_t(1), 0); + v.emplace_back(dash::team_unit_t(1), 1); + v.emplace_back(dash::team_unit_t(1), 2); + v.emplace_back(dash::team_unit_t(1), 3); + + if (dash::myid() == 0) { + g.add_edge(v[0], v[1]); + g.add_edge(v[0], v[4]); + g.add_edge(v[1], v[5]); + g.add_edge(v[1], v[6]); + g.add_edge(v[2], v[6]); + g.add_edge(v[3], v[7]); + } + if (dash::myid() == 1) { + g.add_edge(v[4], v[5]); + g.add_edge(v[5], v[6]); + g.add_edge(v[6], v[7]); + } + + g.barrier(); +} + +void breadth_first_search(graph_t &g, vertex_index_t source) { + dash::Team &team = g.team(); + + std::vector frontier; + std::vector> neighbours(team.size()); + + if (source.unit == team.myid()) { + frontier.push_back(source); + } + + int level = 0; + while (true) { + for (auto f_it = frontier.begin(); f_it != frontier.end(); ++f_it) { + vertex_index_t v_index = *f_it; + vertex_t &v = g.vertices[v_index]; + if (v.properties.level == -1) { + for (auto e_it = g.out_edges.vbegin(v); e_it != g.out_edges.vend(v); + ++e_it) { + edge_t e = *e_it; + auto target = e.target(); + neighbours[target.unit.id].push_back(target); + } + v.properties.level = level; + } + } + std::cout << neighbours[0].size() << " " << neighbours[1].size() + << std::endl; + + std::vector neighbour_data; + std::vector neighbour_count(team.size()); + std::vector neighbour_displs(team.size()); + for (int i = 0; i < neighbours.size(); ++i) { + for (auto &neighbour : neighbours[i]) { + neighbour_data.push_back(neighbour); + neighbour_count[i] += sizeof(vertex_index_t); + } + for (int j = i + 1; j < neighbour_displs.size(); ++j) { + neighbour_displs[j] += neighbour_count[i]; + } + neighbours[i].clear(); + } + std::vector neighbour_recv_count(team.size()); + dart_alltoall(neighbour_count.data(), neighbour_recv_count.data(), + sizeof(std::size_t), DART_TYPE_BYTE, team.dart_id()); + int total = 0; + std::vector neighbour_recv_displs(team.size()); + for (int i = 0; i < neighbour_recv_count.size(); ++i) { + total += neighbour_recv_count[i]; + for (int j = i + 1; j < neighbour_recv_displs.size(); ++j) { + neighbour_recv_displs[j] += neighbour_recv_count[i]; + } + } + int global_count = 0; + dart_allreduce(&total, &global_count, sizeof(int), DART_TYPE_BYTE, + DART_OP_SUM, team.dart_id()); + // frontier is empty on all nodes -> terminate algorithm + if (global_count == 0) { + break; + } + std::vector new_frontier(total / sizeof(vertex_index_t)); + dart_alltoallv(neighbour_data.data(), neighbour_count.data(), + neighbour_displs.data(), DART_TYPE_BYTE, new_frontier.data(), + neighbour_recv_count.data(), neighbour_recv_displs.data(), + team.dart_id()); + + frontier = new_frontier; + + ++level; + } +} + +int main(int argc, char *argv[]) { + dash::init(&argc, &argv); + graph_t g(8); + + create_graph(g); + breadth_first_search(g, vertex_index_t(dash::team_unit_t(0), 0)); + if (dash::myid() == 1) { + std::cout << g.num_vertices() << std::endl; + for (auto it = g.vertices.begin(); it != g.vertices.end(); ++it) { + vertex_t v = *it; + std::cout << "vertex " << v.properties.id << ": level " + << v.properties.level << std::endl; + } + /* + for(auto it = g.out_edges.begin(); it != g.out_edges.end(); ++it) { + edge_t e = *it; + std::cout << "edge: " << e.source().offset << " -> " << e.target().offset + << std::endl; + } + */ + } + dash::barrier(); +} From 328585ab09903a5e784167d732c684b95dce13c4 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Sun, 30 Apr 2017 11:59:44 +0200 Subject: [PATCH 044/102] renamed example main to include it in build --- dash/examples/ex.02.graph/{main.cc => main.cpp} | 2 +- dash/include/dash/Graph.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) rename dash/examples/ex.02.graph/{main.cc => main.cpp} (98%) diff --git a/dash/examples/ex.02.graph/main.cc b/dash/examples/ex.02.graph/main.cpp similarity index 98% rename from dash/examples/ex.02.graph/main.cc rename to dash/examples/ex.02.graph/main.cpp index fc78c3c15..e05deaab7 100644 --- a/dash/examples/ex.02.graph/main.cc +++ b/dash/examples/ex.02.graph/main.cpp @@ -102,7 +102,7 @@ void breadth_first_search(graph_t &g, vertex_index_t source) { } } int global_count = 0; - dart_allreduce(&total, &global_count, sizeof(int), DART_TYPE_BYTE, + dart_allreduce(&total, &global_count, 1, DART_TYPE_INT, DART_OP_SUM, team.dart_id()); // frontier is empty on all nodes -> terminate algorithm if (global_count == 0) { diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 76f0c8726..d43a3a568 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -11,7 +11,6 @@ #include #include #include -#include namespace dash { From 7e67e9751f01ead20aceab6a0043f2de67eefe32 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Sun, 30 Apr 2017 17:13:31 +0200 Subject: [PATCH 045/102] fixed double deallocation bug --- dash/include/dash/Graph.h | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index d43a3a568..4422b9253 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -446,19 +446,28 @@ class Graph { void deallocate() { if(_glob_mem_vertex != nullptr) { delete _glob_mem_vertex; + _glob_mem_vertex = nullptr; } if(_glob_mem_in_edge != nullptr) { delete _glob_mem_in_edge; + if(_glob_mem_in_edge == _glob_mem_out_edge) { + _glob_mem_out_edge = nullptr; + } + _glob_mem_in_edge = nullptr; } if(_glob_mem_out_edge != nullptr && _glob_mem_out_edge != _glob_mem_in_edge) { delete _glob_mem_out_edge; + _glob_mem_out_edge = nullptr; } if(_glob_mem_edge != nullptr) { delete _glob_mem_edge; + _glob_mem_edge = nullptr; + } + if(_team != nullptr) { + // Remove deallocator from the respective team instance + _team->unregister_deallocator(this, std::bind(&Graph::deallocate, this)); } - // Remove deallocator from the respective team instance - _team->unregister_deallocator(this, std::bind(&Graph::deallocate, this)); } /** From e24b1202fb845bfc80ce02edadc30e1dbe443be5 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 1 May 2017 20:19:50 +0200 Subject: [PATCH 046/102] fixed template parameter for underlying container --- dash/include/dash/Graph.h | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 4422b9253..0788deced 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -52,8 +52,8 @@ template< typename EdgeProperties = EmptyProperties, // user-defined struct typename VertexIndexType = int, typename EdgeIndexType = int, - template typename EdgeContainer = std::vector, - template typename VertexContainer = std::vector + template typename EdgeContainer = std::vector, + template typename VertexContainer = std::vector > class Graph { @@ -65,10 +65,8 @@ class Graph { EdgeContainer, VertexContainer> self_t; typedef Vertex vertex_type; typedef Edge edge_type; - typedef VertexContainer> vertex_container_type; - typedef EdgeContainer> edge_container_type; + typedef VertexContainer vertex_container_type; + typedef EdgeContainer edge_container_type; private: From 7513e9cfd4b6e74660f87f0344364395c3ea9c6f Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 1 Jun 2017 14:04:53 +0200 Subject: [PATCH 047/102] added integration test for graph iterators --- dash/include/dash/GlobSharedRef.h | 2 + dash/include/dash/Graph.h | 2 +- dash/test/container/GraphTest.cc | 279 ++++++++++++++++++++++++++++++ dash/test/container/GraphTest.h | 21 +++ 4 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 dash/test/container/GraphTest.cc create mode 100644 dash/test/container/GraphTest.h diff --git a/dash/include/dash/GlobSharedRef.h b/dash/include/dash/GlobSharedRef.h index 132b65ea0..e829afafc 100644 --- a/dash/include/dash/GlobSharedRef.h +++ b/dash/include/dash/GlobSharedRef.h @@ -2,8 +2,10 @@ #define DASH__GLOB_SHARED_EF_H_ #include +/* #include #include +*/ #include diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 0788deced..f111856b8 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -194,7 +194,7 @@ class Graph { * \return Offset of the vertex with the highest index in local address * space. */ - vertex_index_type max_vertex_index() const { + vertex_offset_type max_vertex_index() const { return _local_vertex_max_index; } diff --git a/dash/test/container/GraphTest.cc b/dash/test/container/GraphTest.cc new file mode 100644 index 000000000..f9fdc2200 --- /dev/null +++ b/dash/test/container/GraphTest.cc @@ -0,0 +1,279 @@ + +#include "GraphTest.h" +#include +#include + +struct vertex_prop { + int id; +}; + +struct edge_prop { + int id; +}; + +TEST_F(GraphTest, GlobalIteration) +{ + typedef int value_t; + typedef dash::default_index_t vertex_index_t; + typedef dash::default_index_t edge_index_t; + typedef dash::Graph< + dash::DirectedGraph, + void, + vertex_prop, + edge_prop, + vertex_index_t, + edge_index_t, + std::vector, + std::vector> graph_t; + typedef graph_t::vertex_index_type vertex_index_type; + typedef graph_t::vertex_type vertex_type; + typedef graph_t::edge_type edge_type; + + auto nunits = dash::size(); + auto myid = dash::myid(); + + auto ninsert_vertices_per_unit = 3; + auto ninsert_edges_per_vertex = 3; + ASSERT_LE(ninsert_edges_per_vertex, ninsert_vertices_per_unit); + auto ninsert_edges_per_unit = ninsert_vertices_per_unit * + ninsert_edges_per_vertex; + + auto vertex_cap = 1 * nunits; + auto edge_per_vertex_cap = 1; + + graph_t graph(vertex_cap, edge_per_vertex_cap); + + EXPECT_EQ_U(0, graph.num_vertices()); + EXPECT_EQ_U(0, graph.num_edges()); + EXPECT_EQ_U(-1, graph.max_vertex_index()); + EXPECT_EQ_U(-1, graph.max_edge_index()); + EXPECT_TRUE_U(graph.empty()); + + dash::barrier(); + DASH_LOG_DEBUG("GraphTest.GlobalIteration", "graph initialized"); + int ei = 0; + for(int i = 0; i < ninsert_vertices_per_unit; ++i) { + vertex_prop vprop { nunits * i + myid }; + auto source = graph.add_vertex(vprop); + //TODO: test vertex indeices for already existing vertices + DASH_LOG_DEBUG("GraphTest.GlobalIteration", "vertex inserted"); + auto target_unit = (myid + 1) % nunits; + for(int j = 0; j < ninsert_edges_per_vertex; ++j) { + vertex_index_type target(dash::team_unit_t(target_unit), j); + edge_prop eprop { nunits * ei + myid }; + graph.add_edge(source, target, eprop); + ++ei; + } + DASH_LOG_DEBUG("GraphTest.GlobalIteration", "edge inserted"); + } + + graph.barrier(); + DASH_LOG_DEBUG("GraphTest.GlobalIteration", "elements committed"); + + if(myid == 0) { + int i = 0; + int unit_id = 0; + DASH_LOG_DEBUG("GraphTest.GlobalIteration", "begin vertex iteration"); + for(auto it = graph.vertices.begin(); it != graph.vertices.end(); ++it) { + int id = nunits * i + unit_id; + vertex_type v = *it; + DASH_LOG_DEBUG("GraphTest.GlobalIteration", "vertex", + "it", it, + "value", v.properties.id); + EXPECT_EQ_U(id, v.properties.id); + ++i; + if(i == ninsert_vertices_per_unit) { + i = 0; + ++unit_id; + } + } + + i = 0; + unit_id = 0; + DASH_LOG_DEBUG("GraphTest.GlobalIteration", "begin out-edge iteration"); + for(auto it = graph.out_edges.begin(); it != graph.out_edges.end(); ++it) { + int id = nunits * i + unit_id; + edge_type e = *it; + DASH_LOG_DEBUG("GraphTest.GlobalIteration", "out-edge", + "it", it, + "value", e.properties.id); + EXPECT_EQ_U(e.properties.id, id); + ++i; + if(i == ninsert_edges_per_unit) { + i = 0; + ++unit_id; + } + } + + int j = 0; + int k = 0; + int l = 0; + int unit_id_prev = nunits - 1; + DASH_LOG_DEBUG("GraphTest.GlobalIteration", "begin in-edge iteration"); + for(auto it = graph.in_edges.begin(); it != graph.in_edges.end(); ++it) { + int id = nunits * (j * ninsert_edges_per_vertex) + (nunits * k) + + unit_id_prev; + edge_type e = *it; + DASH_LOG_DEBUG("GraphTest.GlobalIteration", "in-edge", + "it", it, + "value", e.properties.id); + EXPECT_EQ_U(e.properties.id, id); + ++j; + if(j == ninsert_edges_per_vertex) { + j = 0; + k += 1; + } + ++l; + if(l == ninsert_edges_per_unit) { + l = 0; + unit_id_prev = (unit_id_prev + 1) % nunits; + k = 0; + } + } + + i = 0; + j = 0; + k = 0; + l = 0; + int m = 0; + unit_id = 0; + unit_id_prev = nunits - 1; + DASH_LOG_DEBUG("GraphTest.GlobalIteration", "begin edge iteration"); + for(auto it = graph.edges.begin(); it != graph.edges.end(); ++it) { + edge_type e = *it; + int id; + if(m < ninsert_edges_per_vertex) { + id = nunits * i + unit_id; + ++i; + if(i == ninsert_edges_per_unit) { + i = 0; + ++unit_id; + } + } else { + id = nunits * (j * ninsert_edges_per_vertex) + (nunits * k) + + unit_id_prev; + ++j; + if(j == ninsert_edges_per_vertex) { + j = 0; + k += 1; + } + ++l; + if(l == ninsert_edges_per_unit) { + l = 0; + unit_id_prev = (unit_id_prev + 1) % nunits; + k = 0; + } + } + ++m; + if(m == 2 * ninsert_edges_per_vertex) { + m = 0; + } + DASH_LOG_DEBUG("GraphTest.GlobalIteration", "edge", + "it", it, + "value", e.properties.id); + EXPECT_EQ_U(e.properties.id, id); + } + + i = 0; + unit_id = 0; + DASH_LOG_DEBUG("GraphTest.GlobalIteration", + "begin adjacency out-edge iteration"); + for(auto it = graph.vertices.begin(); it != graph.vertices.end(); ++it) { + vertex_type v = *it; + for(auto ait = graph.out_edges.vbegin(v); + ait != graph.out_edges.vend(v); ++ait) { + int id = nunits * i + unit_id; + edge_type e = *ait; + DASH_LOG_DEBUG("GraphTest.GlobalIteration", "out-edge", + "it", ait, + "value", e.properties.id); + EXPECT_EQ_U(e.properties.id, id); + ++i; + if(i == ninsert_edges_per_unit) { + i = 0; + ++unit_id; + } + } + } + + j = 0; + k = 0; + l = 0; + unit_id_prev = nunits - 1; + DASH_LOG_DEBUG("GraphTest.GlobalIteration", + "begin adjacency in-edge iteration"); + for(auto it = graph.vertices.begin(); it != graph.vertices.end(); ++it) { + vertex_type v = *it; + for(auto ait = graph.in_edges.vbegin(v); + ait != graph.in_edges.vend(v); ++ait) { + int id = nunits * (j * ninsert_edges_per_vertex) + (nunits * k) + + unit_id_prev; + edge_type e = *ait; + DASH_LOG_DEBUG("GraphTest.GlobalIteration", "in-edge", + "it", ait, + "value", e.properties.id); + EXPECT_EQ_U(e.properties.id, id); + ++j; + if(j == ninsert_edges_per_vertex) { + j = 0; + k += 1; + } + ++l; + if(l == ninsert_edges_per_unit) { + l = 0; + unit_id_prev = (unit_id_prev + 1) % nunits; + k = 0; + } + } + } + + i = 0; + j = 0; + k = 0; + l = 0; + m = 0; + unit_id = 0; + unit_id_prev = nunits - 1; + DASH_LOG_DEBUG("GraphTest.GlobalIteration", + "begin adjacency edge iteration"); + for(auto it = graph.vertices.begin(); it != graph.vertices.end(); ++it) { + vertex_type v = *it; + for(auto ait = graph.edges.vbegin(v); + ait != graph.edges.vend(v); ++ait) { + edge_type e = *ait; + int id; + if(m < ninsert_edges_per_vertex) { + id = nunits * i + unit_id; + ++i; + if(i == ninsert_edges_per_unit) { + i = 0; + ++unit_id; + } + } else { + id = nunits * (j * ninsert_edges_per_vertex) + (nunits * k) + + unit_id_prev; + ++j; + if(j == ninsert_edges_per_vertex) { + j = 0; + k += 1; + } + ++l; + if(l == ninsert_edges_per_unit) { + l = 0; + unit_id_prev = (unit_id_prev + 1) % nunits; + k = 0; + } + } + ++m; + if(m == 2 * ninsert_edges_per_vertex) { + m = 0; + } + DASH_LOG_DEBUG("GraphTest.GlobalIteration", "edge", + "it", ait, + "value", e.properties.id); + } + } + } + dash::barrier(); +} + diff --git a/dash/test/container/GraphTest.h b/dash/test/container/GraphTest.h new file mode 100644 index 000000000..3e5c060a9 --- /dev/null +++ b/dash/test/container/GraphTest.h @@ -0,0 +1,21 @@ +#ifndef DASH__TEST__GRAPH_TEST_H_ +#define DASH__TEST__GRAPH_TEST_H_ + +#include "../TestBase.h" + +/** + * Test fixture for class dash::List + */ +class GraphTest : public dash::test::TestBase { +protected: + + GraphTest() { + LOG_MESSAGE(">>> Test suite: GraphTest"); + } + + virtual ~GraphTest() { + LOG_MESSAGE("<<< Closing test suite: GraphTest"); + } +}; + +#endif // DASH__TEST__GRAPH_TEST_H_ From 3b7b3448066b6c1f6e096fda5426dce348fa056c Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 30 Oct 2017 15:20:00 +0100 Subject: [PATCH 048/102] exchanged indices with iterators --- dash/include/dash/Graph.h | 231 +++++++++--------- dash/include/dash/Pattern.h | 3 - dash/include/dash/graph/EdgeIterator.h | 64 +---- dash/include/dash/graph/InEdgeIterator.h | 74 +----- dash/include/dash/graph/OutEdgeIterator.h | 74 +----- dash/include/dash/graph/VertexIterator.h | 13 - dash/include/dash/graph/internal/Graph.h | 104 ++------ .../pattern/EdgePartitionedDynamicPattern.h | 83 ------- .../pattern/LoadBalancedDynamicPattern.h | 130 ---------- .../pattern/VertexPartitionedDynamicPattern.h | 108 -------- .../dash/memory/GlobHeapContiguousMem.h | 3 +- 11 files changed, 164 insertions(+), 723 deletions(-) delete mode 100644 dash/include/dash/graph/pattern/EdgePartitionedDynamicPattern.h delete mode 100644 dash/include/dash/graph/pattern/LoadBalancedDynamicPattern.h delete mode 100644 dash/include/dash/graph/pattern/VertexPartitionedDynamicPattern.h diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index f111856b8..5ca7f4b38 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -46,12 +46,10 @@ namespace dash { */ template< GraphDirection Direction = DirectedGraph, - //typename DynamicPattern = dash::graph::VertexPartitionedDynamicPattern, - typename DynamicPattern = void, typename VertexProperties = EmptyProperties, // user-defined struct typename EdgeProperties = EmptyProperties, // user-defined struct - typename VertexIndexType = int, - typename EdgeIndexType = int, + typename VertexSizeType = int, + typename EdgeSizeType = int, template typename EdgeContainer = std::vector, template typename VertexContainer = std::vector > @@ -59,9 +57,9 @@ class Graph { public: - typedef Graph self_t; typedef Vertex vertex_type; typedef Edge edge_type; @@ -95,38 +93,34 @@ class Graph { glob_mem_vert_type::container_list_index vertex_cont_ref_type; typedef typename glob_mem_edge_type::container_list_index edge_cont_ref_type; - typedef VertexIndexType vertex_offset_type; - typedef EdgeIndexType edge_offset_type; - typedef VertexIndex vertex_index_type; - typedef EdgeIndex edge_index_type; - typedef typename - std::make_unsigned::type vertex_size_type; - typedef typename - std::make_unsigned::type edge_size_type; + typedef VertexSizeType vertex_size_type; + typedef EdgeSizeType edge_size_type; - typedef DynamicPattern pattern_type; typedef VertexProperties vertex_properties_type; typedef EdgeProperties edge_properties_type; typedef GlobRef reference; + typedef VertexIndex vertex_index_type; + typedef typename glob_mem_vert_type::local_iterator local_vertex_iterator; typedef typename - glob_mem_edge_type::local_iterator local_edge_iterator; + glob_mem_edge_type::local_iterator local_in_edge_iterator; + typedef typename + glob_mem_edge_type::local_iterator local_out_edge_iterator; + typedef typename + glob_mem_edge_comb_type::local_iterator local_edge_iterator; typedef typename glob_mem_vert_type::global_iterator global_vertex_iterator; typedef typename - glob_mem_edge_type::global_iterator global_edge_iterator; + glob_mem_edge_type::global_iterator global_in_edge_iterator; typedef typename - glob_mem_edge_comb_type::global_iterator global_edge_comb_iterator; + glob_mem_edge_type::global_iterator global_out_edge_iterator; + typedef typename + glob_mem_edge_comb_type::global_iterator global_edge_iterator; - typedef typename vertex_it_wrapper::iterator vertex_iterator; - typedef typename edge_it_wrapper::iterator edge_iterator; - typedef typename in_edge_it_wrapper::iterator in_edge_iterator; - typedef typename out_edge_it_wrapper::iterator out_edge_iterator; - public: vertex_it_wrapper vertices = vertex_it_wrapper(this); @@ -188,16 +182,6 @@ class Graph { return _glob_mem_vertex->size(); } - /** - * Returns the index of the vertex with the highest index in the whole graph. - * - * \return Offset of the vertex with the highest index in local address - * space. - */ - vertex_offset_type max_vertex_index() const { - return _local_vertex_max_index; - } - /** * Returns the number of edges in the whole graph. * @@ -207,15 +191,6 @@ class Graph { return _glob_mem_edge->size(); } - /** - * Returns the index of the edge with the highest index in the whole graph. - * - * \return Offset of the edge with the highest index in local address space. - */ - edge_offset_type max_edge_index() const { - return _local_edge_max_index; - } - /** * Returns, whether the graph is empty. * @@ -230,19 +205,13 @@ class Graph { * * \return Index of the newly created vertex. */ - vertex_index_type add_vertex(const VertexProperties & prop) { - auto out_ref = _glob_mem_out_edge->add_container(_alloc_edges_per_vertex); - auto in_ref = out_ref; + local_vertex_iterator add_vertex(const VertexProperties & prop) { + _glob_mem_out_edge->add_container(_alloc_edges_per_vertex); if(_glob_mem_in_edge != _glob_mem_out_edge) { - in_ref = _glob_mem_in_edge->add_container(_alloc_edges_per_vertex); + _glob_mem_in_edge->add_container(_alloc_edges_per_vertex); } - auto max_index = _glob_mem_vertex->container_local_size( - _vertex_container_ref); - vertex_index_type v_index(_myid, max_index); - vertex_type v(v_index, in_ref, out_ref, prop); - _glob_mem_vertex->push_back(_vertex_container_ref, v); - // TODO: return global index - return vertex_index_type(_myid, max_index); + vertex_type v(prop); + return _glob_mem_vertex->push_back(_vertex_container_ref, v); } /** @@ -250,9 +219,9 @@ class Graph { * * \return Index of the newly created vertex. */ - vertex_index_type add_vertex() { + local_vertex_iterator add_vertex() { VertexProperties prop; - add_vertex(prop); + return add_vertex(prop); } /** @@ -269,6 +238,28 @@ class Graph { } + /** + * Adds an edge between two given vertices with the given properties + * locally. + * + * \return Pair, with pair::first set to the index of the newly created edge + * and pair::second set to a boolean indicating whether the edge has + * actually been added. + */ + std::pair add_edge( + const local_vertex_iterator & source, + const local_vertex_iterator & target, + const EdgeProperties & prop + ) { + local_out_edge_iterator l_it; + l_it = add_local_edge(source, target, prop, _glob_mem_out_edge); + add_local_edge(target, source, prop, _glob_mem_in_edge); + + // currently, double edges are allowed for all cases, and vertex deletion + // is not implemented so we always return true + return std::make_pair(l_it, true); + } + /** * Adds an edge between two given vertices with the given properties * locally. @@ -279,35 +270,25 @@ class Graph { * and pair::second set to a boolean indicating whether the edge has * actually been added. */ - std::pair add_edge( - const vertex_index_type & source, - const vertex_index_type & target, + std::pair add_edge( + const local_vertex_iterator & source, + const global_vertex_iterator & target, const EdgeProperties & prop ) { - edge_index_type local_index; - if(source.unit == _myid) { - local_index = add_local_edge(source, target, prop, _glob_mem_out_edge); - } else { - edge_type edge(source, target, prop); - _remote_edges[source.unit].push_back(edge); - } - if(target.unit == _myid) { + local_out_edge_iterator l_it; + l_it = add_local_edge(source, target, prop, _glob_mem_out_edge); + if(target.is_local()) { // _glob_mem_in_edge == _glob_mem_out_edge for undirected graph types - edge_index_type local_index_tmp = add_local_edge(target, - source, prop, _glob_mem_in_edge); - if(local_index.offset == -1) { - local_index = local_index_tmp; - } + add_local_edge(target.local(), source, prop, _glob_mem_in_edge); // do not double-send edges - } else if(source.unit != target.unit) { - edge_type edge(source, target, prop); - _remote_edges[target.unit].push_back(edge); + } else { + edge_type edge(source, target, prop, _myid); + _remote_edges[target.lpos().unit].push_back(edge); } - //TODO: handle cases, were both vertices reside on a different unit - // currently, double edges are allowed for all cases, so we always return - // true - return std::make_pair(local_index, true); + // currently, double edges are allowed for all cases, and vertex deletion + // is not implemented so we always return true + return std::make_pair(l_it, true); } /** @@ -319,26 +300,58 @@ class Graph { * and pair::second set to a boolean indicating whether the edge has * actually been added. */ - std::pair add_edge( - const vertex_index_type & source, - const vertex_index_type & target + std::pair add_edge( + const local_vertex_iterator & source, + const local_vertex_iterator & target ) { EdgeProperties prop; - add_edge(source, target, prop); + return add_edge(source, target, prop); + } + + /** + * Adds an edge between two given vertices locally. + * Edges that belong to vertices held on a different unit are marked for + * transfer. These edges will be transferred after calling \c barrier(). + * + * \return Pair, with pair::first set to the index of the newly created edge + * and pair::second set to a boolean indicating whether the edge has + * actually been added. + */ + std::pair add_edge( + const local_vertex_iterator & source, + const global_vertex_iterator & target + ) { + EdgeProperties prop; + return add_edge(source, target, prop); } /** * Removes the edges between two given vertices. */ - void remove_edge(const vertex_index_type & v1, - const vertex_index_type & v2) { + void remove_edge(const local_vertex_iterator & v1, + const local_vertex_iterator & v2) { + + } + + /** + * Removes the edges between two given vertices. + */ + void remove_edge(const global_vertex_iterator & v1, + const global_vertex_iterator & v2) { } /** * Removes a given edge. */ - void remove_edge(const edge_index_type & e) { + void remove_edge(const local_out_edge_iterator & e) { + + } + + /** + * Removes a given edge. + */ + void remove_edge(const global_out_edge_iterator & e) { } @@ -408,7 +421,9 @@ class Graph { // commit changes in local memory space globally _glob_mem_vertex->commit(); _glob_mem_out_edge->commit(); - _glob_mem_in_edge->commit(); + if(_glob_mem_out_edge != _glob_mem_in_edge) { + _glob_mem_in_edge->commit(); + } _glob_mem_edge->commit(); } @@ -480,30 +495,32 @@ class Graph { /** * Inserts an edge locally. The source vertex must belong to this unit. * - * \return Index of the newly created edge. + * \return Local iterator to the created edge. */ - edge_index_type add_local_edge( - const vertex_index_type & source, - const vertex_index_type & target, + template + typename glob_mem_edge_type::local_iterator add_local_edge( + const local_vertex_iterator source, + const TargetIterator target, + const EdgeProperties & prop, + glob_mem_edge_type * glob_mem_edge + ) { + auto edge = edge_type(source, target, prop, _myid); + return glob_mem_edge->push_back(source.pos(), edge); + } + + /** + * Inserts an edge locally. The source vertex must belong to this unit. + * + * \return Local iterator to the created edge. + */ + typename glob_mem_edge_type::local_iterator add_local_edge( + const vertex_index_type source, + const vertex_index_type target, const EdgeProperties & prop, glob_mem_edge_type * glob_mem_edge ) { - edge_index_type local_index; auto edge = edge_type(source, target, prop); - auto vertex = _glob_mem_vertex->get(_vertex_container_ref, - source.offset); - auto edge_ref = vertex._out_edge_ref; - if(glob_mem_edge == _glob_mem_in_edge) { - edge_ref = vertex._in_edge_ref; - } - local_index = edge_index_type( - _myid, - edge_ref, - glob_mem_edge->container_local_size(edge_ref) - ); - edge._index = local_index; - glob_mem_edge->push_back(edge_ref, edge); - return local_index; + return glob_mem_edge->push_back(source.offset, edge); } private: @@ -520,10 +537,6 @@ class Graph { glob_mem_edge_comb_type * _glob_mem_edge = nullptr; /** Unit ID of the current unit */ team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; - /** Index of last added vertex */ - vertex_offset_type _local_vertex_max_index = -1; - /** Index of last added edge */ - edge_offset_type _local_edge_max_index = -1; /** Index of the vertex container in _glob_mem_vertex */ vertex_cont_ref_type _vertex_container_ref; /** Amount of edge elements to be pre-allocated for every vertex */ diff --git a/dash/include/dash/Pattern.h b/dash/include/dash/Pattern.h index d30163842..79dce5cf7 100644 --- a/dash/include/dash/Pattern.h +++ b/dash/include/dash/Pattern.h @@ -654,7 +654,4 @@ using Pattern = dash::BlockPattern; #include #include -// dynamic graph pattern types: -#include - #endif // DASH__PATTERN_H__INCLUDED diff --git a/dash/include/dash/graph/EdgeIterator.h b/dash/include/dash/graph/EdgeIterator.h index 7add8087b..607f44f9f 100644 --- a/dash/include/dash/graph/EdgeIterator.h +++ b/dash/include/dash/graph/EdgeIterator.h @@ -10,14 +10,11 @@ template struct EdgeIteratorWrapper { typedef Graph graph_type; - typedef typename Graph::global_edge_comb_iterator iterator; + typedef typename Graph::global_edge_iterator iterator; typedef const iterator const_iterator; typedef typename Graph::local_edge_iterator local_iterator; typedef const local_iterator const_local_iterator; - typedef typename Graph::edge_index_type edge_index_type; typedef typename Graph::edge_properties_type edge_properties_type; - typedef typename Graph::vertex_type vertex_type; - typedef typename Graph::vertex_index_type vertex_index_type; /** * Constructs the wrapper. @@ -26,13 +23,6 @@ struct EdgeIteratorWrapper { : _graph(graph) { } - /** - * Returns a property object for the given edge. - */ - edge_properties_type & operator[](const edge_index_type & v) const { - - } - /** * Returns global iterator to the beginning of the edge list. */ @@ -93,58 +83,6 @@ struct EdgeIteratorWrapper { return _graph->_glob_mem_edge->lend(); } - /** - * Returns global iterator to the beginning of the edge list of the given - * vertex - */ - iterator vbegin(const vertex_type & v) { - return iterator( - _graph->_glob_mem_edge, - v._index.unit, - v._in_edge_ref + v._out_edge_ref, - 0 - ); - } - - /** - * Returns global iterator to the end of the edge list of the given vertex - */ - iterator vend(const vertex_type & v) { - auto edge_ref = v._in_edge_ref + v._out_edge_ref; - return iterator( - _graph->_glob_mem_edge, - v._index.unit, - edge_ref, - _graph->_glob_mem_edge->container_size(v._index.unit, edge_ref) - ); - } - - /** - * Returns global iterator to the beginning of the edge list of the given - * vertex - */ - const_iterator vbegin(const vertex_type & v) const { - return iterator( - _graph->_glob_mem_edge, - v._index.unit, - v._in_edge_ref + v._out_edge_ref, - 0 - ); - } - - /** - * Returns global iterator to the end of the edge list of the given vertex - */ - const_iterator vend(const vertex_type & v) const { - auto edge_ref = v._in_edge_ref + v._out_edge_ref; - return iterator( - _graph->_glob_mem_edge, - v._index.unit, - edge_ref, - _graph->_glob_mem_edge->container_size(v._index.unit, edge_ref) - ); - } - private: graph_type * _graph; diff --git a/dash/include/dash/graph/InEdgeIterator.h b/dash/include/dash/graph/InEdgeIterator.h index 77490567a..f1d9438e8 100644 --- a/dash/include/dash/graph/InEdgeIterator.h +++ b/dash/include/dash/graph/InEdgeIterator.h @@ -9,15 +9,12 @@ namespace dash { template struct InEdgeIteratorWrapper { - typedef Graph graph_type; - typedef typename Graph::global_edge_iterator iterator; - typedef const iterator const_iterator; - typedef typename Graph::local_edge_iterator local_iterator; - typedef const local_iterator const_local_iterator; - typedef typename Graph::edge_index_type edge_index_type; - typedef typename Graph::edge_properties_type edge_properties_type; - typedef typename Graph::vertex_type vertex_type; - typedef typename Graph::vertex_index_type vertex_index_type; + typedef Graph graph_type; + typedef typename Graph::global_in_edge_iterator iterator; + typedef const iterator const_iterator; + typedef typename Graph::local_in_edge_iterator local_iterator; + typedef const local_iterator const_local_iterator; + typedef typename Graph::edge_properties_type edge_properties_type; /** * Constructs the wrapper. @@ -26,13 +23,6 @@ struct InEdgeIteratorWrapper { : _graph(graph) { } - /** - * Returns a property object for the given edge. - */ - edge_properties_type & operator[](const edge_index_type & v) const { - - } - /** * Returns global iterator to the beginning of the edge list. */ @@ -93,58 +83,6 @@ struct InEdgeIteratorWrapper { return _graph->_glob_mem_in_edge->lend(); } - /** - * Returns global iterator to the beginning of the edge list of the given - * vertex - */ - iterator vbegin(const vertex_type & v) { - return iterator( - _graph->_glob_mem_in_edge, - v._index.unit, - v._in_edge_ref, - 0 - ); - } - - /** - * Returns global iterator to the end of the edge list of the given vertex - */ - iterator vend(const vertex_type & v) { - return iterator( - _graph->_glob_mem_in_edge, - v._index.unit, - v._in_edge_ref, - _graph->_glob_mem_in_edge->container_size(v._index.unit, - v._in_edge_ref) - ); - } - - /** - * Returns global iterator to the beginning of the edge list of the given - * vertex - */ - const_iterator vbegin(const vertex_type & v) const { - return iterator( - _graph->_glob_mem_in_edge, - v._index.unit, - v._in_edge_ref, - 0 - ); - } - - /** - * Returns global iterator to the end of the edge list of the given vertex - */ - const_iterator vend(const vertex_type & v) const { - return iterator( - _graph->_glob_mem_in_edge, - v._index.unit, - v._in_edge_ref, - _graph->_glob_mem_in_edge->container_size(v._index.unit, - v._in_edge_ref) - ); - } - private: graph_type * _graph; diff --git a/dash/include/dash/graph/OutEdgeIterator.h b/dash/include/dash/graph/OutEdgeIterator.h index 86c5f2546..97d6db2ec 100644 --- a/dash/include/dash/graph/OutEdgeIterator.h +++ b/dash/include/dash/graph/OutEdgeIterator.h @@ -9,15 +9,12 @@ namespace dash { template struct OutEdgeIteratorWrapper { - typedef Graph graph_type; - typedef typename Graph::global_edge_iterator iterator; - typedef const iterator const_iterator; - typedef typename Graph::local_edge_iterator local_iterator; - typedef const local_iterator const_local_iterator; - typedef typename Graph::edge_index_type edge_index_type; - typedef typename Graph::edge_properties_type edge_properties_type; - typedef typename Graph::vertex_type vertex_type; - typedef typename Graph::vertex_index_type vertex_index_type; + typedef Graph graph_type; + typedef typename Graph::global_out_edge_iterator iterator; + typedef const iterator const_iterator; + typedef typename Graph::local_out_edge_iterator local_iterator; + typedef const local_iterator const_local_iterator; + typedef typename Graph::edge_properties_type edge_properties_type; /** * Constructs the wrapper. @@ -26,13 +23,6 @@ struct OutEdgeIteratorWrapper { : _graph(graph) { } - /** - * Returns a property object for the given edge. - */ - edge_properties_type & operator[](const edge_index_type & v) const { - - } - /** * Returns global iterator to the beginning of the edge list. */ @@ -93,58 +83,6 @@ struct OutEdgeIteratorWrapper { return _graph->_glob_mem_out_edge->lend(); } - /** - * Returns global iterator to the beginning of the edge list of the given - * vertex - */ - iterator vbegin(const vertex_type & v) { - return iterator( - _graph->_glob_mem_out_edge, - v._index.unit, - v._out_edge_ref, - 0 - ); - } - - /** - * Returns global iterator to the end of the edge list of the given vertex - */ - iterator vend(const vertex_type & v) { - return iterator( - _graph->_glob_mem_out_edge, - v._index.unit, - v._out_edge_ref, - _graph->_glob_mem_out_edge->container_size(v._index.unit, - v._out_edge_ref) - ); - } - - /** - * Returns global iterator to the beginning of the edge list of the given - * vertex - */ - const_iterator vbegin(const vertex_type & v) const { - return iterator( - _graph->_glob_mem_out_edge, - v._index.unit, - v._out_edge_ref, - 0 - ); - } - - /** - * Returns global iterator to the end of the edge list of the given vertex - */ - const_iterator vend(const vertex_type & v) const { - return iterator( - _graph->_glob_mem_out_edge, - v._index.unit, - v._out_edge_ref, - _graph->_glob_mem_out_edge->container_size(v._index.unit, - v._out_edge_ref) - ); - } - private: graph_type * _graph; diff --git a/dash/include/dash/graph/VertexIterator.h b/dash/include/dash/graph/VertexIterator.h index 9a6b7c84c..735dde4a3 100644 --- a/dash/include/dash/graph/VertexIterator.h +++ b/dash/include/dash/graph/VertexIterator.h @@ -14,8 +14,6 @@ struct VertexIteratorWrapper { typedef const iterator const_iterator; typedef typename Graph::local_vertex_iterator local_iterator; typedef const local_iterator const_local_iterator; - typedef typename Graph::vertex_index_type vertex_index_type; - typedef typename Graph::vertex_type vertex_type; /** * Constructs the wrapper. @@ -24,17 +22,6 @@ struct VertexIteratorWrapper { : _graph(graph) { } - /** - * Returns a property object for the given vertex. - */ - vertex_type & operator[](const vertex_index_type & v) const { - if(_graph->_myid == v.unit) { - return _graph->_glob_mem_vertex->get(_graph->_vertex_container_ref, - v.offset); - } - // TODO: handle error here - } - /** * Returns global iterator to the beginning of the vertex list. */ diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index 361a1545c..3f12d9270 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -38,35 +38,6 @@ struct VertexIndex { }; -/** - * Index type for edges. - */ -template -struct EdgeIndex { - - /** - * Default constructor. - */ - EdgeIndex() = default; - - /** - * Index Constructor. - */ - EdgeIndex(team_unit_t u, IndexType c, IndexType o) - : unit(u), - container(c), - offset(o) - { } - - /** The unit holding the referenced edge */ - team_unit_t unit; - /** The container in globale memory space of unit containing the edge */ - IndexType container; - /** The offset in the referenced container */ - IndexType offset; - -}; - /** * Vertex type holding properties and references to its corresponding edge * lists. @@ -74,8 +45,6 @@ struct EdgeIndex { template class Vertex { - typedef typename GraphType::edge_cont_ref_type edge_container_ref; - typedef typename GraphType::vertex_index_type index_type; typedef typename GraphType::vertex_properties_type properties_type; friend GraphType; @@ -94,15 +63,9 @@ class Vertex { * Creates a vertex with given properties. */ Vertex( - const index_type & index, - const edge_container_ref & in_edge_ref, - const edge_container_ref & out_edge_ref, const properties_type & properties ) - : _index(index), - _in_edge_ref(in_edge_ref), - _out_edge_ref(out_edge_ref), - properties(properties) + : properties(properties) { } public: @@ -110,15 +73,6 @@ class Vertex { /** Properties of this vertex */ properties_type properties; -private: - - /** Index of the vertex in local index space */ - index_type _index; - /** index of the in-edge list belonging to this vertex */ - edge_container_ref _in_edge_ref; - /** index of the out-edge list belonging to this vertex */ - edge_container_ref _out_edge_ref; - }; /** @@ -127,8 +81,9 @@ class Vertex { template class Edge { + typedef typename GraphType::local_vertex_iterator local_vertex_iterator; + typedef typename GraphType::global_vertex_iterator global_vertex_iterator; typedef typename GraphType::vertex_index_type vertex_index_type; - typedef typename GraphType::edge_index_type index_type; typedef typename GraphType::edge_properties_type properties_type; friend GraphType; @@ -141,31 +96,42 @@ class Edge { Edge() = default; /** - * Creates an edge with index. + * Creates an edge. */ Edge( - const index_type & index, - const vertex_index_type & source, - const vertex_index_type & target, - const properties_type & properties + const local_vertex_iterator & source, + const local_vertex_iterator & target, + const properties_type & properties, + const team_unit_t owner ) - : _index(index), - _source(source), - _target(target), + : _source(owner, source.pos()), + _target(owner, target.pos()), + properties(properties) + { } + + /** + * Creates an edge. + */ + Edge( + const local_vertex_iterator & source, + const global_vertex_iterator & target, + const properties_type & properties, + const team_unit_t owner + ) + : _source(owner, source.pos()), + _target(target.lpos().unit, target.lpos().index), properties(properties) { } /** - * Creates an edge without index. Needed for edges that belong to other - * units. + * Creates an edge. */ Edge( const vertex_index_type & source, const vertex_index_type & target, const properties_type & properties - ) - : _index(), - _source(source), + ) + : _source(source), _target(target), properties(properties) { } @@ -175,20 +141,6 @@ class Edge { /** Properties of this edge */ properties_type properties; - /** - * Returns the source vertex of the edge - */ - vertex_index_type source() const { - return _source; - } - - /** - * Returns the target vertex of the edge - */ - vertex_index_type target() const { - return _target; - } - private: //TODO: Examine, if saving source can be avoided @@ -196,8 +148,6 @@ class Edge { vertex_index_type _source; /** Target vertex the edge is pointing to */ vertex_index_type _target; - /** Index of the edge in local index space */ - index_type _index; }; diff --git a/dash/include/dash/graph/pattern/EdgePartitionedDynamicPattern.h b/dash/include/dash/graph/pattern/EdgePartitionedDynamicPattern.h deleted file mode 100644 index 66ab20087..000000000 --- a/dash/include/dash/graph/pattern/EdgePartitionedDynamicPattern.h +++ /dev/null @@ -1,83 +0,0 @@ -#ifndef DASH__EDGE_PARTITIONED_DYNAMIC_PATTERN_H__INCLUDED -#define DASH__EDGE_PARTITIONED_DYNAMIC_PATTERN_H__INCLUDED -namespace dash { - -namespace graph { - -/** - * Pattern for graph data. Partitions the (logical) adjecency matrix of the - * graph into 2-dimensional blocks. Assigns one block to each unit. - * NOTE: The number of units in the assigned team must be a power of 2. - * NOTE: This pattern is intended for static graphs only. - * - * \concept{DashDynamicPatternConcept} - */ -template< - typename IndexType = std::size_t> -class EdgePartitionedDynamicPattern { - - typedef IndexType index_type; - typedef typename std::make_unsigned::type size_type; - - typedef struct { - dart_unit_t unit; - index_type index; - } local_index_type; - - /** - * Constructs the partitioning scheme. - */ - EdgePartitionedDynamicPattern(Team & team); - - /** - * Converts the given global vertex index to the owning unit. - */ - dart_unit_t unit_at(index_type global_index) const; - - /** - * Converts the given global vertex index to the owning unit and its - * local index. - */ - local_index_type local(index_type global_index) const; - - /** - * Converts the goiven local index of the calling unit to the respective - * global index. - */ - index_type global(index_type local_index) const; - - /** - * Returns, whether the element at the given global index is locak to - * the given unit. - */ - bool is_local(index_type global_index, dart_unit_t unit) const; - - /** - * Returns the amount of elements for which memory has been allocated. - */ - size_type capacity() const; - - /** - * Returns the amount of elements for which memory has been allocated at the - * calling unit. - */ - size_type local_capacity() const; - - /** - * Returns the amount of elements currently indexed in the pattern. - */ - size_type size() const; - - /** - * Returns the amount of elements currently indexed in the pattern local to - * the calling unit. - */ - size_type local_size() const; - -}; - -} - -} - -#endif // DASH__EDGE_PARTITIONED_DYNAMIC_PATTERN_H__INCLUDED diff --git a/dash/include/dash/graph/pattern/LoadBalancedDynamicPattern.h b/dash/include/dash/graph/pattern/LoadBalancedDynamicPattern.h deleted file mode 100644 index 50a340311..000000000 --- a/dash/include/dash/graph/pattern/LoadBalancedDynamicPattern.h +++ /dev/null @@ -1,130 +0,0 @@ -#ifndef DASH__LOAD_BALANCED_DYNAMIC_PATTERN_H__INCLUDED -#define DASH__LOAD_BALANCED_DYNAMIC_PATTERN_H__INCLUDED -namespace dash { - -namespace graph { - -namespace detail { - -/** - * Functor that computes the unit ID for a given vertex ID in a team. - */ -class VertexHashDistribution { - - /** - * Constructs the Functor. - */ - VertexHashDistribution(Team & team); - - /** - * Maps a vertex ID to the corresponding unit. - * TODO: argument should be Graph::size_type - */ - dart_unit_t operator()(std::size_t id); - -}; - -/** - * Functor that computes an integer value depicting the current workload of - * the calling unit. - */ -class CostFunction { - - typedef local_edge_iterator; - - /** - * Constructs the functor. - */ - CostFunction(); - - /** - * Computes an integer value depicting the current workload of the calling - * uniti. - * TODO: What happens if the main workload source are vertices? - */ - int operator()(local_edge_iterator first, local_edge_iterator last); -}; - -} - - -/** - * Pattern for graph data. Partitions the data by vertex and - * distributes these vertices to units using some hash function. - * Additionally redistributes vertices according to some cost funtion. - * NOTE: Redistribution allocates additional memory for a distributed - * directory. - * - * \concept{DashDynamicPatternConcept} - */ -template< - class DistributionMethod = dash::graph::detail::VertexHashDistribution, - class CostFunction = dash::graph::detail::CostFunction, - typename IndexType = std::size_t> -class LoadBalancedDynamicPattern { - - typedef IndexType index_type; - typedef typename std::make_unsigned::type size_type; - - typedef struct { - dart_unit_t unit; - index_type index; - } local_index_type; - - /** - * Constructs the partitioning scheme. - */ - LoadBalancedDynamicPattern(Team & team); - - /** - * Converts the given global vertex index to the owning unit. - */ - dart_unit_t unit_at(index_type global_index) const; - - /** - * Converts the given global vertex index to the owning unit and its - * local index. - */ - local_index_type local(index_type global_index) const; - - /** - * Converts the goiven local index of the calling unit to the respective - * global index. - */ - index_type global(index_type local_index) const; - - /** - * Returns, whether the element at the given global index is locak to - * the given unit. - */ - bool is_local(index_type global_index, dart_unit_t unit) const; - - /** - * Returns the amount of elements for which memory has been allocated. - */ - size_type capacity() const; - - /** - * Returns the amount of elements for which memory has been allocated at the - * calling unit. - */ - size_type local_capacity() const; - - /** - * Returns the amount of elements currently indexed in the pattern. - */ - size_type size() const; - - /** - * Returns the amount of elements currently indexed in the pattern local to - * the calling unit. - */ - size_type local_size() const; - -}; - -} - -} - -#endif // DASH__LOAD_BALANCED_DYNAMIC_PATTERN_H__INCLUDED diff --git a/dash/include/dash/graph/pattern/VertexPartitionedDynamicPattern.h b/dash/include/dash/graph/pattern/VertexPartitionedDynamicPattern.h deleted file mode 100644 index c0d6b4eb4..000000000 --- a/dash/include/dash/graph/pattern/VertexPartitionedDynamicPattern.h +++ /dev/null @@ -1,108 +0,0 @@ -#ifndef DASH__VERTEX_PARTITIONED_DYNAMIC_PATTERN_H__INCLUDED -#define DASH__VERTEX_PARTITIONED_DYNAMIC_PATTERN_H__INCLUDED - -#include - -namespace dash { - -namespace graph { - -namespace detail { - -/** - * Functor that computes the unit ID for a given vertex ID in a team. - */ -class VertexHashDistribution { - - /** - * Constructs the Functor. - */ - VertexHashDistribution(Team & team); - - /** - * Maps a vertex ID to the corresponding unit. - * TODO: argument should be Graph::size_type - */ - dart_unit_t operator()(int id); - -}; - -} - - -/** - * Pattern for graph data. Partitions the data by vertex and - * distributes these vertices to units using some hash function. - * - * \concept{DashDynamicPatternConcept} - */ -template< - class DistributionMethod = dash::graph::detail::VertexHashDistribution, - typename IndexType = int> -class VertexPartitionedDynamicPattern { - - typedef IndexType index_type; - typedef typename std::make_unsigned::type size_type; - - typedef struct { - dart_unit_t unit; - index_type index; - } local_index_type; - - /** - * Constructs the partitioning scheme. - */ - VertexPartitionedDynamicPattern(Team & team); - - /** - * Converts the given global vertex index to the owning unit. - */ - dart_unit_t unit_at(index_type global_index) const; - - /** - * Converts the given global vertex index to the owning unit and its - * local index. - */ - local_index_type local(index_type global_index) const; - - /** - * Converts the goiven local index of the calling unit to the respective - * global index. - */ - index_type global(index_type local_index) const; - - /** - * Returns, whether the element at the given global index is locak to - * the given unit. - */ - bool is_local(index_type global_index, dart_unit_t unit) const; - - /** - * Returns the amount of elements for which memory has been allocated. - */ - size_type capacity() const; - - /** - * Returns the amount of elements for which memory has been allocated at the - * calling unit. - */ - size_type local_capacity() const; - - /** - * Returns the amount of elements currently indexed in the pattern. - */ - size_type size() const; - - /** - * Returns the amount of elements currently indexed in the pattern local to - * the calling unit. - */ - size_type local_size() const; - -}; - -} - -} - -#endif // DASH__VERTEX_PARTITIONED_DYNAMIC_PATTERN_H__INCLUDED diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h index 8d104a603..1358f110f 100644 --- a/dash/include/dash/memory/GlobHeapContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -342,7 +342,7 @@ class GlobHeapContiguousMem { /** * Insert value at the end of the given bucket. */ - void push_back(container_list_index cont, value_type & val) { + local_iterator push_back(container_list_index cont, value_type & val) { auto cont_it = _container_list->begin(); std::advance(cont_it, cont); auto c_data = *cont_it; @@ -362,6 +362,7 @@ class GlobHeapContiguousMem { update_lbegin(); update_lend(); + return _lend; } /** From cdc9a590c6440e713384bdef8253f5a44b6f8056 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 6 Nov 2017 17:41:06 +0100 Subject: [PATCH 049/102] adapted code to concept change --- dash/include/dash/Graph.h | 191 +++++++++++++----- dash/include/dash/graph/internal/Graph.h | 171 +++++++++++++++- .../dash/memory/GlobHeapContiguousMem.h | 2 +- 3 files changed, 311 insertions(+), 53 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 5ca7f4b38..6afaf4626 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -80,53 +80,70 @@ class Graph { friend out_edge_it_wrapper; typedef GlobHeapContiguousMem< - vertex_container_type> glob_mem_vert_type; - typedef GlobHeapContiguousMem< - edge_container_type> glob_mem_edge_type; - typedef - GlobHeapCombinedMem glob_mem_edge_comb_type; - typedef std::vector> edge_list_type; - -public: - - typedef typename - glob_mem_vert_type::container_list_index vertex_cont_ref_type; - typedef typename - glob_mem_edge_type::container_list_index edge_cont_ref_type; - typedef VertexSizeType vertex_size_type; - typedef EdgeSizeType edge_size_type; - - typedef VertexProperties vertex_properties_type; - typedef EdgeProperties edge_properties_type; - - typedef GlobRef reference; - - typedef VertexIndex vertex_index_type; - - typedef typename - glob_mem_vert_type::local_iterator local_vertex_iterator; - typedef typename - glob_mem_edge_type::local_iterator local_in_edge_iterator; - typedef typename - glob_mem_edge_type::local_iterator local_out_edge_iterator; - typedef typename - glob_mem_edge_comb_type::local_iterator local_edge_iterator; - - typedef typename - glob_mem_vert_type::global_iterator global_vertex_iterator; - typedef typename - glob_mem_edge_type::global_iterator global_in_edge_iterator; - typedef typename - glob_mem_edge_type::global_iterator global_out_edge_iterator; - typedef typename - glob_mem_edge_comb_type::global_iterator global_edge_iterator; + vertex_container_type> glob_mem_vert_type; + typedef GlobHeapContiguousMem< + edge_container_type> glob_mem_edge_type; + typedef + GlobHeapCombinedMem glob_mem_edge_comb_type; + typedef std::vector> edge_list_type; + +public: + + typedef typename + glob_mem_vert_type::container_list_index vertex_cont_ref_type; + typedef typename + glob_mem_edge_type::container_list_index edge_cont_ref_type; + typedef VertexSizeType vertex_size_type; + typedef EdgeSizeType edge_size_type; + + typedef VertexProperties vertex_properties_type; + typedef EdgeProperties edge_properties_type; + + typedef GlobRef reference; + + typedef VertexIndex vertex_index_type; + + typedef typename + glob_mem_vert_type::local_iterator local_vertex_iterator; + typedef typename + glob_mem_edge_type::local_iterator local_in_edge_iterator; + typedef typename + glob_mem_edge_type::local_iterator local_out_edge_iterator; + typedef local_out_edge_iterator local_inout_edge_iterator; + typedef typename + glob_mem_edge_comb_type::local_iterator local_edge_iterator; + + typedef typename + glob_mem_vert_type::global_iterator global_vertex_iterator; + typedef typename + glob_mem_edge_type::global_iterator global_in_edge_iterator; + typedef typename + glob_mem_edge_type::global_iterator global_out_edge_iterator; + typedef global_out_edge_iterator global_inout_edge_iterator; + typedef typename + glob_mem_edge_comb_type::global_iterator global_edge_iterator; + + + typedef VertexProxy + local_vertex_proxy_type; + typedef VertexProxy + global_vertex_proxy_type; + typedef EdgeProxy + local_inout_edge_proxy_type; + typedef EdgeProxy + global_inout_edge_proxy_type; + typedef EdgeProxy + local_edge_proxy_type; + typedef EdgeProxy + global_edge_proxy_type; + + friend local_vertex_proxy_type; + friend global_vertex_proxy_type; + friend local_edge_proxy_type; + friend global_edge_proxy_type; public: - vertex_it_wrapper vertices = vertex_it_wrapper(this); - edge_it_wrapper edges = edge_it_wrapper(this); - in_edge_it_wrapper in_edges = in_edge_it_wrapper(this); - out_edge_it_wrapper out_edges = out_edge_it_wrapper(this); public: @@ -173,6 +190,74 @@ class Graph { */ self_t & operator=(self_t &&) = delete; + /** + * Returns an object handling interactions with a vertex pointed to by + * the given iterator. + */ + local_vertex_proxy_type operator[](local_vertex_iterator & it) { + return local_vertex_proxy_type(it, this); + } + + /** + * Returns an object handling interactions with a vertex pointed to by + * the given iterator. + */ + global_vertex_proxy_type operator[](global_vertex_iterator & it) { + return global_vertex_proxy_type(it, this); + } + + /** + * Returns an object handling interactions with an edge pointed to by + * the given iterator. + */ + local_inout_edge_proxy_type operator[](local_inout_edge_iterator & it) { + return local_inout_edge_proxy_type(it, this); + } + + /** + * Returns an object handling interactions with an edge pointed to by + * the given iterator. + */ + global_inout_edge_proxy_type operator[](global_inout_edge_iterator & it) { + return global_inout_edge_proxy_type(it, this); + } + + /** + * Returns an object handling interactions with an edge pointed to by + * the given iterator. + */ + global_edge_proxy_type operator[](global_edge_iterator & it) { + return global_edge_proxy_type(it, this); + } + + /** + * Returns a vertex iterator range object. + */ + vertex_it_wrapper & vertices() { + return _vertices; + } + + /** + * Returns an edge iterator range object. + */ + edge_it_wrapper & edges() { + return _edges; + } + + /** + * Returns an in-edge iterator range object. + */ + in_edge_it_wrapper & in_edges() { + return _in_edges; + } + + /** + * Returns an out-edge iterator range object. + */ + out_edge_it_wrapper & out_edges() { + return _out_edges; + } + /** * Returns the number of vertices in the whole graph. * @@ -227,15 +312,15 @@ class Graph { /** * Removes a given vertex. */ - void remove_vertex(const vertex_index_type & v) { + void remove_vertex(const local_vertex_iterator & v) { } /** - * Removes all edges (in & out) from the given vertex). + * Removes a given vertex. */ - void clear_vertex(const vertex_index_type & v) { - + void remove_vertex(const global_vertex_iterator & v) { + } /** @@ -373,8 +458,8 @@ class Graph { for(int j = i + 1; j < remote_edges_displs.size(); ++j) { remote_edges_displs[j] += remote_edges_count[i]; } + _remote_edges[i].clear(); } - _remote_edges.clear(); // exchange amount of edges to be transferred with other units std::vector edge_count(_team->size()); DASH_ASSERT_RETURNS( @@ -543,6 +628,14 @@ class Graph { edge_size_type _alloc_edges_per_vertex; /** Edges that have to be added to vertices on another unit in next commit */ edge_list_type _remote_edges; + /** wrapper for vertex iterator ranges */ + vertex_it_wrapper _vertices = vertex_it_wrapper(this); + /** wrapper for edge iterator ranges */ + edge_it_wrapper _edges = edge_it_wrapper(this); + /** wrapper for in-edge iterator ranges */ + in_edge_it_wrapper _in_edges = in_edge_it_wrapper(this); + /** wrapper for out-edge iterator ranges */ + out_edge_it_wrapper _out_edges = out_edge_it_wrapper(this); }; diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index 3f12d9270..e273f29ed 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -48,9 +48,6 @@ class Vertex { typedef typename GraphType::vertex_properties_type properties_type; friend GraphType; - friend InEdgeIteratorWrapper; - friend OutEdgeIteratorWrapper; - friend EdgeIteratorWrapper; public: @@ -75,6 +72,139 @@ class Vertex { }; +template +class VertexProxy { + + typedef GraphType graph_type; + typedef IteratorType iterator_type; + typedef VertexProxy self_t; + typedef typename GraphType::vertex_type vertex_type; + typedef typename GraphType::vertex_properties_type properties_type; + + template + class edge_range { + + typedef Parent parent_type; + typedef GlobMemType glob_mem_type; + typedef typename glob_mem_type::global_iterator global_iterator; + typedef typename glob_mem_type::local_iterator local_iterator; + typedef typename parent_type::graph_type::local_vertex_iterator + local_vertex_iterator; + + public: + + edge_range(parent_type & p, glob_mem_type * gmem) + : _parent(p), + _glob_mem(gmem) + { } + + global_iterator begin() { + return begin(_parent._iterator); + } + + global_iterator end() { + return end(_parent._iterator); + } + + private: + + template + global_iterator begin(VertexIteratorType it) { + auto lpos = _parent._iterator.lpos(); + return global_iterator( + _glob_mem, + lpos.unit, + lpos.index, + 0 + ); + } + + global_iterator begin(local_vertex_iterator it) { + return global_iterator( + _glob_mem, + _parent._graph->_myid, + it.pos(), + 0 + ); + } + + template + global_iterator end(VertexIteratorType it) { + auto lpos = _parent._iterator.lpos(); + return global_iterator( + _glob_mem, + lpos.unit, + lpos.index, + _glob_mem->container_size(lpos.unit, lpos.index) + ); + } + + global_iterator end(local_vertex_iterator it) { + return global_iterator( + _glob_mem, + _parent._graph->_myid, + it.pos(), + _glob_mem->container_size(_parent._graph->_myid, it.pos()) + ); + } + + private: + + parent_type & _parent; + glob_mem_type * _glob_mem; + + }; + + typedef edge_range + inout_edge_range_type; + typedef edge_range + edge_range_type; + +public: + + VertexProxy() = delete; + + VertexProxy(iterator_type it, graph_type * g) + : _iterator(it), + _graph(g), + _out_edges(*this, g->_glob_mem_out_edge), + _in_edges(*this, g->_glob_mem_in_edge), + _edges(*this, g->_glob_mem_edge) + { } + + inout_edge_range_type & out_edges() { + return _out_edges; + } + + inout_edge_range_type & in_edges() { + return _in_edges; + } + + edge_range_type & edges() { + return _edges; + } + + properties_type & properties() { + // load properties lazily + if(!_vertex_loaded) { + _vertex = *_iterator; + _vertex_loaded = true; + } + return _vertex.properties; + } + +private: + + iterator_type _iterator; + vertex_type _vertex; + bool _vertex_loaded = false; + graph_type * _graph; + inout_edge_range_type _out_edges; + inout_edge_range_type _in_edges; + edge_range_type _edges; + +}; + /** * Edge type holding properties and references to the vertices it belongs to. */ @@ -151,6 +281,41 @@ class Edge { }; +template +class EdgeProxy { + + typedef GraphType graph_type; + typedef IteratorType iterator_type; + typedef typename GraphType::edge_type edge_type; + typedef typename GraphType::edge_properties_type properties_type; + +public: + + EdgeProxy() = delete; + + EdgeProxy(iterator_type it, graph_type * g) + : _iterator(it), + _graph(g) + { } + + properties_type & properties() { + // load properties lazily + if(!_edge_loaded) { + _edge = *_iterator; + _edge_loaded = true; + } + return _edge.properties; + } + +private: + + iterator_type _iterator; + edge_type _edge; + bool _edge_loaded = false; + graph_type * _graph; + +}; + /** * Default property type holding no data. */ diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h index 1358f110f..331b462e6 100644 --- a/dash/include/dash/memory/GlobHeapContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -362,7 +362,7 @@ class GlobHeapContiguousMem { update_lbegin(); update_lend(); - return _lend; + return _lend - 1; } /** From ac85bd3608252970fb4e19ad4f616b099171c651 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Tue, 7 Nov 2017 11:37:43 +0100 Subject: [PATCH 050/102] fixed combined edge adjacency iteration --- dash/include/dash/graph/internal/Graph.h | 28 +++++++++++++++++++----- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index e273f29ed..ce35451e0 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -86,10 +86,13 @@ class VertexProxy { typedef Parent parent_type; typedef GlobMemType glob_mem_type; + typedef typename glob_mem_type::index_type index_type; typedef typename glob_mem_type::global_iterator global_iterator; typedef typename glob_mem_type::local_iterator local_iterator; typedef typename parent_type::graph_type::local_vertex_iterator local_vertex_iterator; + typedef typename parent_type::graph_type::glob_mem_edge_comb_type + glob_mem_edge_comb_type; public: @@ -111,19 +114,21 @@ class VertexProxy { template global_iterator begin(VertexIteratorType it) { auto lpos = _parent._iterator.lpos(); + auto index = it_position(_glob_mem, lpos.index); return global_iterator( _glob_mem, lpos.unit, - lpos.index, + index, 0 ); } global_iterator begin(local_vertex_iterator it) { + auto index = it_position(_glob_mem, it.pos()); return global_iterator( _glob_mem, _parent._graph->_myid, - it.pos(), + index, 0 ); } @@ -131,23 +136,34 @@ class VertexProxy { template global_iterator end(VertexIteratorType it) { auto lpos = _parent._iterator.lpos(); + auto index = it_position(_glob_mem, lpos.index); return global_iterator( _glob_mem, lpos.unit, - lpos.index, - _glob_mem->container_size(lpos.unit, lpos.index) + index, + _glob_mem->container_size(lpos.unit, index) ); } global_iterator end(local_vertex_iterator it) { + auto index = it_position(_glob_mem, it.pos()); return global_iterator( _glob_mem, _parent._graph->_myid, - it.pos(), - _glob_mem->container_size(_parent._graph->_myid, it.pos()) + index, + _glob_mem->container_size(_parent._graph->_myid, index) ); } + template + index_type it_position(_GMem * gmem, index_type pos) { + return pos; + } + + index_type it_position(glob_mem_edge_comb_type * gmem, index_type pos) { + return pos * 2; + } + private: parent_type & _parent; From 04e98862397b520c99b48f4c0451bf41c9ce5312 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Tue, 7 Nov 2017 12:25:46 +0100 Subject: [PATCH 051/102] finished proxy types --- dash/include/dash/Graph.h | 18 +-- dash/include/dash/graph/internal/Graph.h | 173 ++++++++++++++++++++--- 2 files changed, 159 insertions(+), 32 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 6afaf4626..2c19ab7d7 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -194,7 +194,7 @@ class Graph { * Returns an object handling interactions with a vertex pointed to by * the given iterator. */ - local_vertex_proxy_type operator[](local_vertex_iterator & it) { + local_vertex_proxy_type operator[](local_vertex_iterator it) { return local_vertex_proxy_type(it, this); } @@ -202,7 +202,7 @@ class Graph { * Returns an object handling interactions with a vertex pointed to by * the given iterator. */ - global_vertex_proxy_type operator[](global_vertex_iterator & it) { + global_vertex_proxy_type operator[](global_vertex_iterator it) { return global_vertex_proxy_type(it, this); } @@ -210,7 +210,7 @@ class Graph { * Returns an object handling interactions with an edge pointed to by * the given iterator. */ - local_inout_edge_proxy_type operator[](local_inout_edge_iterator & it) { + local_inout_edge_proxy_type operator[](local_inout_edge_iterator it) { return local_inout_edge_proxy_type(it, this); } @@ -218,7 +218,7 @@ class Graph { * Returns an object handling interactions with an edge pointed to by * the given iterator. */ - global_inout_edge_proxy_type operator[](global_inout_edge_iterator & it) { + global_inout_edge_proxy_type operator[](global_inout_edge_iterator it) { return global_inout_edge_proxy_type(it, this); } @@ -226,7 +226,7 @@ class Graph { * Returns an object handling interactions with an edge pointed to by * the given iterator. */ - global_edge_proxy_type operator[](global_edge_iterator & it) { + global_edge_proxy_type operator[](global_edge_iterator it) { return global_edge_proxy_type(it, this); } @@ -491,13 +491,13 @@ class Graph { ); // add missing edges to local memory space for(auto edge : edges) { - if(edge._source.unit == _myid) { - add_local_edge(edge._source, edge._target, edge.properties, + if(edge.source.unit == _myid) { + add_local_edge(edge.source, edge.target, edge.properties, _glob_mem_out_edge); } - if(edge._target.unit == _myid) { + if(edge.target.unit == _myid) { // _glob_mem_in_edge == _glob_mem_out_edge for undirected graph types - add_local_edge(edge._target, edge._source, edge.properties, + add_local_edge(edge.target, edge.source, edge.properties, _glob_mem_in_edge); //TODO: for directed graphs, should target and source really be mutated // in the in-edge list? diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index ce35451e0..b600f88b3 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -72,6 +72,10 @@ class Vertex { }; +/** + * Proxy type for vertices. Encapsulates logic for data retrieval, data manipulation + * and iterator retrieval. + */ template class VertexProxy { @@ -81,6 +85,10 @@ class VertexProxy { typedef typename GraphType::vertex_type vertex_type; typedef typename GraphType::vertex_properties_type properties_type; + /** + * Handles the iterator ranges for adjacency iteration of the respective + * vertex. + */ template class edge_range { @@ -96,21 +104,35 @@ class VertexProxy { public: + /* + * Constructs the range handler object. + */ edge_range(parent_type & p, glob_mem_type * gmem) : _parent(p), _glob_mem(gmem) { } + /** + * Returns global iterator to the first element in the edge list of + * the given vertex. + */ global_iterator begin() { return begin(_parent._iterator); } + /** + * Returns global iterator past the last element in the edge list of + * the given vertex. + */ global_iterator end() { return end(_parent._iterator); } private: + /** + * Begin iterator for global vertex iterators. + */ template global_iterator begin(VertexIteratorType it) { auto lpos = _parent._iterator.lpos(); @@ -123,6 +145,9 @@ class VertexProxy { ); } + /** + * Begin iterator for local vertex iterators. + */ global_iterator begin(local_vertex_iterator it) { auto index = it_position(_glob_mem, it.pos()); return global_iterator( @@ -133,6 +158,9 @@ class VertexProxy { ); } + /** + * End iterator for global vertex iterators. + */ template global_iterator end(VertexIteratorType it) { auto lpos = _parent._iterator.lpos(); @@ -145,6 +173,9 @@ class VertexProxy { ); } + /** + * End iterator for local vertex iterators. + */ global_iterator end(local_vertex_iterator it) { auto index = it_position(_glob_mem, it.pos()); return global_iterator( @@ -155,18 +186,28 @@ class VertexProxy { ); } + /** + * Determines the position of the edge-list for inbound and outbound + * edge memory spaces. + */ template index_type it_position(_GMem * gmem, index_type pos) { return pos; } + /** + * Determines the position of the edge-list for the combined edge memory + * space. + */ index_type it_position(glob_mem_edge_comb_type * gmem, index_type pos) { return pos * 2; } private: + /** Reference to the corresponding VertexProxy object */ parent_type & _parent; + /** Reference to the GlobMem object of the targeted memory space */ glob_mem_type * _glob_mem; }; @@ -178,8 +219,14 @@ class VertexProxy { public: + /** + * Default constructor. Explicitly deleted. + */ VertexProxy() = delete; + /** + * Constructs the object with a vertex iterator. + */ VertexProxy(iterator_type it, graph_type * g) : _iterator(it), _graph(g), @@ -188,19 +235,31 @@ class VertexProxy { _edges(*this, g->_glob_mem_edge) { } + /** + * Returns a range for adjacent outbound edges of the referenced vertex. + */ inout_edge_range_type & out_edges() { return _out_edges; } + /** + * Returns a range for adjacent inbound edges of the referenced vertex. + */ inout_edge_range_type & in_edges() { return _in_edges; } + /** + * Returns a range for adjacent edges of the referenced vertex. + */ edge_range_type & edges() { return _edges; } - properties_type & properties() { + /** + * Returns the properties of the referenced vertex. Data is loaded lazily. + */ + properties_type & attributes() { // load properties lazily if(!_vertex_loaded) { _vertex = *_iterator; @@ -209,14 +268,30 @@ class VertexProxy { return _vertex.properties; } + /** + * Sets the attribute data for the referenced vertex. + */ + void set_attributes(properties_type & prop) { + _vertex.properties = prop; + auto ref = *_iterator; + ref = _vertex; + } + private: + /** Iterator to the referenced vertex */ iterator_type _iterator; + /** data of the referenced vertex */ vertex_type _vertex; + /** Whether the vertex data has already been loaded */ bool _vertex_loaded = false; + /** Pointer to the graph container */ graph_type * _graph; + /** Range object for outbound edges */ inout_edge_range_type _out_edges; + /** Range object for inbbound edges */ inout_edge_range_type _in_edges; + /** Range object for edges */ edge_range_type _edges; }; @@ -232,8 +307,6 @@ class Edge { typedef typename GraphType::vertex_index_type vertex_index_type; typedef typename GraphType::edge_properties_type properties_type; - friend GraphType; - public: /** @@ -250,8 +323,8 @@ class Edge { const properties_type & properties, const team_unit_t owner ) - : _source(owner, source.pos()), - _target(owner, target.pos()), + : source(owner, source.pos()), + target(owner, target.pos()), properties(properties) { } @@ -264,8 +337,8 @@ class Edge { const properties_type & properties, const team_unit_t owner ) - : _source(owner, source.pos()), - _target(target.lpos().unit, target.lpos().index), + : source(owner, source.pos()), + target(target.lpos().unit, target.lpos().index), properties(properties) { } @@ -277,8 +350,8 @@ class Edge { const vertex_index_type & target, const properties_type & properties ) - : _source(source), - _target(target), + : source(source), + target(target), properties(properties) { } @@ -286,48 +359,102 @@ class Edge { /** Properties of this edge */ properties_type properties; - -private: - - //TODO: Examine, if saving source can be avoided /** Source vertex the edge is pointing from */ - vertex_index_type _source; + vertex_index_type source; /** Target vertex the edge is pointing to */ - vertex_index_type _target; + vertex_index_type target; }; +/** + * Proxy type for edges. Encapsulates logic for data retrieval and + * manipulation. + */ template class EdgeProxy { - typedef GraphType graph_type; - typedef IteratorType iterator_type; - typedef typename GraphType::edge_type edge_type; - typedef typename GraphType::edge_properties_type properties_type; + typedef GraphType graph_type; + typedef IteratorType iterator_type; + typedef typename graph_type::edge_type edge_type; + typedef typename graph_type::edge_properties_type properties_type; + typedef typename graph_type::global_vertex_iterator vertex_iterator; public: - + /** + * Default constructor. Explicitly deleted. + */ EdgeProxy() = delete; + /** + * Constructs the edge proxy from a given edge iterator. + */ EdgeProxy(iterator_type it, graph_type * g) : _iterator(it), _graph(g) { } - properties_type & properties() { - // load properties lazily + /** + * Returns the properties of the referenced edge. Data is loaded lazily. + */ + properties_type & attributes() { + lazy_load(); + return _edge.properties; + } + + /** + * Returns iglobal iterator to the source vertex. + */ + vertex_iterator source() { + lazy_load(); + return vertex_iterator( + _graph->_glob_mem_vertex, + _edge.source.unit, + _edge.source.offset + ); + } + + /** + * Returns iglobal iterator to the target vertex. + */ + vertex_iterator target() { + lazy_load(); + return vertex_iterator( + _graph->_glob_mem_vertex, + _edge.target.unit, + _edge.target.offset + ); + } + + /** + * Sets the attribute data for the referenced edge. + */ + void set_attributes(properties_type & prop) { + _edge.properties = prop; + auto ref = *_iterator; + ref = _edge; + } + +private: + + /** + * Loads edge data lazily. + */ + void lazy_load() { if(!_edge_loaded) { _edge = *_iterator; _edge_loaded = true; } - return _edge.properties; } private: + /** Iterator to the referenced edge */ iterator_type _iterator; + /** Data of the referenced edge */ edge_type _edge; + /** Whether edge data has already been loaded */ bool _edge_loaded = false; + /** Pointer to the graph container */ graph_type * _graph; }; From 4d3e858c28290ff1de85ebb0746bd799ff28c9ca Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Fri, 10 Nov 2017 13:51:37 +0100 Subject: [PATCH 052/102] fixed dart_storage call --- dash/include/dash/memory/GlobHeapContiguousMem.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h index 331b462e6..34997d5b9 100644 --- a/dash/include/dash/memory/GlobHeapContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -260,7 +260,7 @@ class GlobHeapContiguousMem { // attach new container location to global memory space dart_gptr_t gptr = DART_GPTR_NULL; - dart_storage_t ds = dart_storage(c_data->container->size()); + dash::dart_storage ds(c_data->container->size()); DASH_ASSERT_RETURNS( dart_team_memregister( _team->dart_id(), From 14fb545417eda23b24987d67865db4c784096f84 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 13 Nov 2017 12:33:50 +0100 Subject: [PATCH 053/102] fixed set_attributes for local pointers --- dash/include/dash/graph/internal/Graph.h | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index b600f88b3..52045adf4 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -82,8 +82,10 @@ class VertexProxy { typedef GraphType graph_type; typedef IteratorType iterator_type; typedef VertexProxy self_t; - typedef typename GraphType::vertex_type vertex_type; - typedef typename GraphType::vertex_properties_type properties_type; + typedef typename graph_type::vertex_type vertex_type; + typedef typename graph_type::vertex_properties_type properties_type; + typedef typename graph_type::global_vertex_iterator global_vertex_iterator; + typedef typename graph_type::local_vertex_iterator local_vertex_iterator; /** * Handles the iterator ranges for adjacency iteration of the respective @@ -272,11 +274,29 @@ class VertexProxy { * Sets the attribute data for the referenced vertex. */ void set_attributes(properties_type & prop) { + set_attributes(prop, _iterator); + } + +private: + + /** + * Overload of set_aatributes for global pointers. + */ + void set_attributes(properties_type & prop, global_vertex_iterator it) { _vertex.properties = prop; auto ref = *_iterator; ref = _vertex; } + /** + * Overload of set_aatributes for local pointers. + */ + void set_attributes(properties_type & prop, local_vertex_iterator it) { + _vertex.properties = prop; + typename iterator_type::pointer ptr = _iterator; + *ptr = _vertex; + } + private: /** Iterator to the referenced vertex */ From 0642e9463d6742fccac486c616dbb349074daead Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Wed, 6 Dec 2017 17:24:02 +0100 Subject: [PATCH 054/102] fixed dart_alltoallv --- dart-impl/mpi/src/dart_communication.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dart-impl/mpi/src/dart_communication.c b/dart-impl/mpi/src/dart_communication.c index 00a1ece4b..67b803a4c 100644 --- a/dart-impl/mpi/src/dart_communication.c +++ b/dart-impl/mpi/src/dart_communication.c @@ -1891,7 +1891,7 @@ dart_ret_t dart_alltoallv( "dart_adapt_teamlist_convert failed", teamid); return DART_ERR_INVAL; } - if (sendbuf == recvbuf || NULL == sendbuf) { + if (sendbuf == recvbuf) { sendbuf = MPI_IN_PLACE; } comm = team_data->comm; From 062a8a431a1c48ce0b77305cfcb4d81de6ec9a8a Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Wed, 6 Dec 2017 17:24:41 +0100 Subject: [PATCH 055/102] added graph constructor with edge-list initialization --- dash/include/dash/Graph.h | 156 +++++++++++++++++++++++++++++++++++--- 1 file changed, 147 insertions(+), 9 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 2c19ab7d7..f9b997741 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -2,6 +2,8 @@ #define DASH__GRAPH_H__INCLUDED #include +#include +#include #include #include #include @@ -139,16 +141,13 @@ class Graph { friend local_vertex_proxy_type; friend global_vertex_proxy_type; + friend local_inout_edge_proxy_type; + friend global_inout_edge_proxy_type; friend local_edge_proxy_type; friend global_edge_proxy_type; - -public: - public: - //TODO: add constructor that can construct a graph with edge iterators - /** * Constructs an empty graph. */ @@ -164,6 +163,140 @@ class Graph { allocate(n_vertices, n_vertex_edges); } + /** + * Constructs a graph from an iterator range pointing to elements of type + * std::pair. + * Assumes vertex IDs are taken from a contiguous range [0...n] and n is + * divisible by the number of units in the Team of the container. + * Partitions vertices based on their id: + * vertex_id / (n / num_units) = owner + * + * /todo Add + */ + template + Graph( + InputIterator begin, + InputIterator end, + vertex_size_type n_vertices, + Team & team = dash::Team::All() + ) + : _team(&team), + _myid(team.myid()), + _remote_edges(team.size()) + { + // TODO: find heuristic to allocate a reasonable amount of edge memory + // per vertex + allocate(n_vertices, 0); + + std::unordered_map lvertices; + std::unordered_map gvertices; + // TODO: find method that uses less memory + std::unordered_set remote_vertices_set; + std::vector> remote_vertices(_team->size()); + for(auto it = begin; it != end; ++it) { + auto v = it->first; + auto u = it->second; + + if(lvertices.find(v) == lvertices.end()) { + lvertices[v] = add_vertex(); + } + auto target_owner = vertex_owner(u, n_vertices); + if(target_owner == _myid) { + if(lvertices.find(u) == lvertices.end()) { + lvertices[u] = add_vertex(); + } + } else { + // collect vertices for remote nodes and prevent adding vertices more + // than once + bool inserted; + std::tie(std::ignore, inserted) = remote_vertices_set.insert(u); + if(inserted) { + remote_vertices[target_owner].push_back(u); + } + } + } + + // send vertices to their owner units and receive their local index + { + std::vector sizes_send(remote_vertices.size()); + std::vector displs_send(remote_vertices.size()); + std::vector remote_vertices_send; + int total_send = 0; + for(int i = 0; i < remote_vertices.size(); ++i) { + sizes_send[i] = remote_vertices[i].size() * sizeof(vertex_size_type); + displs_send[i] = total_send * sizeof(vertex_size_type); + total_send += remote_vertices[i].size(); + } + remote_vertices_send.reserve(total_send); + for(auto & vertex_set : remote_vertices) { + remote_vertices_send.insert(remote_vertices_send.end(), + vertex_set.begin(), vertex_set.end()); + } + std::vector sizes_recv(remote_vertices.size()); + std::vector displs_recv(remote_vertices.size()); + dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), + DART_TYPE_BYTE, _team->dart_id()); + int total_recv = 0; + for(int i = 0; i < sizes_recv.size(); ++i) { + displs_recv[i] = total_recv; + total_recv += sizes_recv[i] / sizeof(vertex_size_type); + } + std::vector remote_vertices_recv(total_recv); + if(total_send > 0 || total_recv > 0) { + dart_alltoallv(remote_vertices_send.data(), sizes_send.data(), + displs_send.data(), DART_TYPE_BYTE, remote_vertices_recv.data(), + sizes_recv.data(), displs_recv.data(), _team->dart_id()); + } + + // exchange data + for(auto & index : remote_vertices_recv) { + if(lvertices.find(index) == lvertices.end()) { + auto v = add_vertex(); + index = v.pos(); + } else { + index = lvertices[index].pos(); + } + } + if(total_send > 0 || total_recv > 0) { + dart_alltoallv(remote_vertices_recv.data(), sizes_recv.data(), + displs_recv.data(), DART_TYPE_BYTE, remote_vertices_send.data(), + sizes_send.data(), displs_send.data(), _team->dart_id()); + } + // all vertices have been added - commit changes to global memory space + commit(); + // remote_vertices_send now contains the local indices in the iteration + // space of the corresponding unit + team_unit_t unit { 0 }; + int index = 0; + for(auto & lindex : remote_vertices_send) { + if(index >= sizes_send[unit]) { + ++unit; + index = 0; + } + gvertices[remote_vertices[unit][index]] = global_vertex_iterator( + _glob_mem_vertex, unit, lindex); + ++index; + } + } + + // finally add edges with the vertex iterators gained from the previous + // steps + for(auto it = begin; it != end; ++it) { + auto v_it = lvertices[it->first]; + auto u = it->second; + + if(vertex_owner(u, n_vertices) == _myid) { + auto u_it = lvertices[u]; + add_edge(v_it, u_it); + } else { + auto u_it = gvertices[u]; + add_edge(v_it, u_it); + } + } + // commit edges + commit(); + } + /** Destructs the graph. */ ~Graph() { @@ -349,7 +482,7 @@ class Graph { * Adds an edge between two given vertices with the given properties * locally. * Edges that belong to vertices held on a different unit are marked for - * transfer. These edges will be transferred after calling \c barrier(). + * transfer. These edges will be transferred after calling \c commit(). * * \return Pair, with pair::first set to the index of the newly created edge * and pair::second set to a boolean indicating whether the edge has @@ -379,7 +512,7 @@ class Graph { /** * Adds an edge between two given vertices locally. * Edges that belong to vertices held on a different unit are marked for - * transfer. These edges will be transferred after calling \c barrier(). + * transfer. These edges will be transferred after calling \c commit(). * * \return Pair, with pair::first set to the index of the newly created edge * and pair::second set to a boolean indicating whether the edge has @@ -396,7 +529,7 @@ class Graph { /** * Adds an edge between two given vertices locally. * Edges that belong to vertices held on a different unit are marked for - * transfer. These edges will be transferred after calling \c barrier(). + * transfer. These edges will be transferred after calling \c commit(). * * \return Pair, with pair::first set to the index of the newly created edge * and pair::second set to a boolean indicating whether the edge has @@ -444,7 +577,7 @@ class Graph { * Commits local changes of the graph to global memory space since the last * call of this method. */ - void barrier() { + void commit() { // move all edges that have to be added by other units in a contiguous // memory region std::vector remote_edges; @@ -608,6 +741,11 @@ class Graph { return glob_mem_edge->push_back(source.offset, edge); } + team_unit_t vertex_owner(vertex_size_type v, vertex_size_type n_vertices) { + team_unit_t owner { v / (n_vertices / _team->size()) }; + return owner; + } + private: /** the team containing all units using the container */ From c5688e16355b779c4ddf63dadd96dcf92448c116 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 11 Dec 2017 15:21:52 +0100 Subject: [PATCH 056/102] fixed constructor for edge-list initialization --- dash/include/dash/Graph.h | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index f9b997741..d8f6bb7f3 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -173,10 +173,10 @@ class Graph { * * /todo Add */ - template + template Graph( - InputIterator begin, - InputIterator end, + ForwardIterator begin, + ForwardIterator end, vertex_size_type n_vertices, Team & team = dash::Team::All() ) @@ -219,11 +219,13 @@ class Graph { // send vertices to their owner units and receive their local index { std::vector sizes_send(remote_vertices.size()); + std::vector sizes_send_n(remote_vertices.size()); std::vector displs_send(remote_vertices.size()); std::vector remote_vertices_send; int total_send = 0; for(int i = 0; i < remote_vertices.size(); ++i) { sizes_send[i] = remote_vertices[i].size() * sizeof(vertex_size_type); + sizes_send_n[i] = remote_vertices[i].size(); displs_send[i] = total_send * sizeof(vertex_size_type); total_send += remote_vertices[i].size(); } @@ -239,9 +241,10 @@ class Graph { int total_recv = 0; for(int i = 0; i < sizes_recv.size(); ++i) { displs_recv[i] = total_recv; - total_recv += sizes_recv[i] / sizeof(vertex_size_type); + total_recv += sizes_recv[i]; } - std::vector remote_vertices_recv(total_recv); + std::vector remote_vertices_recv(total_recv / + sizeof(vertex_size_type)); if(total_send > 0 || total_recv > 0) { dart_alltoallv(remote_vertices_send.data(), sizes_send.data(), displs_send.data(), DART_TYPE_BYTE, remote_vertices_recv.data(), @@ -269,9 +272,15 @@ class Graph { team_unit_t unit { 0 }; int index = 0; for(auto & lindex : remote_vertices_send) { - if(index >= sizes_send[unit]) { + while(index >= sizes_send_n[unit]) { ++unit; index = 0; + if(unit >= _team->size()) { + break; + } + } + if(unit >= _team->size()) { + break; } gvertices[remote_vertices[unit][index]] = global_vertex_iterator( _glob_mem_vertex, unit, lindex); From e148ac2a615a10de59cd7543aad60e1117ae6d62 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 11 Dec 2017 15:23:26 +0100 Subject: [PATCH 057/102] fixed wrong size type in GlobHeapContiguousMem --- dash/include/dash/memory/GlobHeapContiguousMem.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h index 34997d5b9..ad2fbc203 100644 --- a/dash/include/dash/memory/GlobHeapContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -201,7 +201,7 @@ class GlobHeapContiguousMem { // Gather information about the max amount of containers a single unit // currently holds std::vector container_count(_team->size()); - int my_container_count = _container_list->size(); + size_type my_container_count = _container_list->size(); DASH_ASSERT_RETURNS( dart_allgather(&my_container_count, container_count.data(), sizeof(size_type), DART_TYPE_BYTE, _team->dart_id()), From 2fc13a298f44396353f49bf8739ade18a0855513 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 14 Dec 2017 15:11:58 +0100 Subject: [PATCH 058/102] optimized performance --- dash/include/dash/Graph.h | 4 +- .../dash/memory/GlobHeapContiguousMem.h | 370 ++++++++---------- 2 files changed, 173 insertions(+), 201 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index d8f6bb7f3..8e059eba8 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -92,9 +92,9 @@ class Graph { public: typedef typename - glob_mem_vert_type::container_list_index vertex_cont_ref_type; + glob_mem_vert_type::bucket_index_type vertex_cont_ref_type; typedef typename - glob_mem_edge_type::container_list_index edge_cont_ref_type; + glob_mem_edge_type::bucket_index_type edge_cont_ref_type; typedef VertexSizeType vertex_size_type; typedef EdgeSizeType edge_size_type; diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h index ad2fbc203..83a381980 100644 --- a/dash/include/dash/memory/GlobHeapContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -22,46 +22,21 @@ namespace dash { namespace internal { -/** - * Helper class encapsulating single bucket container. - */ -template +template struct container_data { - typedef typename GlobMemType::container_type container_type; - typedef typename GlobMemType::value_type value_type; - typedef typename GlobMemType::index_type index_type; - typedef typename GlobMemType::size_type size_type; - typedef typename GlobMemType::local_iterator local_iterator; - typedef typename local_iterator::bucket_type bucket_type; - typedef typename std::list bucket_list; + typedef ContainerType container_type; + typedef typename ContainerType::value_type value_type; + typedef typename ContainerType::difference_type index_type; typedef typename - GlobMemType::bucket_cumul_sizes_map::difference_type bucket_sizes_index; + GlobHeapLocalPtr::bucket_type bucket_type; - /** - * Constructor. Creates containers for globally available elements and - * locally available (unattached) elements. - */ - container_data(size_type n_local_elem) - : container(new container_type()), - unattached_container(new container_type()) - { - container->reserve(n_local_elem); - } - - /** - * Default constructor. Explicitly deleted. - */ - container_data() = delete; + container_data(bucket_type & bkt) + : bucket(bkt) + { } - /** container for globally available elements */ - std::shared_ptr container; - /** container for locally available (unattached) elements */ - std::shared_ptr unattached_container; - /** pointer to corresponding bucket in GlobHeapContiguousMem object */ - bucket_type * container_bucket; - /** pointer to corresponding bucket in GlobHeapContiguousMem object */ - bucket_type * unattached_container_bucket; + container_type container; + bucket_type & bucket; }; @@ -76,25 +51,26 @@ template class GlobHeapContiguousMem { private: - typedef GlobHeapContiguousMem self_t; + typedef GlobHeapContiguousMem self_t; public: - typedef ContainerType container_type; - typedef internal::container_data data_type; - typedef std::list container_list_type; - typedef typename - container_list_type::difference_type container_list_index; - typedef typename ContainerType::value_type value_type; - typedef typename ContainerType::difference_type index_type; - typedef GlobHeapLocalPtr local_iterator; - typedef GlobPtr global_iterator; - typedef typename ContainerType::size_type size_type; - typedef typename local_iterator::bucket_type bucket_type; - typedef typename std::list bucket_list; - typedef typename std::list bucket_ptr_list; - typedef local_iterator local_pointer; - typedef local_iterator const_local_pointer; - typedef std::vector> bucket_cumul_sizes_map; + typedef ContainerType container_type; + typedef internal::container_data data_type; + typedef std::vector container_list_type; + typedef typename ContainerType::value_type value_type; + typedef typename ContainerType::difference_type index_type; + typedef GlobHeapLocalPtr local_iterator; + typedef GlobPtr global_iterator; + typedef typename ContainerType::size_type size_type; + typedef typename local_iterator::bucket_type bucket_type; + // must be List because of GlobHeapLocalPtr + typedef typename std::list bucket_list_type; + typedef typename std::list bucket_ptr_list; + typedef typename bucket_ptr_list::difference_type bucket_index_type; + typedef typename bucket_list_type::difference_type local_bucket_index_type; + typedef local_iterator local_pointer; + typedef local_iterator const_local_pointer; + typedef std::vector> bucket_cumul_sizes_map; template friend class dash::GlobPtr; @@ -106,8 +82,7 @@ class GlobHeapContiguousMem { * Constructor. */ GlobHeapContiguousMem(Team & team = dash::Team::All()) - : _container_list(new container_list_type()), - _buckets(), + : _buckets(), _global_buckets(), _team(&team), _teamid(team.dart_id()), @@ -119,39 +94,40 @@ class GlobHeapContiguousMem { /** * Adds a new bucket into memory space. */ - container_list_index add_container(size_type n_elements) { + bucket_index_type add_container(size_type n_elements) { + // TODO: set capacity increment_bucket_sizes(); - auto c_data = data_type(n_elements); // create bucket data and add to bucket list bucket_type cont_bucket { 0, - c_data.container->data(), + nullptr, DART_GPTR_NULL, false }; bucket_type unattached_cont_bucket { 0, - c_data.unattached_container->data(), + nullptr, DART_GPTR_NULL, false }; _buckets.push_back(cont_bucket); - // add a pointer to the corresponding bucket data - c_data.container_bucket = &(_buckets.back()); // for global iteration, only _container's bucket is needed _global_buckets.push_back(&(_buckets.back())); - _buckets.push_back(unattached_cont_bucket); - c_data.unattached_container_bucket = &(_buckets.back()); - _container_list->push_back(c_data); - return _container_list->size() - 1; + _unattached_containers.emplace_back(_buckets.back()); + auto & unattached_container = _unattached_containers.back().container; + unattached_container.reserve(n_elements); + + _buckets.push_back(unattached_cont_bucket); + return _global_buckets.size() - 1; } /** * Returns a reference to the requested element. */ - value_type & get(container_list_index cont, index_type pos) { + /* + value_type & get(bucket_index bucket, index_type index) { auto it = _container_list->begin(); std::advance(it, cont); auto c_data = *it; @@ -161,6 +137,7 @@ class GlobHeapContiguousMem { pos -= c_data.container->size(); return c_data.unattached_container->operator[](pos); } + */ /** * Destructor, collectively frees underlying global memory. @@ -200,113 +177,118 @@ class GlobHeapContiguousMem { void commit() { // Gather information about the max amount of containers a single unit // currently holds - std::vector container_count(_team->size()); - size_type my_container_count = _container_list->size(); + std::vector bucket_count(_team->size()); + size_type my_bucket_count = _unattached_containers.size(); DASH_ASSERT_RETURNS( - dart_allgather(&my_container_count, container_count.data(), + dart_allgather(&my_bucket_count, bucket_count.data(), sizeof(size_type), DART_TYPE_BYTE, _team->dart_id()), DART_OK ); - auto max_containers = std::max_element(container_count.begin(), - container_count.end()); + auto max_buckets = std::max_element(bucket_count.begin(), + bucket_count.end()); + container_type * new_container = new container_type(); + new_container->reserve(_local_size); + int count = 0; + int elements = 0; int bucket_num = 0; - int bucket_cumul = 0; - - // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - // Important for performance: - // TODO: put multiple containers into one bucket - // TODO: update only containers with unattached_container.size() > 0 - // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - // attach buckets for the maximum amount of containers, even if this unit - // holds less containers - { - data_type * c_data; - auto c_data_it = _container_list->begin(); - auto current_size = _container_list->size(); - for(int i = 0; i < *max_containers; ++i) { - if(i < current_size) { - c_data = &(*c_data_it); - // merge public and local containers - c_data->container->insert(c_data->container->end(), - c_data->unattached_container->begin(), - c_data->unattached_container->end()); - c_data->unattached_container->clear(); - // update memory location & size of _container - c_data->container_bucket->lptr = c_data->container->data(); - c_data->container_bucket->size = c_data->container->size(); - // update memory location & size of _unattached_container - c_data->unattached_container_bucket->lptr - = c_data->unattached_container->data(); - c_data->unattached_container_bucket->size = 0; - ++c_data_it; - } else { - // if other units add more containers, create an empty container to - // store a dart_gptr for the collective allocation - auto cont_it = _container_list->begin(); - auto cont_index = add_container(0); - std::advance(cont_it, cont_index); - c_data = &(*cont_it); + size_type bucket_cumul = 0; + auto unattached_container_it = _unattached_containers.begin(); + bucket_type * last_bucket; + int elements_before = 0; + for(auto & bucket : _buckets) { + elements_before = elements; + // move data to new range + if(bucket.lptr != nullptr) { + // TODO: optimize memory usage (delete already moved elements) + new_container->insert(new_container->end(), bucket.lptr, + bucket.lptr + bucket.size); + elements += bucket.size; + } + // bucket list alternates between attached and unattached buckets. + // if bucket is already attached: + if(count % 2 == 0) { + if(bucket.size > 0) { + bucket.lptr = new_container->data() + elements_before; } - - // detach old container location from global memory space, if it has - // been attached before - if(c_data->container_bucket->gptr != DART_GPTR_NULL) { - DASH_ASSERT_RETURNS( - dart_team_memderegister(c_data->container_bucket->gptr), - DART_OK - ); + last_bucket = &bucket; + } else { + // if bucket is unattached: + if(bucket.size > 0) { + if(last_bucket->size == 0) { + last_bucket->lptr = new_container->data() + elements_before; + } + last_bucket->size += bucket.size; + bucket.size = 0; + bucket.lptr = nullptr; + unattached_container_it->container.clear(); + ++unattached_container_it; } - - // attach new container location to global memory space - dart_gptr_t gptr = DART_GPTR_NULL; - dash::dart_storage ds(c_data->container->size()); - DASH_ASSERT_RETURNS( - dart_team_memregister( - _team->dart_id(), - ds.nelem, - ds.dtype, - c_data->container->data(), - &gptr), - DART_OK - ); - // no need to update gptr of local bucket list in c_data - c_data->container_bucket->gptr = gptr; - // update cumulated bucket sizes - bucket_cumul += c_data->container->size(); + bucket_cumul += last_bucket->size; _bucket_cumul_sizes[_myid][bucket_num] = bucket_cumul; ++bucket_num; } - _team->barrier(); + ++count; + } + // 2 local buckets per global bucket + count /= 2; + for(int i = count; i < *max_buckets; ++i) { + add_container(0); + if(count > 0) { + _bucket_cumul_sizes[_myid][count] = + _bucket_cumul_sizes[_myid][count - 1]; + } else { + _bucket_cumul_sizes[_myid][count] = 0; + } } - // distribute bucket sizes between all units - // TODO: use one allgather for all buckets - // TODO: make it work for unevenly distributed amount of buckets - auto bucket_count = _bucket_cumul_sizes[_myid].size(); - for(auto c_data : *_container_list) { - std::vector bucket_sizes(bucket_count * _team->size()); - std::vector local_buckets(_bucket_cumul_sizes[_myid]); + // detach old container location from global memory space, if it has + // been attached before + if(_dart_gptr != DART_GPTR_NULL) { DASH_ASSERT_RETURNS( - dart_allgather(local_buckets.data(), bucket_sizes.data(), - sizeof(size_type) * local_buckets.size(), DART_TYPE_BYTE, _team->dart_id()), + dart_team_memderegister(_dart_gptr), DART_OK ); - _size = 0; - auto begin = bucket_sizes.begin(); - for(int i = 0; i < _team->size(); ++i) { - auto end = begin + bucket_count - 1; - std::copy(begin, end + 1, _bucket_cumul_sizes[i].begin()); - begin = end + 1; - _size += *end; - } } + _container.reset(new_container); + + // attach new container location to global memory space + dash::dart_storage ds(_container->size()); + DASH_ASSERT_RETURNS( + dart_team_memregister( + _team->dart_id(), + ds.nelem, + ds.dtype, + _container->data(), + &_dart_gptr), + DART_OK + ); + + // distribute bucket sizes between all units + auto bucket_amount = _bucket_cumul_sizes[_myid].size(); + std::vector bucket_sizes(bucket_amount * _team->size()); + std::vector local_buckets(_bucket_cumul_sizes[_myid]); + DASH_ASSERT_RETURNS( + dart_allgather(local_buckets.data(), bucket_sizes.data(), + sizeof(size_type) * local_buckets.size(), DART_TYPE_BYTE, _team->dart_id()), + DART_OK + ); + _size = 0; + auto begin = bucket_sizes.begin(); + for(int i = 0; i < _team->size(); ++i) { + auto end = begin + bucket_amount; + std::copy(begin, end, _bucket_cumul_sizes[i].begin()); + begin = end; + _size += *(end - 1); + } + + // update local iterators update_lbegin(); update_lend(); - int b_num = 0; + // update global iterators _begin = global_iterator(this, 0); _end = global_iterator(this, _size); } @@ -342,22 +324,13 @@ class GlobHeapContiguousMem { /** * Insert value at the end of the given bucket. */ - local_iterator push_back(container_list_index cont, value_type & val) { - auto cont_it = _container_list->begin(); - std::advance(cont_it, cont); - auto c_data = *cont_it; - // use _unattached container, if _container is full + local_iterator push_back(bucket_index_type index, value_type & val) { // we don't want a realloc of _container because this changes the memory // location, which invalidates global pointers of other units - if(c_data.container->capacity() == c_data.container->size()) { - c_data.unattached_container->push_back(val); - c_data.unattached_container_bucket->lptr - = c_data.unattached_container->data(); - ++(c_data.unattached_container_bucket->size); - } else { - c_data.container->push_back(val); - ++(c_data.container_bucket->size); - } + auto & unatt = _unattached_containers[index]; + unatt.container.push_back(val); + unatt.bucket.size = unatt.container.size(); + unatt.bucket.lptr = unatt.container.data(); ++_local_size; update_lbegin(); @@ -368,12 +341,14 @@ class GlobHeapContiguousMem { /** * Returns the local size of a given bucket. */ + /* size_type container_local_size(container_list_index index) const { auto cont_it = _container_list->begin(); std::advance(cont_it, index); auto c_data = *cont_it; return c_data.container->size() + c_data.unattached_container->size(); } + */ /** * Returns the global size of a given bucket. @@ -408,7 +383,7 @@ class GlobHeapContiguousMem { /// Unit id mapped to address in global memory space. team_unit_t unit, /// Index of bucket containing the referenced address. - index_type bucket_index, + bucket_index_type bucket_index, /// Offset of the referenced address in the bucket's memory space. index_type bucket_phase) const { @@ -417,23 +392,18 @@ class GlobHeapContiguousMem { if (_nunits == 0) { DASH_THROW(dash::exception::RuntimeError, "No units in team"); } - // Get the referenced bucket's dart_gptr: - auto bucket_it = _global_buckets.begin(); - std::advance(bucket_it, bucket_index); - auto dart_gptr = (*bucket_it)->gptr; - DASH_LOG_TRACE_VAR("GlobDynamicMem.dart_gptr_at", (*bucket_it)->attached); - DASH_LOG_TRACE_VAR("GlobDynamicMem.dart_gptr_at", (*bucket_it)->gptr); - if (unit == _myid) { - DASH_LOG_TRACE_VAR("GlobDynamicMem.dart_gptr_at", (*bucket_it)->lptr); - DASH_LOG_TRACE_VAR("GlobDynamicMem.dart_gptr_at", (*bucket_it)->size); - DASH_ASSERT_LT(bucket_phase, (*bucket_it)->size, - "bucket phase out of bounds"); - } + auto dart_gptr = _dart_gptr; if (DART_GPTR_ISNULL(dart_gptr)) { DASH_LOG_TRACE("GlobDynamicMem.dart_gptr_at", "bucket.gptr is DART_GPTR_NULL"); dart_gptr = DART_GPTR_NULL; } else { + size_type bucket_start; + if(bucket_index > 0) { + bucket_start = _bucket_cumul_sizes[unit][bucket_index - 1]; + } else { + bucket_start = 0; + } // Move dart_gptr to unit and local offset: DASH_ASSERT_RETURNS( dart_gptr_setunit(&dart_gptr, unit), @@ -441,7 +411,7 @@ class GlobHeapContiguousMem { DASH_ASSERT_RETURNS( dart_gptr_incaddr( &dart_gptr, - bucket_phase * sizeof(value_type)), + (bucket_start + bucket_phase) * sizeof(value_type)), DART_OK); } DASH_LOG_DEBUG("GlobDynamicMem.dart_gptr_at >", dart_gptr); @@ -450,7 +420,17 @@ class GlobHeapContiguousMem { private: - // NOTE: method copied from GlobHeapMem.h + /** + * Returns indices of attached and unattached containers from _buckets given + * a bucket index from _global_buckets + */ + /* + std::pair + local_buckets(bucket_index_type index) { + return std::make_pair(index * 2, (index * 2) + 1); + } + */ + /** * Native pointer of the initial address of the local memory of * a unit. @@ -458,32 +438,20 @@ class GlobHeapContiguousMem { */ void update_lbegin() noexcept { - local_iterator unit_lbegin( - // iteration space - _buckets.begin(), _buckets.end(), - // position in iteration space - 0, - // bucket at position in iteration space, - // offset in bucket - _buckets.begin(), 0); + // cannot use lightweight constructor here, because the first bucket might + // be empty + local_iterator unit_lbegin(_buckets.begin(), _buckets.end(), 0); _lbegin = unit_lbegin; } - // NOTE: method copied from GlobHeapMem.h /** * Update internal native pointer of the final address of the local memory * of a unit. */ void update_lend() noexcept { - local_iterator unit_lend( - // iteration space - _buckets.begin(), _buckets.end(), - // position in iteration space - _local_size, - // bucket at position in iteration space, - // offset in bucket - _buckets.end(), 0); + local_iterator unit_lend(_buckets.begin(), _buckets.end(), _local_size, + _buckets.end(), 0); _lend = unit_lend; } @@ -500,12 +468,16 @@ class GlobHeapContiguousMem { private: - /** List of all attached container */ - std::shared_ptr _container_list; /** List of buckets for GlobHeapLocalPtr */ - bucket_list _buckets; + bucket_list_type _buckets; /** list of buckets available for global iteration */ bucket_ptr_list _global_buckets; + /** Container holding globally visible data */ + std::unique_ptr _container; + /** List of containers holding locally visible data of each bucket */ + container_list_type _unattached_containers; + /** DART gptr of _container */ + dart_gptr_t _dart_gptr = DART_GPTR_NULL; /** Team associated with this memory space */ Team * _team; /** ID of the team */ From b471e8355bb6ee06e40e279dd6278be198a9c998 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 14 Dec 2017 15:26:01 +0100 Subject: [PATCH 059/102] removed unnecessary code --- .../dash/memory/GlobHeapContiguousMem.h | 41 +------------------ 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h index 83a381980..b9fd155e7 100644 --- a/dash/include/dash/memory/GlobHeapContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -123,22 +123,6 @@ class GlobHeapContiguousMem { return _global_buckets.size() - 1; } - /** - * Returns a reference to the requested element. - */ - /* - value_type & get(bucket_index bucket, index_type index) { - auto it = _container_list->begin(); - std::advance(it, cont); - auto c_data = *it; - if(c_data.container->size() > pos) { - return c_data.container->operator[](pos); - } - pos -= c_data.container->size(); - return c_data.unattached_container->operator[](pos); - } - */ - /** * Destructor, collectively frees underlying global memory. */ @@ -311,6 +295,8 @@ class GlobHeapContiguousMem { * Iterator to the beginning of the memory space's local portion. */ local_iterator lbegin() const { + // TODO: use iterator of _container, if _unattached_containers do not + // contain any data return _lbegin; } @@ -338,18 +324,6 @@ class GlobHeapContiguousMem { return _lend - 1; } - /** - * Returns the local size of a given bucket. - */ - /* - size_type container_local_size(container_list_index index) const { - auto cont_it = _container_list->begin(); - std::advance(cont_it, index); - auto c_data = *cont_it; - return c_data.container->size() + c_data.unattached_container->size(); - } - */ - /** * Returns the global size of a given bucket. */ @@ -420,17 +394,6 @@ class GlobHeapContiguousMem { private: - /** - * Returns indices of attached and unattached containers from _buckets given - * a bucket index from _global_buckets - */ - /* - std::pair - local_buckets(bucket_index_type index) { - return std::make_pair(index * 2, (index * 2) + 1); - } - */ - /** * Native pointer of the initial address of the local memory of * a unit. From d6af3828d8bfb49889c0be22f75583a301f81016 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 14 Dec 2017 16:00:01 +0100 Subject: [PATCH 060/102] fixed dart_allgatherv --- dart-impl/mpi/src/dart_communication.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dart-impl/mpi/src/dart_communication.c b/dart-impl/mpi/src/dart_communication.c index 67b803a4c..06b075fa2 100644 --- a/dart-impl/mpi/src/dart_communication.c +++ b/dart-impl/mpi/src/dart_communication.c @@ -1764,7 +1764,7 @@ dart_ret_t dart_allgatherv( DART_LOG_ERROR("dart_allgatherv ! unknown teamid %d", teamid); return DART_ERR_INVAL; } - if (sendbuf == recvbuf || NULL == sendbuf) { + if (sendbuf == recvbuf) { sendbuf = MPI_IN_PLACE; } MPI_Comm comm = team_data->comm; From e27c4ff8a021d27c57536b9c7a559d0c40601ddb Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 8 Jan 2018 16:29:22 +0100 Subject: [PATCH 061/102] optimized iterator performance --- .../include/dash/memory/GlobHeapCombinedMem.h | 2 +- .../dash/memory/GlobHeapContiguousLocalPtr.h | 509 ++++++++++++++++++ .../dash/memory/GlobHeapContiguousMem.h | 37 +- .../dash/memory/GlobHeapContiguousPtr.h | 16 +- 4 files changed, 547 insertions(+), 17 deletions(-) create mode 100644 dash/include/dash/memory/GlobHeapContiguousLocalPtr.h diff --git a/dash/include/dash/memory/GlobHeapCombinedMem.h b/dash/include/dash/memory/GlobHeapCombinedMem.h index 85e28f5a8..a8d7e3af4 100644 --- a/dash/include/dash/memory/GlobHeapCombinedMem.h +++ b/dash/include/dash/memory/GlobHeapCombinedMem.h @@ -21,7 +21,7 @@ class GlobHeapCombinedMem { typedef GlobPtr global_iterator; typedef typename glob_mem_type::local_iterator local_iterator; typedef typename local_iterator::bucket_type bucket_type; - typedef typename std::list bucket_list; + typedef typename std::vector bucket_list; typedef local_iterator local_pointer; typedef local_iterator const_local_pointer; diff --git a/dash/include/dash/memory/GlobHeapContiguousLocalPtr.h b/dash/include/dash/memory/GlobHeapContiguousLocalPtr.h new file mode 100644 index 000000000..b5907a2e4 --- /dev/null +++ b/dash/include/dash/memory/GlobHeapContiguousLocalPtr.h @@ -0,0 +1,509 @@ +#ifndef DASH__MEMORY__GLOB_HEAP_CONTIUGOUS_LOCAL_PTR_H__INCLUDED +#define DASH__MEMORY__GLOB_HEAP_CONTIGUOUS_LOCAL_PTR_H__INCLUDED + +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + + +namespace dash { + +// TODO: Remove this class and add template parameter for bucket list type to +// GlobHeapLocalPtr +/** + * Iterator on local buckets. Represents local pointer type. + */ +template< + typename ElementType, + typename IndexType, + typename PointerType = ElementType *, + typename ReferenceType = ElementType & > +class GlobHeapContiguousLocalPtr +: public std::iterator< + std::random_access_iterator_tag, + ElementType, + IndexType, + PointerType, + ReferenceType > +{ + template + friend class GlobHeapContiguousLocalPtr; + + template + friend std::ostream & dash::operator<<( + std::ostream & os, + const dash::GlobHeapContiguousLocalPtr & it); + +private: + typedef GlobHeapContiguousLocalPtr + self_t; + +public: + typedef IndexType index_type; + typedef typename std::make_unsigned::type size_type; + +/// Type definitions required for std::iterator_traits: +public: + typedef std::random_access_iterator_tag iterator_category; + typedef IndexType difference_type; + typedef ElementType value_type; + typedef ElementType * pointer; + typedef ElementType & reference; + + typedef internal::glob_dynamic_mem_bucket_type + bucket_type; + + typedef typename std::vector + bucket_list; + +private: + + typedef typename bucket_list::iterator + bucket_iterator; + + typedef typename bucket_list::const_iterator + const_bucket_iterator; + +public: + template + GlobHeapContiguousLocalPtr( + const BucketIter & bucket_first, + const BucketIter & bucket_last, + index_type position, + const BucketIter & bucket_it, + index_type bucket_phase) + : _bucket_first(bucket_first), + _bucket_last(bucket_last), + _idx(position), + _bucket_it(bucket_it), + _bucket_phase(bucket_phase), + _is_nullptr(false) + { } + + template + GlobHeapContiguousLocalPtr( + const BucketIter & bucket_first, + const BucketIter & bucket_last, + index_type position) + : _bucket_first(bucket_first), + _bucket_last(bucket_last), + _idx(position), + _bucket_it(bucket_first), + _bucket_phase(0), + _is_nullptr(false) + { + DASH_LOG_TRACE_VAR("GlobHeapContiguousLocalPtr(idx)", position); +#ifdef DASH_ENABLE_TRACE_LOGGING + index_type bucket_idx = 0; +#endif + for (_bucket_it = _bucket_first; + _bucket_it != _bucket_last; ++_bucket_it) { + if (position >= _bucket_it->size) { + position -= _bucket_it->size; + } else { + _bucket_phase = position; + break; + } +#ifdef DASH_ENABLE_TRACE_LOGGING + ++bucket_idx; +#endif + } + DASH_LOG_TRACE("GlobHeapContiguousLocalPtr(idx) >", + "bucket:", bucket_idx, + "phase:", _bucket_phase); + } + + GlobHeapContiguousLocalPtr() = default; + + GlobHeapContiguousLocalPtr(const self_t & other) + : _bucket_first(other._bucket_first), + _bucket_last(other._bucket_last), + _idx(other._idx), + _bucket_it(other._bucket_it), + _bucket_phase(other._bucket_phase), + _is_nullptr(other._is_nullptr) + { } + + self_t & operator=(const self_t & rhs) + { + if (this != std::addressof(rhs)) { + _bucket_first = rhs._bucket_first; + _bucket_last = rhs._bucket_last; + _idx = rhs._idx; + _bucket_it = rhs._bucket_it; + _bucket_phase = rhs._bucket_phase; + _is_nullptr = rhs._is_nullptr; + } + return *this; + } + + /** + * Conversion to const iterator. + */ + template + operator GlobHeapContiguousLocalPtr() const + { + if (_is_nullptr) { + return GlobHeapContiguousLocalPtr(nullptr); + } + return GlobHeapContiguousLocalPtr( + _bucket_first, + _bucket_last, + _idx, + _bucket_it, + _bucket_phase); + } + + GlobHeapContiguousLocalPtr(std::nullptr_t) + : _is_nullptr(true) + { } + + self_t & operator=(std::nullptr_t) + { + _is_nullptr = true; + return *this; + } + + inline bool operator==(std::nullptr_t) const + { + return _is_nullptr; + } + + inline bool operator!=(std::nullptr_t) const + { + return !_is_nullptr; + } + + /** + * Dereference operator. + */ + reference operator*() + { + DASH_ASSERT(!_is_nullptr); + if (_bucket_phase > _bucket_it->size) { + DASH_THROW(dash::exception::OutOfRange, + "dereferenced position " << _idx << " is out of range: " << + "bucket phase: " << _bucket_phase << ", " << + "bucket size: " << _bucket_it->size); + } + return _bucket_it->lptr[_bucket_phase]; + } + + /** + * Random access operator. + */ + reference operator[](index_type offset) + { + DASH_ASSERT(!_is_nullptr); + if (_bucket_phase + offset < _bucket_it->size) { + // element is in bucket currently referenced by this iterator: + return _bucket_it->lptr[_bucket_phase + offset]; + } else { + // find bucket containing element at given offset: + for (auto b_it = _bucket_it; b_it != _bucket_last; ++b_it) { + if (offset >= b_it->size) { + offset -= b_it->size; + } else if (offset < b_it->size) { + return b_it->lptr[offset]; + } + } + } + DASH_THROW(dash::exception::OutOfRange, + "dereferenced position " << _idx + offset << " " << + "is out of range: pointer position: " << _idx << ", " << + "offset: " << offset); + } + + /** + * Conversion to native pointer. + * + * Use with caution: This conversion returns a pointer a that does not + * iterate over buckets, pointer arithmetics may lead to undefined + * behaviour. + */ + explicit operator pointer() const + { + DASH_LOG_TRACE("GlobHeapContiguousLocalPtr.pointer()"); + pointer lptr = nullptr; + if (_is_nullptr) { + DASH_LOG_TRACE("GlobHeapContiguousLocalPtr.pointer", "is nullptr"); + } else { + auto bucket_size = _bucket_it->size; + // This iterator type represents a local pointer so no bounds checks + // have to be performed in pointer arithmetics. + // Moving a pointer to out-of-bounds address is allowed, however + // dereferencing it will lead to segfault. This is a prerequisite for + // many common pointer arithmetic use cases. + // Example: + // value = *((globmem.lend() + 2) - 3); + // is a valid operation and equivalent to + // value = *(globmem.lend() + (2 - 3)); + // as it creates a temporary pointer to an address beyond _lend (+2) + // which is then moved back into valid memory range (-3). + if (_bucket_it == _bucket_last) { + DASH_LOG_TRACE("GlobHeapContiguousLocalPtr.pointer", "position at lend"); + } else if (_bucket_phase >= bucket_size) { + DASH_LOG_TRACE("GlobHeapContiguousLocalPtr.pointer", + "bucket size:", bucket_size, ",", + "bucket phase:", _bucket_phase); + DASH_LOG_TRACE("GlobHeapContiguousLocalPtr.pointer", + "note: iterator position out of bounds (lend?)"); + } + lptr = _bucket_it->lptr + _bucket_phase; + } + DASH_LOG_TRACE_VAR("GlobHeapContiguousLocalPtr.pointer >", lptr); + return lptr; + } + + self_t & operator++() + { + increment(1); + return *this; + } + + self_t & operator--() + { + decrement(1); + return *this; + } + + self_t & operator++(int) + { + auto res = *this; + increment(1); + return res; + } + + self_t & operator--(int) + { + auto res = *this; + decrement(1); + return res; + } + + self_t & operator+=(int offset) + { + increment(offset); + return *this; + } + + self_t & operator-=(int offset) + { + decrement(offset); + return *this; + } + + self_t operator+(int offset) const + { + auto res = *this; + res += offset; + return res; + } + + self_t operator-(int offset) const + { + auto res = *this; + res -= offset; + return res; + } + + inline index_type operator+( + const self_t & other) const + { + return _idx + other._idx; + } + + inline index_type operator-( + const self_t & other) const + { + return _idx - other._idx; + } + + template + inline bool operator<(const GlobHeapContiguousLocalPtr & other) const + { + return (_idx < other._idx); + } + + template + inline bool operator<=(const GlobHeapContiguousLocalPtr & other) const + { + return (_idx <= other._idx); + } + + template + inline bool operator>(const GlobHeapContiguousLocalPtr & other) const + { + return (_idx > other._idx); + } + + template + inline bool operator>=(const GlobHeapContiguousLocalPtr & other) const + { + return (_idx >= other._idx); + } + + template + inline bool operator==(const GlobHeapContiguousLocalPtr & other) const + { + return (this == std::addressof(other) || _idx == other._idx); + } + + template + inline bool operator!=(const GlobHeapContiguousLocalPtr & other) const + { + return !(*this == other); + } + + /** + * Whether the pointer references an element in local memory space. + * + * \return true + */ + constexpr bool is_local() const + { + return true; + } + + /** + * Position of the pointer relative to its referenced memory space. + */ + index_type pos() const + { + return _idx; + } + +private: + /** + * Advance pointer by specified position offset. + */ + void increment(int offset) + { + DASH_ASSERT(!_is_nullptr); + _idx += offset; + if (_bucket_phase + offset < _bucket_it->size) { + // element is in bucket currently referenced by this iterator: + _bucket_phase += offset; + } else { + // find bucket containing element at given offset: + for (; _bucket_it != _bucket_last; ++_bucket_it) { + if (offset + _bucket_phase >= _bucket_it->size) { + offset -= _bucket_it->size - _bucket_phase; + _bucket_phase = 0; + } else if (offset < _bucket_it->size) { + _bucket_phase = offset; + break; + } + } + } + // end iterator + if (_bucket_it == _bucket_last) { + _bucket_phase = offset; + } + } + + /** + * Decrement pointer by specified position offset. + */ + void decrement(int offset) + { + DASH_ASSERT(!_is_nullptr); + if (offset > _idx) { + DASH_THROW(dash::exception::OutOfRange, + "offset " << offset << " is out of range"); + } + _idx -= offset; + if (offset <= _bucket_phase) { + // element is in bucket currently referenced by this iterator: + _bucket_phase -= offset; + } else { + offset -= _bucket_phase; + // find bucket containing element at given offset: + for (; _bucket_it != _bucket_first; --_bucket_it) { + if (offset >= _bucket_it->size) { + offset -= _bucket_it->size; + } else if (offset < _bucket_it->size) { + _bucket_phase = _bucket_it->size - offset; + break; + } + } + } + if (_bucket_it == _bucket_first) { + _bucket_phase = _bucket_it->size - offset; + } + if (false) { + DASH_THROW(dash::exception::OutOfRange, + "offset " << offset << " is out of range"); + } + } + +private: + bucket_iterator _bucket_first; + bucket_iterator _bucket_last; + index_type _idx = 0; + bucket_iterator _bucket_it; + index_type _bucket_phase = 0; + bool _is_nullptr = false; + +}; // class GlobHeapContiguousLocalPtr + +/** + * Resolve the number of elements between two local bucket iterators. + * + * \complexity O(1) + * + * \ingroup Algorithms + */ +template< + typename ElementType, + typename IndexType, + class Pointer, + class Reference> +auto distance( + /// Global iterator to the first position in the global sequence + const dash::GlobHeapContiguousLocalPtr< + ElementType, IndexType, Pointer, Reference> & first, + /// Global iterator to the final position in the global sequence + const dash::GlobHeapContiguousLocalPtr< + ElementType, IndexType, Pointer, Reference> & last) +-> IndexType +{ + return last - first; +} + +template< + typename ElementType, + typename IndexType, + class Pointer, + class Reference> +std::ostream & operator<<( + std::ostream & os, + const dash::GlobHeapContiguousLocalPtr< + ElementType, IndexType, Pointer, Reference> & it) +{ + std::ostringstream ss; + ElementType * lptr = static_cast(it); + ss << "dash::GlobHeapContiguousLocalPtr<" + << typeid(ElementType).name() << ">" + << "(" + << "idx:" << it._idx << ", " + << "bp:" << it._bucket_phase << ", " + << "lptr:" << lptr + << ")"; + return operator<<(os, ss.str()); +} + +} // namespace dash + +#endif // DASH__MEMORY__GLOB_HEAP_CONTIGUOUS_LOCAL_PTR_H__INCLUDED diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h index b9fd155e7..b1426f781 100644 --- a/dash/include/dash/memory/GlobHeapContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include #include @@ -28,15 +28,15 @@ struct container_data { typedef ContainerType container_type; typedef typename ContainerType::value_type value_type; typedef typename ContainerType::difference_type index_type; - typedef typename - GlobHeapLocalPtr::bucket_type bucket_type; + typedef typename GlobHeapContiguousLocalPtr::bucket_list::difference_type bucket_index_type; - container_data(bucket_type & bkt) + container_data(bucket_index_type bkt) : bucket(bkt) { } - container_type container; - bucket_type & bucket; + container_type container; + bucket_index_type bucket; }; @@ -59,12 +59,13 @@ class GlobHeapContiguousMem { typedef std::vector container_list_type; typedef typename ContainerType::value_type value_type; typedef typename ContainerType::difference_type index_type; - typedef GlobHeapLocalPtr local_iterator; + typedef GlobHeapContiguousLocalPtr local_iterator; typedef GlobPtr global_iterator; typedef typename ContainerType::size_type size_type; typedef typename local_iterator::bucket_type bucket_type; // must be List because of GlobHeapLocalPtr - typedef typename std::list bucket_list_type; + typedef typename std::vector bucket_list_type; typedef typename std::list bucket_ptr_list; typedef typename bucket_ptr_list::difference_type bucket_index_type; typedef typename bucket_list_type::difference_type local_bucket_index_type; @@ -75,6 +76,8 @@ class GlobHeapContiguousMem { template friend class dash::GlobPtr; friend class GlobHeapCombinedMem; + template + friend class VertexProxy; public: @@ -115,7 +118,7 @@ class GlobHeapContiguousMem { // for global iteration, only _container's bucket is needed _global_buckets.push_back(&(_buckets.back())); - _unattached_containers.emplace_back(_buckets.back()); + _unattached_containers.emplace_back(_buckets.size() - 1); auto & unattached_container = _unattached_containers.back().container; unattached_container.reserve(n_elements); @@ -315,13 +318,20 @@ class GlobHeapContiguousMem { // location, which invalidates global pointers of other units auto & unatt = _unattached_containers[index]; unatt.container.push_back(val); - unatt.bucket.size = unatt.container.size(); - unatt.bucket.lptr = unatt.container.data(); + auto & bucket = _buckets[unatt.bucket]; + bucket.size = unatt.container.size(); + bucket.lptr = unatt.container.data(); ++_local_size; update_lbegin(); update_lend(); - return _lend - 1; + return local_iterator( + _buckets.begin(), + _buckets.end(), + _local_size - 1, + _buckets.begin() + _local_size, + bucket.size - 1 + ); } /** @@ -413,7 +423,8 @@ class GlobHeapContiguousMem { */ void update_lend() noexcept { - local_iterator unit_lend(_buckets.begin(), _buckets.end(), _local_size, + local_iterator unit_lend( + _buckets.begin(), _buckets.end(), _local_size, _buckets.end(), 0); _lend = unit_lend; } diff --git a/dash/include/dash/memory/GlobHeapContiguousPtr.h b/dash/include/dash/memory/GlobHeapContiguousPtr.h index a1e7af3aa..bc9f3c697 100644 --- a/dash/include/dash/memory/GlobHeapContiguousPtr.h +++ b/dash/include/dash/memory/GlobHeapContiguousPtr.h @@ -203,6 +203,9 @@ class GlobPtr< DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx)", "unit:", unit, "lidx:", local_index); + if(unit > _bucket_cumul_sizes->size()) { + std::cout << std::endl; + } DASH_ASSERT_LT(unit, _bucket_cumul_sizes->size(), "invalid unit id"); for (size_type unit = 0; unit < _idx_unit_id; ++unit) { @@ -237,8 +240,8 @@ class GlobPtr< _myid(gmem->team().myid()), _idx_unit_id(unit), _idx_local_idx(0), - _idx_bucket_idx(0), - _idx_bucket_phase(0) + _idx_bucket_idx(bucket_index), + _idx_bucket_phase(local_index) { DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx)", "unit:", unit, @@ -252,7 +255,14 @@ class GlobPtr< if(bucket_index > 0) { local_index += (*_bucket_cumul_sizes)[_idx_unit_id.id][bucket_index - 1]; } + _idx_local_idx += local_index; + _idx += local_index; + /* + if(bucket_index > 0) { + local_index += (*_bucket_cumul_sizes)[_idx_unit_id.id][bucket_index - 1]; + } increment(local_index); + */ DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx) >", "gidx:", _idx, "maxidx:", _max_idx, @@ -320,7 +330,7 @@ class GlobPtr< * * \return A global reference to the element at the pointer's position. */ - reference operator*() const + reference operator*() const { auto lptr = local(); if (lptr != nullptr) { From f2cfbc6ca894ef02718f14007f9537ff3def31df Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 8 Jan 2018 16:30:07 +0100 Subject: [PATCH 062/102] added local iterator range for adjacency iteration --- dash/include/dash/Graph.h | 4 + dash/include/dash/graph/internal/Graph.h | 120 +++++++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 8e059eba8..7a2f6e5d2 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -717,6 +717,10 @@ class Graph { return *_team; } + global_vertex_iterator vertex_gptr(local_vertex_iterator it) { + return global_vertex_iterator(_glob_mem_vertex, _myid, it.pos()); + } + private: /** diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index 52045adf4..05337b1e1 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -130,6 +130,14 @@ class VertexProxy { return end(_parent._iterator); } + local_iterator lbegin() { + return lbegin(_parent._iterator); + } + + local_iterator lend() { + return lend(_parent._iterator); + } + private: /** @@ -188,6 +196,118 @@ class VertexProxy { ); } + /** + * Local begin iterator for global vertex iterators. + */ + template + local_iterator lbegin(VertexIteratorType it) { + auto lpos = _parent._iterator.lpos(); + auto index = it_position(_glob_mem, lpos.index); + auto lindex = index * 2; + auto bucket_it = _glob_mem->_buckets.begin(); + std::advance(bucket_it, lindex); + // if bucket empty, advance to end iterator + if(bucket_it->size == 0) { + std::advance(bucket_it, 2); + } + auto position = 0; + // TODO: position WRONG, if there are uncommitted changes + // save cumulated sizes of unattached containers somewhere + if(index > 0) { + position = + _glob_mem->_bucket_cumul_sizes[_parent._graph->_myid][index - 1]; + } + // use lightweight constructor to avoid costly increment operations + return local_iterator( + _glob_mem->_buckets.begin(), + _glob_mem->_buckets.end(), + position, + bucket_it, + 0 + ); + } + + /** + * Local begin iterator for local vertex iterators. + */ + local_iterator lbegin(local_vertex_iterator it) { + auto index = it_position(_glob_mem, it.pos()); + auto lindex = index * 2; + auto bucket_it = _glob_mem->_buckets.begin(); + std::advance(bucket_it, lindex); + // if bucket empty, advance to end iterator + if(bucket_it->size == 0) { + std::advance(bucket_it, 2); + } + auto position = 0; + // TODO: position WRONG, if there are uncommitted changes + // save cumulated sizes of unattached containers somewhere + if(index > 0) { + position = + _glob_mem->_bucket_cumul_sizes[_parent._graph->_myid][index - 1]; + } + // use lightweight constructor to avoid costly increment operations + return local_iterator( + _glob_mem->_buckets.begin(), + _glob_mem->_buckets.end(), + position, + bucket_it, + 0 + ); + } + + /** + * Local end iterator for global vertex iterators. + */ + template + local_iterator lend(VertexIteratorType it) { + auto lpos = _parent._iterator.lpos(); + auto index = it_position(_glob_mem, lpos.index) + 1; + auto lindex = index * 2; + auto bucket_it = _glob_mem->_buckets.begin(); + //std::advance(bucket_it, lindex); + auto position = 0; + // TODO: position WRONG, if there are uncommitted changes + // save cumulated sizes of unattached containers somewhere + if(index > 0) { + position = + _glob_mem->_bucket_cumul_sizes[_parent._graph->_myid][index - 1]; + } + // use lightweight constructor to avoid costly increment operations + return local_iterator( + _glob_mem->_buckets.begin(), + _glob_mem->_buckets.end(), + position, + bucket_it, + 0 + ); + } + + /** + * Local end iterator for local vertex iterators. + */ + local_iterator lend(local_vertex_iterator it) { + auto index = it_position(_glob_mem, it.pos()) + 1; + auto lindex = index * 2; + auto bucket_it = _glob_mem->_buckets.begin(); + //std::advance(bucket_it, index); + auto position = 0; + // TODO: position WRONG, if there are uncommitted changes + // save cumulated sizes of unattached containers somewhere + if(index > 0) { + position = + _glob_mem->_bucket_cumul_sizes[_parent._graph->_myid][index - 1]; + } + // use lightweight constructor to avoid costly increment operations + return local_iterator( + _glob_mem->_buckets.begin(), + _glob_mem->_buckets.end(), + position, + bucket_it, + 0 + ); + } + /** * Determines the position of the edge-list for inbound and outbound * edge memory spaces. From 86becf7d6a8caeb5b08ec384dafc749268e1cb23 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Sun, 28 Jan 2018 16:26:49 +0100 Subject: [PATCH 063/102] fixed wrong pointer type --- dash/include/dash/graph/internal/Graph.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index 05337b1e1..fd5352a4a 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -413,7 +413,7 @@ class VertexProxy { */ void set_attributes(properties_type & prop, local_vertex_iterator it) { _vertex.properties = prop; - typename iterator_type::pointer ptr = _iterator; + auto ptr = _iterator; *ptr = _vertex; } From 213c78eb4e61227ef0d58c852e1b140b85a14108 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 1 Feb 2018 15:18:18 +0100 Subject: [PATCH 064/102] fixed bottleneck in GlobPtr --- dash/include/dash/memory/GlobHeapContiguousPtr.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dash/include/dash/memory/GlobHeapContiguousPtr.h b/dash/include/dash/memory/GlobHeapContiguousPtr.h index bc9f3c697..70fb49d3c 100644 --- a/dash/include/dash/memory/GlobHeapContiguousPtr.h +++ b/dash/include/dash/memory/GlobHeapContiguousPtr.h @@ -577,7 +577,7 @@ class GlobPtr< if (offset == 0) { break; } - auto unit_bkt_sizes = (*_bucket_cumul_sizes)[_idx_unit_id]; + auto & unit_bkt_sizes = (*_bucket_cumul_sizes)[_idx_unit_id]; auto unit_bkt_sizes_total = unit_bkt_sizes.back(); auto unit_num_bkts = unit_bkt_sizes.size(); DASH_LOG_TRACE("GlobPtr.increment", From c636f79ca07f1752e20ac01b898f34c9a6d4768e Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Tue, 6 Feb 2018 18:48:07 +0100 Subject: [PATCH 065/102] cleaned up old cold fragments --- dash/include/dash/Graph.h | 3 ++- .../dash/memory/GlobHeapContiguousLocalPtr.h | 2 +- .../dash/memory/GlobHeapContiguousMem.h | 18 ++++-------------- .../dash/memory/internal/GlobHeapMemTypes.h | 9 +++++++++ 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 7a2f6e5d2..912430d05 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -755,7 +755,8 @@ class Graph { } team_unit_t vertex_owner(vertex_size_type v, vertex_size_type n_vertices) { - team_unit_t owner { v / (n_vertices / _team->size()) }; + int owner_id = static_cast(v) / (static_cast(n_vertices) / _team->size()); + team_unit_t owner { owner_id }; return owner; } diff --git a/dash/include/dash/memory/GlobHeapContiguousLocalPtr.h b/dash/include/dash/memory/GlobHeapContiguousLocalPtr.h index b5907a2e4..aaf85f235 100644 --- a/dash/include/dash/memory/GlobHeapContiguousLocalPtr.h +++ b/dash/include/dash/memory/GlobHeapContiguousLocalPtr.h @@ -61,7 +61,7 @@ class GlobHeapContiguousLocalPtr typedef ElementType * pointer; typedef ElementType & reference; - typedef internal::glob_dynamic_mem_bucket_type + typedef internal::glob_dynamic_contiguous_mem_bucket_type bucket_type; typedef typename std::vector diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h index b1426f781..b867d26b0 100644 --- a/dash/include/dash/memory/GlobHeapContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -66,9 +66,7 @@ class GlobHeapContiguousMem { typedef typename local_iterator::bucket_type bucket_type; // must be List because of GlobHeapLocalPtr typedef typename std::vector bucket_list_type; - typedef typename std::list bucket_ptr_list; - typedef typename bucket_ptr_list::difference_type bucket_index_type; - typedef typename bucket_list_type::difference_type local_bucket_index_type; + typedef typename container_list_type::difference_type bucket_index_type; typedef local_iterator local_pointer; typedef local_iterator const_local_pointer; typedef std::vector> bucket_cumul_sizes_map; @@ -86,7 +84,6 @@ class GlobHeapContiguousMem { */ GlobHeapContiguousMem(Team & team = dash::Team::All()) : _buckets(), - _global_buckets(), _team(&team), _teamid(team.dart_id()), _nunits(team.size()), @@ -104,26 +101,21 @@ class GlobHeapContiguousMem { // create bucket data and add to bucket list bucket_type cont_bucket { 0, - nullptr, - DART_GPTR_NULL, - false + nullptr }; bucket_type unattached_cont_bucket { 0, - nullptr, - DART_GPTR_NULL, - false + nullptr }; _buckets.push_back(cont_bucket); // for global iteration, only _container's bucket is needed - _global_buckets.push_back(&(_buckets.back())); _unattached_containers.emplace_back(_buckets.size() - 1); auto & unattached_container = _unattached_containers.back().container; unattached_container.reserve(n_elements); _buckets.push_back(unattached_cont_bucket); - return _global_buckets.size() - 1; + return _unattached_containers.size() - 1; } /** @@ -444,8 +436,6 @@ class GlobHeapContiguousMem { /** List of buckets for GlobHeapLocalPtr */ bucket_list_type _buckets; - /** list of buckets available for global iteration */ - bucket_ptr_list _global_buckets; /** Container holding globally visible data */ std::unique_ptr _container; /** List of containers holding locally visible data of each bucket */ diff --git a/dash/include/dash/memory/internal/GlobHeapMemTypes.h b/dash/include/dash/memory/internal/GlobHeapMemTypes.h index 09d935248..df5159460 100644 --- a/dash/include/dash/memory/internal/GlobHeapMemTypes.h +++ b/dash/include/dash/memory/internal/GlobHeapMemTypes.h @@ -15,6 +15,15 @@ struct glob_dynamic_mem_bucket_type bool attached; }; +template< + typename SizeType, + typename ElementType > +struct glob_dynamic_contiguous_mem_bucket_type +{ + SizeType size; + ElementType * lptr; +}; + } // namespace internal } // namespace dash From a74525d0038c2d4574836fbf5f6bb056a8bddd7b Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Tue, 6 Feb 2018 19:54:02 +0100 Subject: [PATCH 066/102] added graph algorithms --- dash/include/dash/Algorithm.h | 3 + .../algorithm/graph/ConnectedComponents.h | 222 ++++++++++++++ .../algorithm/graph/MinimumSpanningTree.h | 275 ++++++++++++++++++ 3 files changed, 500 insertions(+) create mode 100644 dash/include/dash/algorithm/graph/ConnectedComponents.h create mode 100644 dash/include/dash/algorithm/graph/MinimumSpanningTree.h diff --git a/dash/include/dash/Algorithm.h b/dash/include/dash/Algorithm.h index f7a779cbd..b7812b044 100644 --- a/dash/include/dash/Algorithm.h +++ b/dash/include/dash/Algorithm.h @@ -27,4 +27,7 @@ #include +#include +#include + #endif // DASH__ALGORITHM_H_ diff --git a/dash/include/dash/algorithm/graph/ConnectedComponents.h b/dash/include/dash/algorithm/graph/ConnectedComponents.h new file mode 100644 index 000000000..69178c7fe --- /dev/null +++ b/dash/include/dash/algorithm/graph/ConnectedComponents.h @@ -0,0 +1,222 @@ +#ifndef DASH__ALGORITHM____GRAPH__CONNECTED_COMPONENTS_H__ +#define DASH__ALGORITHM____GRAPH__CONNECTED_COMPONENTS_H__ + +#include + +namespace dash { + +typedef std::vector> matrix_t; +typedef std::vector>> matrix_pair_t; + +namespace internal { + +template +std::vector cc_get_data( + matrix_t & indices, + matrix_t & permutations, + GraphType & graph) +{ + // exchange indices to get + std::vector sizes_send(indices.size()); + std::vector displs_send(indices.size()); + std::vector indices_send; + int total_send = 0; + for(int i = 0; i < indices.size(); ++i) { + sizes_send[i] = indices[i].size(); + displs_send[i] = total_send; + total_send += indices[i].size(); + } + indices_send.reserve(total_send); + for(auto & index_set : indices) { + indices_send.insert(indices_send.end(), index_set.begin(), + index_set.end()); + } + std::vector sizes_recv(indices.size()); + std::vector displs_recv(indices.size()); + dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), + DART_TYPE_BYTE, graph.team().dart_id()); + int total_recv = 0; + for(int i = 0; i < sizes_recv.size(); ++i) { + displs_recv[i] = total_recv; + total_recv += sizes_recv[i]; + } + std::vector indices_recv(total_recv); + if(total_send > 0 || total_recv > 0) { + dart_alltoallv(indices_send.data(), sizes_send.data(), displs_send.data(), + DART_TYPE_INT, indices_recv.data(), sizes_recv.data(), + displs_recv.data(), graph.team().dart_id()); + } + + // exchange data + for(auto & index : indices_recv) { + // TODO: optimize cache performance + index = graph[graph.vertices().begin() + index].attributes().comp; + } + if(total_send > 0 || total_recv > 0) { + dart_alltoallv(indices_recv.data(), sizes_recv.data(), displs_recv.data(), + DART_TYPE_INT, indices_send.data(), sizes_send.data(), + displs_send.data(), graph.team().dart_id()); + } + + // restore original order + // TODO: use more sophisticated ordering mechanism + std::vector output(indices_send.size()); + int index = 0; + for(int i = 0; i < permutations.size(); ++i) { + for(int j = 0; j < permutations[i].size(); ++j) { + output[index] = indices_send[permutations[i][j]]; + ++index; + } + } + return output; +} + +template +void cc_set_data(matrix_pair_t & data_pairs, GraphType & graph) { + std::vector sizes_send(data_pairs.size()); + std::vector displs_send(data_pairs.size()); + std::vector> pairs_send; + int total_send = 0; + for(int i = 0; i < data_pairs.size(); ++i) { + sizes_send[i] = data_pairs[i].size() * sizeof(std::pair); + displs_send[i] = total_send * sizeof(std::pair); + total_send += data_pairs[i].size(); + } + pairs_send.reserve(total_send); + for(auto & pair_set : data_pairs) { + pairs_send.insert(pairs_send.end(), pair_set.begin(), pair_set.end()); + } + std::vector sizes_recv(data_pairs.size()); + std::vector displs_recv(data_pairs.size()); + dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), + DART_TYPE_BYTE, graph.team().dart_id()); + int total_recv = 0; + for(int i = 0; i < sizes_recv.size(); ++i) { + displs_recv[i] = total_recv; + total_recv += sizes_recv[i]; + } + std::vector> pairs_recv(total_recv / + sizeof(std::pair)); + if(total_send > 0 || total_recv > 0) { + dart_alltoallv(pairs_send.data(), sizes_send.data(), displs_send.data(), + DART_TYPE_BYTE, pairs_recv.data(), sizes_recv.data(), + displs_recv.data(), graph.team().dart_id()); + } + for(auto & pair : pairs_recv) { + typename GraphType::vertex_properties_type prop { pair.second }; + graph[graph.vertices().begin() + pair.first].set_attributes(prop); + } +} + +} // namespace internal + +// TODO: component type should be Graph::vertex_size_type +/** + * Computes connected components on a graph. + * Requires the graph's vertices to store the following attributes: + * - (int) comp + * Requires the graph's edges to store the following attributep: + * none + * + * Output: + * The vertex component attributes store the respective components after + */ +template +void connected_components(GraphType & g) { + // set component to global index in iteration space + // TODO: find faster method for this + int i = 0; + for(auto it = g.vertices().begin(); it != g.vertices().end(); ++it) { + if(it.is_local()) { + typename GraphType::vertex_properties_type prop { it.pos() }; + g[it].set_attributes(prop); + } + ++i; + } + + dash::barrier(); + + while(1) { + int gr = 0; + std::vector data; + { + matrix_t indices(g.team().size()); + matrix_t permutations(g.team().size()); + int i = 0; + for(auto it = g.out_edges().lbegin(); it != g.out_edges().lend(); ++it) { + auto trg = g[it].target(); + auto lpos = trg.lpos(); + // TODO: use more sophsticated sorting mechanism + indices[lpos.unit].push_back(trg.pos()); + permutations[lpos.unit].push_back(i); + ++i; + } + data = internal::cc_get_data(indices, permutations, g); + } + + { + matrix_pair_t data_pairs(g.team().size()); + int i = 0; + for(auto it = g.out_edges().lbegin(); it != g.out_edges().lend(); ++it) { + auto e = g[it]; + auto src = g[e.source()]; + auto src_comp = src.attributes().comp; + auto trg_comp = data[i]; + if(src_comp < trg_comp) { + auto trg_comp_it = g.vertices().begin() + trg_comp; + data_pairs[trg_comp_it.lpos().unit].push_back( + std::make_pair(trg_comp_it.pos(), src_comp) + ); + gr = 1; + } + ++i; + } + internal::cc_set_data(data_pairs, g); + } + int gr_all = 0; + dart_allreduce(&gr, &gr_all, 1, DART_TYPE_INT, DART_OP_SUM, + g.team().dart_id()); + if(gr_all == 0) break; + while(1) { + int pj = 0; + { + matrix_t indices(g.team().size()); + matrix_t permutations(g.team().size()); + int i = 0; + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto v = g[it]; + auto comp = v.attributes().comp; + auto next_it = g.vertices().begin() + comp; + auto lpos = next_it.lpos(); + indices[lpos.unit].push_back(next_it.pos()); + permutations[lpos.unit].push_back(i); + ++i; + } + data = internal::cc_get_data(indices, permutations, g); + } + + int i = 0; + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto v = g[it]; + auto comp = v.attributes().comp; + auto comp_next = data[i]; + if(comp != comp_next) { + typename GraphType::vertex_properties_type prop { comp_next }; + v.set_attributes(prop); + pj = 1; + } + ++i; + } + int pj_all = 0; + dart_allreduce(&pj, &pj_all, 1, DART_TYPE_INT, DART_OP_SUM, + g.team().dart_id()); + if(pj_all == 0) break; + } + dash::barrier(); + } + dash::barrier(); +} + +} // namespace dash + +#endif // DASH__ALGORITHM____GRAPH__CONNECTED_COMPONENTS_H__ diff --git a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h new file mode 100644 index 000000000..d77e8ef8d --- /dev/null +++ b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h @@ -0,0 +1,275 @@ +#ifndef DASH__ALGORITHM____GRAPH__MINIMUM_SPANNING_TREE_H__ +#define DASH__ALGORITHM____GRAPH__MINIMUM_SPANNING_TREE_H__ + +#include +#include + +namespace dash { + +typedef std::vector> matrix_t; +typedef std::vector>> matrix_pair_t; +// (supervertex, new component, vertex src, vertex trg, weight) +typedef std::tuple tuple_t; +typedef std::vector> matrix_min_pairs_t; + +namespace internal { + +template +std::vector mst_get_data( + matrix_t & indices, + matrix_t & permutations, + GraphType & graph) +{ + // exchange indices to get + std::vector sizes_send(indices.size()); + std::vector displs_send(indices.size()); + std::vector indices_send; + int total_send = 0; + for(int i = 0; i < indices.size(); ++i) { + sizes_send[i] = indices[i].size(); + displs_send[i] = total_send; + total_send += indices[i].size(); + } + indices_send.reserve(total_send); + for(auto & index_set : indices) { + indices_send.insert(indices_send.end(), index_set.begin(), + index_set.end()); + } + std::vector sizes_recv(indices.size()); + std::vector displs_recv(indices.size()); + dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), + DART_TYPE_BYTE, graph.team().dart_id()); + int total_recv = 0; + for(int i = 0; i < sizes_recv.size(); ++i) { + displs_recv[i] = total_recv; + total_recv += sizes_recv[i]; + } + std::vector indices_recv(total_recv); + if(total_send > 0 || total_recv > 0) { + dart_alltoallv(indices_send.data(), sizes_send.data(), displs_send.data(), + DART_TYPE_INT, indices_recv.data(), sizes_recv.data(), + displs_recv.data(), graph.team().dart_id()); + } + + // exchange data + for(auto & index : indices_recv) { + // TODO: optimize cache performance + index = graph[graph.vertices().begin() + index].attributes().comp; + } + if(total_send > 0 || total_recv > 0) { + dart_alltoallv(indices_recv.data(), sizes_recv.data(), displs_recv.data(), + DART_TYPE_INT, indices_send.data(), sizes_send.data(), + displs_send.data(), graph.team().dart_id()); + } + + // restore original order + // TODO: use more sophisticated ordering mechanism + std::vector output(indices_send.size()); + int index = 0; + for(int i = 0; i < permutations.size(); ++i) { + for(int j = 0; j < permutations[i].size(); ++j) { + output[permutations[i][j]] = indices_send[index]; + ++index; + } + } + return output; +} + +template +void mst_set_data_min(matrix_min_pairs_t & data_pairs, GraphType & graph) { + std::vector sizes_send(data_pairs.size()); + std::vector displs_send(data_pairs.size()); + std::vector pairs_send; + int total_send = 0; + for(int i = 0; i < data_pairs.size(); ++i) { + sizes_send[i] = data_pairs[i].size() * sizeof(tuple_t); + displs_send[i] = total_send * sizeof(tuple_t); + total_send += data_pairs[i].size(); + } + pairs_send.reserve(total_send); + for(auto & pair_set : data_pairs) { + pairs_send.insert(pairs_send.end(), pair_set.begin(), pair_set.end()); + } + std::vector sizes_recv(data_pairs.size()); + std::vector displs_recv(data_pairs.size()); + dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), + DART_TYPE_BYTE, graph.team().dart_id()); + int total_recv = 0; + for(int i = 0; i < sizes_recv.size(); ++i) { + displs_recv[i] = total_recv; + total_recv += sizes_recv[i]; + } + std::vector pairs_recv(total_recv / + sizeof(tuple_t)); + if(total_send > 0 || total_recv > 0) { + dart_alltoallv(pairs_send.data(), sizes_send.data(), displs_send.data(), + DART_TYPE_BYTE, pairs_recv.data(), sizes_recv.data(), + displs_recv.data(), graph.team().dart_id()); + } + std::unordered_map mapping; + for(auto & pair : pairs_recv) { + auto it = mapping.find(std::get<0>(pair)); + if(it != mapping.end()) { + auto weight = std::get<4>(it->second); + if(weight > std::get<4>(pair)) { + it->second = pair; + } + } else { + mapping[std::get<0>(pair)] = pair; + } + } + for(auto & pair : mapping) { + typename GraphType::vertex_properties_type prop { std::get<1>(pair.second) }; + graph[graph.vertices().begin() + pair.first].set_attributes(prop); + // save edges here (src = get<2>(pair.second), trg = get<3>(pair.second) + } +} + +} // namespace internal + +// TODO: component type should be Graph::vertex_size_type +/** + * Computes minimum spanning tree on a graph. + * Requires the graph's vertices to store the following attributes: + * - (int) comp + * Requires the graph's edges to store the following attributep: + * - (int) weight + * + * Output: + * TODO: store the tree information in the edges OR in a separate data + * structure + */ +template +void minimum_spanning_tree(GraphType & g) { + // set component to global index in iteration space + int i = 0; + for(auto it = g.vertices().begin(); it != g.vertices().end(); ++it) { + if(it.is_local()) { + typename GraphType::vertex_properties_type prop { it.pos() }; + g[it].set_attributes(prop); + } + ++i; + } + + /* + // set random edge weight + std::uniform_int_distribution dist(0, 10); + std::minstd_rand rng((std::random_device())()); + i = 0; + for(auto it = g.out_edges().begin(); it != g.out_edges().end(); ++it) { + if(it.is_local()) { + typename GraphType::edge_properties_type prop { dist(rng) }; + g[it].set_attributes(prop); + } + ++i; + } + */ + + dash::barrier(); + + while(1) { + int gr = 0; + std::vector data; + { + int i = 0; + matrix_t indices(g.team().size()); + matrix_t permutations(g.team().size()); + + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto v = g[it]; + for(auto e_it = v.out_edges().lbegin(); e_it != v.out_edges().lend(); + ++e_it) { + auto trg = g[e_it].target(); + auto lpos = trg.lpos(); + indices[lpos.unit].push_back(trg.pos()); + permutations[lpos.unit].push_back(i); + ++i; + } + } + data = internal::mst_get_data(indices, permutations, g); + } + + { + int i = 0; + matrix_min_pairs_t data_pairs(g.team().size()); + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto v = g[it]; + auto src_comp = v.attributes().comp; + int min_weight = std::numeric_limits::max(); + int trg_comp_min = -1; + int trg_min = -1; + for(auto e_it = v.out_edges().lbegin(); e_it != v.out_edges().lend(); + ++e_it) { + auto e = g[e_it]; + auto e_weight = e.attributes().weight; + auto trg_comp = data[i]; + if(src_comp != trg_comp && min_weight > e_weight) { + min_weight = e_weight; + trg_comp_min = trg_comp; + trg_min = e.target().pos(); + } + ++i; + } + if(trg_comp_min >= 0) { + auto src = g.vertex_gptr(it).pos(); + auto src_comp_it = g.vertices().begin() + src_comp; + auto trg_comp_it = g.vertices().begin() + trg_comp_min; + data_pairs[src_comp_it.lpos().unit].push_back( + std::make_tuple(src_comp_it.pos(), trg_comp_min, src, trg_min, + min_weight)); + data_pairs[trg_comp_it.lpos().unit].push_back( + std::make_tuple(trg_comp_it.pos(), src_comp, src, trg_min, + min_weight)); + gr = 1; + } + } + internal::mst_set_data_min(data_pairs, g); + } + + int gr_all = 0; + dart_allreduce(&gr, &gr_all, 1, DART_TYPE_INT, DART_OP_SUM, + g.team().dart_id()); + if(gr_all == 0) break; + while(1) { + int pj = 0; + { + matrix_t indices(g.team().size()); + matrix_t permutations(g.team().size()); + int i = 0; + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto v = g[it]; + auto comp = v.attributes().comp; + auto next_it = g.vertices().begin() + comp; + auto lpos = next_it.lpos(); + indices[lpos.unit].push_back(next_it.pos()); + permutations[lpos.unit].push_back(i); + ++i; + } + data = internal::mst_get_data(indices, permutations, g); + } + + int i = 0; + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto v = g[it]; + auto comp = v.attributes().comp; + auto comp_next = data[i]; + if(comp > comp_next) { + typename GraphType::vertex_properties_type prop { comp_next }; + v.set_attributes(prop); + pj = 1; + } + ++i; + } + int pj_all = 0; + dart_allreduce(&pj, &pj_all, 1, DART_TYPE_INT, DART_OP_SUM, + g.team().dart_id()); + if(pj_all == 0) break; + } + } + + dash::barrier(); + +} +} // namespace dash + +#endif // DASH__ALGORITHM____GRAPH__MINIMUM_SPANNING_TREE_H__ From 9ccd6faba0ddc50303135df420a05de5c5ce1eba Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 8 Feb 2018 14:11:23 +0100 Subject: [PATCH 067/102] added support for construction with whole edge list per unit --- dash/include/dash/Graph.h | 76 +++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 912430d05..5b138b262 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -188,30 +189,36 @@ class Graph { // per vertex allocate(n_vertices, 0); - std::unordered_map lvertices; - std::unordered_map gvertices; + std::map lvertices; + std::map gvertices; // TODO: find method that uses less memory std::unordered_set remote_vertices_set; std::vector> remote_vertices(_team->size()); for(auto it = begin; it != end; ++it) { auto v = it->first; - auto u = it->second; + if(vertex_owner(v, n_vertices) == _myid) { + auto u = it->second; - if(lvertices.find(v) == lvertices.end()) { - lvertices[v] = add_vertex(); - } - auto target_owner = vertex_owner(u, n_vertices); - if(target_owner == _myid) { - if(lvertices.find(u) == lvertices.end()) { - lvertices[u] = add_vertex(); + if(lvertices.find(v) == lvertices.end()) { + // add dummy first, more vertices are added later and they have + // to be in order + lvertices[v] = local_vertex_iterator(); } - } else { - // collect vertices for remote nodes and prevent adding vertices more - // than once - bool inserted; - std::tie(std::ignore, inserted) = remote_vertices_set.insert(u); - if(inserted) { - remote_vertices[target_owner].push_back(u); + auto target_owner = vertex_owner(u, n_vertices); + if(target_owner == _myid) { + if(lvertices.find(u) == lvertices.end()) { + // add dummy first, more vertices are added later and they have + // to be in order + lvertices[u] = local_vertex_iterator(); + } + } else { + // collect vertices for remote nodes and prevent adding vertices more + // than once + bool inserted; + std::tie(std::ignore, inserted) = remote_vertices_set.insert(u); + if(inserted) { + remote_vertices[target_owner].push_back(u); + } } } } @@ -253,10 +260,16 @@ class Graph { // exchange data for(auto & index : remote_vertices_recv) { - if(lvertices.find(index) == lvertices.end()) { - auto v = add_vertex(); - index = v.pos(); - } else { + lvertices[index] = local_vertex_iterator(); + } + // add vertices in order + for(auto & lvertex : lvertices) { + lvertex.second = add_vertex(); + } + // send local position of added vertices that are referenced by edges + // on other units + for(auto & index : remote_vertices_recv) { + if(lvertices.find(index) != lvertices.end()) { index = lvertices[index].pos(); } } @@ -291,15 +304,18 @@ class Graph { // finally add edges with the vertex iterators gained from the previous // steps for(auto it = begin; it != end; ++it) { - auto v_it = lvertices[it->first]; - auto u = it->second; - - if(vertex_owner(u, n_vertices) == _myid) { - auto u_it = lvertices[u]; - add_edge(v_it, u_it); - } else { - auto u_it = gvertices[u]; - add_edge(v_it, u_it); + auto v = it->first; + if(vertex_owner(v, n_vertices) == _myid) { + auto v_it = lvertices[it->first]; + auto u = it->second; + + if(vertex_owner(u, n_vertices) == _myid) { + auto u_it = lvertices[u]; + add_edge(v_it, u_it); + } else { + auto u_it = gvertices[u]; + add_edge(v_it, u_it); + } } } // commit edges From db8d7dc57d6eea1de1982a5599c342dd1fa01a37 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 8 Feb 2018 14:11:48 +0100 Subject: [PATCH 068/102] fixed connected components algorithm --- dash/include/dash/algorithm/graph/ConnectedComponents.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dash/include/dash/algorithm/graph/ConnectedComponents.h b/dash/include/dash/algorithm/graph/ConnectedComponents.h index 69178c7fe..1ddbfd66a 100644 --- a/dash/include/dash/algorithm/graph/ConnectedComponents.h +++ b/dash/include/dash/algorithm/graph/ConnectedComponents.h @@ -64,7 +64,7 @@ std::vector cc_get_data( int index = 0; for(int i = 0; i < permutations.size(); ++i) { for(int j = 0; j < permutations[i].size(); ++j) { - output[index] = indices_send[permutations[i][j]]; + output[permutations[i][j]] = indices_send[index]; ++index; } } From 0537f487145fc7e5b0984a8c99abaff7dc8d35fe Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 8 Feb 2018 14:49:05 +0100 Subject: [PATCH 069/102] fixed alltoallv --- dart-impl/mpi/src/dart_communication.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/dart-impl/mpi/src/dart_communication.c b/dart-impl/mpi/src/dart_communication.c index 06b075fa2..c3e8f55ec 100644 --- a/dart-impl/mpi/src/dart_communication.c +++ b/dart-impl/mpi/src/dart_communication.c @@ -1891,9 +1891,6 @@ dart_ret_t dart_alltoallv( "dart_adapt_teamlist_convert failed", teamid); return DART_ERR_INVAL; } - if (sendbuf == recvbuf) { - sendbuf = MPI_IN_PLACE; - } comm = team_data->comm; // convert nrecvcounts and recvdispls From 8d15d25be1feff73161b45435563bc8fad8056c7 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 8 Feb 2018 15:17:24 +0100 Subject: [PATCH 070/102] fixed dart_alltoall --- dart-impl/mpi/src/dart_communication.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dart-impl/mpi/src/dart_communication.c b/dart-impl/mpi/src/dart_communication.c index 4b1ce7e80..ddee3c96e 100644 --- a/dart-impl/mpi/src/dart_communication.c +++ b/dart-impl/mpi/src/dart_communication.c @@ -1915,7 +1915,7 @@ dart_ret_t dart_alltoall( dart_datatype_t dtype, dart_team_t teamid) { - MPI_Datatype mpi_dtype = dart__mpi__datatype(dtype); + MPI_Datatype mpi_dtype = dart__mpi__datatype_struct(dtype)->basic.mpi_type; MPI_Comm comm; DART_LOG_TRACE("dart_alltoall() team:%d nelem:%"PRIu64"", teamid, nelem); @@ -1970,7 +1970,7 @@ dart_ret_t dart_alltoallv( const size_t * recvdispls, dart_team_t teamid) { - MPI_Datatype mpi_dtype = dart__mpi__datatype(dtype); + MPI_Datatype mpi_dtype = dart__mpi__datatype_struct(dtype)->basic.mpi_type; MPI_Comm comm; int comm_size; DART_LOG_TRACE("dart_alltoallv() team:%d nsendelem:%"PRIu64"", From 24fcec2070f6a760cc42dc933ac7db312d5f99de Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 8 Feb 2018 15:17:49 +0100 Subject: [PATCH 071/102] added graph constructor test --- dash/test/container/GraphTest.cc | 912 ++++++++++++++++++++++--------- 1 file changed, 651 insertions(+), 261 deletions(-) diff --git a/dash/test/container/GraphTest.cc b/dash/test/container/GraphTest.cc index f9fdc2200..61b6df150 100644 --- a/dash/test/container/GraphTest.cc +++ b/dash/test/container/GraphTest.cc @@ -1,278 +1,668 @@ - #include "GraphTest.h" -#include #include -struct vertex_prop { - int id; +struct vprop { + int comp; }; -struct edge_prop { - int id; -}; +typedef dash::Graph graph_t; -TEST_F(GraphTest, GlobalIteration) +TEST_F(GraphTest, Construction) { - typedef int value_t; - typedef dash::default_index_t vertex_index_t; - typedef dash::default_index_t edge_index_t; - typedef dash::Graph< - dash::DirectedGraph, - void, - vertex_prop, - edge_prop, - vertex_index_t, - edge_index_t, - std::vector, - std::vector> graph_t; - typedef graph_t::vertex_index_type vertex_index_type; - typedef graph_t::vertex_type vertex_type; - typedef graph_t::edge_type edge_type; - - auto nunits = dash::size(); - auto myid = dash::myid(); - - auto ninsert_vertices_per_unit = 3; - auto ninsert_edges_per_vertex = 3; - ASSERT_LE(ninsert_edges_per_vertex, ninsert_vertices_per_unit); - auto ninsert_edges_per_unit = ninsert_vertices_per_unit * - ninsert_edges_per_vertex; - - auto vertex_cap = 1 * nunits; - auto edge_per_vertex_cap = 1; - - graph_t graph(vertex_cap, edge_per_vertex_cap); - - EXPECT_EQ_U(0, graph.num_vertices()); - EXPECT_EQ_U(0, graph.num_edges()); - EXPECT_EQ_U(-1, graph.max_vertex_index()); - EXPECT_EQ_U(-1, graph.max_edge_index()); - EXPECT_TRUE_U(graph.empty()); - - dash::barrier(); - DASH_LOG_DEBUG("GraphTest.GlobalIteration", "graph initialized"); - int ei = 0; - for(int i = 0; i < ninsert_vertices_per_unit; ++i) { - vertex_prop vprop { nunits * i + myid }; - auto source = graph.add_vertex(vprop); - //TODO: test vertex indeices for already existing vertices - DASH_LOG_DEBUG("GraphTest.GlobalIteration", "vertex inserted"); - auto target_unit = (myid + 1) % nunits; - for(int j = 0; j < ninsert_edges_per_vertex; ++j) { - vertex_index_type target(dash::team_unit_t(target_unit), j); - edge_prop eprop { nunits * ei + myid }; - graph.add_edge(source, target, eprop); - ++ei; - } - DASH_LOG_DEBUG("GraphTest.GlobalIteration", "edge inserted"); - } + std::list> edge_list = { + {433, 280}, {471, 192}, {738, 907}, {128, 664}, {999, 933}, {984, 440}, + {247, 310}, {565, 909}, {700, 727}, {408, 406}, {265, 550}, {411, 899}, + {630, 170}, {179, 57}, {733, 727}, {543, 943}, {475, 141}, {632, 394}, + {859, 581}, {203, 609}, {796, 589}, {494, 784}, {947, 420}, {354, 889}, + {184, 492}, {939, 587}, {598, 623}, {110, 971}, {296, 560}, {507, 881}, + {310, 674}, {280, 274}, {829, 845}, {273, 673}, {851, 52}, {814, 902}, + {995, 527}, {389, 611}, {537, 387}, {521, 350}, {849, 934}, {208, 57}, + {227, 962}, {30, 617}, {617, 962}, {512, 167}, {325, 1}, {232, 645}, {676, + 963}, {89, 242}, {57, 237}, {118, 252}, {807, 913}, {75, 425}, {782, 63}, + {901, 394}, {917, 23}, {923, 686}, {569, 451}, {217, 309}, {54, 950}, {508, + 185}, {642, 135}, {548, 436}, {495, 332}, {93, 368}, {483, 300}, {537, + 504}, {330, 526}, {29, 220}, {705, 598}, {468, 861}, {140, 13}, {192, 598}, + {347, 504}, {859, 182}, {369, 826}, {405, 979}, {694, 603}, {138, 269}, + {24, 364}, {256, 442}, {983, 657}, {946, 896}, {449, 915}, {6, 838}, {673, + 773}, {24, 231}, {235, 574}, {226, 875}, {327, 722}, {474, 311}, {501, + 904}, {291, 525}, {410, 937}, {804, 981}, {367, 113}, {449, 281}, {390, + 841}, {148, 698}, {396, 122}, {470, 726}, {442, 23}, {627, 762}, {493, + 406}, {710, 914}, {600, 289}, {182, 684}, {800, 569}, {965, 896}, {953, + 509}, {173, 25}, {671, 289}, {682, 205}, {71, 683}, {666, 286}, {414, 381}, + {23, 568}, {951, 45}, {987, 833}, {196, 225}, {686, 689}, {708, 87}, {954, + 715}, {209, 238}, {735, 700}, {357, 571}, {352, 466}, {373, 730}, {441, + 701}, {349, 224}, {289, 890}, {234, 680}, {721, 974}, {698, 801}, {181, + 460}, {47, 444}, {446, 142}, {262, 236}, {27, 494}, {958, 229}, {283, 210}, + {191, 279}, {359, 6}, {166, 810}, {808, 281}, {11, 801}, {484, 134}, {207, + 78}, {97, 128}, {182, 655}, {658, 329}, {926, 225}, {677, 285}, {183, 468}, + {685, 750}, {200, 999}, {871, 48}, {832, 515}, {485, 859}, {485, 75}, {13, + 435}, {612, 658}, {138, 796}, {919, 949}, {910, 747}, {975, 691}, {410, + 123}, {143, 788}, {535, 650}, {289, 386}, {48, 466}, {625, 783}, {690, + 851}, {761, 689}, {638, 964}, {106, 327}, {886, 578}, {223, 840}, {83, + 572}, {484, 802}, {703, 337}, {139, 522}, {234, 937}, {73, 555}, {15, 789}, + {335, 444}, {159, 468}, {747, 900}, {202, 563}, {880, 741}, {227, 307}, + {37, 594}, {380, 88}, {827, 286}, {747, 679}, {351, 208}, {374, 482}, {989, + 684}, {157, 691}, {207, 681}, {213, 302}, {372, 741}, {260, 231}, {710, + 110}, {529, 723}, {876, 580}, {156, 344}, {503, 467}, {887, 439}, {798, + 53}, {312, 342}, {890, 237}, {973, 738}, {36, 726}, {581, 386}, {153, 333}, + {341, 992}, {210, 389}, {551, 194}, {402, 407}, {875, 538}, {16, 873}, + {180, 47}, {119, 360}, {402, 996}, {422, 30}, {380, 939}, {722, 24}, {663, + 827}, {621, 266}, {113, 815}, {887, 976}, {859, 285}, {527, 646}, {971, + 190}, {84, 915}, {110, 953}, {934, 716}, {304, 316}, {413, 419}, {32, 373}, + {730, 936}, {478, 235}, {976, 947}, {859, 384}, {94, 33}, {912, 253}, {277, + 39}, {980, 696}, {91, 692}, {160, 263}, {190, 707}, {459, 166}, {221, 537}, + {849, 375}, {997, 679}, {57, 591}, {601, 48}, {393, 78}, {771, 839}, {254, + 485}, {939, 809}, {478, 167}, {881, 502}, {81, 317}, {76, 31}, {614, 887}, + {578, 417}, {970, 582}, {237, 389}, {951, 228}, {783, 626}, {146, 834}, + {233, 138}, {458, 861}, {167, 957}, {391, 619}, {880, 742}, {769, 210}, + {595, 786}, {708, 823}, {574, 159}, {663, 907}, {383, 781}, {861, 971}, + {395, 450}, {942, 183}, {558, 883}, {827, 609}, {23, 561}, {479, 386}, {68, + 915}, {75, 926}, {874, 927}, {15, 344}, {308, 359}, {788, 210}, {771, 65}, + {18, 494}, {760, 102}, {218, 602}, {53, 478}, {577, 476}, {794, 736}, {633, + 394}, {102, 941}, {686, 188}, {19, 326}, {626, 42}, {580, 305}, {304, 691}, + {569, 294}, {184, 239}, {698, 274}, {84, 168}, {994, 35}, {415, 591}, {139, + 416}, {152, 790}, {414, 123}, {987, 740}, {230, 973}, {206, 337}, {8, 329}, + {661, 477}, {514, 582}, {646, 944}, {32, 887}, {79, 655}, {749, 541}, {831, + 479}, {697, 443}, {691, 497}, {531, 432}, {595, 315}, {440, 663}, {567, + 705}, {710, 788}, {659, 805}, {112, 585}, {633, 934}, {358, 108}, {170, + 381}, {360, 876}, {155, 383}, {496, 69}, {104, 259}, {579, 513}, {907, 90}, + {438, 791}, {296, 322}, {382, 380}, {788, 196}, {667, 833}, {632, 598}, + {834, 396}, {45, 579}, {809, 724}, {470, 139}, {122, 686}, {667, 858}, + {860, 724}, {971, 644}, {838, 708}, {525, 660}, {94, 971}, {248, 531}, + {381, 380}, {178, 225}, {388, 291}, {218, 876}, {893, 177}, {896, 985}, + {930, 37}, {718, 804}, {553, 972}, {789, 952}, {367, 343}, {674, 387}, + {556, 160}, {985, 256}, {443, 917}, {377, 25}, {113, 453}, {43, 271}, {802, + 567}, {79, 199}, {713, 108}, {793, 517}, {860, 488}, {232, 686}, {983, + 451}, {360, 106}, {328, 853}, {190, 654}, {853, 10}, {580, 896}, {125, + 334}, {570, 449}, {192, 702}, {884, 146}, {209, 918}, {383, 810}, {186, + 460}, {137, 104}, {144, 845}, {775, 91}, {296, 805}, {773, 742}, {2, 136}, + {204, 752}, {370, 227}, {638, 957}, {192, 298}, {471, 309}, {718, 2}, {964, + 584}, {770, 1}, {232, 333}, {836, 921}, {263, 900}, {414, 629}, {824, 304}, + {379, 575}, {233, 160}, {547, 70}, {978, 603}, {55, 803}, {89, 160}, {490, + 598}, {888, 100}, {475, 973}, {880, 544}, {231, 894}, {546, 77}, {612, + 810}, {410, 882}, {964, 539}, {816, 920}, {809, 528}, {848, 472}, {202, + 291}, {724, 5}, {594, 131}, {385, 738}, {95, 277}, {432, 829}, {218, 434}, + {21, 127}, {926, 930}, {522, 650}, {563, 734}, {940, 927}, {733, 830}, + {494, 847}, {193, 290}, {492, 456}, {961, 47}, {876, 891}, {711, 827}, + {715, 474}, {489, 253}, {678, 289}, {398, 385}, {433, 146}, {173, 189}, + {985, 296}, {401, 357}, {592, 386}, {725, 243}, {308, 332}, {244, 637}, + {253, 938}, {852, 905}, {366, 712}, {560, 854}, {669, 270}, {666, 915}, + {362, 933}, {344, 832}, {255, 526}, {735, 438}, {400, 804}, {144, 230}, + {304, 338}, {287, 577}, {842, 8}, {327, 383}, {700, 41}, {499, 713}, {343, + 629}, {969, 654}, {24, 916}, {686, 276}, {525, 722}, {769, 302}, {240, 56}, + {828, 236}, {554, 486}, {727, 969}, {775, 356}, {411, 369}, {42, 943}, {70, + 429}, {824, 292}, {747, 577}, {359, 193}, {869, 301}, {390, 377}, {763, + 507}, {489, 526}, {63, 869}, {504, 647}, {525, 315}, {330, 65}, {818, 970}, + {976, 690}, {593, 324}, {242, 378}, {734, 271}, {120, 111}, {891, 898}, + {70, 675}, {929, 823}, {742, 330}, {347, 904}, {142, 496}, {353, 505}, + {448, 352}, {325, 794}, {54, 485}, {543, 70}, {763, 423}, {142, 846}, {112, + 7}, {652, 82}, {999, 939}, {402, 293}, {378, 300}, {717, 731}, {412, 885}, + {827, 595}, {179, 823}, {389, 424}, {353, 533}, {229, 511}, {889, 743}, + {235, 394}, {717, 22}, {121, 962}, {31, 991}, {843, 687}, {904, 89}, {695, + 705}, {624, 573}, {869, 643}, {887, 495}, {340, 827}, {833, 219}, {746, + 804}, {547, 350}, {127, 206}, {755, 827}, {108, 539}, {586, 321}, {451, + 251}, {860, 409}, {505, 953}, {713, 909}, {177, 226}, {15, 708}, {425, + 339}, {80, 63}, {181, 26}, {419, 139}, {298, 680}, {805, 607}, {259, 545}, + {451, 440}, {880, 225}, {255, 259}, {337, 637}, {974, 363}, {849, 850}, + {905, 206}, {868, 820}, {972, 972}, {736, 724}, {305, 420}, {63, 848}, + {947, 836}, {39, 947}, {355, 636}, {418, 13}, {816, 641}, {845, 374}, {359, + 921}, {322, 65}, {165, 728}, {997, 813}, {77, 344}, {340, 390}, {264, 367}, + {261, 712}, {953, 838}, {204, 743}, {48, 398}, {801, 61}, {186, 700}, {854, + 109}, {624, 172}, {304, 915}, {426, 711}, {45, 117}, {321, 549}, {387, + 555}, {206, 970}, {791, 317}, {786, 20}, {343, 594}, {361, 382}, {278, + 277}, {331, 225}, {510, 807}, {441, 493}, {396, 271}, {797, 678}, {254, + 123}, {539, 172}, {380, 267}, {420, 211}, {96, 247}, {407, 644}, {869, + 814}, {202, 338}, {327, 759}, {734, 713}, {18, 699}, {827, 103}, {953, + 551}, {206, 734}, {499, 98}, {66, 784}, {346, 444}, {824, 654}, {3, 224}, + {519, 231}, {319, 627}, {816, 292}, {81, 9}, {926, 899}, {27, 659}, {432, + 183}, {170, 709}, {522, 394}, {165, 264}, {469, 54}, {838, 705}, {101, + 415}, {713, 54}, {828, 650}, {480, 333}, {899, 162}, {830, 709}, {93, 789}, + {192, 173}, {720, 943}, {921, 758}, {100, 616}, {611, 62}, {168, 702}, + {776, 468}, {417, 79}, {420, 73}, {415, 149}, {383, 193}, {258, 597}, {871, + 677}, {327, 545}, {693, 93}, {714, 7}, {313, 135}, {788, 457}, {976, 961}, + {680, 113}, {422, 998}, {162, 301}, {269, 440}, {834, 25}, {559, 853}, + {787, 947}, {919, 930}, {910, 932}, {970, 417}, {846, 238}, {252, 741}, + {119, 401}, {136, 438}, {829, 432}, {614, 245}, {890, 7}, {303, 367}, {498, + 31}, {363, 384}, {805, 632}, {476, 821}, {940, 345}, {526, 932}, {966, + 643}, {572, 542}, {504, 255}, {841, 653}, {63, 91}, {97, 440}, {885, 550}, + {156, 851}, {190, 241}, {105, 859}, {264, 18}, {704, 965}, {598, 654}, + {922, 197}, {18, 543}, {677, 871}, {168, 514}, {334, 157}, {294, 553}, + {466, 973}, {792, 673}, {541, 331}, {307, 943}, {640, 551}, {947, 250}, + {390, 780}, {343, 379}, {80, 340}, {838, 145}, {634, 45}, {410, 2}, {934, + 338}, {621, 206}, {606, 977}, {735, 258}, {373, 274}, {631, 574}, {120, + 947}, {882, 725}, {139, 294}, {694, 450}, {476, 201}, {815, 305}, {234, + 563}, {305, 214}, {498, 39}, {753, 93}, {863, 81}, {202, 617}, {341, 719}, + {118, 153}, {586, 342}, {624, 806}, {498, 730}, {64, 211}, {858, 569}, + {541, 202}, {204, 259}, {517, 187}, {3, 682}, {36, 760}, {300, 287}, {980, + 873}, {260, 430}, {427, 411}, {677, 520}, {119, 515}, {18, 823}, {530, 39}, + {728, 185}, {537, 856}, {190, 883}, {409, 157}, {442, 271}, {347, 185}, + {884, 749}, {757, 55}, {394, 98}, {662, 829}, {32, 751}, {685, 491}, {937, + 571}, {624, 613}, {3, 417}, {90, 932}, {48, 377}, {541, 231}, {498, 940}, + {573, 755}, {157, 40}, {888, 205}, {161, 489}, {344, 601}, {300, 543}, + {319, 378}, {257, 918}, {536, 295}, {738, 305}, {970, 797}, {609, 698}, + {90, 668}, {114, 527}, {116, 267}, {429, 953}, {273, 401}, {76, 772}, {806, + 158}, {411, 627}, {271, 128}, {177, 72}, {995, 652}, {574, 35}, {250, 284}, + {283, 339}, {154, 304}, {602, 904}, {465, 102}, {709, 747}, {952, 735}, + {12, 418}, {473, 24}, {24, 880}, {38, 195}, {34, 797}, {647, 687}, {34, + 367}, {165, 62}, {454, 192}, {949, 492}, {671, 677}, {868, 896}, {333, + 965}, {846, 833}, {486, 905}, {123, 273}, {495, 697}, {319, 687}, {20, + 827}, {863, 991}, {530, 940}, {934, 56}, {2, 922}, {423, 716}, {863, 560}, + {447, 208}, {300, 597}, {941, 567}, {652, 665}, {668, 666}, {962, 732}, + {221, 61}, {108, 983}, {359, 374}, {589, 882}, {545, 460}, {728, 281}, + {101, 286}, {913, 474}, {819, 438}, {189, 863}, {377, 662}, {715, 322}, + {851, 766}, {957, 400}, {948, 312}, {444, 219}, {708, 763}, {19, 939}, + {390, 433}, {629, 689}, {55, 92}, {854, 656}, {67, 592}, {104, 334}, {578, + 863}, {961, 905}, {607, 412}, {177, 53}, {799, 76}, {891, 446}, {343, 78}, + {112, 79}, {872, 242}, {604, 137}, {606, 267}, {158, 640}, {559, 979}, + {908, 154}, {532, 630}, {519, 234}, {647, 247}, {690, 376}, {193, 268}, + {617, 89}, {561, 501}, {719, 455}, {820, 890}, {568, 977}, {363, 416}, + {234, 833}, {851, 107}, {213, 609}, {140, 436}, {229, 40}, {438, 262}, {77, + 404}, {686, 9}, {42, 545}, {398, 433}, {759, 105}, {13, 808}, {727, 495}, + {918, 189}, {879, 817}, {45, 747}, {335, 886}, {266, 985}, {708, 643}, + {200, 613}, {855, 834}, {980, 575}, {756, 71}, {609, 504}, {711, 333}, + {798, 973}, {316, 946}, {444, 449}, {213, 929}, {860, 579}, {683, 38}, {8, + 863}, {992, 618}, {407, 715}, {791, 526}, {848, 85}, {30, 690}, {468, 208}, + {238, 404}, {989, 480}, {662, 156}, {670, 160}, {691, 172}, {16, 903}, + {620, 93}, {856, 96}, {308, 571}, {479, 881}, {44, 903}, {621, 540}, {9, + 168}, {481, 405}, {551, 995}, {172, 260}, {369, 659}, {110, 855}, {880, + 298}, {687, 388}, {95, 828}, {960, 512}, {968, 6}, {878, 909}, {138, 124}, + {751, 408}, {252, 243}, {683, 119}, {573, 282}, {768, 782}, {415, 691}, + {272, 643}, {108, 966}, {657, 186}, {486, 665}, {889, 927}, {565, 826}, + {901, 858}, {99, 999}, {748, 415}, {571, 472}, {910, 484}, {905, 735}, + {894, 229}, {381, 908}, {944, 320}, {815, 85}, {177, 915}, {669, 415}, + {686, 425}, {19, 242}, {985, 845}, {213, 277}, {955, 349}, {940, 804}, + {954, 198}, {403, 329}, {87, 855}, {239, 563}, {152, 898}, {544, 643}, + {760, 95}, {780, 501}, {470, 475}, {301, 646}, {298, 5}, {795, 899}, {82, + 558}, {778, 360}, {209, 706}, {589, 301}, {719, 798}, {268, 205}, {327, + 460}, {470, 22}, {660, 372}, {96, 497}, {485, 760}, {168, 821}, {215, 332}, + {276, 112}, {332, 804}, {808, 747}, {673, 35}, {605, 230}, {987, 466}, + {179, 383}, {705, 165}, {356, 404}, {943, 208}, {14, 50}, {776, 346}, {789, + 358}, {751, 233}, {303, 306}, {654, 843}, {841, 547}, {263, 35}, {218, + 227}, {462, 727}, {52, 64}, {220, 173}, {675, 635}, {856, 409}, {495, 696}, + {664, 132}, {388, 849}, {766, 515}, {781, 228}, {145, 633}, {405, 645}, + {426, 954}, {793, 318}, {46, 30}, {848, 998}, {907, 612}, {163, 173}, {670, + 712}, {356, 239}, {966, 345}, {341, 388}, {435, 237}, {459, 470}, {466, + 310}, {380, 128}, {174, 263}, {28, 118}, {119, 782}, {531, 8}, {930, 160}, + {604, 165}, {131, 773}, {949, 844}, {753, 316}, {740, 917}, {78, 608}, + {917, 51}, {923, 824}, {381, 895}, {61, 109}, {959, 224}, {357, 393}, {756, + 289}, {826, 231}, {23, 535}, {416, 756}, {812, 411}, {1, 12}, {340, 746}, + {472, 109}, {467, 595}, {218, 179}, {582, 756}, {801, 138}, {698, 895}, + {43, 359}, {999, 378}, {624, 595}, {37, 23}, {454, 226}, {70, 213}, {676, + 673}, {654, 62}, {249, 809}, {150, 776}, {192, 442}, {875, 325}, {706, + 628}, {212, 638}, {597, 149}, {205, 604}, {951, 189}, {641, 944}, {915, + 487}, {944, 532}, {460, 48}, {95, 482}, {735, 61}, {136, 970}, {788, 733}, + {533, 485}, {533, 660}, {758, 626}, {777, 313}, {702, 720}, {709, 926}, + {777, 297}, {332, 698}, {552, 27}, {465, 725}, {330, 473}, {938, 20}, {276, + 605}, {454, 844}, {950, 805}, {276, 408}, {391, 773}, {297, 949}, {64, 38}, + {177, 236}, {76, 498}, {218, 521}, {582, 428}, {205, 175}, {987, 756}, + {247, 115}, {815, 970}, {860, 775}, {319, 15}, {860, 36}, {326, 228}, {966, + 293}, {257, 159}, {935, 552}, {34, 735}, {39, 165}, {878, 215}, {405, 352}, + {72, 298}, {927, 771}, {896, 66}, {116, 677}, {36, 268}, {546, 11}, {742, + 766}, {42, 783}, {774, 864}, {245, 4}, {932, 149}, {77, 509}, {714, 65}, + {839, 333}, {204, 656}, {345, 375}, {855, 807}, {466, 109}, {727, 880}, + {948, 413}, {5, 541}, {355, 443}, {583, 629}, {350, 688}, {739, 416}, {329, + 182}, {185, 140}, {713, 911}, {950, 346}, {927, 204}, {375, 753}, {974, + 735}, {28, 783}, {706, 207}, {0, 21}, {842, 758}, {743, 924}, {966, 312}, + {329, 675}, {291, 490}, {329, 493}, {523, 862}, {916, 834}, {860, 424}, + {861, 46}, {617, 363}, {844, 174}, {309, 106}, {45, 899}, {353, 567}, {783, + 689}, {681, 155}, {524, 933}, {980, 142}, {69, 794}, {417, 89}, {562, 922}, + {741, 86}, {74, 996}, {813, 989}, {694, 73}, {680, 415}, {772, 797}, {982, + 101}, {283, 936}, {825, 226}, {945, 394}, {740, 599}, {68, 519}, {979, + 192}, {138, 4}, {201, 791}, {595, 210}, {215, 215}, {802, 488}, {539, 277}, + {919, 844}, {746, 495}, {604, 1}, {951, 456}, {51, 346}, {214, 108}, {587, + 565}, {464, 939}, {109, 945}, {691, 274}, {878, 197}, {311, 534}, {458, + 992}, {737, 696}, {853, 295}, {156, 139}, {285, 518}, {92, 169}, {590, + 142}, {263, 792}, {109, 656}, {406, 132}, {610, 734}, {359, 367}, {206, + 953}, {883, 324}, {669, 123}, {878, 598}, {569, 32}, {422, 527}, {957, + 964}, {700, 522}, {333, 719}, {647, 832}, {269, 457}, {703, 110}, {512, + 238}, {233, 121}, {710, 179}, {802, 93}, {715, 825}, {321, 307}, {866, + 980}, {231, 586}, {845, 456}, {26, 265}, {656, 883}, {221, 951}, {131, 80}, + {629, 876}, {736, 293}, {171, 526}, {121, 189}, {269, 962}, {709, 939}, + {800, 320}, {874, 235}, {728, 670}, {821, 325}, {149, 445}, {17, 593}, + {573, 443}, {879, 321}, {718, 148}, {282, 455}, {420, 144}, {449, 303}, + {404, 53}, {958, 594}, {855, 155}, {33, 537}, {179, 325}, {907, 621}, {26, + 951}, {722, 289}, {213, 846}, {141, 106}, {530, 683}, {905, 934}, {425, + 686}, {486, 256}, {47, 82}, {133, 45}, {526, 690}, {733, 856}, {144, 172}, + {69, 853}, {658, 302}, {64, 165}, {957, 613}, {29, 136}, {464, 346}, {114, + 87}, {832, 654}, {631, 900}, {96, 461}, {657, 458}, {863, 372}, {217, 614}, + {352, 705}, {791, 378}, {617, 550}, {222, 578}, {86, 907}, {163, 593}, + {410, 307}, {603, 403}, {335, 892}, {20, 926}, {400, 745}, {747, 190}, + {443, 698}, {705, 735}, {169, 30}, {921, 106}, {167, 498}, {11, 339}, {89, + 578}, {470, 594}, {325, 50}, {127, 952}, {767, 463}, {887, 49}, {193, 249}, + {69, 487}, {723, 280}, {294, 744}, {242, 384}, {685, 92}, {975, 970}, {824, + 253}, {592, 419}, {278, 481}, {361, 162}, {439, 17}, {881, 221}, {4, 615}, + {399, 211}, {546, 45}, {834, 438}, {553, 879}, {603, 895}, {412, 366}, {63, + 697}, {16, 246}, {808, 2}, {73, 225}, {288, 607}, {475, 616}, {428, 999}, + {560, 595}, {421, 404}, {563, 899}, {126, 138}, {388, 419}, {47, 376}, + {305, 1}, {420, 381}, {501, 447}, {378, 720}, {469, 301}, {584, 853}, {659, + 498}, {849, 517}, {970, 851}, {840, 212}, {612, 508}, {38, 471}, {98, 504}, + {143, 368}, {846, 492}, {760, 788}, {720, 353}, {115, 858}, {389, 911}, + {142, 719}, {960, 273}, {97, 714}, {306, 744}, {336, 321}, {684, 493}, + {951, 464}, {985, 894}, {627, 860}, {550, 559}, {503, 840}, {473, 170}, + {322, 672}, {521, 717}, {329, 566}, {703, 582}, {934, 969}, {313, 635}, + {698, 984}, {128, 880}, {658, 154}, {524, 145}, {9, 315}, {71, 425}, {565, + 104}, {817, 84}, {429, 976}, {601, 279}, {695, 146}, {210, 144}, {740, + 609}, {872, 201}, {163, 52}, {857, 27}, {699, 499}, {921, 545}, {730, 762}, + {850, 326}, {609, 673}, {699, 453}, {910, 238}, {592, 215}, {573, 850}, + {749, 217}, {449, 592}, {468, 331}, {476, 543}, {16, 800}, {361, 250}, + {995, 363}, {288, 858}, {971, 118}, {541, 583}, {561, 587}, {832, 293}, + {660, 60}, {247, 685}, {799, 279}, {169, 913}, {115, 746}, {91, 341}, {153, + 758}, {985, 628}, {116, 170}, {791, 712}, {449, 731}, {4, 719}, {975, 88}, + {292, 114}, {670, 607}, {4, 178}, {8, 389}, {492, 51}, {183, 606}, {314, + 888}, {571, 887}, {182, 275}, {572, 352}, {903, 459}, {131, 558}, {681, + 288}, {40, 333}, {587, 267}, {483, 764}, {966, 348}, {676, 279}, {141, + 409}, {904, 760}, {740, 406}, {474, 565}, {932, 428}, {40, 192}, {539, + 232}, {613, 528}, {207, 63}, {321, 714}, {872, 570}, {862, 809}, {882, + 348}, {594, 603}, {756, 279}, {531, 767}, {993, 556}, {666, 276}, {961, + 176}, {573, 800}, {417, 146}, {863, 327}, {627, 118}, {692, 250}, {272, + 244}, {587, 159}, {22, 704}, {263, 515}, {492, 19}, {728, 979}, {257, 917}, + {383, 409}, {58, 423}, {480, 417}, {399, 716}, {374, 633}, {408, 265}, + {762, 749}, {695, 629}, {893, 272}, {285, 719}, {673, 456}, {690, 491}, + {243, 595}, {425, 955}, {558, 459}, {485, 562}, {47, 996}, {902, 486}, + {581, 716}, {525, 812}, {519, 945}, {923, 918}, {607, 572}, {882, 719}, + {544, 313}, {136, 36}, {977, 567}, {23, 244}, {777, 198}, {515, 590}, {349, + 90}, {858, 183}, {271, 996}, {116, 821}, {267, 829}, {576, 681}, {718, + 619}, {614, 393}, {571, 478}, {647, 840}, {843, 691}, {387, 201}, {330, + 391}, {667, 494}, {504, 437}, {111, 560}, {800, 103}, {931, 618}, {322, + 595}, {634, 848}, {96, 443}, {804, 14}, {647, 636}, {496, 521}, {529, 633}, + {556, 117}, {600, 162}, {211, 432}, {550, 909}, {890, 45}, {725, 61}, {147, + 626}, {482, 894}, {755, 159}, {55, 731}, {352, 488}, {934, 163}, {93, 653}, + {524, 36}, {607, 528}, {341, 126}, {425, 87}, {354, 490}, {22, 72}, {869, + 869}, {671, 203}, {181, 372}, {105, 40}, {634, 342}, {229, 923}, {271, + 472}, {325, 837}, {929, 218}, {968, 410}, {144, 628}, {716, 344}, {983, + 457}, {237, 209}, {400, 562}, {271, 227}, {134, 461}, {587, 903}, {658, + 519}, {172, 354}, {530, 739}, {244, 175}, {883, 940}, {786, 638}, {725, + 141}, {316, 830}, {584, 514}, {729, 900}, {481, 737}, {800, 228}, {478, 8}, + {36, 395}, {684, 178}, {369, 397}, {824, 109}, {792, 796}, {756, 679}, + {699, 241}, {227, 789}, {510, 521}, {665, 375}, {107, 595}, {565, 386}, + {687, 129}, {498, 585}, {897, 718}, {275, 767}, {538, 469}, {849, 208}, + {15, 349}, {518, 750}, {416, 158}, {577, 390}, {751, 420}, {892, 233}, + {793, 895}, {870, 691}, {339, 125}, {41, 403}, {784, 7}, {426, 181}, {197, + 430}, {71, 474}, {950, 597}, {183, 147}, {186, 939}, {679, 571}, {209, + 878}, {836, 395}, {391, 472}, {934, 326}, {536, 151}, {238, 925}, {38, + 763}, {425, 185}, {308, 254}, {110, 47}, {599, 993}, {236, 918}, {64, 337}, + {41, 57}, {676, 439}, {378, 388}, {845, 335}, {130, 115}, {662, 594}, {281, + 557}, {867, 935}, {236, 982}, {896, 905}, {952, 302}, {440, 186}, {46, + 229}, {301, 727}, {634, 140}, {85, 411}, {256, 571}, {16, 616}, {729, 321}, + {316, 102}, {756, 885}, {482, 617}, {98, 90}, {550, 914}, {651, 19}, {174, + 642}, {484, 245}, {949, 551}, {130, 581}, {745, 75}, {799, 498}, {375, 30}, + {30, 333}, {195, 199}, {80, 717}, {133, 852}, {464, 906}, {137, 642}, {108, + 772}, {833, 352}, {975, 236}, {799, 557}, {571, 106}, {237, 379}, {698, + 712}, {618, 735}, {352, 682}, {754, 799}, {981, 502}, {976, 721}, {216, + 281}, {470, 956}, {322, 777}, {387, 286}, {682, 601}, {351, 208}, {662, + 912}, {812, 226}, {806, 450}, {560, 469}, {94, 346}, {350, 347}, {625, + 116}, {21, 708}, {246, 770}, {17, 194}, {696, 357}, {397, 824}, {337, 647}, + {347, 913}, {989, 966}, {865, 180}, {669, 562}, {996, 627}, {455, 299}, + {672, 664}, {127, 377}, {646, 630}, {688, 232}, {584, 944}, {359, 678}, + {669, 61}, {628, 693}, {407, 293}, {206, 220}, {921, 248}, {750, 96}, {635, + 432}, {197, 304}, {692, 814}, {90, 506}, {407, 172}, {999, 392}, {851, + 698}, {479, 678}, {559, 484}, {837, 77}, {625, 13}, {837, 672}, {990, 558}, + {176, 124}, {366, 101}, {624, 558}, {69, 197}, {715, 502}, {186, 413}, + {614, 999}, {994, 827}, {964, 576}, {520, 539}, {599, 5}, {501, 329}, {954, + 376}, {276, 766}, {586, 487}, {765, 221}, {381, 549}, {257, 476}, {267, + 772}, {44, 362}, {961, 300}, {811, 52}, {264, 264}, {365, 994}, {776, 246}, + {943, 689}, {53, 545}, {93, 179}, {586, 443}, {861, 552}, {677, 338}, {448, + 496}, {740, 410}, {488, 537}, {588, 939}, {988, 592}, {448, 853}, {939, + 535}, {481, 302}, {241, 570}, {257, 231}, {706, 565}, {161, 725}, {509, + 477}, {555, 367}, {892, 476}, {287, 492}, {756, 49}, {305, 257}, {976, + 210}, {630, 406}, {792, 899}, {566, 152}, {801, 657}, {707, 759}, {49, + 246}, {812, 444}, {800, 615}, {484, 520}, {912, 857}, {26, 649}, {699, + 806}, {219, 916}, {300, 9}, {972, 176}, {447, 722}, {622, 791}, {806, 7}, + {824, 747}, {572, 233}, {741, 433}, {201, 402}, {262, 236}, {685, 390}, + {246, 64}, {190, 960}, {199, 936}, {984, 415}, {671, 121}, {739, 895}, + {239, 647}, {439, 129}, {6, 796}, {623, 682}, {429, 487}, {348, 180}, {299, + 899}, {834, 206}, {657, 78}, {155, 359}, {9, 884}, {660, 712}, {622, 243}, + {861, 605}, {570, 666}, {202, 614}, {933, 742}, {946, 580}, {797, 961}, + {376, 382}, {159, 153}, {925, 179}, {639, 942}, {233, 640}, {895, 229}, + {37, 552}, {850, 532}, {133, 500}, {740, 928}, {418, 742}, {994, 330}, + {694, 468}, {9, 545}, {171, 926}, {369, 674}, {333, 905}, {675, 27}, {479, + 266}, {122, 397}, {724, 366}, {363, 633}, {974, 268}, {243, 38}, {519, + 977}, {183, 256}, {80, 356}, {688, 363}, {964, 704}, {39, 806}, {102, 218}, + {865, 619}, {406, 385}, {354, 605}, {694, 495}, {159, 269}, {425, 290}, + {858, 232}, {338, 45}, {420, 337}, {404, 153}, {471, 485}, {639, 532}, + {628, 607}, {980, 680}, {826, 492}, {382, 321}, {33, 366}, {709, 250}, + {644, 122}, {763, 175}, {86, 126}, {606, 304}, {689, 490}, {791, 427}, + {732, 659}, {443, 40}, {773, 967}, {223, 400}, {473, 851}, {294, 983}, {76, + 705}, {310, 932}, {908, 862}, {451, 940}, {660, 670}, {250, 128}, {315, + 347}, {383, 576}, {121, 152}, {373, 334}, {963, 908}, {912, 936}, {974, + 563}, {989, 207}, {76, 699}, {239, 737}, {921, 325}, {819, 508}, {313, + 871}, {640, 904}, {679, 409}, {272, 429}, {793, 494}, {281, 143}, {446, + 929}, {961, 471}, {688, 492}, {519, 461}, {492, 569}, {258, 430}, {46, + 387}, {265, 162}, {746, 960}, {729, 795}, {304, 753}, {691, 833}, {941, + 109}, {97, 570}, {26, 652}, {842, 661}, {510, 778}, {883, 982}, {711, 605}, + {209, 160}, {182, 364}, {149, 82}, {245, 435}, {247, 818}, {499, 301}, + {820, 669}, {381, 89}, {899, 165}, {398, 969}, {22, 147}, {0, 1}, {0, 3}, + {0, 4}, {0, 7}, {0, 8}, {0, 10}, {0, 15}, {0, 16}, {0, 18}, {0, 31}, {0, + 32}, {0, 34}, {0, 35}, {0, 38}, {0, 41}, {0, 46}, {0, 57}, {0, 62}, {0, + 63}, {0, 65}, {0, 69}, {0, 72}, {0, 77}, {0, 85}, {0, 88}, {0, 94}, {0, + 96}, {0, 97}, {0, 108}, {0, 125}, {0, 126}, {0, 128}, {0, 132}, {0, 140}, + {0, 147}, {0, 156}, {0, 157}, {0, 160}, {0, 163}, {0, 166}, {0, 187}, {0, + 191}, {0, 195}, {0, 203}, {0, 205}, {0, 213}, {0, 218}, {0, 221}, {0, 250}, + {0, 253}, {0, 257}, {0, 260}, {0, 265}, {0, 266}, {0, 268}, {0, 269}, {0, + 281}, {0, 285}, {0, 306}, {0, 312}, {0, 313}, {0, 316}, {0, 327}, {0, 343}, + {0, 378}, {0, 382}, {0, 401}, {0, 406}, {0, 444}, {0, 471}, {0, 483}, {0, + 484}, {0, 500}, {0, 501}, {0, 503}, {0, 504}, {0, 507}, {0, 515}, {0, 518}, + {0, 522}, {0, 532}, {0, 542}, {0, 562}, {0, 565}, {0, 569}, {0, 612}, {0, + 625}, {0, 626}, {0, 628}, {0, 632}, {0, 650}, {0, 697}, {0, 750}, {0, 751}, + {0, 754}, {0, 757}, {0, 760}, {0, 781}, {0, 784}, {0, 785}, {0, 797}, {0, + 813}, {0, 837}, {0, 898}, {1, 0}, {1, 3}, {1, 4}, {1, 7}, {1, 15}, {1, 18}, + {1, 19}, {1, 25}, {1, 31}, {1, 34}, {1, 35}, {1, 46}, {1, 62}, {1, 63}, {1, + 65}, {1, 69}, {1, 80}, {1, 93}, {1, 125}, {1, 132}, {1, 135}, {1, 144}, {1, + 156}, {1, 171}, {1, 194}, {1, 233}, {1, 250}, {1, 254}, {1, 266}, {1, 281}, + {1, 285}, {1, 313}, {1, 327}, {1, 344}, {1, 409}, {1, 437}, {1, 452}, {1, + 475}, {1, 500}, {1, 515}, {1, 531}, {1, 565}, {1, 593}, {1, 600}, {1, 628}, + {1, 632}, {3, 0}, {3, 1}, {3, 7}, {3, 15}, {3, 26}, {3, 31}, {3, 32}, {3, + 34}, {3, 46}, {3, 62}, {3, 63}, {3, 65}, {3, 69}, {3, 111}, {3, 125}, {3, + 132}, {3, 140}, {3, 141}, {3, 143}, {3, 156}, {3, 187}, {3, 250}, {3, 269}, + {3, 281}, {3, 312}, {3, 327}, {3, 328}, {3, 346}, {3, 437}, {3, 500}, {3, + 501}, {3, 508}, {3, 515}, {3, 519}, {3, 547}, {3, 562}, {3, 594}, {3, 663}, + {3, 698}, {3, 750}, {3, 765}, {3, 885}, {4, 0}, {4, 7}, {4, 16}, {4, 23}, + {4, 26}, {4, 31}, {4, 57}, {4, 125}, {4, 150}, {4, 229}, {4, 236}, {4, + 258}, {4, 625}, {4, 671}, {4, 688}, {4, 843}, {7, 0}, {7, 1}, {7, 3}, {7, + 15}, {7, 16}, {7, 18}, {7, 22}, {7, 31}, {7, 62}, {7, 69}, {7, 93}, {7, + 125}, {7, 132}, {7, 135}, {7, 140}, {7, 159}, {7, 187}, {7, 202}, {7, 236}, + {7, 257}, {7, 265}, {7, 273}, {7, 284}, {7, 313}, {7, 315}, {7, 382}, {7, + 383}, {7, 390}, {7, 452}, {7, 503}, {7, 515}, {7, 518}, {7, 531}, {7, 538}, + {7, 549}, {7, 562}, {7, 565}, {7, 628}, {7, 750}, {7, 843}, {7, 876}, {8, + 3}, {8, 4}, {8, 7}, {8, 15}, {8, 22}, {8, 31}, {8, 47}, {8, 49}, {8, 135}, + {8, 159}, {8, 228}, {8, 250}, {8, 437}, {8, 508}, {8, 566}, {10, 0}, {10, + 1}, {10, 15}, {10, 16}, {10, 31}, {10, 41}, {10, 62}, {10, 125}, {10, 140}, + {10, 205}, {10, 250}, {10, 375}, {10, 562}, {10, 656}, {10, 882}, {11, 0}, + {11, 62}, {11, 265}, {11, 281}, {11, 625}, {15, 0}, {15, 1}, {15, 4}, {15, + 7}, {15, 10}, {15, 16}, {15, 18}, {15, 31}, {15, 38}, {15, 46}, {15, 54}, + {15, 62}, {15, 63}, {15, 69}, {15, 73}, {15, 93}, {15, 94}, {15, 109}, {15, + 125}, {15, 175}, {15, 190}, {15, 225}, {15, 254}, {15, 281}, {15, 284}, + {15, 316}, {15, 346}, {15, 406}, {15, 422}, {15, 500}, {15, 501}, {15, + 510}, {15, 515}, {15, 523}, {15, 562}, {15, 625}, {15, 702}, {15, 753}, + {15, 781}, {15, 791}, {16, 0}, {16, 1}, {16, 3}, {16, 10}, {16, 18}, {16, + 22}, {16, 62}, {16, 65}, {16, 140}, {16, 167}, {16, 282}, {16, 330}, {16, + 500}, {16, 503}, {16, 516}, {16, 562}, {16, 570}, {16, 641}, {16, 750}, + {16, 784}, {18, 1}, {18, 7}, {18, 22}, {18, 65}, {18, 132}, {18, 147}, {18, + 171}, {18, 187}, {18, 222}, {18, 251}, {18, 253}, {18, 268}, {18, 312}, + {18, 413}, {18, 422}, {18, 515}, {18, 516}, {18, 633}, {18, 758}, {19, 0}, + {19, 3}, {19, 4}, {19, 46}, {19, 273}, {19, 289}, {19, 750}, {22, 0}, {22, + 31}, {22, 34}, {22, 38}, {22, 46}, {22, 69}, {22, 70}, {22, 187}, {22, + 222}, {22, 250}, {22, 291}, {22, 312}, {22, 313}, {22, 359}, {22, 382}, + {22, 390}, {22, 437}, {22, 626}, {22, 641}, {23, 0}, {23, 41}, {23, 78}, + {23, 100}, {23, 281}, {23, 375}, {25, 0}, {25, 62}, {25, 140}, {25, 187}, + {25, 531}, {25, 844}, {26, 108}, {26, 394}, {31, 0}, {31, 1}, {31, 3}, {31, + 7}, {31, 15}, {31, 18}, {31, 47}, {31, 53}, {31, 65}, {31, 93}, {31, 108}, + {31, 112}, {31, 126}, {31, 140}, {31, 156}, {31, 157}, {31, 163}, {31, + 187}, {31, 202}, {31, 253}, {31, 265}, {31, 268}, {31, 312}, {31, 375}, + {31, 393}, {31, 468}, {31, 500}, {31, 503}, {31, 504}, {31, 507}, {31, + 510}, {31, 516}, {31, 538}, {31, 569}, {31, 687}, {31, 750}, {31, 812}, + {31, 876}, {32, 0}, {32, 3}, {32, 4}, {32, 18}, {32, 25}, {32, 26}, {32, + 46}, {32, 62}, {32, 140}, {32, 257}, {32, 272}, {32, 327}, {32, 382}, {32, + 500}, {32, 587}, {34, 0}, {34, 15}, {34, 16}, {34, 38}, {34, 46}, {34, 63}, + {34, 115}, {34, 126}, {34, 141}, {34, 265}, {34, 501}, {34, 508}, {34, + 538}, {34, 597}, {34, 750}, {35, 0}, {35, 3}, {35, 31}, {35, 34}, {35, 62}, + {35, 69}, {35, 80}, {35, 171}, {35, 508}, {38, 0}, {38, 7}, {38, 16}, {38, + 18}, {38, 25}, {38, 63}, {38, 66}, {38, 70}, {38, 96}, {38, 125}, {38, + 251}, {38, 265}, {38, 291}, {38, 313}, {38, 315}, {38, 316}, {38, 376}, + {38, 569}, {38, 625}, {38, 750}, {38, 757}, {38, 785}, {38, 875}, {39, 34}, + {39, 69}, {39, 125}, {39, 132}, {39, 266}, {39, 319}, {41, 50}, {41, 250}, + {41, 750}, {41, 765}, {42, 3}, {42, 312}, {46, 0}, {46, 1}, {46, 3}, {46, + 4}, {46, 7}, {46, 10}, {46, 15}, {46, 16}, {46, 25}, {46, 77}, {46, 163}, + {46, 265}, {46, 500}, {46, 501}, {46, 515}, {47, 101}, {47, 143}, {47, + 265}, {49, 1}, {49, 7}, {49, 125}, {49, 202}, {49, 257}, {49, 656}, {49, + 907}, {50, 22}, {53, 11}, {53, 228}, {53, 688}, {54, 62}, {54, 554}, {56, + 0}, {56, 1}, {56, 10}, {56, 31}, {56, 128}, {62, 0}, {62, 1}, {62, 3}, {62, + 4}, {62, 7}, {62, 10}, {62, 15}, {62, 16}, {62, 18}, {62, 25}, {62, 31}, + {62, 32}, {62, 35}, {62, 38}, {62, 46}, {62, 47}, {62, 65}, {62, 66}, {62, + 69}, {62, 96}, {62, 108}, {62, 125}, {62, 126}, {62, 135}, {62, 140}, {62, + 144}, {62, 147}, {62, 159}, {62, 187}, {62, 188}, {62, 250}, {62, 251}, + {62, 253}, {62, 257}, {62, 265}, {62, 288}, {62, 312}, {62, 319}, {62, + 353}, {62, 375}, {62, 376}, {62, 500}, {62, 504}, {62, 508}, {62, 510}, + {62, 525}, {62, 534}, {62, 538}, {62, 569}, {62, 625}, {62, 635}, {62, + 750}, {62, 812}, {62, 850}, {62, 851}, {63, 0}, {63, 1}, {63, 8}, {63, 46}, + {63, 65}, {63, 140}, {63, 171}, {63, 202}, {63, 250}, {63, 281}, {63, 284}, + {63, 503}, {63, 515}, {63, 518}, {63, 538}, {63, 562}, {63, 593}, {63, + 734}, {65, 0}, {65, 3}, {65, 4}, {65, 15}, {65, 16}, {65, 38}, {65, 87}, + {65, 112}, {65, 125}, {65, 126}, {65, 132}, {65, 133}, {65, 140}, {65, + 174}, {65, 266}, {65, 312}, {65, 375}, {65, 437}, {65, 503}, {65, 508}, + {65, 562}, {65, 626}, {66, 0}, {66, 46}, {66, 93}, {66, 257}, {66, 500}, + {66, 772}, {69, 1}, {69, 7}, {69, 11}, {69, 18}, {69, 62}, {69, 132}, {69, + 234}, {69, 257}, {69, 281}, {69, 296}, {69, 504}, {69, 532}, {69, 562}, + {69, 572}, {69, 694}, {69, 750}, {69, 757}, {70, 0}, {70, 15}, {70, 18}, + {70, 63}, {70, 157}, {70, 171}, {70, 250}, {70, 757}, {72, 32}, {72, 190}, + {72, 218}, {72, 291}, {72, 659}, {73, 0}, {73, 187}, {73, 250}, {73, 281}, + {73, 328}, {73, 500}, {77, 1}, {77, 3}, {77, 31}, {77, 38}, {77, 78}, {77, + 129}, {77, 171}, {77, 198}, {77, 250}, {77, 281}, {77, 508}, {77, 510}, + {77, 538}, {77, 563}, {77, 750}, {77, 754}, {77, 765}, {77, 768}, {77, + 937}, {78, 0}, {78, 1}, {78, 31}, {78, 218}, {78, 269}, {78, 376}, {80, 0}, + {80, 8}, {80, 10}, {80, 31}, {80, 69}, {80, 250}, {80, 269}, {80, 285}, + {80, 500}, {80, 547}, {81, 3}, {81, 31}, {81, 515}, {81, 531}, {84, 10}, + {84, 15}, {84, 132}, {84, 190}, {84, 250}, {84, 507}, {84, 532}, {84, 534}, + {84, 878}, {85, 126}, {85, 312}, {87, 140}, {93, 0}, {93, 3}, {93, 7}, {93, + 15}, {93, 18}, {93, 46}, {93, 53}, {93, 63}, {93, 132}, {93, 228}, {93, + 250}, {93, 261}, {93, 379}, {93, 383}, {93, 444}, {93, 507}, {93, 515}, + {93, 534}, {93, 562}, {94, 3}, {94, 34}, {94, 62}, {94, 500}, {94, 515}, + {94, 632}, {96, 38}, {96, 46}, {96, 53}, {96, 128}, {96, 250}, {96, 313}, + {100, 1}, {100, 128}, {101, 7}, {108, 0}, {108, 70}, {108, 254}, {108, + 272}, {108, 515}, {108, 565}, {109, 656}, {112, 500}, {115, 77}, {115, + 250}, {125, 0}, {125, 1}, {125, 3}, {125, 4}, {125, 7}, {125, 8}, {125, + 11}, {125, 15}, {125, 16}, {125, 23}, {125, 31}, {125, 32}, {125, 38}, + {125, 49}, {125, 62}, {125, 126}, {125, 156}, {125, 164}, {125, 187}, {125, + 194}, {125, 198}, {125, 205}, {125, 250}, {125, 253}, {125, 266}, {125, + 281}, {125, 284}, {125, 296}, {125, 347}, {125, 382}, {125, 501}, {125, + 507}, {125, 515}, {125, 523}, {125, 538}, {125, 554}, {125, 569}, {125, + 577}, {125, 593}, {125, 608}, {125, 647}, {125, 710}, {125, 718}, {125, + 750}, {125, 765}, {125, 766}, {126, 0}, {126, 1}, {126, 15}, {126, 25}, + {126, 32}, {126, 63}, {126, 80}, {126, 125}, {126, 128}, {126, 147}, {126, + 268}, {126, 284}, {126, 312}, {126, 437}, {126, 515}, {126, 531}, {128, 3}, + {128, 7}, {128, 15}, {128, 22}, {128, 62}, {128, 63}, {128, 73}, {128, 77}, + {128, 140}, {128, 147}, {128, 209}, {128, 269}, {128, 272}, {128, 281}, + {128, 296}, {128, 331}, {128, 515}, {128, 626}, {128, 632}, {129, 0}, {129, + 250}, {132, 0}, {132, 7}, {132, 62}, {132, 70}, {132, 77}, {132, 125}, + {132, 141}, {132, 160}, {132, 281}, {132, 500}, {132, 510}, {132, 625}, + {132, 626}, {132, 635}, {133, 18}, {133, 25}, {133, 125}, {133, 150}, {133, + 156}, {133, 281}, {133, 343}, {133, 500}, {135, 0}, {135, 1}, {135, 10}, + {135, 250}, {135, 531}, {136, 219}, {136, 375}, {136, 625}, {140, 0}, {140, + 3}, {140, 7}, {140, 31}, {140, 46}, {140, 56}, {140, 87}, {140, 250}, {140, + 257}, {140, 265}, {140, 282}, {140, 312}, {140, 500}, {140, 501}, {140, + 562}, {140, 565}, {140, 609}, {140, 875}, {141, 0}, {141, 140}, {141, 143}, + {141, 254}, {141, 281}, {141, 500}, {141, 522}, {141, 882}, {143, 4}, {143, + 10}, {143, 15}, {143, 62}, {143, 93}, {143, 226}, {143, 350}, {143, 584}, + {143, 633}, {143, 690}, {144, 0}, {144, 34}, {144, 375}, {144, 501}, {147, + 0}, {147, 22}, {147, 257}, {150, 265}, {156, 1}, {156, 8}, {156, 15}, {156, + 23}, {156, 31}, {156, 39}, {156, 125}, {156, 140}, {156, 203}, {156, 289}, + {156, 327}, {156, 376}, {156, 566}, {156, 577}, {156, 578}, {156, 596}, + {156, 907}, {157, 132}, {157, 156}, {157, 163}, {157, 437}, {157, 503}, + {157, 515}, {157, 593}, {157, 702}, {159, 1}, {159, 7}, {159, 32}, {159, + 62}, {159, 65}, {159, 77}, {159, 108}, {159, 132}, {159, 140}, {159, 233}, + {159, 501}, {160, 0}, {160, 72}, {160, 77}, {163, 0}, {163, 1}, {163, 125}, + {163, 126}, {163, 133}, {163, 625}, {163, 626}, {164, 1}, {164, 156}, {164, + 250}, {164, 365}, {166, 0}, {166, 78}, {171, 0}, {171, 1}, {171, 54}, {171, + 65}, {171, 94}, {171, 261}, {171, 500}, {172, 319}, {174, 77}, {175, 3}, + {178, 1}, {178, 11}, {178, 62}, {179, 500}, {187, 0}, {187, 3}, {187, 4}, + {187, 80}, {187, 126}, {187, 250}, {187, 281}, {187, 500}, {187, 518}, + {187, 534}, {187, 632}, {187, 702}, {187, 750}, {187, 785}, {188, 8}, {188, + 125}, {188, 147}, {188, 179}, {188, 253}, {188, 688}, {190, 0}, {190, 1}, + {190, 38}, {190, 766}, {191, 0}, {191, 140}, {194, 0}, {194, 3}, {194, 7}, + {194, 10}, {194, 35}, {194, 47}, {194, 93}, {194, 265}, {194, 410}, {194, + 846}, {195, 500}, {197, 15}, {197, 500}, {198, 0}, {202, 7}, {202, 62}, + {202, 125}, {202, 187}, {202, 453}, {202, 503}, {205, 69}, {206, 0}, {210, + 0}, {210, 789}, {212, 751}, {218, 15}, {218, 46}, {218, 63}, {218, 125}, + {218, 174}, {218, 257}, {218, 261}, {218, 265}, {219, 7}, {221, 0}, {221, + 7}, {221, 281}, {221, 508}, {225, 46}, {225, 516}, {241, 31}, {241, 63}, + {250, 0}, {250, 1}, {250, 3}, {250, 7}, {250, 8}, {250, 16}, {250, 18}, + {250, 19}, {250, 31}, {250, 32}, {250, 49}, {250, 53}, {250, 62}, {250, + 63}, {250, 69}, {250, 70}, {250, 77}, {250, 84}, {250, 108}, {250, 125}, + {250, 128}, {250, 129}, {250, 135}, {250, 140}, {250, 156}, {250, 167}, + {250, 187}, {250, 190}, {250, 209}, {250, 234}, {250, 257}, {250, 281}, + {250, 285}, {250, 313}, {250, 315}, {250, 343}, {250, 376}, {250, 452}, + {250, 500}, {250, 503}, {250, 510}, {250, 515}, {250, 562}, {250, 563}, + {250, 565}, {250, 572}, {250, 577}, {250, 593}, {250, 608}, {250, 629}, + {250, 640}, {250, 753}, {250, 769}, {250, 812}, {250, 827}, {250, 890}, + {250, 921}, {251, 3}, {251, 10}, {251, 34}, {251, 38}, {251, 125}, {251, + 126}, {251, 140}, {251, 171}, {251, 190}, {251, 202}, {251, 222}, {251, + 241}, {251, 284}, {251, 441}, {251, 500}, {251, 515}, {251, 556}, {251, + 728}, {251, 751}, {251, 827}, {253, 0}, {253, 1}, {253, 10}, {253, 22}, + {253, 25}, {253, 31}, {253, 47}, {253, 62}, {253, 125}, {253, 132}, {253, + 163}, {253, 195}, {253, 222}, {253, 315}, {253, 320}, {253, 375}, {253, + 500}, {253, 501}, {253, 507}, {253, 635}, {253, 651}, {253, 812}, {254, 0}, + {254, 135}, {254, 266}, {254, 281}, {254, 959}, {257, 10}, {257, 22}, {257, + 31}, {257, 62}, {257, 63}, {257, 125}, {257, 147}, {257, 157}, {257, 250}, + {257, 288}, {257, 297}, {257, 334}, {257, 353}, {257, 375}, {257, 501}, + {257, 694}, {257, 757}, {258, 65}, {258, 522}, {258, 593}, {258, 625}, + {258, 827}, {260, 0}, {260, 3}, {260, 254}, {260, 312}, {260, 330}, {260, + 516}, {260, 781}, {261, 10}, {261, 251}, {265, 0}, {265, 1}, {265, 3}, + {265, 8}, {265, 10}, {265, 26}, {265, 65}, {265, 77}, {265, 125}, {265, + 188}, {265, 257}, {265, 312}, {265, 409}, {265, 500}, {265, 503}, {265, + 570}, {266, 3}, {266, 156}, {266, 233}, {266, 694}, {268, 3}, {268, 8}, + {268, 16}, {268, 77}, {268, 125}, {268, 140}, {268, 503}, {268, 584}, {269, + 8}, {269, 31}, {272, 3}, {272, 77}, {272, 500}, {272, 632}, {275, 69}, + {275, 534}, {281, 0}, {281, 1}, {281, 8}, {281, 11}, {281, 39}, {281, 46}, + {281, 63}, {281, 70}, {281, 93}, {281, 128}, {281, 251}, {281, 507}, {281, + 516}, {281, 569}, {281, 671}, {281, 687}, {281, 768}, {282, 1}, {282, 23}, + {282, 31}, {282, 96}, {282, 132}, {282, 205}, {282, 288}, {282, 390}, {282, + 503}, {282, 788}, {284, 0}, {284, 31}, {284, 258}, {284, 500}, {288, 0}, + {288, 80}, {288, 140}, {288, 538}, {291, 7}, {291, 31}, {296, 0}, {296, 4}, + {296, 7}, {296, 16}, {296, 194}, {296, 359}, {296, 500}, {296, 501}, {296, + 508}, {296, 629}, {296, 687}, {296, 751}, {296, 812}, {297, 500}, {299, 0}, + {299, 1}, {299, 539}, {303, 22}, {303, 32}, {303, 656}, {306, 42}, {312, + 0}, {312, 7}, {312, 31}, {312, 128}, {312, 156}, {312, 229}, {312, 378}, + {312, 385}, {312, 578}, {312, 751}, {312, 788}, {312, 910}, {313, 0}, {313, + 15}, {313, 100}, {313, 125}, {313, 128}, {315, 49}, {315, 202}, {315, 593}, + {315, 909}, {316, 31}, {316, 38}, {316, 327}, {316, 625}, {319, 0}, {319, + 34}, {319, 125}, {319, 500}, {319, 625}, {319, 718}, {320, 19}, {320, 225}, + {320, 609}, {322, 0}, {322, 10}, {322, 140}, {322, 315}, {322, 625}, {323, + 538}, {327, 1}, {327, 3}, {327, 31}, {327, 69}, {328, 0}, {328, 77}, {328, + 132}, {330, 0}, {334, 156}, {334, 313}, {334, 376}, {334, 508}, {334, 546}, + {334, 648}, {337, 500}, {337, 503}, {338, 3}, {338, 126}, {343, 0}, {343, + 534}, {344, 34}, {344, 226}, {346, 125}, {346, 500}, {350, 562}, {350, + 587}, {353, 15}, {353, 500}, {353, 750}, {354, 500}, {358, 1}, {358, 299}, + {359, 501}, {365, 7}, {375, 0}, {375, 1}, {375, 35}, {375, 190}, {375, + 312}, {376, 15}, {376, 69}, {376, 78}, {376, 93}, {376, 250}, {376, 781}, + {378, 63}, {378, 69}, {378, 382}, {379, 15}, {382, 31}, {382, 531}, {382, + 640}, {385, 35}, {386, 15}, {386, 62}, {386, 159}, {386, 515}, {390, 94}, + {390, 125}, {390, 140}, {390, 260}, {390, 569}, {390, 609}, {390, 753}, + {391, 0}, {391, 18}, {391, 78}, {391, 876}, {393, 251}, {393, 625}, {398, + 897}, {400, 3}, {406, 34}, {406, 65}, {406, 144}, {406, 327}, {406, 515}, + {406, 531}, {406, 785}, {407, 3}, {409, 15}, {409, 46}, {409, 437}, {410, + 3}, {413, 15}, {413, 656}, {414, 49}, {421, 109}, {421, 197}, {422, 100}, + {424, 315}, {437, 15}, {437, 62}, {437, 125}, {437, 500}, {440, 7}, {440, + 500}, {440, 562}, {440, 775}, {448, 0}, {448, 438}, {452, 16}, {452, 31}, + {452, 250}, {452, 562}, {459, 62}, {459, 268}, {459, 796}, {468, 0}, {471, + 7}, {471, 538}, {475, 109}, {476, 63}, {490, 7}, {500, 0}, {500, 1}, {500, + 4}, {500, 7}, {500, 10}, {500, 11}, {500, 15}, {500, 18}, {500, 22}, {500, + 25}, {500, 31}, {500, 34}, {500, 38}, {500, 42}, {500, 46}, {500, 47}, + {500, 50}, {500, 62}, {500, 63}, {500, 69}, {500, 73}, {500, 77}, {500, + 80}, {500, 93}, {500, 94}, {500, 125}, {500, 126}, {500, 144}, {500, 166}, + {500, 171}, {500, 178}, {500, 250}, {500, 257}, {500, 265}, {500, 281}, + {500, 315}, {500, 316}, {500, 376}, {500, 386}, {500, 413}, {500, 437}, + {500, 503}, {500, 507}, {500, 508}, {500, 510}, {500, 516}, {500, 542}, + {500, 674}, {500, 687}, {500, 718}, {500, 750}, {500, 819}, {500, 875}, + {500, 906}, {501, 0}, {501, 7}, {501, 18}, {501, 22}, {501, 31}, {501, 47}, + {501, 125}, {501, 141}, {501, 150}, {501, 171}, {501, 190}, {501, 203}, + {501, 209}, {501, 260}, {501, 288}, {501, 500}, {501, 510}, {501, 534}, + {501, 656}, {501, 875}, {503, 0}, {503, 3}, {503, 16}, {503, 42}, {503, + 46}, {503, 63}, {503, 66}, {503, 147}, {503, 250}, {503, 261}, {503, 322}, + {503, 350}, {503, 378}, {503, 468}, {503, 500}, {503, 625}, {504, 1}, {504, + 3}, {504, 22}, {504, 77}, {504, 115}, {504, 500}, {504, 600}, {504, 789}, + {504, 796}, {507, 0}, {507, 3}, {507, 15}, {507, 18}, {507, 35}, {507, 38}, + {507, 125}, {507, 250}, {507, 500}, {507, 501}, {507, 547}, {508, 0}, {508, + 32}, {508, 132}, {508, 140}, {508, 382}, {508, 501}, {510, 66}, {510, 210}, + {510, 303}, {511, 268}, {511, 299}, {511, 500}, {515, 0}, {515, 1}, {515, + 3}, {515, 53}, {515, 69}, {515, 93}, {515, 128}, {515, 132}, {515, 156}, + {515, 167}, {515, 171}, {515, 188}, {515, 206}, {515, 250}, {515, 257}, + {515, 281}, {515, 375}, {515, 410}, {515, 437}, {515, 516}, {515, 751}, + {516, 0}, {516, 15}, {516, 35}, {516, 77}, {516, 103}, {516, 156}, {516, + 296}, {516, 300}, {516, 315}, {516, 459}, {516, 656}, {516, 834}, {516, + 865}, {518, 0}, {518, 7}, {518, 15}, {518, 65}, {518, 194}, {518, 250}, + {518, 265}, {518, 268}, {518, 577}, {518, 625}, {518, 812}, {519, 62}, + {522, 0}, {522, 31}, {522, 65}, {522, 80}, {522, 315}, {522, 376}, {522, + 531}, {522, 781}, {522, 897}, {523, 150}, {523, 282}, {525, 22}, {525, 31}, + {525, 251}, {526, 49}, {531, 0}, {531, 1}, {531, 7}, {531, 10}, {531, 49}, + {531, 93}, {531, 126}, {531, 132}, {531, 156}, {531, 202}, {531, 218}, + {531, 258}, {531, 315}, {531, 375}, {531, 503}, {531, 593}, {531, 765}, + {531, 772}, {532, 0}, {532, 3}, {532, 15}, {532, 31}, {532, 32}, {532, 62}, + {532, 73}, {532, 132}, {532, 562}, {534, 15}, {534, 300}, {535, 0}, {535, + 163}, {538, 0}, {538, 7}, {538, 15}, {538, 629}, {539, 62}, {539, 468}, + {541, 891}, {546, 1}, {546, 125}, {546, 281}, {546, 288}, {546, 319}, {546, + 327}, {546, 500}, {546, 532}, {546, 539}, {546, 640}, {547, 1}, {547, 250}, + {547, 784}, {549, 128}, {550, 0}, {550, 221}, {556, 15}, {562, 0}, {562, + 3}, {562, 4}, {562, 8}, {562, 31}, {562, 34}, {562, 46}, {562, 69}, {562, + 93}, {562, 125}, {562, 143}, {562, 319}, {562, 500}, {562, 501}, {562, + 504}, {562, 508}, {562, 510}, {563, 0}, {563, 39}, {563, 73}, {563, 257}, + {563, 376}, {563, 500}, {565, 0}, {565, 15}, {565, 296}, {565, 390}, {566, + 0}, {566, 815}, {569, 3}, {569, 125}, {569, 126}, {569, 175}, {569, 190}, + {569, 313}, {569, 538}, {570, 65}, {572, 65}, {572, 251}, {573, 500}, {577, + 0}, {577, 1}, {577, 15}, {577, 18}, {577, 159}, {577, 501}, {577, 518}, + {577, 584}, {578, 0}, {578, 385}, {580, 500}, {584, 625}, {593, 0}, {593, + 1}, {593, 3}, {593, 31}, {593, 41}, {593, 53}, {593, 281}, {593, 633}, + {593, 878}, {593, 882}, {594, 32}, {594, 93}, {594, 784}, {596, 66}, {600, + 0}, {600, 253}, {601, 577}, {603, 62}, {608, 0}, {611, 18}, {616, 375}, + {625, 0}, {625, 1}, {625, 3}, {625, 7}, {625, 34}, {625, 108}, {625, 126}, + {625, 129}, {625, 190}, {625, 254}, {625, 275}, {625, 284}, {625, 344}, + {625, 346}, {625, 390}, {625, 425}, {625, 532}, {625, 538}, {625, 750}, + {625, 968}, {626, 31}, {626, 32}, {626, 34}, {626, 63}, {626, 100}, {626, + 111}, {626, 500}, {626, 508}, {626, 643}, {628, 0}, {628, 46}, {629, 62}, + {629, 625}, {632, 8}, {632, 81}, {632, 93}, {632, 156}, {632, 202}, {632, + 547}, {633, 15}, {633, 268}, {633, 516}, {640, 0}, {640, 23}, {640, 39}, + {640, 251}, {640, 319}, {640, 358}, {640, 522}, {641, 8}, {641, 343}, {641, + 751}, {643, 265}, {644, 18}, {644, 41}, {648, 19}, {656, 3}, {656, 7}, + {656, 62}, {656, 382}, {656, 507}, {656, 757}, {657, 0}, {657, 81}, {657, + 894}, {659, 65}, {659, 476}, {659, 750}, {663, 125}, {671, 78}, {671, 253}, + {672, 375}, {687, 0}, {687, 15}, {687, 18}, {687, 31}, {687, 32}, {687, + 108}, {687, 125}, {687, 296}, {687, 500}, {688, 0}, {688, 38}, {690, 265}, + {690, 757}, {694, 0}, {694, 65}, {694, 250}, {695, 0}, {697, 93}, {702, 3}, + {702, 156}, {702, 282}, {703, 281}, {705, 46}, {705, 250}, {706, 94}, {709, + 628}, {718, 7}, {718, 31}, {718, 62}, {725, 0}, {725, 18}, {733, 376}, + {736, 784}, {740, 77}, {740, 126}, {750, 0}, {750, 16}, {750, 22}, {750, + 31}, {750, 32}, {750, 62}, {750, 63}, {750, 115}, {750, 128}, {750, 159}, + {750, 250}, {750, 315}, {750, 343}, {750, 344}, {750, 424}, {750, 562}, + {750, 569}, {750, 626}, {751, 46}, {751, 93}, {751, 300}, {751, 522}, {753, + 0}, {753, 80}, {753, 171}, {753, 250}, {753, 406}, {753, 562}, {757, 46}, + {757, 47}, {757, 209}, {757, 265}, {757, 312}, {758, 266}, {760, 0}, {760, + 8}, {761, 62}, {761, 600}, {765, 62}, {765, 148}, {765, 516}, {765, 534}, + {765, 823}, {766, 141}, {766, 500}, {768, 0}, {768, 141}, {768, 251}, {768, + 781}, {772, 375}, {772, 500}, {773, 0}, {781, 62}, {781, 63}, {781, 147}, + {781, 157}, {781, 187}, {782, 3}, {782, 15}, {782, 34}, {782, 125}, {782, + 253}, {784, 516}, {784, 569}, {788, 111}, {788, 116}, {789, 7}, {789, 140}, + {789, 376}, {796, 906}, {797, 1}, {812, 18}, {812, 126}, {812, 750}, {812, + 796}, {813, 0}, {813, 7}, {813, 500}, {815, 49}, {815, 96}, {819, 565}, + {822, 0}, {827, 4}, {827, 125}, {827, 383}, {830, 31}, {843, 4}, {843, + 281}, {846, 0}, {846, 69}, {859, 260}, {875, 35}, {875, 80}, {875, 148}, + {875, 206}, {875, 250}, {875, 312}, {875, 500}, {875, 503}, {875, 702}, + {875, 782}, {875, 878}, {876, 0}, {878, 31}, {878, 78}, {879, 515}, {890, + 163}, {890, 816}, {891, 553}, {897, 46}, {897, 265}, {906, 0}, {906, 65}, + {906, 518}, {913, 69}, {913, 197}, {937, 413}, {937, 531}, {937, 565}, + {940, 16}, {941, 250}, {944, 53}, {944, 346}, {947, 1}, {952, 375}, {59, 371}, + {371, 431}, {431, 564}, {564, 779}, {779, 835}, {835, 877}, {877, 986} + }; - graph.barrier(); - DASH_LOG_DEBUG("GraphTest.GlobalIteration", "elements committed"); + DASH_LOG_DEBUG("GraphTest.Construction", "construction started"); + graph_t g(edge_list.begin(), edge_list.end(), 1000); + DASH_LOG_DEBUG("GraphTest.Construction", "construction finished"); - if(myid == 0) { - int i = 0; - int unit_id = 0; - DASH_LOG_DEBUG("GraphTest.GlobalIteration", "begin vertex iteration"); - for(auto it = graph.vertices.begin(); it != graph.vertices.end(); ++it) { - int id = nunits * i + unit_id; - vertex_type v = *it; - DASH_LOG_DEBUG("GraphTest.GlobalIteration", "vertex", - "it", it, - "value", v.properties.id); - EXPECT_EQ_U(id, v.properties.id); - ++i; - if(i == ninsert_vertices_per_unit) { - i = 0; - ++unit_id; - } - } - - i = 0; - unit_id = 0; - DASH_LOG_DEBUG("GraphTest.GlobalIteration", "begin out-edge iteration"); - for(auto it = graph.out_edges.begin(); it != graph.out_edges.end(); ++it) { - int id = nunits * i + unit_id; - edge_type e = *it; - DASH_LOG_DEBUG("GraphTest.GlobalIteration", "out-edge", - "it", it, - "value", e.properties.id); - EXPECT_EQ_U(e.properties.id, id); - ++i; - if(i == ninsert_edges_per_unit) { - i = 0; - ++unit_id; - } - } - - int j = 0; - int k = 0; - int l = 0; - int unit_id_prev = nunits - 1; - DASH_LOG_DEBUG("GraphTest.GlobalIteration", "begin in-edge iteration"); - for(auto it = graph.in_edges.begin(); it != graph.in_edges.end(); ++it) { - int id = nunits * (j * ninsert_edges_per_vertex) + (nunits * k) + - unit_id_prev; - edge_type e = *it; - DASH_LOG_DEBUG("GraphTest.GlobalIteration", "in-edge", - "it", it, - "value", e.properties.id); - EXPECT_EQ_U(e.properties.id, id); - ++j; - if(j == ninsert_edges_per_vertex) { - j = 0; - k += 1; - } - ++l; - if(l == ninsert_edges_per_unit) { - l = 0; - unit_id_prev = (unit_id_prev + 1) % nunits; - k = 0; - } - } - - i = 0; - j = 0; - k = 0; - l = 0; - int m = 0; - unit_id = 0; - unit_id_prev = nunits - 1; - DASH_LOG_DEBUG("GraphTest.GlobalIteration", "begin edge iteration"); - for(auto it = graph.edges.begin(); it != graph.edges.end(); ++it) { - edge_type e = *it; - int id; - if(m < ninsert_edges_per_vertex) { - id = nunits * i + unit_id; - ++i; - if(i == ninsert_edges_per_unit) { - i = 0; - ++unit_id; - } - } else { - id = nunits * (j * ninsert_edges_per_vertex) + (nunits * k) + - unit_id_prev; - ++j; - if(j == ninsert_edges_per_vertex) { - j = 0; - k += 1; - } - ++l; - if(l == ninsert_edges_per_unit) { - l = 0; - unit_id_prev = (unit_id_prev + 1) % nunits; - k = 0; - } - } - ++m; - if(m == 2 * ninsert_edges_per_vertex) { - m = 0; - } - DASH_LOG_DEBUG("GraphTest.GlobalIteration", "edge", - "it", it, - "value", e.properties.id); - EXPECT_EQ_U(e.properties.id, id); - } - - i = 0; - unit_id = 0; - DASH_LOG_DEBUG("GraphTest.GlobalIteration", - "begin adjacency out-edge iteration"); - for(auto it = graph.vertices.begin(); it != graph.vertices.end(); ++it) { - vertex_type v = *it; - for(auto ait = graph.out_edges.vbegin(v); - ait != graph.out_edges.vend(v); ++ait) { - int id = nunits * i + unit_id; - edge_type e = *ait; - DASH_LOG_DEBUG("GraphTest.GlobalIteration", "out-edge", - "it", ait, - "value", e.properties.id); - EXPECT_EQ_U(e.properties.id, id); - ++i; - if(i == ninsert_edges_per_unit) { - i = 0; - ++unit_id; - } - } - } - - j = 0; - k = 0; - l = 0; - unit_id_prev = nunits - 1; - DASH_LOG_DEBUG("GraphTest.GlobalIteration", - "begin adjacency in-edge iteration"); - for(auto it = graph.vertices.begin(); it != graph.vertices.end(); ++it) { - vertex_type v = *it; - for(auto ait = graph.in_edges.vbegin(v); - ait != graph.in_edges.vend(v); ++ait) { - int id = nunits * (j * ninsert_edges_per_vertex) + (nunits * k) + - unit_id_prev; - edge_type e = *ait; - DASH_LOG_DEBUG("GraphTest.GlobalIteration", "in-edge", - "it", ait, - "value", e.properties.id); - EXPECT_EQ_U(e.properties.id, id); - ++j; - if(j == ninsert_edges_per_vertex) { - j = 0; - k += 1; - } - ++l; - if(l == ninsert_edges_per_unit) { - l = 0; - unit_id_prev = (unit_id_prev + 1) % nunits; - k = 0; - } - } - } - - i = 0; - j = 0; - k = 0; - l = 0; - m = 0; - unit_id = 0; - unit_id_prev = nunits - 1; - DASH_LOG_DEBUG("GraphTest.GlobalIteration", - "begin adjacency edge iteration"); - for(auto it = graph.vertices.begin(); it != graph.vertices.end(); ++it) { - vertex_type v = *it; - for(auto ait = graph.edges.vbegin(v); - ait != graph.edges.vend(v); ++ait) { - edge_type e = *ait; - int id; - if(m < ninsert_edges_per_vertex) { - id = nunits * i + unit_id; - ++i; - if(i == ninsert_edges_per_unit) { - i = 0; - ++unit_id; - } - } else { - id = nunits * (j * ninsert_edges_per_vertex) + (nunits * k) + - unit_id_prev; - ++j; - if(j == ninsert_edges_per_vertex) { - j = 0; - k += 1; - } - ++l; - if(l == ninsert_edges_per_unit) { - l = 0; - unit_id_prev = (unit_id_prev + 1) % nunits; - k = 0; + int unrecognized_edges = 0; + if(dash::myid() == 0) { + for(auto it = g.vertices().begin(); it != g.vertices().end(); ++it) { + auto v = g[it]; + for(auto e_it = v.out_edges().begin(); e_it != v.out_edges().end(); + ++e_it) { + auto e = g[e_it]; + //TODO: use map for faster runtime + bool unrecognized = true; + for(auto el_it = edge_list.begin(); el_it != edge_list.end(); + ++el_it) { + if(el_it->first == e.source().pos() + && el_it->second == e.target().pos()) { + edge_list.erase(el_it); + unrecognized = false; + break; } } - ++m; - if(m == 2 * ninsert_edges_per_vertex) { - m = 0; + if(unrecognized) { + ++unrecognized_edges; } - DASH_LOG_DEBUG("GraphTest.GlobalIteration", "edge", - "it", ait, - "value", e.properties.id); } } + // all edges from the edge list in the graph + EXPECT_EQ_U(0, edge_list.size()); + // no edges in the graph that are not in the edge list + EXPECT_EQ_U(0, unrecognized_edges); } dash::barrier(); } From 72b2a4c2320dc71bc56675ef7f16a3c4e85e26fe Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 8 Feb 2018 15:45:44 +0100 Subject: [PATCH 072/102] added connected components test --- .../test/algorithm/ConnectedComponentsTest.cc | 710 ++++++++++++++++++ dash/test/algorithm/ConnectedComponentsTest.h | 21 + 2 files changed, 731 insertions(+) create mode 100644 dash/test/algorithm/ConnectedComponentsTest.cc create mode 100644 dash/test/algorithm/ConnectedComponentsTest.h diff --git a/dash/test/algorithm/ConnectedComponentsTest.cc b/dash/test/algorithm/ConnectedComponentsTest.cc new file mode 100644 index 000000000..5031a1760 --- /dev/null +++ b/dash/test/algorithm/ConnectedComponentsTest.cc @@ -0,0 +1,710 @@ +#include "ConnectedComponentsTest.h" +#include +#include + +struct vprop { + int comp; +}; + +typedef dash::Graph graph_t; + +TEST_F(ConnectedComponentsTest, AlgorithmRun) +{ + std::list> edge_list = { + {433, 280}, {471, 192}, {738, 907}, {128, 664}, {999, 933}, {984, 440}, + {247, 310}, {565, 909}, {700, 727}, {408, 406}, {265, 550}, {411, 899}, + {630, 170}, {179, 57}, {733, 727}, {543, 943}, {475, 141}, {632, 394}, + {859, 581}, {203, 609}, {796, 589}, {494, 784}, {947, 420}, {354, 889}, + {184, 492}, {939, 587}, {598, 623}, {110, 971}, {296, 560}, {507, 881}, + {310, 674}, {280, 274}, {829, 845}, {273, 673}, {851, 52}, {814, 902}, + {995, 527}, {389, 611}, {537, 387}, {521, 350}, {849, 934}, {208, 57}, + {227, 962}, {30, 617}, {617, 962}, {512, 167}, {325, 1}, {232, 645}, {676, + 963}, {89, 242}, {57, 237}, {118, 252}, {807, 913}, {75, 425}, {782, 63}, + {901, 394}, {917, 23}, {923, 686}, {569, 451}, {217, 309}, {54, 950}, {508, + 185}, {642, 135}, {548, 436}, {495, 332}, {93, 368}, {483, 300}, {537, + 504}, {330, 526}, {29, 220}, {705, 598}, {468, 861}, {140, 13}, {192, 598}, + {347, 504}, {859, 182}, {369, 826}, {405, 979}, {694, 603}, {138, 269}, + {24, 364}, {256, 442}, {983, 657}, {946, 896}, {449, 915}, {6, 838}, {673, + 773}, {24, 231}, {235, 574}, {226, 875}, {327, 722}, {474, 311}, {501, + 904}, {291, 525}, {410, 937}, {804, 981}, {367, 113}, {449, 281}, {390, + 841}, {148, 698}, {396, 122}, {470, 726}, {442, 23}, {627, 762}, {493, + 406}, {710, 914}, {600, 289}, {182, 684}, {800, 569}, {965, 896}, {953, + 509}, {173, 25}, {671, 289}, {682, 205}, {71, 683}, {666, 286}, {414, 381}, + {23, 568}, {951, 45}, {987, 833}, {196, 225}, {686, 689}, {708, 87}, {954, + 715}, {209, 238}, {735, 700}, {357, 571}, {352, 466}, {373, 730}, {441, + 701}, {349, 224}, {289, 890}, {234, 680}, {721, 974}, {698, 801}, {181, + 460}, {47, 444}, {446, 142}, {262, 236}, {27, 494}, {958, 229}, {283, 210}, + {191, 279}, {359, 6}, {166, 810}, {808, 281}, {11, 801}, {484, 134}, {207, + 78}, {97, 128}, {182, 655}, {658, 329}, {926, 225}, {677, 285}, {183, 468}, + {685, 750}, {200, 999}, {871, 48}, {832, 515}, {485, 859}, {485, 75}, {13, + 435}, {612, 658}, {138, 796}, {919, 949}, {910, 747}, {975, 691}, {410, + 123}, {143, 788}, {535, 650}, {289, 386}, {48, 466}, {625, 783}, {690, + 851}, {761, 689}, {638, 964}, {106, 327}, {886, 578}, {223, 840}, {83, + 572}, {484, 802}, {703, 337}, {139, 522}, {234, 937}, {73, 555}, {15, 789}, + {335, 444}, {159, 468}, {747, 900}, {202, 563}, {880, 741}, {227, 307}, + {37, 594}, {380, 88}, {827, 286}, {747, 679}, {351, 208}, {374, 482}, {989, + 684}, {157, 691}, {207, 681}, {213, 302}, {372, 741}, {260, 231}, {710, + 110}, {529, 723}, {876, 580}, {156, 344}, {503, 467}, {887, 439}, {798, + 53}, {312, 342}, {890, 237}, {973, 738}, {36, 726}, {581, 386}, {153, 333}, + {341, 992}, {210, 389}, {551, 194}, {402, 407}, {875, 538}, {16, 873}, + {180, 47}, {119, 360}, {402, 996}, {422, 30}, {380, 939}, {722, 24}, {663, + 827}, {621, 266}, {113, 815}, {887, 976}, {859, 285}, {527, 646}, {971, + 190}, {84, 915}, {110, 953}, {934, 716}, {304, 316}, {413, 419}, {32, 373}, + {730, 936}, {478, 235}, {976, 947}, {859, 384}, {94, 33}, {912, 253}, {277, + 39}, {980, 696}, {91, 692}, {160, 263}, {190, 707}, {459, 166}, {221, 537}, + {849, 375}, {997, 679}, {57, 591}, {601, 48}, {393, 78}, {771, 839}, {254, + 485}, {939, 809}, {478, 167}, {881, 502}, {81, 317}, {76, 31}, {614, 887}, + {578, 417}, {970, 582}, {237, 389}, {951, 228}, {783, 626}, {146, 834}, + {233, 138}, {458, 861}, {167, 957}, {391, 619}, {880, 742}, {769, 210}, + {595, 786}, {708, 823}, {574, 159}, {663, 907}, {383, 781}, {861, 971}, + {395, 450}, {942, 183}, {558, 883}, {827, 609}, {23, 561}, {479, 386}, {68, + 915}, {75, 926}, {874, 927}, {15, 344}, {308, 359}, {788, 210}, {771, 65}, + {18, 494}, {760, 102}, {218, 602}, {53, 478}, {577, 476}, {794, 736}, {633, + 394}, {102, 941}, {686, 188}, {19, 326}, {626, 42}, {580, 305}, {304, 691}, + {569, 294}, {184, 239}, {698, 274}, {84, 168}, {994, 35}, {415, 591}, {139, + 416}, {152, 790}, {414, 123}, {987, 740}, {230, 973}, {206, 337}, {8, 329}, + {661, 477}, {514, 582}, {646, 944}, {32, 887}, {79, 655}, {749, 541}, {831, + 479}, {697, 443}, {691, 497}, {531, 432}, {595, 315}, {440, 663}, {567, + 705}, {710, 788}, {659, 805}, {112, 585}, {633, 934}, {358, 108}, {170, + 381}, {360, 876}, {155, 383}, {496, 69}, {104, 259}, {579, 513}, {907, 90}, + {438, 791}, {296, 322}, {382, 380}, {788, 196}, {667, 833}, {632, 598}, + {834, 396}, {45, 579}, {809, 724}, {470, 139}, {122, 686}, {667, 858}, + {860, 724}, {971, 644}, {838, 708}, {525, 660}, {94, 971}, {248, 531}, + {381, 380}, {178, 225}, {388, 291}, {218, 876}, {893, 177}, {896, 985}, + {930, 37}, {718, 804}, {553, 972}, {789, 952}, {367, 343}, {674, 387}, + {556, 160}, {985, 256}, {443, 917}, {377, 25}, {113, 453}, {43, 271}, {802, + 567}, {79, 199}, {713, 108}, {793, 517}, {860, 488}, {232, 686}, {983, + 451}, {360, 106}, {328, 853}, {190, 654}, {853, 10}, {580, 896}, {125, + 334}, {570, 449}, {192, 702}, {884, 146}, {209, 918}, {383, 810}, {186, + 460}, {137, 104}, {144, 845}, {775, 91}, {296, 805}, {773, 742}, {2, 136}, + {204, 752}, {370, 227}, {638, 957}, {192, 298}, {471, 309}, {718, 2}, {964, + 584}, {770, 1}, {232, 333}, {836, 921}, {263, 900}, {414, 629}, {824, 304}, + {379, 575}, {233, 160}, {547, 70}, {978, 603}, {55, 803}, {89, 160}, {490, + 598}, {888, 100}, {475, 973}, {880, 544}, {231, 894}, {546, 77}, {612, + 810}, {410, 882}, {964, 539}, {816, 920}, {809, 528}, {848, 472}, {202, + 291}, {724, 5}, {594, 131}, {385, 738}, {95, 277}, {432, 829}, {218, 434}, + {21, 127}, {926, 930}, {522, 650}, {563, 734}, {940, 927}, {733, 830}, + {494, 847}, {193, 290}, {492, 456}, {961, 47}, {876, 891}, {711, 827}, + {715, 474}, {489, 253}, {678, 289}, {398, 385}, {433, 146}, {173, 189}, + {985, 296}, {401, 357}, {592, 386}, {725, 243}, {308, 332}, {244, 637}, + {253, 938}, {852, 905}, {366, 712}, {560, 854}, {669, 270}, {666, 915}, + {362, 933}, {344, 832}, {255, 526}, {735, 438}, {400, 804}, {144, 230}, + {304, 338}, {287, 577}, {842, 8}, {327, 383}, {700, 41}, {499, 713}, {343, + 629}, {969, 654}, {24, 916}, {686, 276}, {525, 722}, {769, 302}, {240, 56}, + {828, 236}, {554, 486}, {727, 969}, {775, 356}, {411, 369}, {42, 943}, {70, + 429}, {824, 292}, {747, 577}, {359, 193}, {869, 301}, {390, 377}, {763, + 507}, {489, 526}, {63, 869}, {504, 647}, {525, 315}, {330, 65}, {818, 970}, + {976, 690}, {593, 324}, {242, 378}, {734, 271}, {120, 111}, {891, 898}, + {70, 675}, {929, 823}, {742, 330}, {347, 904}, {142, 496}, {353, 505}, + {448, 352}, {325, 794}, {54, 485}, {543, 70}, {763, 423}, {142, 846}, {112, + 7}, {652, 82}, {999, 939}, {402, 293}, {378, 300}, {717, 731}, {412, 885}, + {827, 595}, {179, 823}, {389, 424}, {353, 533}, {229, 511}, {889, 743}, + {235, 394}, {717, 22}, {121, 962}, {31, 991}, {843, 687}, {904, 89}, {695, + 705}, {624, 573}, {869, 643}, {887, 495}, {340, 827}, {833, 219}, {746, + 804}, {547, 350}, {127, 206}, {755, 827}, {108, 539}, {586, 321}, {451, + 251}, {860, 409}, {505, 953}, {713, 909}, {177, 226}, {15, 708}, {425, + 339}, {80, 63}, {181, 26}, {419, 139}, {298, 680}, {805, 607}, {259, 545}, + {451, 440}, {880, 225}, {255, 259}, {337, 637}, {974, 363}, {849, 850}, + {905, 206}, {868, 820}, {972, 972}, {736, 724}, {305, 420}, {63, 848}, + {947, 836}, {39, 947}, {355, 636}, {418, 13}, {816, 641}, {845, 374}, {359, + 921}, {322, 65}, {165, 728}, {997, 813}, {77, 344}, {340, 390}, {264, 367}, + {261, 712}, {953, 838}, {204, 743}, {48, 398}, {801, 61}, {186, 700}, {854, + 109}, {624, 172}, {304, 915}, {426, 711}, {45, 117}, {321, 549}, {387, + 555}, {206, 970}, {791, 317}, {786, 20}, {343, 594}, {361, 382}, {278, + 277}, {331, 225}, {510, 807}, {441, 493}, {396, 271}, {797, 678}, {254, + 123}, {539, 172}, {380, 267}, {420, 211}, {96, 247}, {407, 644}, {869, + 814}, {202, 338}, {327, 759}, {734, 713}, {18, 699}, {827, 103}, {953, + 551}, {206, 734}, {499, 98}, {66, 784}, {346, 444}, {824, 654}, {3, 224}, + {519, 231}, {319, 627}, {816, 292}, {81, 9}, {926, 899}, {27, 659}, {432, + 183}, {170, 709}, {522, 394}, {165, 264}, {469, 54}, {838, 705}, {101, + 415}, {713, 54}, {828, 650}, {480, 333}, {899, 162}, {830, 709}, {93, 789}, + {192, 173}, {720, 943}, {921, 758}, {100, 616}, {611, 62}, {168, 702}, + {776, 468}, {417, 79}, {420, 73}, {415, 149}, {383, 193}, {258, 597}, {871, + 677}, {327, 545}, {693, 93}, {714, 7}, {313, 135}, {788, 457}, {976, 961}, + {680, 113}, {422, 998}, {162, 301}, {269, 440}, {834, 25}, {559, 853}, + {787, 947}, {919, 930}, {910, 932}, {970, 417}, {846, 238}, {252, 741}, + {119, 401}, {136, 438}, {829, 432}, {614, 245}, {890, 7}, {303, 367}, {498, + 31}, {363, 384}, {805, 632}, {476, 821}, {940, 345}, {526, 932}, {966, + 643}, {572, 542}, {504, 255}, {841, 653}, {63, 91}, {97, 440}, {885, 550}, + {156, 851}, {190, 241}, {105, 859}, {264, 18}, {704, 965}, {598, 654}, + {922, 197}, {18, 543}, {677, 871}, {168, 514}, {334, 157}, {294, 553}, + {466, 973}, {792, 673}, {541, 331}, {307, 943}, {640, 551}, {947, 250}, + {390, 780}, {343, 379}, {80, 340}, {838, 145}, {634, 45}, {410, 2}, {934, + 338}, {621, 206}, {606, 977}, {735, 258}, {373, 274}, {631, 574}, {120, + 947}, {882, 725}, {139, 294}, {694, 450}, {476, 201}, {815, 305}, {234, + 563}, {305, 214}, {498, 39}, {753, 93}, {863, 81}, {202, 617}, {341, 719}, + {118, 153}, {586, 342}, {624, 806}, {498, 730}, {64, 211}, {858, 569}, + {541, 202}, {204, 259}, {517, 187}, {3, 682}, {36, 760}, {300, 287}, {980, + 873}, {260, 430}, {427, 411}, {677, 520}, {119, 515}, {18, 823}, {530, 39}, + {728, 185}, {537, 856}, {190, 883}, {409, 157}, {442, 271}, {347, 185}, + {884, 749}, {757, 55}, {394, 98}, {662, 829}, {32, 751}, {685, 491}, {937, + 571}, {624, 613}, {3, 417}, {90, 932}, {48, 377}, {541, 231}, {498, 940}, + {573, 755}, {157, 40}, {888, 205}, {161, 489}, {344, 601}, {300, 543}, + {319, 378}, {257, 918}, {536, 295}, {738, 305}, {970, 797}, {609, 698}, + {90, 668}, {114, 527}, {116, 267}, {429, 953}, {273, 401}, {76, 772}, {806, + 158}, {411, 627}, {271, 128}, {177, 72}, {995, 652}, {574, 35}, {250, 284}, + {283, 339}, {154, 304}, {602, 904}, {465, 102}, {709, 747}, {952, 735}, + {12, 418}, {473, 24}, {24, 880}, {38, 195}, {34, 797}, {647, 687}, {34, + 367}, {165, 62}, {454, 192}, {949, 492}, {671, 677}, {868, 896}, {333, + 965}, {846, 833}, {486, 905}, {123, 273}, {495, 697}, {319, 687}, {20, + 827}, {863, 991}, {530, 940}, {934, 56}, {2, 922}, {423, 716}, {863, 560}, + {447, 208}, {300, 597}, {941, 567}, {652, 665}, {668, 666}, {962, 732}, + {221, 61}, {108, 983}, {359, 374}, {589, 882}, {545, 460}, {728, 281}, + {101, 286}, {913, 474}, {819, 438}, {189, 863}, {377, 662}, {715, 322}, + {851, 766}, {957, 400}, {948, 312}, {444, 219}, {708, 763}, {19, 939}, + {390, 433}, {629, 689}, {55, 92}, {854, 656}, {67, 592}, {104, 334}, {578, + 863}, {961, 905}, {607, 412}, {177, 53}, {799, 76}, {891, 446}, {343, 78}, + {112, 79}, {872, 242}, {604, 137}, {606, 267}, {158, 640}, {559, 979}, + {908, 154}, {532, 630}, {519, 234}, {647, 247}, {690, 376}, {193, 268}, + {617, 89}, {561, 501}, {719, 455}, {820, 890}, {568, 977}, {363, 416}, + {234, 833}, {851, 107}, {213, 609}, {140, 436}, {229, 40}, {438, 262}, {77, + 404}, {686, 9}, {42, 545}, {398, 433}, {759, 105}, {13, 808}, {727, 495}, + {918, 189}, {879, 817}, {45, 747}, {335, 886}, {266, 985}, {708, 643}, + {200, 613}, {855, 834}, {980, 575}, {756, 71}, {609, 504}, {711, 333}, + {798, 973}, {316, 946}, {444, 449}, {213, 929}, {860, 579}, {683, 38}, {8, + 863}, {992, 618}, {407, 715}, {791, 526}, {848, 85}, {30, 690}, {468, 208}, + {238, 404}, {989, 480}, {662, 156}, {670, 160}, {691, 172}, {16, 903}, + {620, 93}, {856, 96}, {308, 571}, {479, 881}, {44, 903}, {621, 540}, {9, + 168}, {481, 405}, {551, 995}, {172, 260}, {369, 659}, {110, 855}, {880, + 298}, {687, 388}, {95, 828}, {960, 512}, {968, 6}, {878, 909}, {138, 124}, + {751, 408}, {252, 243}, {683, 119}, {573, 282}, {768, 782}, {415, 691}, + {272, 643}, {108, 966}, {657, 186}, {486, 665}, {889, 927}, {565, 826}, + {901, 858}, {99, 999}, {748, 415}, {571, 472}, {910, 484}, {905, 735}, + {894, 229}, {381, 908}, {944, 320}, {815, 85}, {177, 915}, {669, 415}, + {686, 425}, {19, 242}, {985, 845}, {213, 277}, {955, 349}, {940, 804}, + {954, 198}, {403, 329}, {87, 855}, {239, 563}, {152, 898}, {544, 643}, + {760, 95}, {780, 501}, {470, 475}, {301, 646}, {298, 5}, {795, 899}, {82, + 558}, {778, 360}, {209, 706}, {589, 301}, {719, 798}, {268, 205}, {327, + 460}, {470, 22}, {660, 372}, {96, 497}, {485, 760}, {168, 821}, {215, 332}, + {276, 112}, {332, 804}, {808, 747}, {673, 35}, {605, 230}, {987, 466}, + {179, 383}, {705, 165}, {356, 404}, {943, 208}, {14, 50}, {776, 346}, {789, + 358}, {751, 233}, {303, 306}, {654, 843}, {841, 547}, {263, 35}, {218, + 227}, {462, 727}, {52, 64}, {220, 173}, {675, 635}, {856, 409}, {495, 696}, + {664, 132}, {388, 849}, {766, 515}, {781, 228}, {145, 633}, {405, 645}, + {426, 954}, {793, 318}, {46, 30}, {848, 998}, {907, 612}, {163, 173}, {670, + 712}, {356, 239}, {966, 345}, {341, 388}, {435, 237}, {459, 470}, {466, + 310}, {380, 128}, {174, 263}, {28, 118}, {119, 782}, {531, 8}, {930, 160}, + {604, 165}, {131, 773}, {949, 844}, {753, 316}, {740, 917}, {78, 608}, + {917, 51}, {923, 824}, {381, 895}, {61, 109}, {959, 224}, {357, 393}, {756, + 289}, {826, 231}, {23, 535}, {416, 756}, {812, 411}, {1, 12}, {340, 746}, + {472, 109}, {467, 595}, {218, 179}, {582, 756}, {801, 138}, {698, 895}, + {43, 359}, {999, 378}, {624, 595}, {37, 23}, {454, 226}, {70, 213}, {676, + 673}, {654, 62}, {249, 809}, {150, 776}, {192, 442}, {875, 325}, {706, + 628}, {212, 638}, {597, 149}, {205, 604}, {951, 189}, {641, 944}, {915, + 487}, {944, 532}, {460, 48}, {95, 482}, {735, 61}, {136, 970}, {788, 733}, + {533, 485}, {533, 660}, {758, 626}, {777, 313}, {702, 720}, {709, 926}, + {777, 297}, {332, 698}, {552, 27}, {465, 725}, {330, 473}, {938, 20}, {276, + 605}, {454, 844}, {950, 805}, {276, 408}, {391, 773}, {297, 949}, {64, 38}, + {177, 236}, {76, 498}, {218, 521}, {582, 428}, {205, 175}, {987, 756}, + {247, 115}, {815, 970}, {860, 775}, {319, 15}, {860, 36}, {326, 228}, {966, + 293}, {257, 159}, {935, 552}, {34, 735}, {39, 165}, {878, 215}, {405, 352}, + {72, 298}, {927, 771}, {896, 66}, {116, 677}, {36, 268}, {546, 11}, {742, + 766}, {42, 783}, {774, 864}, {245, 4}, {932, 149}, {77, 509}, {714, 65}, + {839, 333}, {204, 656}, {345, 375}, {855, 807}, {466, 109}, {727, 880}, + {948, 413}, {5, 541}, {355, 443}, {583, 629}, {350, 688}, {739, 416}, {329, + 182}, {185, 140}, {713, 911}, {950, 346}, {927, 204}, {375, 753}, {974, + 735}, {28, 783}, {706, 207}, {0, 21}, {842, 758}, {743, 924}, {966, 312}, + {329, 675}, {291, 490}, {329, 493}, {523, 862}, {916, 834}, {860, 424}, + {861, 46}, {617, 363}, {844, 174}, {309, 106}, {45, 899}, {353, 567}, {783, + 689}, {681, 155}, {524, 933}, {980, 142}, {69, 794}, {417, 89}, {562, 922}, + {741, 86}, {74, 996}, {813, 989}, {694, 73}, {680, 415}, {772, 797}, {982, + 101}, {283, 936}, {825, 226}, {945, 394}, {740, 599}, {68, 519}, {979, + 192}, {138, 4}, {201, 791}, {595, 210}, {215, 215}, {802, 488}, {539, 277}, + {919, 844}, {746, 495}, {604, 1}, {951, 456}, {51, 346}, {214, 108}, {587, + 565}, {464, 939}, {109, 945}, {691, 274}, {878, 197}, {311, 534}, {458, + 992}, {737, 696}, {853, 295}, {156, 139}, {285, 518}, {92, 169}, {590, + 142}, {263, 792}, {109, 656}, {406, 132}, {610, 734}, {359, 367}, {206, + 953}, {883, 324}, {669, 123}, {878, 598}, {569, 32}, {422, 527}, {957, + 964}, {700, 522}, {333, 719}, {647, 832}, {269, 457}, {703, 110}, {512, + 238}, {233, 121}, {710, 179}, {802, 93}, {715, 825}, {321, 307}, {866, + 980}, {231, 586}, {845, 456}, {26, 265}, {656, 883}, {221, 951}, {131, 80}, + {629, 876}, {736, 293}, {171, 526}, {121, 189}, {269, 962}, {709, 939}, + {800, 320}, {874, 235}, {728, 670}, {821, 325}, {149, 445}, {17, 593}, + {573, 443}, {879, 321}, {718, 148}, {282, 455}, {420, 144}, {449, 303}, + {404, 53}, {958, 594}, {855, 155}, {33, 537}, {179, 325}, {907, 621}, {26, + 951}, {722, 289}, {213, 846}, {141, 106}, {530, 683}, {905, 934}, {425, + 686}, {486, 256}, {47, 82}, {133, 45}, {526, 690}, {733, 856}, {144, 172}, + {69, 853}, {658, 302}, {64, 165}, {957, 613}, {29, 136}, {464, 346}, {114, + 87}, {832, 654}, {631, 900}, {96, 461}, {657, 458}, {863, 372}, {217, 614}, + {352, 705}, {791, 378}, {617, 550}, {222, 578}, {86, 907}, {163, 593}, + {410, 307}, {603, 403}, {335, 892}, {20, 926}, {400, 745}, {747, 190}, + {443, 698}, {705, 735}, {169, 30}, {921, 106}, {167, 498}, {11, 339}, {89, + 578}, {470, 594}, {325, 50}, {127, 952}, {767, 463}, {887, 49}, {193, 249}, + {69, 487}, {723, 280}, {294, 744}, {242, 384}, {685, 92}, {975, 970}, {824, + 253}, {592, 419}, {278, 481}, {361, 162}, {439, 17}, {881, 221}, {4, 615}, + {399, 211}, {546, 45}, {834, 438}, {553, 879}, {603, 895}, {412, 366}, {63, + 697}, {16, 246}, {808, 2}, {73, 225}, {288, 607}, {475, 616}, {428, 999}, + {560, 595}, {421, 404}, {563, 899}, {126, 138}, {388, 419}, {47, 376}, + {305, 1}, {420, 381}, {501, 447}, {378, 720}, {469, 301}, {584, 853}, {659, + 498}, {849, 517}, {970, 851}, {840, 212}, {612, 508}, {38, 471}, {98, 504}, + {143, 368}, {846, 492}, {760, 788}, {720, 353}, {115, 858}, {389, 911}, + {142, 719}, {960, 273}, {97, 714}, {306, 744}, {336, 321}, {684, 493}, + {951, 464}, {985, 894}, {627, 860}, {550, 559}, {503, 840}, {473, 170}, + {322, 672}, {521, 717}, {329, 566}, {703, 582}, {934, 969}, {313, 635}, + {698, 984}, {128, 880}, {658, 154}, {524, 145}, {9, 315}, {71, 425}, {565, + 104}, {817, 84}, {429, 976}, {601, 279}, {695, 146}, {210, 144}, {740, + 609}, {872, 201}, {163, 52}, {857, 27}, {699, 499}, {921, 545}, {730, 762}, + {850, 326}, {609, 673}, {699, 453}, {910, 238}, {592, 215}, {573, 850}, + {749, 217}, {449, 592}, {468, 331}, {476, 543}, {16, 800}, {361, 250}, + {995, 363}, {288, 858}, {971, 118}, {541, 583}, {561, 587}, {832, 293}, + {660, 60}, {247, 685}, {799, 279}, {169, 913}, {115, 746}, {91, 341}, {153, + 758}, {985, 628}, {116, 170}, {791, 712}, {449, 731}, {4, 719}, {975, 88}, + {292, 114}, {670, 607}, {4, 178}, {8, 389}, {492, 51}, {183, 606}, {314, + 888}, {571, 887}, {182, 275}, {572, 352}, {903, 459}, {131, 558}, {681, + 288}, {40, 333}, {587, 267}, {483, 764}, {966, 348}, {676, 279}, {141, + 409}, {904, 760}, {740, 406}, {474, 565}, {932, 428}, {40, 192}, {539, + 232}, {613, 528}, {207, 63}, {321, 714}, {872, 570}, {862, 809}, {882, + 348}, {594, 603}, {756, 279}, {531, 767}, {993, 556}, {666, 276}, {961, + 176}, {573, 800}, {417, 146}, {863, 327}, {627, 118}, {692, 250}, {272, + 244}, {587, 159}, {22, 704}, {263, 515}, {492, 19}, {728, 979}, {257, 917}, + {383, 409}, {58, 423}, {480, 417}, {399, 716}, {374, 633}, {408, 265}, + {762, 749}, {695, 629}, {893, 272}, {285, 719}, {673, 456}, {690, 491}, + {243, 595}, {425, 955}, {558, 459}, {485, 562}, {47, 996}, {902, 486}, + {581, 716}, {525, 812}, {519, 945}, {923, 918}, {607, 572}, {882, 719}, + {544, 313}, {136, 36}, {977, 567}, {23, 244}, {777, 198}, {515, 590}, {349, + 90}, {858, 183}, {271, 996}, {116, 821}, {267, 829}, {576, 681}, {718, + 619}, {614, 393}, {571, 478}, {647, 840}, {843, 691}, {387, 201}, {330, + 391}, {667, 494}, {504, 437}, {111, 560}, {800, 103}, {931, 618}, {322, + 595}, {634, 848}, {96, 443}, {804, 14}, {647, 636}, {496, 521}, {529, 633}, + {556, 117}, {600, 162}, {211, 432}, {550, 909}, {890, 45}, {725, 61}, {147, + 626}, {482, 894}, {755, 159}, {55, 731}, {352, 488}, {934, 163}, {93, 653}, + {524, 36}, {607, 528}, {341, 126}, {425, 87}, {354, 490}, {22, 72}, {869, + 869}, {671, 203}, {181, 372}, {105, 40}, {634, 342}, {229, 923}, {271, + 472}, {325, 837}, {929, 218}, {968, 410}, {144, 628}, {716, 344}, {983, + 457}, {237, 209}, {400, 562}, {271, 227}, {134, 461}, {587, 903}, {658, + 519}, {172, 354}, {530, 739}, {244, 175}, {883, 940}, {786, 638}, {725, + 141}, {316, 830}, {584, 514}, {729, 900}, {481, 737}, {800, 228}, {478, 8}, + {36, 395}, {684, 178}, {369, 397}, {824, 109}, {792, 796}, {756, 679}, + {699, 241}, {227, 789}, {510, 521}, {665, 375}, {107, 595}, {565, 386}, + {687, 129}, {498, 585}, {897, 718}, {275, 767}, {538, 469}, {849, 208}, + {15, 349}, {518, 750}, {416, 158}, {577, 390}, {751, 420}, {892, 233}, + {793, 895}, {870, 691}, {339, 125}, {41, 403}, {784, 7}, {426, 181}, {197, + 430}, {71, 474}, {950, 597}, {183, 147}, {186, 939}, {679, 571}, {209, + 878}, {836, 395}, {391, 472}, {934, 326}, {536, 151}, {238, 925}, {38, + 763}, {425, 185}, {308, 254}, {110, 47}, {599, 993}, {236, 918}, {64, 337}, + {41, 57}, {676, 439}, {378, 388}, {845, 335}, {130, 115}, {662, 594}, {281, + 557}, {867, 935}, {236, 982}, {896, 905}, {952, 302}, {440, 186}, {46, + 229}, {301, 727}, {634, 140}, {85, 411}, {256, 571}, {16, 616}, {729, 321}, + {316, 102}, {756, 885}, {482, 617}, {98, 90}, {550, 914}, {651, 19}, {174, + 642}, {484, 245}, {949, 551}, {130, 581}, {745, 75}, {799, 498}, {375, 30}, + {30, 333}, {195, 199}, {80, 717}, {133, 852}, {464, 906}, {137, 642}, {108, + 772}, {833, 352}, {975, 236}, {799, 557}, {571, 106}, {237, 379}, {698, + 712}, {618, 735}, {352, 682}, {754, 799}, {981, 502}, {976, 721}, {216, + 281}, {470, 956}, {322, 777}, {387, 286}, {682, 601}, {351, 208}, {662, + 912}, {812, 226}, {806, 450}, {560, 469}, {94, 346}, {350, 347}, {625, + 116}, {21, 708}, {246, 770}, {17, 194}, {696, 357}, {397, 824}, {337, 647}, + {347, 913}, {989, 966}, {865, 180}, {669, 562}, {996, 627}, {455, 299}, + {672, 664}, {127, 377}, {646, 630}, {688, 232}, {584, 944}, {359, 678}, + {669, 61}, {628, 693}, {407, 293}, {206, 220}, {921, 248}, {750, 96}, {635, + 432}, {197, 304}, {692, 814}, {90, 506}, {407, 172}, {999, 392}, {851, + 698}, {479, 678}, {559, 484}, {837, 77}, {625, 13}, {837, 672}, {990, 558}, + {176, 124}, {366, 101}, {624, 558}, {69, 197}, {715, 502}, {186, 413}, + {614, 999}, {994, 827}, {964, 576}, {520, 539}, {599, 5}, {501, 329}, {954, + 376}, {276, 766}, {586, 487}, {765, 221}, {381, 549}, {257, 476}, {267, + 772}, {44, 362}, {961, 300}, {811, 52}, {264, 264}, {365, 994}, {776, 246}, + {943, 689}, {53, 545}, {93, 179}, {586, 443}, {861, 552}, {677, 338}, {448, + 496}, {740, 410}, {488, 537}, {588, 939}, {988, 592}, {448, 853}, {939, + 535}, {481, 302}, {241, 570}, {257, 231}, {706, 565}, {161, 725}, {509, + 477}, {555, 367}, {892, 476}, {287, 492}, {756, 49}, {305, 257}, {976, + 210}, {630, 406}, {792, 899}, {566, 152}, {801, 657}, {707, 759}, {49, + 246}, {812, 444}, {800, 615}, {484, 520}, {912, 857}, {26, 649}, {699, + 806}, {219, 916}, {300, 9}, {972, 176}, {447, 722}, {622, 791}, {806, 7}, + {824, 747}, {572, 233}, {741, 433}, {201, 402}, {262, 236}, {685, 390}, + {246, 64}, {190, 960}, {199, 936}, {984, 415}, {671, 121}, {739, 895}, + {239, 647}, {439, 129}, {6, 796}, {623, 682}, {429, 487}, {348, 180}, {299, + 899}, {834, 206}, {657, 78}, {155, 359}, {9, 884}, {660, 712}, {622, 243}, + {861, 605}, {570, 666}, {202, 614}, {933, 742}, {946, 580}, {797, 961}, + {376, 382}, {159, 153}, {925, 179}, {639, 942}, {233, 640}, {895, 229}, + {37, 552}, {850, 532}, {133, 500}, {740, 928}, {418, 742}, {994, 330}, + {694, 468}, {9, 545}, {171, 926}, {369, 674}, {333, 905}, {675, 27}, {479, + 266}, {122, 397}, {724, 366}, {363, 633}, {974, 268}, {243, 38}, {519, + 977}, {183, 256}, {80, 356}, {688, 363}, {964, 704}, {39, 806}, {102, 218}, + {865, 619}, {406, 385}, {354, 605}, {694, 495}, {159, 269}, {425, 290}, + {858, 232}, {338, 45}, {420, 337}, {404, 153}, {471, 485}, {639, 532}, + {628, 607}, {980, 680}, {826, 492}, {382, 321}, {33, 366}, {709, 250}, + {644, 122}, {763, 175}, {86, 126}, {606, 304}, {689, 490}, {791, 427}, + {732, 659}, {443, 40}, {773, 967}, {223, 400}, {473, 851}, {294, 983}, {76, + 705}, {310, 932}, {908, 862}, {451, 940}, {660, 670}, {250, 128}, {315, + 347}, {383, 576}, {121, 152}, {373, 334}, {963, 908}, {912, 936}, {974, + 563}, {989, 207}, {76, 699}, {239, 737}, {921, 325}, {819, 508}, {313, + 871}, {640, 904}, {679, 409}, {272, 429}, {793, 494}, {281, 143}, {446, + 929}, {961, 471}, {688, 492}, {519, 461}, {492, 569}, {258, 430}, {46, + 387}, {265, 162}, {746, 960}, {729, 795}, {304, 753}, {691, 833}, {941, + 109}, {97, 570}, {26, 652}, {842, 661}, {510, 778}, {883, 982}, {711, 605}, + {209, 160}, {182, 364}, {149, 82}, {245, 435}, {247, 818}, {499, 301}, + {820, 669}, {381, 89}, {899, 165}, {398, 969}, {22, 147}, {0, 1}, {0, 3}, + {0, 4}, {0, 7}, {0, 8}, {0, 10}, {0, 15}, {0, 16}, {0, 18}, {0, 31}, {0, + 32}, {0, 34}, {0, 35}, {0, 38}, {0, 41}, {0, 46}, {0, 57}, {0, 62}, {0, + 63}, {0, 65}, {0, 69}, {0, 72}, {0, 77}, {0, 85}, {0, 88}, {0, 94}, {0, + 96}, {0, 97}, {0, 108}, {0, 125}, {0, 126}, {0, 128}, {0, 132}, {0, 140}, + {0, 147}, {0, 156}, {0, 157}, {0, 160}, {0, 163}, {0, 166}, {0, 187}, {0, + 191}, {0, 195}, {0, 203}, {0, 205}, {0, 213}, {0, 218}, {0, 221}, {0, 250}, + {0, 253}, {0, 257}, {0, 260}, {0, 265}, {0, 266}, {0, 268}, {0, 269}, {0, + 281}, {0, 285}, {0, 306}, {0, 312}, {0, 313}, {0, 316}, {0, 327}, {0, 343}, + {0, 378}, {0, 382}, {0, 401}, {0, 406}, {0, 444}, {0, 471}, {0, 483}, {0, + 484}, {0, 500}, {0, 501}, {0, 503}, {0, 504}, {0, 507}, {0, 515}, {0, 518}, + {0, 522}, {0, 532}, {0, 542}, {0, 562}, {0, 565}, {0, 569}, {0, 612}, {0, + 625}, {0, 626}, {0, 628}, {0, 632}, {0, 650}, {0, 697}, {0, 750}, {0, 751}, + {0, 754}, {0, 757}, {0, 760}, {0, 781}, {0, 784}, {0, 785}, {0, 797}, {0, + 813}, {0, 837}, {0, 898}, {1, 0}, {1, 3}, {1, 4}, {1, 7}, {1, 15}, {1, 18}, + {1, 19}, {1, 25}, {1, 31}, {1, 34}, {1, 35}, {1, 46}, {1, 62}, {1, 63}, {1, + 65}, {1, 69}, {1, 80}, {1, 93}, {1, 125}, {1, 132}, {1, 135}, {1, 144}, {1, + 156}, {1, 171}, {1, 194}, {1, 233}, {1, 250}, {1, 254}, {1, 266}, {1, 281}, + {1, 285}, {1, 313}, {1, 327}, {1, 344}, {1, 409}, {1, 437}, {1, 452}, {1, + 475}, {1, 500}, {1, 515}, {1, 531}, {1, 565}, {1, 593}, {1, 600}, {1, 628}, + {1, 632}, {3, 0}, {3, 1}, {3, 7}, {3, 15}, {3, 26}, {3, 31}, {3, 32}, {3, + 34}, {3, 46}, {3, 62}, {3, 63}, {3, 65}, {3, 69}, {3, 111}, {3, 125}, {3, + 132}, {3, 140}, {3, 141}, {3, 143}, {3, 156}, {3, 187}, {3, 250}, {3, 269}, + {3, 281}, {3, 312}, {3, 327}, {3, 328}, {3, 346}, {3, 437}, {3, 500}, {3, + 501}, {3, 508}, {3, 515}, {3, 519}, {3, 547}, {3, 562}, {3, 594}, {3, 663}, + {3, 698}, {3, 750}, {3, 765}, {3, 885}, {4, 0}, {4, 7}, {4, 16}, {4, 23}, + {4, 26}, {4, 31}, {4, 57}, {4, 125}, {4, 150}, {4, 229}, {4, 236}, {4, + 258}, {4, 625}, {4, 671}, {4, 688}, {4, 843}, {7, 0}, {7, 1}, {7, 3}, {7, + 15}, {7, 16}, {7, 18}, {7, 22}, {7, 31}, {7, 62}, {7, 69}, {7, 93}, {7, + 125}, {7, 132}, {7, 135}, {7, 140}, {7, 159}, {7, 187}, {7, 202}, {7, 236}, + {7, 257}, {7, 265}, {7, 273}, {7, 284}, {7, 313}, {7, 315}, {7, 382}, {7, + 383}, {7, 390}, {7, 452}, {7, 503}, {7, 515}, {7, 518}, {7, 531}, {7, 538}, + {7, 549}, {7, 562}, {7, 565}, {7, 628}, {7, 750}, {7, 843}, {7, 876}, {8, + 3}, {8, 4}, {8, 7}, {8, 15}, {8, 22}, {8, 31}, {8, 47}, {8, 49}, {8, 135}, + {8, 159}, {8, 228}, {8, 250}, {8, 437}, {8, 508}, {8, 566}, {10, 0}, {10, + 1}, {10, 15}, {10, 16}, {10, 31}, {10, 41}, {10, 62}, {10, 125}, {10, 140}, + {10, 205}, {10, 250}, {10, 375}, {10, 562}, {10, 656}, {10, 882}, {11, 0}, + {11, 62}, {11, 265}, {11, 281}, {11, 625}, {15, 0}, {15, 1}, {15, 4}, {15, + 7}, {15, 10}, {15, 16}, {15, 18}, {15, 31}, {15, 38}, {15, 46}, {15, 54}, + {15, 62}, {15, 63}, {15, 69}, {15, 73}, {15, 93}, {15, 94}, {15, 109}, {15, + 125}, {15, 175}, {15, 190}, {15, 225}, {15, 254}, {15, 281}, {15, 284}, + {15, 316}, {15, 346}, {15, 406}, {15, 422}, {15, 500}, {15, 501}, {15, + 510}, {15, 515}, {15, 523}, {15, 562}, {15, 625}, {15, 702}, {15, 753}, + {15, 781}, {15, 791}, {16, 0}, {16, 1}, {16, 3}, {16, 10}, {16, 18}, {16, + 22}, {16, 62}, {16, 65}, {16, 140}, {16, 167}, {16, 282}, {16, 330}, {16, + 500}, {16, 503}, {16, 516}, {16, 562}, {16, 570}, {16, 641}, {16, 750}, + {16, 784}, {18, 1}, {18, 7}, {18, 22}, {18, 65}, {18, 132}, {18, 147}, {18, + 171}, {18, 187}, {18, 222}, {18, 251}, {18, 253}, {18, 268}, {18, 312}, + {18, 413}, {18, 422}, {18, 515}, {18, 516}, {18, 633}, {18, 758}, {19, 0}, + {19, 3}, {19, 4}, {19, 46}, {19, 273}, {19, 289}, {19, 750}, {22, 0}, {22, + 31}, {22, 34}, {22, 38}, {22, 46}, {22, 69}, {22, 70}, {22, 187}, {22, + 222}, {22, 250}, {22, 291}, {22, 312}, {22, 313}, {22, 359}, {22, 382}, + {22, 390}, {22, 437}, {22, 626}, {22, 641}, {23, 0}, {23, 41}, {23, 78}, + {23, 100}, {23, 281}, {23, 375}, {25, 0}, {25, 62}, {25, 140}, {25, 187}, + {25, 531}, {25, 844}, {26, 108}, {26, 394}, {31, 0}, {31, 1}, {31, 3}, {31, + 7}, {31, 15}, {31, 18}, {31, 47}, {31, 53}, {31, 65}, {31, 93}, {31, 108}, + {31, 112}, {31, 126}, {31, 140}, {31, 156}, {31, 157}, {31, 163}, {31, + 187}, {31, 202}, {31, 253}, {31, 265}, {31, 268}, {31, 312}, {31, 375}, + {31, 393}, {31, 468}, {31, 500}, {31, 503}, {31, 504}, {31, 507}, {31, + 510}, {31, 516}, {31, 538}, {31, 569}, {31, 687}, {31, 750}, {31, 812}, + {31, 876}, {32, 0}, {32, 3}, {32, 4}, {32, 18}, {32, 25}, {32, 26}, {32, + 46}, {32, 62}, {32, 140}, {32, 257}, {32, 272}, {32, 327}, {32, 382}, {32, + 500}, {32, 587}, {34, 0}, {34, 15}, {34, 16}, {34, 38}, {34, 46}, {34, 63}, + {34, 115}, {34, 126}, {34, 141}, {34, 265}, {34, 501}, {34, 508}, {34, + 538}, {34, 597}, {34, 750}, {35, 0}, {35, 3}, {35, 31}, {35, 34}, {35, 62}, + {35, 69}, {35, 80}, {35, 171}, {35, 508}, {38, 0}, {38, 7}, {38, 16}, {38, + 18}, {38, 25}, {38, 63}, {38, 66}, {38, 70}, {38, 96}, {38, 125}, {38, + 251}, {38, 265}, {38, 291}, {38, 313}, {38, 315}, {38, 316}, {38, 376}, + {38, 569}, {38, 625}, {38, 750}, {38, 757}, {38, 785}, {38, 875}, {39, 34}, + {39, 69}, {39, 125}, {39, 132}, {39, 266}, {39, 319}, {41, 50}, {41, 250}, + {41, 750}, {41, 765}, {42, 3}, {42, 312}, {46, 0}, {46, 1}, {46, 3}, {46, + 4}, {46, 7}, {46, 10}, {46, 15}, {46, 16}, {46, 25}, {46, 77}, {46, 163}, + {46, 265}, {46, 500}, {46, 501}, {46, 515}, {47, 101}, {47, 143}, {47, + 265}, {49, 1}, {49, 7}, {49, 125}, {49, 202}, {49, 257}, {49, 656}, {49, + 907}, {50, 22}, {53, 11}, {53, 228}, {53, 688}, {54, 62}, {54, 554}, {56, + 0}, {56, 1}, {56, 10}, {56, 31}, {56, 128}, {62, 0}, {62, 1}, {62, 3}, {62, + 4}, {62, 7}, {62, 10}, {62, 15}, {62, 16}, {62, 18}, {62, 25}, {62, 31}, + {62, 32}, {62, 35}, {62, 38}, {62, 46}, {62, 47}, {62, 65}, {62, 66}, {62, + 69}, {62, 96}, {62, 108}, {62, 125}, {62, 126}, {62, 135}, {62, 140}, {62, + 144}, {62, 147}, {62, 159}, {62, 187}, {62, 188}, {62, 250}, {62, 251}, + {62, 253}, {62, 257}, {62, 265}, {62, 288}, {62, 312}, {62, 319}, {62, + 353}, {62, 375}, {62, 376}, {62, 500}, {62, 504}, {62, 508}, {62, 510}, + {62, 525}, {62, 534}, {62, 538}, {62, 569}, {62, 625}, {62, 635}, {62, + 750}, {62, 812}, {62, 850}, {62, 851}, {63, 0}, {63, 1}, {63, 8}, {63, 46}, + {63, 65}, {63, 140}, {63, 171}, {63, 202}, {63, 250}, {63, 281}, {63, 284}, + {63, 503}, {63, 515}, {63, 518}, {63, 538}, {63, 562}, {63, 593}, {63, + 734}, {65, 0}, {65, 3}, {65, 4}, {65, 15}, {65, 16}, {65, 38}, {65, 87}, + {65, 112}, {65, 125}, {65, 126}, {65, 132}, {65, 133}, {65, 140}, {65, + 174}, {65, 266}, {65, 312}, {65, 375}, {65, 437}, {65, 503}, {65, 508}, + {65, 562}, {65, 626}, {66, 0}, {66, 46}, {66, 93}, {66, 257}, {66, 500}, + {66, 772}, {69, 1}, {69, 7}, {69, 11}, {69, 18}, {69, 62}, {69, 132}, {69, + 234}, {69, 257}, {69, 281}, {69, 296}, {69, 504}, {69, 532}, {69, 562}, + {69, 572}, {69, 694}, {69, 750}, {69, 757}, {70, 0}, {70, 15}, {70, 18}, + {70, 63}, {70, 157}, {70, 171}, {70, 250}, {70, 757}, {72, 32}, {72, 190}, + {72, 218}, {72, 291}, {72, 659}, {73, 0}, {73, 187}, {73, 250}, {73, 281}, + {73, 328}, {73, 500}, {77, 1}, {77, 3}, {77, 31}, {77, 38}, {77, 78}, {77, + 129}, {77, 171}, {77, 198}, {77, 250}, {77, 281}, {77, 508}, {77, 510}, + {77, 538}, {77, 563}, {77, 750}, {77, 754}, {77, 765}, {77, 768}, {77, + 937}, {78, 0}, {78, 1}, {78, 31}, {78, 218}, {78, 269}, {78, 376}, {80, 0}, + {80, 8}, {80, 10}, {80, 31}, {80, 69}, {80, 250}, {80, 269}, {80, 285}, + {80, 500}, {80, 547}, {81, 3}, {81, 31}, {81, 515}, {81, 531}, {84, 10}, + {84, 15}, {84, 132}, {84, 190}, {84, 250}, {84, 507}, {84, 532}, {84, 534}, + {84, 878}, {85, 126}, {85, 312}, {87, 140}, {93, 0}, {93, 3}, {93, 7}, {93, + 15}, {93, 18}, {93, 46}, {93, 53}, {93, 63}, {93, 132}, {93, 228}, {93, + 250}, {93, 261}, {93, 379}, {93, 383}, {93, 444}, {93, 507}, {93, 515}, + {93, 534}, {93, 562}, {94, 3}, {94, 34}, {94, 62}, {94, 500}, {94, 515}, + {94, 632}, {96, 38}, {96, 46}, {96, 53}, {96, 128}, {96, 250}, {96, 313}, + {100, 1}, {100, 128}, {101, 7}, {108, 0}, {108, 70}, {108, 254}, {108, + 272}, {108, 515}, {108, 565}, {109, 656}, {112, 500}, {115, 77}, {115, + 250}, {125, 0}, {125, 1}, {125, 3}, {125, 4}, {125, 7}, {125, 8}, {125, + 11}, {125, 15}, {125, 16}, {125, 23}, {125, 31}, {125, 32}, {125, 38}, + {125, 49}, {125, 62}, {125, 126}, {125, 156}, {125, 164}, {125, 187}, {125, + 194}, {125, 198}, {125, 205}, {125, 250}, {125, 253}, {125, 266}, {125, + 281}, {125, 284}, {125, 296}, {125, 347}, {125, 382}, {125, 501}, {125, + 507}, {125, 515}, {125, 523}, {125, 538}, {125, 554}, {125, 569}, {125, + 577}, {125, 593}, {125, 608}, {125, 647}, {125, 710}, {125, 718}, {125, + 750}, {125, 765}, {125, 766}, {126, 0}, {126, 1}, {126, 15}, {126, 25}, + {126, 32}, {126, 63}, {126, 80}, {126, 125}, {126, 128}, {126, 147}, {126, + 268}, {126, 284}, {126, 312}, {126, 437}, {126, 515}, {126, 531}, {128, 3}, + {128, 7}, {128, 15}, {128, 22}, {128, 62}, {128, 63}, {128, 73}, {128, 77}, + {128, 140}, {128, 147}, {128, 209}, {128, 269}, {128, 272}, {128, 281}, + {128, 296}, {128, 331}, {128, 515}, {128, 626}, {128, 632}, {129, 0}, {129, + 250}, {132, 0}, {132, 7}, {132, 62}, {132, 70}, {132, 77}, {132, 125}, + {132, 141}, {132, 160}, {132, 281}, {132, 500}, {132, 510}, {132, 625}, + {132, 626}, {132, 635}, {133, 18}, {133, 25}, {133, 125}, {133, 150}, {133, + 156}, {133, 281}, {133, 343}, {133, 500}, {135, 0}, {135, 1}, {135, 10}, + {135, 250}, {135, 531}, {136, 219}, {136, 375}, {136, 625}, {140, 0}, {140, + 3}, {140, 7}, {140, 31}, {140, 46}, {140, 56}, {140, 87}, {140, 250}, {140, + 257}, {140, 265}, {140, 282}, {140, 312}, {140, 500}, {140, 501}, {140, + 562}, {140, 565}, {140, 609}, {140, 875}, {141, 0}, {141, 140}, {141, 143}, + {141, 254}, {141, 281}, {141, 500}, {141, 522}, {141, 882}, {143, 4}, {143, + 10}, {143, 15}, {143, 62}, {143, 93}, {143, 226}, {143, 350}, {143, 584}, + {143, 633}, {143, 690}, {144, 0}, {144, 34}, {144, 375}, {144, 501}, {147, + 0}, {147, 22}, {147, 257}, {150, 265}, {156, 1}, {156, 8}, {156, 15}, {156, + 23}, {156, 31}, {156, 39}, {156, 125}, {156, 140}, {156, 203}, {156, 289}, + {156, 327}, {156, 376}, {156, 566}, {156, 577}, {156, 578}, {156, 596}, + {156, 907}, {157, 132}, {157, 156}, {157, 163}, {157, 437}, {157, 503}, + {157, 515}, {157, 593}, {157, 702}, {159, 1}, {159, 7}, {159, 32}, {159, + 62}, {159, 65}, {159, 77}, {159, 108}, {159, 132}, {159, 140}, {159, 233}, + {159, 501}, {160, 0}, {160, 72}, {160, 77}, {163, 0}, {163, 1}, {163, 125}, + {163, 126}, {163, 133}, {163, 625}, {163, 626}, {164, 1}, {164, 156}, {164, + 250}, {164, 365}, {166, 0}, {166, 78}, {171, 0}, {171, 1}, {171, 54}, {171, + 65}, {171, 94}, {171, 261}, {171, 500}, {172, 319}, {174, 77}, {175, 3}, + {178, 1}, {178, 11}, {178, 62}, {179, 500}, {187, 0}, {187, 3}, {187, 4}, + {187, 80}, {187, 126}, {187, 250}, {187, 281}, {187, 500}, {187, 518}, + {187, 534}, {187, 632}, {187, 702}, {187, 750}, {187, 785}, {188, 8}, {188, + 125}, {188, 147}, {188, 179}, {188, 253}, {188, 688}, {190, 0}, {190, 1}, + {190, 38}, {190, 766}, {191, 0}, {191, 140}, {194, 0}, {194, 3}, {194, 7}, + {194, 10}, {194, 35}, {194, 47}, {194, 93}, {194, 265}, {194, 410}, {194, + 846}, {195, 500}, {197, 15}, {197, 500}, {198, 0}, {202, 7}, {202, 62}, + {202, 125}, {202, 187}, {202, 453}, {202, 503}, {205, 69}, {206, 0}, {210, + 0}, {210, 789}, {212, 751}, {218, 15}, {218, 46}, {218, 63}, {218, 125}, + {218, 174}, {218, 257}, {218, 261}, {218, 265}, {219, 7}, {221, 0}, {221, + 7}, {221, 281}, {221, 508}, {225, 46}, {225, 516}, {241, 31}, {241, 63}, + {250, 0}, {250, 1}, {250, 3}, {250, 7}, {250, 8}, {250, 16}, {250, 18}, + {250, 19}, {250, 31}, {250, 32}, {250, 49}, {250, 53}, {250, 62}, {250, + 63}, {250, 69}, {250, 70}, {250, 77}, {250, 84}, {250, 108}, {250, 125}, + {250, 128}, {250, 129}, {250, 135}, {250, 140}, {250, 156}, {250, 167}, + {250, 187}, {250, 190}, {250, 209}, {250, 234}, {250, 257}, {250, 281}, + {250, 285}, {250, 313}, {250, 315}, {250, 343}, {250, 376}, {250, 452}, + {250, 500}, {250, 503}, {250, 510}, {250, 515}, {250, 562}, {250, 563}, + {250, 565}, {250, 572}, {250, 577}, {250, 593}, {250, 608}, {250, 629}, + {250, 640}, {250, 753}, {250, 769}, {250, 812}, {250, 827}, {250, 890}, + {250, 921}, {251, 3}, {251, 10}, {251, 34}, {251, 38}, {251, 125}, {251, + 126}, {251, 140}, {251, 171}, {251, 190}, {251, 202}, {251, 222}, {251, + 241}, {251, 284}, {251, 441}, {251, 500}, {251, 515}, {251, 556}, {251, + 728}, {251, 751}, {251, 827}, {253, 0}, {253, 1}, {253, 10}, {253, 22}, + {253, 25}, {253, 31}, {253, 47}, {253, 62}, {253, 125}, {253, 132}, {253, + 163}, {253, 195}, {253, 222}, {253, 315}, {253, 320}, {253, 375}, {253, + 500}, {253, 501}, {253, 507}, {253, 635}, {253, 651}, {253, 812}, {254, 0}, + {254, 135}, {254, 266}, {254, 281}, {254, 959}, {257, 10}, {257, 22}, {257, + 31}, {257, 62}, {257, 63}, {257, 125}, {257, 147}, {257, 157}, {257, 250}, + {257, 288}, {257, 297}, {257, 334}, {257, 353}, {257, 375}, {257, 501}, + {257, 694}, {257, 757}, {258, 65}, {258, 522}, {258, 593}, {258, 625}, + {258, 827}, {260, 0}, {260, 3}, {260, 254}, {260, 312}, {260, 330}, {260, + 516}, {260, 781}, {261, 10}, {261, 251}, {265, 0}, {265, 1}, {265, 3}, + {265, 8}, {265, 10}, {265, 26}, {265, 65}, {265, 77}, {265, 125}, {265, + 188}, {265, 257}, {265, 312}, {265, 409}, {265, 500}, {265, 503}, {265, + 570}, {266, 3}, {266, 156}, {266, 233}, {266, 694}, {268, 3}, {268, 8}, + {268, 16}, {268, 77}, {268, 125}, {268, 140}, {268, 503}, {268, 584}, {269, + 8}, {269, 31}, {272, 3}, {272, 77}, {272, 500}, {272, 632}, {275, 69}, + {275, 534}, {281, 0}, {281, 1}, {281, 8}, {281, 11}, {281, 39}, {281, 46}, + {281, 63}, {281, 70}, {281, 93}, {281, 128}, {281, 251}, {281, 507}, {281, + 516}, {281, 569}, {281, 671}, {281, 687}, {281, 768}, {282, 1}, {282, 23}, + {282, 31}, {282, 96}, {282, 132}, {282, 205}, {282, 288}, {282, 390}, {282, + 503}, {282, 788}, {284, 0}, {284, 31}, {284, 258}, {284, 500}, {288, 0}, + {288, 80}, {288, 140}, {288, 538}, {291, 7}, {291, 31}, {296, 0}, {296, 4}, + {296, 7}, {296, 16}, {296, 194}, {296, 359}, {296, 500}, {296, 501}, {296, + 508}, {296, 629}, {296, 687}, {296, 751}, {296, 812}, {297, 500}, {299, 0}, + {299, 1}, {299, 539}, {303, 22}, {303, 32}, {303, 656}, {306, 42}, {312, + 0}, {312, 7}, {312, 31}, {312, 128}, {312, 156}, {312, 229}, {312, 378}, + {312, 385}, {312, 578}, {312, 751}, {312, 788}, {312, 910}, {313, 0}, {313, + 15}, {313, 100}, {313, 125}, {313, 128}, {315, 49}, {315, 202}, {315, 593}, + {315, 909}, {316, 31}, {316, 38}, {316, 327}, {316, 625}, {319, 0}, {319, + 34}, {319, 125}, {319, 500}, {319, 625}, {319, 718}, {320, 19}, {320, 225}, + {320, 609}, {322, 0}, {322, 10}, {322, 140}, {322, 315}, {322, 625}, {323, + 538}, {327, 1}, {327, 3}, {327, 31}, {327, 69}, {328, 0}, {328, 77}, {328, + 132}, {330, 0}, {334, 156}, {334, 313}, {334, 376}, {334, 508}, {334, 546}, + {334, 648}, {337, 500}, {337, 503}, {338, 3}, {338, 126}, {343, 0}, {343, + 534}, {344, 34}, {344, 226}, {346, 125}, {346, 500}, {350, 562}, {350, + 587}, {353, 15}, {353, 500}, {353, 750}, {354, 500}, {358, 1}, {358, 299}, + {359, 501}, {365, 7}, {375, 0}, {375, 1}, {375, 35}, {375, 190}, {375, + 312}, {376, 15}, {376, 69}, {376, 78}, {376, 93}, {376, 250}, {376, 781}, + {378, 63}, {378, 69}, {378, 382}, {379, 15}, {382, 31}, {382, 531}, {382, + 640}, {385, 35}, {386, 15}, {386, 62}, {386, 159}, {386, 515}, {390, 94}, + {390, 125}, {390, 140}, {390, 260}, {390, 569}, {390, 609}, {390, 753}, + {391, 0}, {391, 18}, {391, 78}, {391, 876}, {393, 251}, {393, 625}, {398, + 897}, {400, 3}, {406, 34}, {406, 65}, {406, 144}, {406, 327}, {406, 515}, + {406, 531}, {406, 785}, {407, 3}, {409, 15}, {409, 46}, {409, 437}, {410, + 3}, {413, 15}, {413, 656}, {414, 49}, {421, 109}, {421, 197}, {422, 100}, + {424, 315}, {437, 15}, {437, 62}, {437, 125}, {437, 500}, {440, 7}, {440, + 500}, {440, 562}, {440, 775}, {448, 0}, {448, 438}, {452, 16}, {452, 31}, + {452, 250}, {452, 562}, {459, 62}, {459, 268}, {459, 796}, {468, 0}, {471, + 7}, {471, 538}, {475, 109}, {476, 63}, {490, 7}, {500, 0}, {500, 1}, {500, + 4}, {500, 7}, {500, 10}, {500, 11}, {500, 15}, {500, 18}, {500, 22}, {500, + 25}, {500, 31}, {500, 34}, {500, 38}, {500, 42}, {500, 46}, {500, 47}, + {500, 50}, {500, 62}, {500, 63}, {500, 69}, {500, 73}, {500, 77}, {500, + 80}, {500, 93}, {500, 94}, {500, 125}, {500, 126}, {500, 144}, {500, 166}, + {500, 171}, {500, 178}, {500, 250}, {500, 257}, {500, 265}, {500, 281}, + {500, 315}, {500, 316}, {500, 376}, {500, 386}, {500, 413}, {500, 437}, + {500, 503}, {500, 507}, {500, 508}, {500, 510}, {500, 516}, {500, 542}, + {500, 674}, {500, 687}, {500, 718}, {500, 750}, {500, 819}, {500, 875}, + {500, 906}, {501, 0}, {501, 7}, {501, 18}, {501, 22}, {501, 31}, {501, 47}, + {501, 125}, {501, 141}, {501, 150}, {501, 171}, {501, 190}, {501, 203}, + {501, 209}, {501, 260}, {501, 288}, {501, 500}, {501, 510}, {501, 534}, + {501, 656}, {501, 875}, {503, 0}, {503, 3}, {503, 16}, {503, 42}, {503, + 46}, {503, 63}, {503, 66}, {503, 147}, {503, 250}, {503, 261}, {503, 322}, + {503, 350}, {503, 378}, {503, 468}, {503, 500}, {503, 625}, {504, 1}, {504, + 3}, {504, 22}, {504, 77}, {504, 115}, {504, 500}, {504, 600}, {504, 789}, + {504, 796}, {507, 0}, {507, 3}, {507, 15}, {507, 18}, {507, 35}, {507, 38}, + {507, 125}, {507, 250}, {507, 500}, {507, 501}, {507, 547}, {508, 0}, {508, + 32}, {508, 132}, {508, 140}, {508, 382}, {508, 501}, {510, 66}, {510, 210}, + {510, 303}, {511, 268}, {511, 299}, {511, 500}, {515, 0}, {515, 1}, {515, + 3}, {515, 53}, {515, 69}, {515, 93}, {515, 128}, {515, 132}, {515, 156}, + {515, 167}, {515, 171}, {515, 188}, {515, 206}, {515, 250}, {515, 257}, + {515, 281}, {515, 375}, {515, 410}, {515, 437}, {515, 516}, {515, 751}, + {516, 0}, {516, 15}, {516, 35}, {516, 77}, {516, 103}, {516, 156}, {516, + 296}, {516, 300}, {516, 315}, {516, 459}, {516, 656}, {516, 834}, {516, + 865}, {518, 0}, {518, 7}, {518, 15}, {518, 65}, {518, 194}, {518, 250}, + {518, 265}, {518, 268}, {518, 577}, {518, 625}, {518, 812}, {519, 62}, + {522, 0}, {522, 31}, {522, 65}, {522, 80}, {522, 315}, {522, 376}, {522, + 531}, {522, 781}, {522, 897}, {523, 150}, {523, 282}, {525, 22}, {525, 31}, + {525, 251}, {526, 49}, {531, 0}, {531, 1}, {531, 7}, {531, 10}, {531, 49}, + {531, 93}, {531, 126}, {531, 132}, {531, 156}, {531, 202}, {531, 218}, + {531, 258}, {531, 315}, {531, 375}, {531, 503}, {531, 593}, {531, 765}, + {531, 772}, {532, 0}, {532, 3}, {532, 15}, {532, 31}, {532, 32}, {532, 62}, + {532, 73}, {532, 132}, {532, 562}, {534, 15}, {534, 300}, {535, 0}, {535, + 163}, {538, 0}, {538, 7}, {538, 15}, {538, 629}, {539, 62}, {539, 468}, + {541, 891}, {546, 1}, {546, 125}, {546, 281}, {546, 288}, {546, 319}, {546, + 327}, {546, 500}, {546, 532}, {546, 539}, {546, 640}, {547, 1}, {547, 250}, + {547, 784}, {549, 128}, {550, 0}, {550, 221}, {556, 15}, {562, 0}, {562, + 3}, {562, 4}, {562, 8}, {562, 31}, {562, 34}, {562, 46}, {562, 69}, {562, + 93}, {562, 125}, {562, 143}, {562, 319}, {562, 500}, {562, 501}, {562, + 504}, {562, 508}, {562, 510}, {563, 0}, {563, 39}, {563, 73}, {563, 257}, + {563, 376}, {563, 500}, {565, 0}, {565, 15}, {565, 296}, {565, 390}, {566, + 0}, {566, 815}, {569, 3}, {569, 125}, {569, 126}, {569, 175}, {569, 190}, + {569, 313}, {569, 538}, {570, 65}, {572, 65}, {572, 251}, {573, 500}, {577, + 0}, {577, 1}, {577, 15}, {577, 18}, {577, 159}, {577, 501}, {577, 518}, + {577, 584}, {578, 0}, {578, 385}, {580, 500}, {584, 625}, {593, 0}, {593, + 1}, {593, 3}, {593, 31}, {593, 41}, {593, 53}, {593, 281}, {593, 633}, + {593, 878}, {593, 882}, {594, 32}, {594, 93}, {594, 784}, {596, 66}, {600, + 0}, {600, 253}, {601, 577}, {603, 62}, {608, 0}, {611, 18}, {616, 375}, + {625, 0}, {625, 1}, {625, 3}, {625, 7}, {625, 34}, {625, 108}, {625, 126}, + {625, 129}, {625, 190}, {625, 254}, {625, 275}, {625, 284}, {625, 344}, + {625, 346}, {625, 390}, {625, 425}, {625, 532}, {625, 538}, {625, 750}, + {625, 968}, {626, 31}, {626, 32}, {626, 34}, {626, 63}, {626, 100}, {626, + 111}, {626, 500}, {626, 508}, {626, 643}, {628, 0}, {628, 46}, {629, 62}, + {629, 625}, {632, 8}, {632, 81}, {632, 93}, {632, 156}, {632, 202}, {632, + 547}, {633, 15}, {633, 268}, {633, 516}, {640, 0}, {640, 23}, {640, 39}, + {640, 251}, {640, 319}, {640, 358}, {640, 522}, {641, 8}, {641, 343}, {641, + 751}, {643, 265}, {644, 18}, {644, 41}, {648, 19}, {656, 3}, {656, 7}, + {656, 62}, {656, 382}, {656, 507}, {656, 757}, {657, 0}, {657, 81}, {657, + 894}, {659, 65}, {659, 476}, {659, 750}, {663, 125}, {671, 78}, {671, 253}, + {672, 375}, {687, 0}, {687, 15}, {687, 18}, {687, 31}, {687, 32}, {687, + 108}, {687, 125}, {687, 296}, {687, 500}, {688, 0}, {688, 38}, {690, 265}, + {690, 757}, {694, 0}, {694, 65}, {694, 250}, {695, 0}, {697, 93}, {702, 3}, + {702, 156}, {702, 282}, {703, 281}, {705, 46}, {705, 250}, {706, 94}, {709, + 628}, {718, 7}, {718, 31}, {718, 62}, {725, 0}, {725, 18}, {733, 376}, + {736, 784}, {740, 77}, {740, 126}, {750, 0}, {750, 16}, {750, 22}, {750, + 31}, {750, 32}, {750, 62}, {750, 63}, {750, 115}, {750, 128}, {750, 159}, + {750, 250}, {750, 315}, {750, 343}, {750, 344}, {750, 424}, {750, 562}, + {750, 569}, {750, 626}, {751, 46}, {751, 93}, {751, 300}, {751, 522}, {753, + 0}, {753, 80}, {753, 171}, {753, 250}, {753, 406}, {753, 562}, {757, 46}, + {757, 47}, {757, 209}, {757, 265}, {757, 312}, {758, 266}, {760, 0}, {760, + 8}, {761, 62}, {761, 600}, {765, 62}, {765, 148}, {765, 516}, {765, 534}, + {765, 823}, {766, 141}, {766, 500}, {768, 0}, {768, 141}, {768, 251}, {768, + 781}, {772, 375}, {772, 500}, {773, 0}, {781, 62}, {781, 63}, {781, 147}, + {781, 157}, {781, 187}, {782, 3}, {782, 15}, {782, 34}, {782, 125}, {782, + 253}, {784, 516}, {784, 569}, {788, 111}, {788, 116}, {789, 7}, {789, 140}, + {789, 376}, {796, 906}, {797, 1}, {812, 18}, {812, 126}, {812, 750}, {812, + 796}, {813, 0}, {813, 7}, {813, 500}, {815, 49}, {815, 96}, {819, 565}, + {822, 0}, {827, 4}, {827, 125}, {827, 383}, {830, 31}, {843, 4}, {843, + 281}, {846, 0}, {846, 69}, {859, 260}, {875, 35}, {875, 80}, {875, 148}, + {875, 206}, {875, 250}, {875, 312}, {875, 500}, {875, 503}, {875, 702}, + {875, 782}, {875, 878}, {876, 0}, {878, 31}, {878, 78}, {879, 515}, {890, + 163}, {890, 816}, {891, 553}, {897, 46}, {897, 265}, {906, 0}, {906, 65}, + {906, 518}, {913, 69}, {913, 197}, {937, 413}, {937, 531}, {937, 565}, + {940, 16}, {941, 250}, {944, 53}, {944, 346}, {947, 1}, {952, 375}, {59, 371}, + {371, 431}, {431, 564}, {564, 779}, {779, 835}, {835, 877}, {877, 986} + }; + + DASH_LOG_DEBUG("ConnectedComponentsTest.AlgorithmRun", + "construction started"); + graph_t g(edge_list.begin(), edge_list.end(), 1000); + DASH_LOG_DEBUG("ConnectedComponentsTest.AlgorithmRun", + "construction finished"); + + dash::barrier(); + + DASH_LOG_DEBUG("ConnectedComponentsTest.AlgorithmRun", "algorithm started"); + dash::connected_components(g); + DASH_LOG_DEBUG("ConnectedComponentsTest.AlgorithmRun", "algorithm finished"); + + std::vector results = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 774, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 774, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0 + }; + + int wrong_components = 0; + if(dash::myid() == 0) { + int index = 0; + for(auto it = g.vertices().begin(); it != g.vertices().end(); ++it) { + auto v = g[it]; + if(v.attributes().comp != results[index]) { + ++wrong_components; + break; + } + ++index; + } + + // each component set according to results array + EXPECT_EQ_U(0, wrong_components); + } + + dash::barrier(); +} + diff --git a/dash/test/algorithm/ConnectedComponentsTest.h b/dash/test/algorithm/ConnectedComponentsTest.h new file mode 100644 index 000000000..426160e4b --- /dev/null +++ b/dash/test/algorithm/ConnectedComponentsTest.h @@ -0,0 +1,21 @@ +#ifndef DASH__TEST__CONNECTED_COMPONENTS_TEST_H_ +#define DASH__TEST__CONNECTED_COMPONENTS_TEST_H_ + +#include "../TestBase.h" + +/** + * Test fixture for class dash::List + */ +class ConnectedComponentsTest : public dash::test::TestBase { +protected: + + ConnectedComponentsTest() { + LOG_MESSAGE(">>> Test suite: ConnectedComponentsTest"); + } + + virtual ~ConnectedComponentsTest() { + LOG_MESSAGE("<<< Closing test suite: ConnectedComponentsTest"); + } +}; + +#endif // DASH__TEST__CONNECTED_COMPONENTS_TEST_H_ From ba8fbe2aabe0b43e1c4101a083178d2bf546a647 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 8 Feb 2018 18:21:11 +0100 Subject: [PATCH 073/102] fixed set_attributes for edges --- dash/include/dash/graph/internal/Graph.h | 1 + 1 file changed, 1 insertion(+) diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index fd5352a4a..0bac4bc52 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -569,6 +569,7 @@ class EdgeProxy { * Sets the attribute data for the referenced edge. */ void set_attributes(properties_type & prop) { + lazy_load(); _edge.properties = prop; auto ref = *_iterator; ref = _edge; From 3d0a8b8de36f5369e418e35357c7780bd96a5945 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 8 Feb 2018 18:21:56 +0100 Subject: [PATCH 074/102] added graph constructor for edge lists with edge attributes --- dash/include/dash/Graph.h | 44 ++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 5b138b262..fcdbe124e 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -195,9 +195,10 @@ class Graph { std::unordered_set remote_vertices_set; std::vector> remote_vertices(_team->size()); for(auto it = begin; it != end; ++it) { - auto v = it->first; + auto item = *it; + auto v = edgelist_source(item); if(vertex_owner(v, n_vertices) == _myid) { - auto u = it->second; + auto u = edgelist_target(item); if(lvertices.find(v) == lvertices.end()) { // add dummy first, more vertices are added later and they have @@ -304,17 +305,18 @@ class Graph { // finally add edges with the vertex iterators gained from the previous // steps for(auto it = begin; it != end; ++it) { - auto v = it->first; + auto item = *it; + auto v = edgelist_source(item); if(vertex_owner(v, n_vertices) == _myid) { - auto v_it = lvertices[it->first]; - auto u = it->second; + auto v_it = lvertices[v]; + auto u = edgelist_target(item); if(vertex_owner(u, n_vertices) == _myid) { auto u_it = lvertices[u]; - add_edge(v_it, u_it); + edgelist_add_edge(v_it, u_it, item); } else { auto u_it = gvertices[u]; - add_edge(v_it, u_it); + edgelist_add_edge(v_it, u_it, item); } } } @@ -322,6 +324,18 @@ class Graph { commit(); } + template + void edgelist_add_edge(SourceIterator s, TargetIterator t, + std::pair) { + add_edge(s, t); + } + + template + void edgelist_add_edge(SourceIterator s, TargetIterator t, + std::pair, int>) { + add_edge(s, t); + } + /** Destructs the graph. */ ~Graph() { @@ -776,6 +790,22 @@ class Graph { return owner; } + vertex_size_type edgelist_source(std::pair e) { + return e.first; + } + + vertex_size_type edgelist_source(std::pair, int> e) { + return e.first.first; + } + + vertex_size_type edgelist_target(std::pair e) { + return e.second; + } + + vertex_size_type edgelist_target(std::pair, int> e) { + return e.first.second; + } + private: /** the team containing all units using the container */ From cb1b62218c15db24023073c740add798c61a94e2 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 12 Feb 2018 11:37:41 +0100 Subject: [PATCH 075/102] updated mst algorithm to put mark minimum edges directly in the graph data structure --- dash/include/dash/Graph.h | 28 ++++++++------- .../algorithm/graph/MinimumSpanningTree.h | 35 +++++++++---------- dash/include/dash/graph/internal/Graph.h | 4 +-- 3 files changed, 34 insertions(+), 33 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index fcdbe124e..0f8b71df2 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -324,18 +324,6 @@ class Graph { commit(); } - template - void edgelist_add_edge(SourceIterator s, TargetIterator t, - std::pair) { - add_edge(s, t); - } - - template - void edgelist_add_edge(SourceIterator s, TargetIterator t, - std::pair, int>) { - add_edge(s, t); - } - /** Destructs the graph. */ ~Graph() { @@ -790,6 +778,8 @@ class Graph { return owner; } + // TODO: Generalize following methods for other edge attributes than { int } + vertex_size_type edgelist_source(std::pair e) { return e.first; } @@ -806,6 +796,20 @@ class Graph { return e.first.second; } + template + void edgelist_add_edge(SourceIterator s, TargetIterator t, + std::pair) { + add_edge(s, t); + } + + template + void edgelist_add_edge(SourceIterator s, TargetIterator t, + std::pair, int> item) { + edge_properties_type prop { item.second }; + add_edge(s, t, prop); + } + + private: /** the team containing all units using the container */ diff --git a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h index d77e8ef8d..92ff59bda 100644 --- a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h +++ b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h @@ -8,8 +8,9 @@ namespace dash { typedef std::vector> matrix_t; typedef std::vector>> matrix_pair_t; -// (supervertex, new component, vertex src, vertex trg, weight) -typedef std::tuple tuple_t; +// (supervertex, new component, vertex src, vertex trg, weight, owner, +// local offset) +typedef std::tuple tuple_t; typedef std::vector> matrix_min_pairs_t; namespace internal { @@ -122,6 +123,12 @@ void mst_set_data_min(matrix_min_pairs_t & data_pairs, GraphType & graph) { typename GraphType::vertex_properties_type prop { std::get<1>(pair.second) }; graph[graph.vertices().begin() + pair.first].set_attributes(prop); // save edges here (src = get<2>(pair.second), trg = get<3>(pair.second) + if(dash::myid() == get<5>(pair.second)) { + auto it = graph.out_edges().lbegin() + get<6>(pair.second); + typename GraphType::edge_properties_type eprop { -1 }; + graph[it].set_attributes(eprop); + auto it2 = graph.out_edges().lbegin() + get<6>(pair.second); + } } } @@ -151,20 +158,6 @@ void minimum_spanning_tree(GraphType & g) { ++i; } - /* - // set random edge weight - std::uniform_int_distribution dist(0, 10); - std::minstd_rand rng((std::random_device())()); - i = 0; - for(auto it = g.out_edges().begin(); it != g.out_edges().end(); ++it) { - if(it.is_local()) { - typename GraphType::edge_properties_type prop { dist(rng) }; - g[it].set_attributes(prop); - } - ++i; - } - */ - dash::barrier(); while(1) { @@ -198,15 +191,19 @@ void minimum_spanning_tree(GraphType & g) { int min_weight = std::numeric_limits::max(); int trg_comp_min = -1; int trg_min = -1; + int ledgepos = 0; + int unit = 0; for(auto e_it = v.out_edges().lbegin(); e_it != v.out_edges().lend(); ++e_it) { auto e = g[e_it]; auto e_weight = e.attributes().weight; auto trg_comp = data[i]; - if(src_comp != trg_comp && min_weight > e_weight) { + if(src_comp != trg_comp && e_weight >= 0 && min_weight > e_weight) { min_weight = e_weight; trg_comp_min = trg_comp; trg_min = e.target().pos(); + ledgepos = e_it.pos(); + unit = dash::myid(); } ++i; } @@ -216,10 +213,10 @@ void minimum_spanning_tree(GraphType & g) { auto trg_comp_it = g.vertices().begin() + trg_comp_min; data_pairs[src_comp_it.lpos().unit].push_back( std::make_tuple(src_comp_it.pos(), trg_comp_min, src, trg_min, - min_weight)); + min_weight, unit, ledgepos)); data_pairs[trg_comp_it.lpos().unit].push_back( std::make_tuple(trg_comp_it.pos(), src_comp, src, trg_min, - min_weight)); + min_weight, unit, ledgepos)); gr = 1; } } diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index 0bac4bc52..1cce367d5 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -571,8 +571,8 @@ class EdgeProxy { void set_attributes(properties_type & prop) { lazy_load(); _edge.properties = prop; - auto ref = *_iterator; - ref = _edge; + auto ptr = _iterator; + *ptr = _edge; } private: From 0c409bd378d89113233c2af78d3df652070872c3 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 12 Feb 2018 13:04:59 +0100 Subject: [PATCH 076/102] added minimum spanning tree test --- .../test/algorithm/MinimumSpanningTreeTest.cc | 79 +++++++++++++++++++ dash/test/algorithm/MinimumSpanningTreeTest.h | 21 +++++ 2 files changed, 100 insertions(+) create mode 100644 dash/test/algorithm/MinimumSpanningTreeTest.cc create mode 100644 dash/test/algorithm/MinimumSpanningTreeTest.h diff --git a/dash/test/algorithm/MinimumSpanningTreeTest.cc b/dash/test/algorithm/MinimumSpanningTreeTest.cc new file mode 100644 index 000000000..81056da1d --- /dev/null +++ b/dash/test/algorithm/MinimumSpanningTreeTest.cc @@ -0,0 +1,79 @@ +#include "MinimumSpanningTreeTest.h" +#include +#include + +struct vprop { + int comp; +}; + +struct eprop { + int weight; +}; + +typedef dash::Graph graph_t; + +TEST_F(MinimumSpanningTreeTest, AlgorithmRun) +{ + std::vector, int>> edge_list = { + {{12, 11}, 8}, {{10, 0}, 1}, {{2, 0}, 8}, {{15, 18}, 4}, {{11, 13}, 0}, + {{8, 18}, 3}, {{1, 9}, 10}, {{11, 1}, 9}, {{11, 13}, 9}, {{0, 19}, 7}, + {{19, 17}, 6}, {{2, 5}, 3}, {{18, 16}, 6}, {{10, 9}, 3}, {{16, 11}, 10}, + {{17, 1}, 1}, {{16, 13}, 4}, {{7, 7}, 1}, {{15, 19}, 0}, {{13, 14}, 6}, + {{10, 8}, 4}, {{10, 1}, 3}, {{7, 9}, 7}, {{8, 13}, 7}, {{14, 8}, 7}, + {{16, 11}, 4}, {{0, 3}, 10}, {{13, 10}, 7}, {{17, 7}, 7}, {{15, 10}, 8}, + {{0, 2}, 6}, {{12, 7}, 9}, {{5, 6}, 9}, {{3, 4}, 9}, {{14, 0}, 9}, + {{17, 14}, 6}, {{4, 4}, 5}, {{1, 13}, 2}, {{11, 15}, 6}, {{9, 2}, 2}, + {{0, 1}, 5}, {{0, 2}, 7}, {{0, 3}, 3}, {{0, 5}, 7}, {{0, 6}, 9}, {{0, 7}, + 6}, {{0, 10}, 9}, {{0, 13}, 7}, {{0, 15}, 4}, {{1, 0}, 3}, {{1, 2}, 7}, + {{1, 10}, 2}, {{2, 1}, 4}, {{2, 10}, 2}, {{2, 16}, 9}, {{3, 0}, 4}, {{3, + 2}, 0}, {{5, 0}, 1}, {{5, 2}, 3}, {{5, 7}, 5}, {{5, 10}, 4}, {{5, 12}, 6}, + {{6, 2}, 5}, {{6, 12}, 2}, {{8, 0}, 2}, {{10, 0}, 0}, {{10, 1}, 8}, {{10, + 3}, 10}, {{11, 0}, 9}, {{11, 2}, 8}, {{11, 7}, 10}, {{11, 10}, 10}, {{12, + 0}, 8}, {{12, 1}, 4}, {{13, 1}, 10}, {{13, 3}, 9}, {{15, 0}, 0}, {{15, 1}, + 10}, {{15, 5}, 7}, {{15, 10}, 8} + }; + + DASH_LOG_DEBUG("MinimumSpanningTreeTest.AlgorithmRun", + "construction started"); + graph_t g(edge_list.begin(), edge_list.end(), 20); + DASH_LOG_DEBUG("MinimumSpanningTreeTest.AlgorithmRun", + "construction finished"); + + DASH_LOG_DEBUG("MinimumSpanningTreeTest.AlgorithmRun", "algorithm started"); + dash::minimum_spanning_tree(g); + DASH_LOG_DEBUG("MinimumSpanningTreeTest.AlgorithmRun", "algorithm finished"); + + std::vector> results = { + {0, 10}, {0, 15}, {1, 17}, {1, 13}, {1, 10}, {2, 10}, {2, 3}, {4, 3}, + {5, 0}, {6, 12}, {7, 5}, {8, 0}, {9, 2}, {11,13}, {12, 1}, {14, 13}, + {15, 19}, {16, 13}, {18, 8} + }; + + int wrong_edges = 0; + if(dash::myid() == 0) { + int i = 0; + for(auto it = g.out_edges().begin(); it != g.out_edges().end(); ++it) { + auto e = g[it]; + if(e.attributes().weight == -1) { + bool found = false; + for(auto & result : results ) { + // undirected graph: check edges in both directions + if((result.first == e.source().pos() + && result.second == e.target().pos()) + || (result.first == e.target().pos() + && result.second == e.source().pos())) { + found = true; + } + } + if(!found) { + ++wrong_edges; + } + } + ++i; + } + EXPECT_EQ_U(0, wrong_edges); + } + + dash::barrier(); +} + diff --git a/dash/test/algorithm/MinimumSpanningTreeTest.h b/dash/test/algorithm/MinimumSpanningTreeTest.h new file mode 100644 index 000000000..cc4825155 --- /dev/null +++ b/dash/test/algorithm/MinimumSpanningTreeTest.h @@ -0,0 +1,21 @@ +#ifndef DASH__TEST__MINIMUM_SPANNING_TREE_TEST_H_ +#define DASH__TEST__MINIMUM_SPANNING_TREE_TEST_H_ + +#include "../TestBase.h" + +/** + * Test fixture for class dash::List + */ +class MinimumSpanningTreeTest : public dash::test::TestBase { +protected: + + MinimumSpanningTreeTest() { + LOG_MESSAGE(">>> Test suite: MinimumSpanningTreeTest"); + } + + virtual ~MinimumSpanningTreeTest() { + LOG_MESSAGE("<<< Closing test suite: MinimumSpanningTreeTest"); + } +}; + +#endif // DASH__TEST__MINIMUM_SPANNING_TREE_TEST_H_ From fa6a97f1666f88d729af836ca3378190d928c616 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Wed, 14 Feb 2018 15:55:04 +0100 Subject: [PATCH 077/102] reduced algorithm complexity --- .../algorithm/graph/ConnectedComponents.h | 56 +++++++++++++++---- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/dash/include/dash/algorithm/graph/ConnectedComponents.h b/dash/include/dash/algorithm/graph/ConnectedComponents.h index 1ddbfd66a..c08ff846e 100644 --- a/dash/include/dash/algorithm/graph/ConnectedComponents.h +++ b/dash/include/dash/algorithm/graph/ConnectedComponents.h @@ -2,6 +2,7 @@ #define DASH__ALGORITHM____GRAPH__CONNECTED_COMPONENTS_H__ #include +#include namespace dash { @@ -14,8 +15,10 @@ template std::vector cc_get_data( matrix_t & indices, matrix_t & permutations, - GraphType & graph) + GraphType & graph, + dash::util::Trace & trace) { + trace.enter_state("send indices"); // exchange indices to get std::vector sizes_send(indices.size()); std::vector displs_send(indices.size()); @@ -47,17 +50,23 @@ std::vector cc_get_data( displs_recv.data(), graph.team().dart_id()); } + trace.exit_state("send indices"); // exchange data + trace.enter_state("get components"); for(auto & index : indices_recv) { // TODO: optimize cache performance - index = graph[graph.vertices().begin() + index].attributes().comp; + index = graph[graph.vertices().lbegin() + index].attributes().comp; } + trace.exit_state("get components"); + trace.enter_state("send components"); if(total_send > 0 || total_recv > 0) { dart_alltoallv(indices_recv.data(), sizes_recv.data(), displs_recv.data(), DART_TYPE_INT, indices_send.data(), sizes_send.data(), displs_send.data(), graph.team().dart_id()); } + trace.exit_state("send components"); + trace.enter_state("restore order"); // restore original order // TODO: use more sophisticated ordering mechanism std::vector output(indices_send.size()); @@ -68,11 +77,14 @@ std::vector cc_get_data( ++index; } } + trace.exit_state("restore order"); return output; } template -void cc_set_data(matrix_pair_t & data_pairs, GraphType & graph) { +void cc_set_data(matrix_pair_t & data_pairs, GraphType & graph, + dash::util::Trace & trace) { + trace.enter_state("send pairs"); std::vector sizes_send(data_pairs.size()); std::vector displs_send(data_pairs.size()); std::vector> pairs_send; @@ -102,10 +114,13 @@ void cc_set_data(matrix_pair_t & data_pairs, GraphType & graph) { DART_TYPE_BYTE, pairs_recv.data(), sizes_recv.data(), displs_recv.data(), graph.team().dart_id()); } + trace.exit_state("send pairs"); + trace.enter_state("set components"); for(auto & pair : pairs_recv) { typename GraphType::vertex_properties_type prop { pair.second }; - graph[graph.vertices().begin() + pair.first].set_attributes(prop); + graph[graph.vertices().lbegin() + pair.first].set_attributes(prop); } + trace.exit_state("set components"); } } // namespace internal @@ -123,8 +138,10 @@ void cc_set_data(matrix_pair_t & data_pairs, GraphType & graph) { */ template void connected_components(GraphType & g) { + dash::util::Trace trace("ConnectedComponents"); // set component to global index in iteration space // TODO: find faster method for this + trace.enter_state("vertex setup"); int i = 0; for(auto it = g.vertices().begin(); it != g.vertices().end(); ++it) { if(it.is_local()) { @@ -133,13 +150,17 @@ void connected_components(GraphType & g) { } ++i; } + trace.exit_state("vertex setup"); + trace.enter_state("barrier"); dash::barrier(); + trace.exit_state("barrier"); while(1) { int gr = 0; std::vector data; { + trace.enter_state("compute indices"); matrix_t indices(g.team().size()); matrix_t permutations(g.team().size()); int i = 0; @@ -147,14 +168,16 @@ void connected_components(GraphType & g) { auto trg = g[it].target(); auto lpos = trg.lpos(); // TODO: use more sophsticated sorting mechanism - indices[lpos.unit].push_back(trg.pos()); + indices[lpos.unit].push_back(lpos.index); permutations[lpos.unit].push_back(i); ++i; } - data = internal::cc_get_data(indices, permutations, g); + trace.exit_state("compute indices"); + data = internal::cc_get_data(indices, permutations, g, trace); } { + trace.enter_state("compute pairs"); matrix_pair_t data_pairs(g.team().size()); int i = 0; for(auto it = g.out_edges().lbegin(); it != g.out_edges().lend(); ++it) { @@ -164,22 +187,27 @@ void connected_components(GraphType & g) { auto trg_comp = data[i]; if(src_comp < trg_comp) { auto trg_comp_it = g.vertices().begin() + trg_comp; - data_pairs[trg_comp_it.lpos().unit].push_back( - std::make_pair(trg_comp_it.pos(), src_comp) + auto lpos = trg_comp_it.lpos(); + data_pairs[lpos.unit].push_back( + std::make_pair(lpos.index, src_comp) ); gr = 1; } ++i; } - internal::cc_set_data(data_pairs, g); + trace.exit_state("compute pairs"); + internal::cc_set_data(data_pairs, g, trace); } int gr_all = 0; + trace.enter_state("allreduce data"); dart_allreduce(&gr, &gr_all, 1, DART_TYPE_INT, DART_OP_SUM, g.team().dart_id()); + trace.exit_state("allreduce data"); if(gr_all == 0) break; while(1) { int pj = 0; { + trace.enter_state("compute indices (pj)"); matrix_t indices(g.team().size()); matrix_t permutations(g.team().size()); int i = 0; @@ -188,13 +216,15 @@ void connected_components(GraphType & g) { auto comp = v.attributes().comp; auto next_it = g.vertices().begin() + comp; auto lpos = next_it.lpos(); - indices[lpos.unit].push_back(next_it.pos()); + indices[lpos.unit].push_back(lpos.index); permutations[lpos.unit].push_back(i); ++i; } - data = internal::cc_get_data(indices, permutations, g); + trace.exit_state("compute indices (pj)"); + data = internal::cc_get_data(indices, permutations, g, trace); } + trace.enter_state("set data (pj)"); int i = 0; for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { auto v = g[it]; @@ -207,12 +237,14 @@ void connected_components(GraphType & g) { } ++i; } + trace.exit_state("set data (pj)"); int pj_all = 0; + trace.enter_state("allreduce pointerjumping"); dart_allreduce(&pj, &pj_all, 1, DART_TYPE_INT, DART_OP_SUM, g.team().dart_id()); + trace.exit_state("allreduce pointerjumping"); if(pj_all == 0) break; } - dash::barrier(); } dash::barrier(); } From eda755481dbc3574cf550a9d5272a50185ea94b6 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Wed, 14 Feb 2018 15:56:10 +0100 Subject: [PATCH 078/102] added element accumulation map for whole units --- .../dash/memory/GlobHeapContiguousMem.h | 15 +++++++++++- .../dash/memory/GlobHeapContiguousPtr.h | 23 ++++++++++++++----- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h index b867d26b0..ac85972f1 100644 --- a/dash/include/dash/memory/GlobHeapContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -69,6 +69,7 @@ class GlobHeapContiguousMem { typedef typename container_list_type::difference_type bucket_index_type; typedef local_iterator local_pointer; typedef local_iterator const_local_pointer; + typedef std::vector unit_cumul_sizes_map; typedef std::vector> bucket_cumul_sizes_map; template @@ -88,7 +89,8 @@ class GlobHeapContiguousMem { _teamid(team.dart_id()), _nunits(team.size()), _myid(team.myid()), - _bucket_cumul_sizes(team.size()) + _bucket_cumul_sizes(team.size()), + _unit_cumul_sizes(team.size()) { } /** @@ -263,6 +265,15 @@ class GlobHeapContiguousMem { _size += *(end - 1); } + for(int i = 0; i < _team->size(); ++i) { + if(i == 0) { + _unit_cumul_sizes[i] = 0; + } else { + _unit_cumul_sizes[i] = _unit_cumul_sizes[i - 1] + + _bucket_cumul_sizes[i - 1].back(); + } + } + // update local iterators update_lbegin(); update_lend(); @@ -460,6 +471,8 @@ class GlobHeapContiguousMem { local_iterator _lend; /** Accumulated sizes of the buckets of each unit. See GlobHeapMem */ bucket_cumul_sizes_map _bucket_cumul_sizes; + /** Accumulated sizes of elements per unit */ + unit_cumul_sizes_map _unit_cumul_sizes; /** Global size of the memory space */ size_type _size = 0; /** Local size of the memory space */ diff --git a/dash/include/dash/memory/GlobHeapContiguousPtr.h b/dash/include/dash/memory/GlobHeapContiguousPtr.h index 70fb49d3c..09207bfa6 100644 --- a/dash/include/dash/memory/GlobHeapContiguousPtr.h +++ b/dash/include/dash/memory/GlobHeapContiguousPtr.h @@ -83,14 +83,16 @@ class GlobPtr< */ private: - typedef std::vector > - bucket_cumul_sizes_map; + typedef std::vector> bucket_cumul_sizes_map; + typedef std::vector unit_cumul_sizes_map; private: /// Global memory used to dereference iterated values. const globmem_type * _globmem = nullptr; /// Mapping unit id to buckets in the unit's attached local storage. const bucket_cumul_sizes_map * _bucket_cumul_sizes = nullptr; + /// Mapping unit id element count + const unit_cumul_sizes_map * _unit_cumul_sizes = nullptr; /// Pointer to first element in local data space. local_pointer _lbegin; /// Current position of the pointer in global canonical index space. @@ -115,6 +117,7 @@ class GlobPtr< GlobPtr() : _globmem(nullptr), _bucket_cumul_sizes(nullptr), + _unit_cumul_sizes(nullptr), _idx(0), _max_idx(0), _myid(dash::Team::GlobalUnitID()), @@ -137,6 +140,7 @@ class GlobPtr< index_type position = 0) : _globmem(reinterpret_cast(gmem)), _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), + _unit_cumul_sizes(&_globmem->_unit_cumul_sizes), _lbegin(_globmem->lbegin()), _idx(position), _max_idx(gmem->size() - 1), @@ -191,6 +195,7 @@ class GlobPtr< index_type local_index) : _globmem(reinterpret_cast(gmem)), _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), + _unit_cumul_sizes(&_globmem->_unit_cumul_sizes), _lbegin(_globmem->lbegin()), _idx(0), _max_idx(gmem->size() - 1), @@ -203,15 +208,15 @@ class GlobPtr< DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx)", "unit:", unit, "lidx:", local_index); - if(unit > _bucket_cumul_sizes->size()) { - std::cout << std::endl; - } DASH_ASSERT_LT(unit, _bucket_cumul_sizes->size(), "invalid unit id"); + /* for (size_type unit = 0; unit < _idx_unit_id; ++unit) { auto prec_unit_local_size = (*_bucket_cumul_sizes)[unit].back(); _idx += prec_unit_local_size; } + */ + _idx += (*_unit_cumul_sizes)[unit]; increment(local_index); DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx) >", "gidx:", _idx, @@ -234,6 +239,7 @@ class GlobPtr< index_type local_index) : _globmem(reinterpret_cast(gmem)), _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), + _unit_cumul_sizes(&_globmem->_unit_cumul_sizes), _lbegin(_globmem->lbegin()), _idx(0), _max_idx(gmem->size() - 1), @@ -248,10 +254,13 @@ class GlobPtr< "lidx:", local_index); DASH_ASSERT_LT(unit, _bucket_cumul_sizes->size(), "invalid unit id"); + /* for (size_type unit = 0; unit < _idx_unit_id; ++unit) { auto prec_unit_local_size = (*_bucket_cumul_sizes)[unit].back(); _idx += prec_unit_local_size; } + */ + _idx += (*_unit_cumul_sizes)[unit]; if(bucket_index > 0) { local_index += (*_bucket_cumul_sizes)[_idx_unit_id.id][bucket_index - 1]; } @@ -280,6 +289,7 @@ class GlobPtr< const GlobPtr & other) : _globmem(other._globmem), _bucket_cumul_sizes(other._bucket_cumul_sizes), + _unit_cumul_sizes(&_globmem->_unit_cumul_sizes), _lbegin(other._lbegin), _idx(other._idx), _max_idx(other._max_idx), @@ -298,6 +308,7 @@ class GlobPtr< { _globmem = other._globmem; _bucket_cumul_sizes = other._bucket_cumul_sizes; + _unit_cumul_sizes = other._unit_cumul_sizes; _lbegin = other._lbegin; _idx = other._idx; _max_idx = other._max_idx; @@ -574,7 +585,7 @@ class GlobPtr< // iterate units: auto unit_id_max = _bucket_cumul_sizes->size() - 1; for (; _idx_unit_id <= unit_id_max; ++_idx_unit_id) { - if (offset == 0) { + if(offset == 0) { break; } auto & unit_bkt_sizes = (*_bucket_cumul_sizes)[_idx_unit_id]; From 690af654bcf50d741ed9f49a3a3cef7e23365471 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Sun, 18 Feb 2018 15:59:50 +0100 Subject: [PATCH 079/102] added further algorithm optimizations --- dash/include/dash/Graph.h | 14 +- .../algorithm/graph/ConnectedComponents.h | 162 +++++++++++------- .../algorithm/graph/MinimumSpanningTree.h | 2 +- .../dash/memory/GlobHeapContiguousMem.h | 12 ++ .../test/algorithm/ConnectedComponentsTest.cc | 2 + 5 files changed, 129 insertions(+), 63 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 0f8b71df2..37a5161cd 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -739,6 +739,19 @@ class Graph { return global_vertex_iterator(_glob_mem_vertex, _myid, it.pos()); } + vertex_properties_type & vertex_attributes(vertex_size_type local_vertex) { + return _glob_mem_vertex->get(local_vertex).properties; + } + + void set_vertex_attributes(vertex_size_type local_vertex, + vertex_properties_type prop) { + return _glob_mem_vertex->set(local_vertex, vertex_type(prop)); + } + + vertex_size_type local_vertex_size() { + return _glob_mem_vertex->local_size(); + } + private: /** @@ -809,7 +822,6 @@ class Graph { add_edge(s, t, prop); } - private: /** the team containing all units using the container */ diff --git a/dash/include/dash/algorithm/graph/ConnectedComponents.h b/dash/include/dash/algorithm/graph/ConnectedComponents.h index c08ff846e..c5c1e8abd 100644 --- a/dash/include/dash/algorithm/graph/ConnectedComponents.h +++ b/dash/include/dash/algorithm/graph/ConnectedComponents.h @@ -7,26 +7,32 @@ namespace dash { typedef std::vector> matrix_t; -typedef std::vector>> matrix_pair_t; +template +using matrix_pair_t = std::vector>>; namespace internal { template -std::vector cc_get_data( +auto cc_get_data( matrix_t & indices, matrix_t & permutations, GraphType & graph, - dash::util::Trace & trace) -{ + dash::util::Trace & trace +) -> std::vector { + typedef typename GraphType::vertex_properties_type vprop_t; trace.enter_state("send indices"); // exchange indices to get std::vector sizes_send(indices.size()); std::vector displs_send(indices.size()); + std::vector sizes_recv_data(indices.size()); + std::vector displs_recv_data(indices.size()); std::vector indices_send; int total_send = 0; for(int i = 0; i < indices.size(); ++i) { sizes_send[i] = indices[i].size(); displs_send[i] = total_send; + sizes_recv_data[i] = indices[i].size() * sizeof(vprop_t); + displs_recv_data[i] = total_send * sizeof(vprop_t); total_send += indices[i].size(); } indices_send.reserve(total_send); @@ -36,10 +42,14 @@ std::vector cc_get_data( } std::vector sizes_recv(indices.size()); std::vector displs_recv(indices.size()); + std::vector sizes_send_data(indices.size()); + std::vector displs_send_data(indices.size()); dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), DART_TYPE_BYTE, graph.team().dart_id()); int total_recv = 0; for(int i = 0; i < sizes_recv.size(); ++i) { + sizes_send_data[i] = sizes_recv[i] * sizeof(vprop_t); + displs_send_data[i] = total_recv * sizeof(vprop_t); displs_recv[i] = total_recv; total_recv += sizes_recv[i]; } @@ -50,30 +60,34 @@ std::vector cc_get_data( displs_recv.data(), graph.team().dart_id()); } + std::vector data_send; + data_send.reserve(total_recv); + std::vector data_recv(total_send); + trace.exit_state("send indices"); // exchange data trace.enter_state("get components"); for(auto & index : indices_recv) { - // TODO: optimize cache performance - index = graph[graph.vertices().lbegin() + index].attributes().comp; + data_send.push_back(graph.vertex_attributes(index)); } trace.exit_state("get components"); trace.enter_state("send components"); if(total_send > 0 || total_recv > 0) { - dart_alltoallv(indices_recv.data(), sizes_recv.data(), displs_recv.data(), - DART_TYPE_INT, indices_send.data(), sizes_send.data(), - displs_send.data(), graph.team().dart_id()); + dart_alltoallv(data_send.data(), sizes_send_data.data(), + displs_send_data.data(), DART_TYPE_BYTE, data_recv.data(), + sizes_recv_data.data(), displs_recv_data.data(), + graph.team().dart_id()); } trace.exit_state("send components"); trace.enter_state("restore order"); // restore original order // TODO: use more sophisticated ordering mechanism - std::vector output(indices_send.size()); + std::vector output(data_recv.size()); int index = 0; for(int i = 0; i < permutations.size(); ++i) { for(int j = 0; j < permutations[i].size(); ++j) { - output[permutations[i][j]] = indices_send[index]; + output[permutations[i][j]] = data_recv[index]; ++index; } } @@ -81,17 +95,53 @@ std::vector cc_get_data( return output; } + template -void cc_set_data(matrix_pair_t & data_pairs, GraphType & graph, - dash::util::Trace & trace) { +std::vector cc_get_components( + GraphType & graph, + dash::util::Trace & trace +) { + std::size_t size = graph.local_vertex_size(); + std::vector sizes_recv(graph.team().size()); + std::vector displs_recv(graph.team().size()); + int total_recv = 0; + dart_allgather(&size, sizes_recv.data(), sizeof(std::size_t), + DART_TYPE_BYTE, graph.team().dart_id()); + + for(int i = 0; i < sizes_recv.size(); ++i) { + displs_recv[i] = total_recv; + total_recv += sizes_recv[i]; + } + + std::vector components_send; + components_send.reserve(size); + for(int i = 0; i < size; ++i) { + components_send.push_back(graph.vertex_attributes(i).comp); + } + + std::vector components_recv(total_recv); + dart_allgatherv(components_send.data(), size, DART_TYPE_INT, + components_recv.data(), sizes_recv.data(), displs_recv.data(), + graph.team().dart_id()); + + return components_recv; +} + +template +void cc_set_data( + matrix_pair_t & data_pairs, + GraphType & graph, + dash::util::Trace & trace +) { + typedef typename GraphType::vertex_properties_type vprop_t; trace.enter_state("send pairs"); std::vector sizes_send(data_pairs.size()); std::vector displs_send(data_pairs.size()); - std::vector> pairs_send; + std::vector> pairs_send; int total_send = 0; for(int i = 0; i < data_pairs.size(); ++i) { - sizes_send[i] = data_pairs[i].size() * sizeof(std::pair); - displs_send[i] = total_send * sizeof(std::pair); + sizes_send[i] = data_pairs[i].size() * sizeof(std::pair); + displs_send[i] = total_send * sizeof(std::pair); total_send += data_pairs[i].size(); } pairs_send.reserve(total_send); @@ -107,8 +157,8 @@ void cc_set_data(matrix_pair_t & data_pairs, GraphType & graph, displs_recv[i] = total_recv; total_recv += sizes_recv[i]; } - std::vector> pairs_recv(total_recv / - sizeof(std::pair)); + std::vector> pairs_recv(total_recv / + sizeof(std::pair)); if(total_send > 0 || total_recv > 0) { dart_alltoallv(pairs_send.data(), sizes_send.data(), displs_send.data(), DART_TYPE_BYTE, pairs_recv.data(), sizes_recv.data(), @@ -117,8 +167,7 @@ void cc_set_data(matrix_pair_t & data_pairs, GraphType & graph, trace.exit_state("send pairs"); trace.enter_state("set components"); for(auto & pair : pairs_recv) { - typename GraphType::vertex_properties_type prop { pair.second }; - graph[graph.vertices().lbegin() + pair.first].set_attributes(prop); + graph.set_vertex_attributes(pair.first, pair.second); } trace.exit_state("set components"); } @@ -138,16 +187,20 @@ void cc_set_data(matrix_pair_t & data_pairs, GraphType & graph, */ template void connected_components(GraphType & g) { + typedef typename GraphType::vertex_properties_type vprop_t; dash::util::Trace trace("ConnectedComponents"); // set component to global index in iteration space // TODO: find faster method for this trace.enter_state("vertex setup"); int i = 0; - for(auto it = g.vertices().begin(); it != g.vertices().end(); ++it) { - if(it.is_local()) { - typename GraphType::vertex_properties_type prop { it.pos() }; - g[it].set_attributes(prop); - } + auto myid = dash::myid(); + auto git = g.vertex_gptr(g.vertices().lbegin()); + auto start = git.pos(); + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto index = it.pos(); + auto gindex = index + start; + vprop_t prop { gindex, myid, index }; + g.set_vertex_attributes(index, prop); ++i; } trace.exit_state("vertex setup"); @@ -158,7 +211,7 @@ void connected_components(GraphType & g) { while(1) { int gr = 0; - std::vector data; + std::vector data; { trace.enter_state("compute indices"); matrix_t indices(g.team().size()); @@ -178,20 +231,20 @@ void connected_components(GraphType & g) { { trace.enter_state("compute pairs"); - matrix_pair_t data_pairs(g.team().size()); + matrix_pair_t data_pairs(g.team().size()); int i = 0; for(auto it = g.out_edges().lbegin(); it != g.out_edges().lend(); ++it) { - auto e = g[it]; - auto src = g[e.source()]; - auto src_comp = src.attributes().comp; - auto trg_comp = data[i]; - if(src_comp < trg_comp) { - auto trg_comp_it = g.vertices().begin() + trg_comp; - auto lpos = trg_comp_it.lpos(); - data_pairs[lpos.unit].push_back( - std::make_pair(lpos.index, src_comp) - ); - gr = 1; + if(data[i].comp != 0) { + auto e = g[it]; + auto src = g[e.source()]; + auto src_comp = src.attributes(); + auto trg_comp = data[i]; + if(src_comp.comp < trg_comp.comp) { + data_pairs[trg_comp.unit].push_back( + std::make_pair(trg_comp.index, src_comp) + ); + gr = 1; + } } ++i; } @@ -206,36 +259,23 @@ void connected_components(GraphType & g) { if(gr_all == 0) break; while(1) { int pj = 0; - { - trace.enter_state("compute indices (pj)"); - matrix_t indices(g.team().size()); - matrix_t permutations(g.team().size()); - int i = 0; - for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { - auto v = g[it]; - auto comp = v.attributes().comp; - auto next_it = g.vertices().begin() + comp; - auto lpos = next_it.lpos(); - indices[lpos.unit].push_back(lpos.index); - permutations[lpos.unit].push_back(i); - ++i; - } - trace.exit_state("compute indices (pj)"); - data = internal::cc_get_data(indices, permutations, g, trace); - } + std::vector components = internal::cc_get_components(g, trace); trace.enter_state("set data (pj)"); int i = 0; for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { auto v = g[it]; auto comp = v.attributes().comp; - auto comp_next = data[i]; - if(comp != comp_next) { - typename GraphType::vertex_properties_type prop { comp_next }; - v.set_attributes(prop); - pj = 1; + if(comp != 0) { + auto comp_next = components[comp]; + if(comp != comp_next) { + auto prop = v.attributes(); + prop.comp = comp_next; + v.set_attributes(prop); + pj = 1; + } + ++i; } - ++i; } trace.exit_state("set data (pj)"); int pj_all = 0; diff --git a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h index 92ff59bda..e4ef70ac7 100644 --- a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h +++ b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h @@ -7,7 +7,7 @@ namespace dash { typedef std::vector> matrix_t; -typedef std::vector>> matrix_pair_t; +typedef std::vector>> pair_t; // (supervertex, new component, vertex src, vertex trg, weight, owner, // local offset) typedef std::tuple tuple_t; diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h index ac85972f1..14a14c4f7 100644 --- a/dash/include/dash/memory/GlobHeapContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -405,6 +405,18 @@ class GlobHeapContiguousMem { return dart_gptr; } + value_type & get(index_type index) { + return (*_container)[index]; + } + + size_type local_size() { + return _local_size; + } + + void set(index_type index, value_type value) { + (*_container)[index] = value; + } + private: /** diff --git a/dash/test/algorithm/ConnectedComponentsTest.cc b/dash/test/algorithm/ConnectedComponentsTest.cc index 5031a1760..bed37ac13 100644 --- a/dash/test/algorithm/ConnectedComponentsTest.cc +++ b/dash/test/algorithm/ConnectedComponentsTest.cc @@ -4,6 +4,8 @@ struct vprop { int comp; + int unit; + int index; }; typedef dash::Graph graph_t; From a5b2f071e4ea7fb3713868d63297719f37d351d1 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 19 Feb 2018 14:37:02 +0100 Subject: [PATCH 080/102] removed bucket sizes of other units to reduce memory consumption --- dash/include/dash/Graph.h | 9 + dash/include/dash/graph/internal/Graph.h | 158 ++++++++++-------- .../dash/memory/GlobHeapContiguousMem.h | 51 +++--- .../dash/memory/GlobHeapContiguousPtr.h | 54 ------ 4 files changed, 122 insertions(+), 150 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 37a5161cd..011a4d93f 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -146,6 +146,7 @@ class Graph { friend global_inout_edge_proxy_type; friend local_edge_proxy_type; friend global_edge_proxy_type; + friend vertex_type; public: @@ -670,6 +671,14 @@ class Graph { _glob_mem_in_edge->commit(); } _glob_mem_edge->commit(); + + for(auto it = vertices().lbegin(); it != vertices().lend(); ++it) { + auto & v = *it; + v.in_edge_list.index = _glob_mem_in_edge->container_begin(it.pos()); + v.in_edge_list.size = _glob_mem_in_edge->container_size(it.pos()); + v.out_edge_list.index = _glob_mem_out_edge->container_begin(it.pos()); + v.out_edge_list.size = _glob_mem_out_edge->container_size(it.pos()); + } } /** diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index 1cce367d5..55e3d87e8 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -46,8 +46,14 @@ template class Vertex { typedef typename GraphType::vertex_properties_type properties_type; + typedef typename GraphType::glob_mem_vert_type gmem_type; + typedef typename gmem_type::index_type index_type; - friend GraphType; + template + struct EdgeListIndex { + IndexType index; + IndexType size; + }; public: @@ -68,7 +74,9 @@ class Vertex { public: /** Properties of this vertex */ - properties_type properties; + properties_type properties; + EdgeListIndex in_edge_list; + EdgeListIndex out_edge_list; }; @@ -99,6 +107,7 @@ class VertexProxy { typedef typename glob_mem_type::index_type index_type; typedef typename glob_mem_type::global_iterator global_iterator; typedef typename glob_mem_type::local_iterator local_iterator; + typedef typename parent_type::vertex_type vertex_type; typedef typename parent_type::graph_type::local_vertex_iterator local_vertex_iterator; typedef typename parent_type::graph_type::glob_mem_edge_comb_type @@ -109,9 +118,11 @@ class VertexProxy { /* * Constructs the range handler object. */ - edge_range(parent_type & p, glob_mem_type * gmem) + edge_range(parent_type & p, glob_mem_type * gmem, + bool is_in_edge_list = false) : _parent(p), - _glob_mem(gmem) + _glob_mem(gmem), + _is_in_edge_list(is_in_edge_list) { } /** @@ -145,13 +156,13 @@ class VertexProxy { */ template global_iterator begin(VertexIteratorType it) { + _parent.lazy_load(); auto lpos = _parent._iterator.lpos(); - auto index = it_position(_glob_mem, lpos.index); + auto index = g_it_position(_glob_mem, _parent._vertex); return global_iterator( _glob_mem, lpos.unit, - index, - 0 + index ); } @@ -159,12 +170,12 @@ class VertexProxy { * Begin iterator for local vertex iterators. */ global_iterator begin(local_vertex_iterator it) { - auto index = it_position(_glob_mem, it.pos()); + _parent.lazy_load(); + auto index = g_it_position(_glob_mem, _parent._vertex); return global_iterator( _glob_mem, _parent._graph->_myid, - index, - 0 + index ); } @@ -173,13 +184,14 @@ class VertexProxy { */ template global_iterator end(VertexIteratorType it) { + _parent.lazy_load(); auto lpos = _parent._iterator.lpos(); - auto index = it_position(_glob_mem, lpos.index); + auto index = g_it_position(_glob_mem, _parent._vertex); + auto size = g_it_size(_glob_mem, _parent._vertex); return global_iterator( _glob_mem, lpos.unit, - index, - _glob_mem->container_size(lpos.unit, index) + index + size ); } @@ -187,44 +199,24 @@ class VertexProxy { * End iterator for local vertex iterators. */ global_iterator end(local_vertex_iterator it) { - auto index = it_position(_glob_mem, it.pos()); + _parent.lazy_load(); + auto index = g_it_position(_glob_mem, _parent._vertex); + auto size = g_it_size(_glob_mem, _parent._vertex); return global_iterator( _glob_mem, _parent._graph->_myid, - index, - _glob_mem->container_size(_parent._graph->_myid, index) + index + size ); } /** * Local begin iterator for global vertex iterators. + * Not implemented because a VertexProxy referencing a remote vertex + * does not have any local data. */ template local_iterator lbegin(VertexIteratorType it) { - auto lpos = _parent._iterator.lpos(); - auto index = it_position(_glob_mem, lpos.index); - auto lindex = index * 2; - auto bucket_it = _glob_mem->_buckets.begin(); - std::advance(bucket_it, lindex); - // if bucket empty, advance to end iterator - if(bucket_it->size == 0) { - std::advance(bucket_it, 2); - } - auto position = 0; - // TODO: position WRONG, if there are uncommitted changes - // save cumulated sizes of unattached containers somewhere - if(index > 0) { - position = - _glob_mem->_bucket_cumul_sizes[_parent._graph->_myid][index - 1]; - } - // use lightweight constructor to avoid costly increment operations - return local_iterator( - _glob_mem->_buckets.begin(), - _glob_mem->_buckets.end(), - position, - bucket_it, - 0 - ); + //TODO: Show "not implemented" message } /** @@ -243,8 +235,7 @@ class VertexProxy { // TODO: position WRONG, if there are uncommitted changes // save cumulated sizes of unattached containers somewhere if(index > 0) { - position = - _glob_mem->_bucket_cumul_sizes[_parent._graph->_myid][index - 1]; + position = _glob_mem->_local_bucket_cumul_sizes[index - 1]; } // use lightweight constructor to avoid costly increment operations return local_iterator( @@ -258,29 +249,12 @@ class VertexProxy { /** * Local end iterator for global vertex iterators. + * Not implemented because a VertexProxy referencing a remote vertex + * does not have any local data. */ template local_iterator lend(VertexIteratorType it) { - auto lpos = _parent._iterator.lpos(); - auto index = it_position(_glob_mem, lpos.index) + 1; - auto lindex = index * 2; - auto bucket_it = _glob_mem->_buckets.begin(); - //std::advance(bucket_it, lindex); - auto position = 0; - // TODO: position WRONG, if there are uncommitted changes - // save cumulated sizes of unattached containers somewhere - if(index > 0) { - position = - _glob_mem->_bucket_cumul_sizes[_parent._graph->_myid][index - 1]; - } - // use lightweight constructor to avoid costly increment operations - return local_iterator( - _glob_mem->_buckets.begin(), - _glob_mem->_buckets.end(), - position, - bucket_it, - 0 - ); + //TODO: Show "not implemented" message } /** @@ -295,8 +269,7 @@ class VertexProxy { // TODO: position WRONG, if there are uncommitted changes // save cumulated sizes of unattached containers somewhere if(index > 0) { - position = - _glob_mem->_bucket_cumul_sizes[_parent._graph->_myid][index - 1]; + position = _glob_mem->_local_bucket_cumul_sizes[index - 1]; } // use lightweight constructor to avoid costly increment operations return local_iterator( @@ -325,12 +298,51 @@ class VertexProxy { return pos * 2; } + /** + * Determines the position of the edge-list for inbound and outbound + * edge memory spaces. + */ + template + index_type g_it_position(_GMem * gmem, vertex_type v) { + //indices for in- and out-edge lists are the same + return v.out_edge_list.index; + } + + /** + * Determines the position of the edge-list for the combined edge memory + * space. + */ + index_type g_it_position(glob_mem_edge_comb_type * gmem, vertex_type v) { + return v.out_edge_list.index * 2; + } + + /** + * Determines the position of the edge-list for inbound and outbound + * edge memory spaces. + */ + template + index_type g_it_size(_GMem * gmem, vertex_type v) { + if(_is_in_edge_list) { + return v.in_edge_list.size; + } + return v.out_edge_list.size; + } + + /** + * Determines the position of the edge-list for the combined edge memory + * space. + */ + index_type g_it_size(glob_mem_edge_comb_type * gmem, vertex_type v) { + return v.out_edge_list.size + v.in_edge_list.size; + } + private: /** Reference to the corresponding VertexProxy object */ parent_type & _parent; /** Reference to the GlobMem object of the targeted memory space */ glob_mem_type * _glob_mem; + bool _is_in_edge_list; }; @@ -353,7 +365,7 @@ class VertexProxy { : _iterator(it), _graph(g), _out_edges(*this, g->_glob_mem_out_edge), - _in_edges(*this, g->_glob_mem_in_edge), + _in_edges(*this, g->_glob_mem_in_edge, true), _edges(*this, g->_glob_mem_edge) { } @@ -382,11 +394,7 @@ class VertexProxy { * Returns the properties of the referenced vertex. Data is loaded lazily. */ properties_type & attributes() { - // load properties lazily - if(!_vertex_loaded) { - _vertex = *_iterator; - _vertex_loaded = true; - } + lazy_load(); return _vertex.properties; } @@ -398,6 +406,16 @@ class VertexProxy { } private: + + /** + * Loads vertex data lazily. + */ + void lazy_load() { + if(!_vertex_loaded) { + _vertex = *_iterator; + _vertex_loaded = true; + } + } /** * Overload of set_aatributes for global pointers. diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h index 14a14c4f7..6c5468518 100644 --- a/dash/include/dash/memory/GlobHeapContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -71,6 +71,8 @@ class GlobHeapContiguousMem { typedef local_iterator const_local_pointer; typedef std::vector unit_cumul_sizes_map; typedef std::vector> bucket_cumul_sizes_map; + typedef std::vector + local_bucket_cumul_sizes_type; template friend class dash::GlobPtr; @@ -91,15 +93,19 @@ class GlobHeapContiguousMem { _myid(team.myid()), _bucket_cumul_sizes(team.size()), _unit_cumul_sizes(team.size()) - { } + { + // set 1 bucket for every unit + for(auto & unit_bkt : _bucket_cumul_sizes) { + unit_bkt.resize(1); + } + } + /** * Adds a new bucket into memory space. */ bucket_index_type add_container(size_type n_elements) { - // TODO: set capacity - increment_bucket_sizes(); - + _local_bucket_cumul_sizes.resize(_local_bucket_cumul_sizes.size() + 1); // create bucket data and add to bucket list bucket_type cont_bucket { 0, @@ -207,21 +213,16 @@ class GlobHeapContiguousMem { } // update cumulated bucket sizes bucket_cumul += last_bucket->size; - _bucket_cumul_sizes[_myid][bucket_num] = bucket_cumul; + _local_bucket_cumul_sizes[bucket_num] = bucket_cumul; ++bucket_num; } ++count; } + _bucket_cumul_sizes[_myid][0] = _local_size; // 2 local buckets per global bucket count /= 2; for(int i = count; i < *max_buckets; ++i) { add_container(0); - if(count > 0) { - _bucket_cumul_sizes[_myid][count] = - _bucket_cumul_sizes[_myid][count - 1]; - } else { - _bucket_cumul_sizes[_myid][count] = 0; - } } // detach old container location from global memory space, if it has @@ -247,6 +248,7 @@ class GlobHeapContiguousMem { DART_OK ); + // TODO: can now be simplified // distribute bucket sizes between all units auto bucket_amount = _bucket_cumul_sizes[_myid].size(); std::vector bucket_sizes(bucket_amount * _team->size()); @@ -340,12 +342,19 @@ class GlobHeapContiguousMem { /** * Returns the global size of a given bucket. */ - size_type container_size(team_unit_t unit, size_type index) const { + size_type container_size(index_type index) const { + if(index <= 0) { + return _local_bucket_cumul_sizes[index]; + } + return _local_bucket_cumul_sizes[index] - + _local_bucket_cumul_sizes[index - 1]; + } + + index_type container_begin(index_type index) const { if(index <= 0) { - return _bucket_cumul_sizes[unit][index]; + return 0; } - return _bucket_cumul_sizes[unit][index] - - _bucket_cumul_sizes[unit][index - 1]; + return _local_bucket_cumul_sizes[index - 1]; } /** @@ -444,17 +453,6 @@ class GlobHeapContiguousMem { _lend = unit_lend; } - /** - * Increments the bucket count of each unit by one. - */ - void increment_bucket_sizes() { - for(auto it = _bucket_cumul_sizes.begin(); - it != _bucket_cumul_sizes.end(); ++it) { - // gets initiliazed with 0 automatically - it->resize(it->size() + 1); - } - } - private: /** List of buckets for GlobHeapLocalPtr */ @@ -483,6 +481,7 @@ class GlobHeapContiguousMem { local_iterator _lend; /** Accumulated sizes of the buckets of each unit. See GlobHeapMem */ bucket_cumul_sizes_map _bucket_cumul_sizes; + local_bucket_cumul_sizes_type _local_bucket_cumul_sizes; /** Accumulated sizes of elements per unit */ unit_cumul_sizes_map _unit_cumul_sizes; /** Global size of the memory space */ diff --git a/dash/include/dash/memory/GlobHeapContiguousPtr.h b/dash/include/dash/memory/GlobHeapContiguousPtr.h index 09207bfa6..616f8aa46 100644 --- a/dash/include/dash/memory/GlobHeapContiguousPtr.h +++ b/dash/include/dash/memory/GlobHeapContiguousPtr.h @@ -227,60 +227,6 @@ class GlobPtr< "phase:", _idx_bucket_phase); } - /** - * Constructor, creates a global pointer on global memory from unit, bucket - * index and local offset in logical storage order. - */ - template - GlobPtr( - const MemSpaceT * gmem, - team_unit_t unit, - index_type bucket_index, - index_type local_index) - : _globmem(reinterpret_cast(gmem)), - _bucket_cumul_sizes(&_globmem->_bucket_cumul_sizes), - _unit_cumul_sizes(&_globmem->_unit_cumul_sizes), - _lbegin(_globmem->lbegin()), - _idx(0), - _max_idx(gmem->size() - 1), - _myid(gmem->team().myid()), - _idx_unit_id(unit), - _idx_local_idx(0), - _idx_bucket_idx(bucket_index), - _idx_bucket_phase(local_index) - { - DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx)", - "unit:", unit, - "lidx:", local_index); - DASH_ASSERT_LT(unit, _bucket_cumul_sizes->size(), "invalid unit id"); - - /* - for (size_type unit = 0; unit < _idx_unit_id; ++unit) { - auto prec_unit_local_size = (*_bucket_cumul_sizes)[unit].back(); - _idx += prec_unit_local_size; - } - */ - _idx += (*_unit_cumul_sizes)[unit]; - if(bucket_index > 0) { - local_index += (*_bucket_cumul_sizes)[_idx_unit_id.id][bucket_index - 1]; - } - _idx_local_idx += local_index; - _idx += local_index; - /* - if(bucket_index > 0) { - local_index += (*_bucket_cumul_sizes)[_idx_unit_id.id][bucket_index - 1]; - } - increment(local_index); - */ - DASH_LOG_TRACE("GlobPtr(gmem,unit,lidx) >", - "gidx:", _idx, - "maxidx:", _max_idx, - "unit:", _idx_unit_id, - "lidx:", _idx_local_idx, - "bucket:", _idx_bucket_idx, - "phase:", _idx_bucket_phase); - } - /** * Copy constructor. */ From 16a7ed88013947b296259981510ce7739501bf62 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Tue, 20 Feb 2018 14:53:06 +0100 Subject: [PATCH 081/102] fixed cc algorithm --- dash/include/dash/Graph.h | 3 +- .../algorithm/graph/ConnectedComponents.h | 70 ++++++++++--------- 2 files changed, 38 insertions(+), 35 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 011a4d93f..ccf509d2e 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -754,7 +754,8 @@ class Graph { void set_vertex_attributes(vertex_size_type local_vertex, vertex_properties_type prop) { - return _glob_mem_vertex->set(local_vertex, vertex_type(prop)); + auto & v = _glob_mem_vertex->get(local_vertex); + v.properties = prop; } vertex_size_type local_vertex_size() { diff --git a/dash/include/dash/algorithm/graph/ConnectedComponents.h b/dash/include/dash/algorithm/graph/ConnectedComponents.h index c5c1e8abd..e4b812f8c 100644 --- a/dash/include/dash/algorithm/graph/ConnectedComponents.h +++ b/dash/include/dash/algorithm/graph/ConnectedComponents.h @@ -97,11 +97,13 @@ auto cc_get_data( template -std::vector cc_get_components( +auto cc_get_components( GraphType & graph, dash::util::Trace & trace -) { - std::size_t size = graph.local_vertex_size(); +) -> std::vector { + typedef typename GraphType::vertex_properties_type vprop_t; + std::size_t size = graph.local_vertex_size() * sizeof(vprop_t); + std::size_t size_n = graph.local_vertex_size(); std::vector sizes_recv(graph.team().size()); std::vector displs_recv(graph.team().size()); int total_recv = 0; @@ -113,14 +115,14 @@ std::vector cc_get_components( total_recv += sizes_recv[i]; } - std::vector components_send; - components_send.reserve(size); - for(int i = 0; i < size; ++i) { - components_send.push_back(graph.vertex_attributes(i).comp); + std::vector components_send; + components_send.reserve(size_n); + for(int i = 0; i < size_n; ++i) { + components_send.push_back(graph.vertex_attributes(i)); } - std::vector components_recv(total_recv); - dart_allgatherv(components_send.data(), size, DART_TYPE_INT, + std::vector components_recv(total_recv / sizeof(vprop_t)); + dart_allgatherv(components_send.data(), size, DART_TYPE_BYTE, components_recv.data(), sizes_recv.data(), displs_recv.data(), graph.team().dart_id()); @@ -130,6 +132,7 @@ std::vector cc_get_components( template void cc_set_data( matrix_pair_t & data_pairs, + int start, GraphType & graph, dash::util::Trace & trace ) { @@ -167,7 +170,7 @@ void cc_set_data( trace.exit_state("send pairs"); trace.enter_state("set components"); for(auto & pair : pairs_recv) { - graph.set_vertex_attributes(pair.first, pair.second); + graph.set_vertex_attributes(pair.first - start, pair.second); } trace.exit_state("set components"); } @@ -179,6 +182,7 @@ void cc_set_data( * Computes connected components on a graph. * Requires the graph's vertices to store the following attributes: * - (int) comp + * - (int) unit * Requires the graph's edges to store the following attributep: * none * @@ -199,7 +203,7 @@ void connected_components(GraphType & g) { for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { auto index = it.pos(); auto gindex = index + start; - vprop_t prop { gindex, myid, index }; + vprop_t prop { gindex, myid }; g.set_vertex_attributes(index, prop); ++i; } @@ -211,25 +215,25 @@ void connected_components(GraphType & g) { while(1) { int gr = 0; - std::vector data; { - trace.enter_state("compute indices"); - matrix_t indices(g.team().size()); - matrix_t permutations(g.team().size()); - int i = 0; - for(auto it = g.out_edges().lbegin(); it != g.out_edges().lend(); ++it) { - auto trg = g[it].target(); - auto lpos = trg.lpos(); - // TODO: use more sophsticated sorting mechanism - indices[lpos.unit].push_back(lpos.index); - permutations[lpos.unit].push_back(i); - ++i; + std::vector data; + { + trace.enter_state("compute indices"); + matrix_t indices(g.team().size()); + matrix_t permutations(g.team().size()); + int i = 0; + for(auto it = g.out_edges().lbegin(); it != g.out_edges().lend(); ++it) { + auto trg = g[it].target(); + auto lpos = trg.lpos(); + // TODO: use more sophsticated sorting mechanism + indices[lpos.unit].push_back(lpos.index); + permutations[lpos.unit].push_back(i); + ++i; + } + trace.exit_state("compute indices"); + data = internal::cc_get_data(indices, permutations, g, trace); } - trace.exit_state("compute indices"); - data = internal::cc_get_data(indices, permutations, g, trace); - } - { trace.enter_state("compute pairs"); matrix_pair_t data_pairs(g.team().size()); int i = 0; @@ -241,7 +245,7 @@ void connected_components(GraphType & g) { auto trg_comp = data[i]; if(src_comp.comp < trg_comp.comp) { data_pairs[trg_comp.unit].push_back( - std::make_pair(trg_comp.index, src_comp) + std::make_pair(trg_comp.comp, src_comp) ); gr = 1; } @@ -249,7 +253,7 @@ void connected_components(GraphType & g) { ++i; } trace.exit_state("compute pairs"); - internal::cc_set_data(data_pairs, g, trace); + internal::cc_set_data(data_pairs, start, g, trace); } int gr_all = 0; trace.enter_state("allreduce data"); @@ -259,7 +263,7 @@ void connected_components(GraphType & g) { if(gr_all == 0) break; while(1) { int pj = 0; - std::vector components = internal::cc_get_components(g, trace); + std::vector components = internal::cc_get_components(g, trace); trace.enter_state("set data (pj)"); int i = 0; @@ -268,10 +272,8 @@ void connected_components(GraphType & g) { auto comp = v.attributes().comp; if(comp != 0) { auto comp_next = components[comp]; - if(comp != comp_next) { - auto prop = v.attributes(); - prop.comp = comp_next; - v.set_attributes(prop); + if(comp != comp_next.comp) { + v.set_attributes(comp_next); pj = 1; } ++i; From a6250010bd6e3bce0f69a79db8412958c818cd4b Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Wed, 21 Feb 2018 15:47:36 +0100 Subject: [PATCH 082/102] further optimizations --- dash/include/dash/Graph.h | 13 ++++- .../algorithm/graph/ConnectedComponents.h | 49 ++++++++++--------- 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index ccf509d2e..5ff60dad8 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -758,6 +758,16 @@ class Graph { v.properties = prop; } + edge_properties_type & out_edge_attributes(edge_size_type local_edge) { + return _glob_mem_out_edge->get(local_edge).properties; + } + + void set_out_edge_attributes(edge_size_type local_edge, + edge_properties_type prop) { + auto & e = _glob_mem_out_edge->get(local_edge); + e.properties = prop; + } + vertex_size_type local_vertex_size() { return _glob_mem_vertex->local_size(); } @@ -796,7 +806,8 @@ class Graph { } team_unit_t vertex_owner(vertex_size_type v, vertex_size_type n_vertices) { - int owner_id = static_cast(v) / (static_cast(n_vertices) / _team->size()); + int owner_id = static_cast(v) / (static_cast(n_vertices) / + _team->size()); team_unit_t owner { owner_id }; return owner; } diff --git a/dash/include/dash/algorithm/graph/ConnectedComponents.h b/dash/include/dash/algorithm/graph/ConnectedComponents.h index e4b812f8c..b5aed8b05 100644 --- a/dash/include/dash/algorithm/graph/ConnectedComponents.h +++ b/dash/include/dash/algorithm/graph/ConnectedComponents.h @@ -54,11 +54,9 @@ auto cc_get_data( total_recv += sizes_recv[i]; } std::vector indices_recv(total_recv); - if(total_send > 0 || total_recv > 0) { dart_alltoallv(indices_send.data(), sizes_send.data(), displs_send.data(), DART_TYPE_INT, indices_recv.data(), sizes_recv.data(), displs_recv.data(), graph.team().dart_id()); - } std::vector data_send; data_send.reserve(total_recv); @@ -72,17 +70,14 @@ auto cc_get_data( } trace.exit_state("get components"); trace.enter_state("send components"); - if(total_send > 0 || total_recv > 0) { dart_alltoallv(data_send.data(), sizes_send_data.data(), displs_send_data.data(), DART_TYPE_BYTE, data_recv.data(), sizes_recv_data.data(), displs_recv_data.data(), graph.team().dart_id()); - } trace.exit_state("send components"); trace.enter_state("restore order"); // restore original order - // TODO: use more sophisticated ordering mechanism std::vector output(data_recv.size()); int index = 0; for(int i = 0; i < permutations.size(); ++i) { @@ -101,6 +96,7 @@ auto cc_get_components( GraphType & graph, dash::util::Trace & trace ) -> std::vector { + trace.enter_state("send components"); typedef typename GraphType::vertex_properties_type vprop_t; std::size_t size = graph.local_vertex_size() * sizeof(vprop_t); std::size_t size_n = graph.local_vertex_size(); @@ -126,6 +122,7 @@ auto cc_get_components( components_recv.data(), sizes_recv.data(), displs_recv.data(), graph.team().dart_id()); + trace.exit_state("send components"); return components_recv; } @@ -155,21 +152,20 @@ void cc_set_data( std::vector displs_recv(data_pairs.size()); dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), DART_TYPE_BYTE, graph.team().dart_id()); - int total_recv = 0; + std::size_t total_recv = 0; for(int i = 0; i < sizes_recv.size(); ++i) { displs_recv[i] = total_recv; total_recv += sizes_recv[i]; } std::vector> pairs_recv(total_recv / sizeof(std::pair)); - if(total_send > 0 || total_recv > 0) { dart_alltoallv(pairs_send.data(), sizes_send.data(), displs_send.data(), DART_TYPE_BYTE, pairs_recv.data(), sizes_recv.data(), displs_recv.data(), graph.team().dart_id()); - } trace.exit_state("send pairs"); trace.enter_state("set components"); for(auto & pair : pairs_recv) { + int ind = pair.first - start; graph.set_vertex_attributes(pair.first - start, pair.second); } trace.exit_state("set components"); @@ -233,24 +229,30 @@ void connected_components(GraphType & g) { trace.exit_state("compute indices"); data = internal::cc_get_data(indices, permutations, g, trace); } - trace.enter_state("compute pairs"); matrix_pair_t data_pairs(g.team().size()); - int i = 0; - for(auto it = g.out_edges().lbegin(); it != g.out_edges().lend(); ++it) { - if(data[i].comp != 0) { - auto e = g[it]; - auto src = g[e.source()]; - auto src_comp = src.attributes(); - auto trg_comp = data[i]; - if(src_comp.comp < trg_comp.comp) { - data_pairs[trg_comp.unit].push_back( - std::make_pair(trg_comp.comp, src_comp) - ); - gr = 1; + { + std::vector> pair_set(g.team().size()); + int i = 0; + for(auto it = g.out_edges().lbegin(); it != g.out_edges().lend(); ++it) { + if(data[i].comp != 0) { + auto e = g[it]; + auto src = g[e.source()]; + auto src_comp = src.attributes(); + auto trg_comp = data[i]; + if(src_comp.comp < trg_comp.comp) { + auto pair_it = pair_set[trg_comp.unit].find(trg_comp.comp); + if(pair_it == pair_set[trg_comp.unit].end()) { + data_pairs[trg_comp.unit].push_back( + std::make_pair(trg_comp.comp, src_comp) + ); + pair_set[trg_comp.unit].insert(trg_comp.comp); + } + gr = 1; + } } + ++i; } - ++i; } trace.exit_state("compute pairs"); internal::cc_set_data(data_pairs, start, g, trace); @@ -266,7 +268,6 @@ void connected_components(GraphType & g) { std::vector components = internal::cc_get_components(g, trace); trace.enter_state("set data (pj)"); - int i = 0; for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { auto v = g[it]; auto comp = v.attributes().comp; @@ -276,7 +277,6 @@ void connected_components(GraphType & g) { v.set_attributes(comp_next); pj = 1; } - ++i; } } trace.exit_state("set data (pj)"); @@ -294,3 +294,4 @@ void connected_components(GraphType & g) { } // namespace dash #endif // DASH__ALGORITHM____GRAPH__CONNECTED_COMPONENTS_H__ + From ba45f3431c68efd90e5aef2da627e20352fa5310 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Wed, 21 Feb 2018 15:48:14 +0100 Subject: [PATCH 083/102] optimized MST algorithm --- .../algorithm/graph/MinimumSpanningTree.h | 288 +++++++++++------- 1 file changed, 180 insertions(+), 108 deletions(-) diff --git a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h index e4ef70ac7..2bfd80648 100644 --- a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h +++ b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h @@ -3,32 +3,45 @@ #include #include +#include namespace dash { -typedef std::vector> matrix_t; typedef std::vector>> pair_t; -// (supervertex, new component, vertex src, vertex trg, weight, owner, -// local offset) -typedef std::tuple tuple_t; -typedef std::vector> matrix_min_pairs_t; +typedef std::vector> matrix_t; + +// (supervertex, new component, weight, owner, local offset) +template +using tuple_t = std::tuple; +template +using matrix_min_pairs_t = std::vector>>; + +template +using matrix_pair_t = std::vector>>; namespace internal { template -std::vector mst_get_data( +auto mst_get_data( matrix_t & indices, matrix_t & permutations, - GraphType & graph) -{ + GraphType & graph, + dash::util::Trace & trace +) -> std::vector { + typedef typename GraphType::vertex_properties_type vprop_t; + trace.enter_state("send indices"); // exchange indices to get std::vector sizes_send(indices.size()); std::vector displs_send(indices.size()); + std::vector sizes_recv_data(indices.size()); + std::vector displs_recv_data(indices.size()); std::vector indices_send; - int total_send = 0; + std::size_t total_send = 0; for(int i = 0; i < indices.size(); ++i) { sizes_send[i] = indices[i].size(); displs_send[i] = total_send; + sizes_recv_data[i] = indices[i].size() * sizeof(vprop_t); + displs_recv_data[i] = total_send * sizeof(vprop_t); total_send += indices[i].size(); } indices_send.reserve(total_send); @@ -38,53 +51,107 @@ std::vector mst_get_data( } std::vector sizes_recv(indices.size()); std::vector displs_recv(indices.size()); + std::vector sizes_send_data(indices.size()); + std::vector displs_send_data(indices.size()); dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), DART_TYPE_BYTE, graph.team().dart_id()); - int total_recv = 0; + std::size_t total_recv = 0; for(int i = 0; i < sizes_recv.size(); ++i) { + sizes_send_data[i] = sizes_recv[i] * sizeof(vprop_t); + displs_send_data[i] = total_recv * sizeof(vprop_t); displs_recv[i] = total_recv; total_recv += sizes_recv[i]; } std::vector indices_recv(total_recv); - if(total_send > 0 || total_recv > 0) { - dart_alltoallv(indices_send.data(), sizes_send.data(), displs_send.data(), - DART_TYPE_INT, indices_recv.data(), sizes_recv.data(), - displs_recv.data(), graph.team().dart_id()); - } + dart_alltoallv(indices_send.data(), sizes_send.data(), displs_send.data(), + DART_TYPE_INT, indices_recv.data(), sizes_recv.data(), + displs_recv.data(), graph.team().dart_id()); + + std::vector data_send; + data_send.reserve(total_recv); + std::vector data_recv(total_send); + trace.exit_state("send indices"); // exchange data + trace.enter_state("get components"); for(auto & index : indices_recv) { - // TODO: optimize cache performance - index = graph[graph.vertices().begin() + index].attributes().comp; - } - if(total_send > 0 || total_recv > 0) { - dart_alltoallv(indices_recv.data(), sizes_recv.data(), displs_recv.data(), - DART_TYPE_INT, indices_send.data(), sizes_send.data(), - displs_send.data(), graph.team().dart_id()); + data_send.push_back(graph.vertex_attributes(index)); } + trace.exit_state("get components"); + trace.enter_state("send components"); + dart_alltoallv(data_send.data(), sizes_send_data.data(), + displs_send_data.data(), DART_TYPE_BYTE, data_recv.data(), + sizes_recv_data.data(), displs_recv_data.data(), + graph.team().dart_id()); + trace.exit_state("send components"); + trace.enter_state("restore order"); // restore original order // TODO: use more sophisticated ordering mechanism - std::vector output(indices_send.size()); + std::vector output(data_recv.size()); int index = 0; for(int i = 0; i < permutations.size(); ++i) { for(int j = 0; j < permutations[i].size(); ++j) { - output[permutations[i][j]] = indices_send[index]; + output[permutations[i][j]] = data_recv[index]; ++index; } } + trace.exit_state("restore order"); return output; } + template -void mst_set_data_min(matrix_min_pairs_t & data_pairs, GraphType & graph) { +auto mst_get_components( + GraphType & graph, + dash::util::Trace & trace +) -> std::vector { + trace.enter_state("send components"); + typedef typename GraphType::vertex_properties_type vprop_t; + std::size_t size = graph.local_vertex_size() * sizeof(vprop_t); + std::size_t size_n = graph.local_vertex_size(); + std::vector sizes_recv(graph.team().size()); + std::vector displs_recv(graph.team().size()); + std::size_t total_recv = 0; + dart_allgather(&size, sizes_recv.data(), sizeof(std::size_t), + DART_TYPE_BYTE, graph.team().dart_id()); + + for(int i = 0; i < sizes_recv.size(); ++i) { + displs_recv[i] = total_recv; + total_recv += sizes_recv[i]; + } + + std::vector components_send; + components_send.reserve(size_n); + for(int i = 0; i < size_n; ++i) { + components_send.push_back(graph.vertex_attributes(i)); + } + + std::vector components_recv(total_recv / sizeof(vprop_t)); + dart_allgatherv(components_send.data(), size, DART_TYPE_BYTE, + components_recv.data(), sizes_recv.data(), displs_recv.data(), + graph.team().dart_id()); + + trace.exit_state("send components"); + return components_recv; +} + +template +void mst_set_data_min( + matrix_min_pairs_t & data_pairs, + int start, + GraphType & graph, + dash::util::Trace & trace +) { + trace.enter_state("send pairs"); + typedef typename GraphType::vertex_properties_type vprop_t; std::vector sizes_send(data_pairs.size()); std::vector displs_send(data_pairs.size()); - std::vector pairs_send; - int total_send = 0; + std::vector> pairs_send; + std::size_t total_send = 0; for(int i = 0; i < data_pairs.size(); ++i) { - sizes_send[i] = data_pairs[i].size() * sizeof(tuple_t); - displs_send[i] = total_send * sizeof(tuple_t); + sizes_send[i] = data_pairs[i].size() * sizeof(tuple_t); + displs_send[i] = total_send * sizeof(tuple_t); total_send += data_pairs[i].size(); } pairs_send.reserve(total_send); @@ -95,41 +162,38 @@ void mst_set_data_min(matrix_min_pairs_t & data_pairs, GraphType & graph) { std::vector displs_recv(data_pairs.size()); dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), DART_TYPE_BYTE, graph.team().dart_id()); - int total_recv = 0; + std::size_t total_recv = 0; for(int i = 0; i < sizes_recv.size(); ++i) { displs_recv[i] = total_recv; total_recv += sizes_recv[i]; } - std::vector pairs_recv(total_recv / - sizeof(tuple_t)); - if(total_send > 0 || total_recv > 0) { + std::vector> pairs_recv(total_recv / + sizeof(tuple_t)); dart_alltoallv(pairs_send.data(), sizes_send.data(), displs_send.data(), DART_TYPE_BYTE, pairs_recv.data(), sizes_recv.data(), displs_recv.data(), graph.team().dart_id()); - } - std::unordered_map mapping; + std::unordered_map> mapping; for(auto & pair : pairs_recv) { auto it = mapping.find(std::get<0>(pair)); if(it != mapping.end()) { - auto weight = std::get<4>(it->second); - if(weight > std::get<4>(pair)) { + auto weight = std::get<2>(it->second); + if(weight > std::get<2>(pair)) { it->second = pair; } } else { mapping[std::get<0>(pair)] = pair; } } + trace.exit_state("send pairs"); + trace.enter_state("set components"); for(auto & pair : mapping) { - typename GraphType::vertex_properties_type prop { std::get<1>(pair.second) }; - graph[graph.vertices().begin() + pair.first].set_attributes(prop); - // save edges here (src = get<2>(pair.second), trg = get<3>(pair.second) - if(dash::myid() == get<5>(pair.second)) { - auto it = graph.out_edges().lbegin() + get<6>(pair.second); + graph.set_vertex_attributes(pair.first - start, std::get<1>(pair.second)); + if(dash::myid() == get<3>(pair.second)) { typename GraphType::edge_properties_type eprop { -1 }; - graph[it].set_attributes(eprop); - auto it2 = graph.out_edges().lbegin() + get<6>(pair.second); + graph.set_out_edge_attributes(std::get<4>(pair.second), eprop); } } + trace.exit_state("set components"); } } // namespace internal @@ -148,118 +212,126 @@ void mst_set_data_min(matrix_min_pairs_t & data_pairs, GraphType & graph) { */ template void minimum_spanning_tree(GraphType & g) { + typedef typename GraphType::vertex_properties_type vprop_t; + dash::util::Trace trace("MinimumSpanningTree"); // set component to global index in iteration space + trace.enter_state("vertex setup"); int i = 0; - for(auto it = g.vertices().begin(); it != g.vertices().end(); ++it) { - if(it.is_local()) { - typename GraphType::vertex_properties_type prop { it.pos() }; - g[it].set_attributes(prop); - } + auto myid = dash::myid(); + auto git = g.vertex_gptr(g.vertices().lbegin()); + auto start = git.pos(); + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto index = it.pos(); + auto gindex = index + start; + vprop_t prop { gindex, myid }; + g.set_vertex_attributes(index, prop); ++i; } + trace.exit_state("vertex setup"); + trace.enter_state("barrier"); dash::barrier(); + trace.exit_state("barrier"); while(1) { int gr = 0; - std::vector data; { - int i = 0; - matrix_t indices(g.team().size()); - matrix_t permutations(g.team().size()); - - for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { - auto v = g[it]; - for(auto e_it = v.out_edges().lbegin(); e_it != v.out_edges().lend(); - ++e_it) { - auto trg = g[e_it].target(); - auto lpos = trg.lpos(); - indices[lpos.unit].push_back(trg.pos()); - permutations[lpos.unit].push_back(i); - ++i; + std::vector data; + { + trace.enter_state("compute indices"); + matrix_t indices(g.team().size()); + matrix_t permutations(g.team().size()); + int i = 0; + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto v = g[it]; + for(auto e_it = v.out_edges().lbegin(); e_it != v.out_edges().lend(); + ++e_it) { + auto trg = g[e_it].target(); + auto lpos = trg.lpos(); + indices[lpos.unit].push_back(lpos.index); + permutations[lpos.unit].push_back(i); + ++i; + } } + trace.exit_state("compute indices"); + data = internal::mst_get_data(indices, permutations, g, trace); } - data = internal::mst_get_data(indices, permutations, g); - } - { + trace.enter_state("compute pairs"); + matrix_min_pairs_t data_pairs(g.team().size()); + std::vector> pair_set(g.team().size()); int i = 0; - matrix_min_pairs_t data_pairs(g.team().size()); for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { auto v = g[it]; - auto src_comp = v.attributes().comp; + auto src_comp = v.attributes(); int min_weight = std::numeric_limits::max(); - int trg_comp_min = -1; - int trg_min = -1; - int ledgepos = 0; - int unit = 0; + vprop_t trg_comp_min; + int ledgepos = -1; for(auto e_it = v.out_edges().lbegin(); e_it != v.out_edges().lend(); ++e_it) { auto e = g[e_it]; auto e_weight = e.attributes().weight; auto trg_comp = data[i]; - if(src_comp != trg_comp && e_weight >= 0 && min_weight > e_weight) { + if(src_comp.comp != trg_comp.comp && e_weight >= 0 && + min_weight > e_weight) { min_weight = e_weight; trg_comp_min = trg_comp; - trg_min = e.target().pos(); ledgepos = e_it.pos(); - unit = dash::myid(); } ++i; } - if(trg_comp_min >= 0) { - auto src = g.vertex_gptr(it).pos(); - auto src_comp_it = g.vertices().begin() + src_comp; - auto trg_comp_it = g.vertices().begin() + trg_comp_min; - data_pairs[src_comp_it.lpos().unit].push_back( - std::make_tuple(src_comp_it.pos(), trg_comp_min, src, trg_min, - min_weight, unit, ledgepos)); - data_pairs[trg_comp_it.lpos().unit].push_back( - std::make_tuple(trg_comp_it.pos(), src_comp, src, trg_min, - min_weight, unit, ledgepos)); + if(ledgepos >= 0) { + auto & set = pair_set[src_comp.unit]; + if(set.find(src_comp.comp) == set.end()) { + data_pairs[src_comp.unit].push_back( + std::make_tuple(src_comp.comp, trg_comp_min, min_weight, myid, + ledgepos) + ); + set.insert(src_comp.comp); + } + set = pair_set[trg_comp_min.unit]; + if(set.find(trg_comp_min.comp) == set.end()) { + data_pairs[trg_comp_min.unit].push_back( + std::make_tuple(trg_comp_min.comp, src_comp, min_weight, myid, + ledgepos) + ); + set.insert(trg_comp_min.comp); + } gr = 1; } } - internal::mst_set_data_min(data_pairs, g); + trace.exit_state("compute pairs"); + internal::mst_set_data_min(data_pairs, start, g, trace); } int gr_all = 0; + trace.enter_state("allreduce data"); dart_allreduce(&gr, &gr_all, 1, DART_TYPE_INT, DART_OP_SUM, g.team().dart_id()); + trace.exit_state("allreduce data"); if(gr_all == 0) break; while(1) { int pj = 0; - { - matrix_t indices(g.team().size()); - matrix_t permutations(g.team().size()); - int i = 0; - for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { - auto v = g[it]; - auto comp = v.attributes().comp; - auto next_it = g.vertices().begin() + comp; - auto lpos = next_it.lpos(); - indices[lpos.unit].push_back(next_it.pos()); - permutations[lpos.unit].push_back(i); - ++i; - } - data = internal::mst_get_data(indices, permutations, g); - } + std::vector components = internal::mst_get_components(g, trace); - int i = 0; + trace.enter_state("set data (pj)"); for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { auto v = g[it]; auto comp = v.attributes().comp; - auto comp_next = data[i]; - if(comp > comp_next) { - typename GraphType::vertex_properties_type prop { comp_next }; - v.set_attributes(prop); - pj = 1; + if(comp != 0) { + auto comp_next = components[comp]; + if(comp > comp_next.comp) { + v.set_attributes(comp_next); + pj = 1; + } } - ++i; } + trace.exit_state("set data (pj)"); int pj_all = 0; + trace.enter_state("allreduce pointerjumping"); dart_allreduce(&pj, &pj_all, 1, DART_TYPE_INT, DART_OP_SUM, g.team().dart_id()); + trace.exit_state("allreduce pointerjumping"); if(pj_all == 0) break; } } From 5aed324c9823582f3e3ce2c1a3338b64554cea53 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Wed, 21 Feb 2018 15:49:20 +0100 Subject: [PATCH 084/102] adapted tests --- dash/test/algorithm/ConnectedComponentsTest.cc | 1 - dash/test/algorithm/MinimumSpanningTreeTest.cc | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/dash/test/algorithm/ConnectedComponentsTest.cc b/dash/test/algorithm/ConnectedComponentsTest.cc index bed37ac13..53e9a9eb2 100644 --- a/dash/test/algorithm/ConnectedComponentsTest.cc +++ b/dash/test/algorithm/ConnectedComponentsTest.cc @@ -5,7 +5,6 @@ struct vprop { int comp; int unit; - int index; }; typedef dash::Graph graph_t; diff --git a/dash/test/algorithm/MinimumSpanningTreeTest.cc b/dash/test/algorithm/MinimumSpanningTreeTest.cc index 81056da1d..6e5d6afa0 100644 --- a/dash/test/algorithm/MinimumSpanningTreeTest.cc +++ b/dash/test/algorithm/MinimumSpanningTreeTest.cc @@ -4,6 +4,7 @@ struct vprop { int comp; + short unit; }; struct eprop { From 2f7f5ad610a36b0324caf3fc7d432bfb3ad3f041 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Fri, 23 Feb 2018 11:20:37 +0100 Subject: [PATCH 085/102] updated cc algorithm --- .../algorithm/graph/ConnectedComponents.h | 89 ++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/dash/include/dash/algorithm/graph/ConnectedComponents.h b/dash/include/dash/algorithm/graph/ConnectedComponents.h index b5aed8b05..f1a62e55a 100644 --- a/dash/include/dash/algorithm/graph/ConnectedComponents.h +++ b/dash/include/dash/algorithm/graph/ConnectedComponents.h @@ -90,7 +90,81 @@ auto cc_get_data( return output; } +template +auto cc_get_components( + matrix_t & indices, + int start, + GraphType & graph, + dash::util::Trace & trace +) -> std::unordered_map { + typedef typename GraphType::vertex_properties_type vprop_t; + trace.enter_state("send indices"); + // exchange indices to get + std::vector sizes_send(indices.size()); + std::vector displs_send(indices.size()); + std::vector sizes_recv_data(indices.size()); + std::vector displs_recv_data(indices.size()); + std::vector indices_send; + int total_send = 0; + for(int i = 0; i < indices.size(); ++i) { + sizes_send[i] = indices[i].size(); + displs_send[i] = total_send; + sizes_recv_data[i] = indices[i].size() * sizeof(vprop_t); + displs_recv_data[i] = total_send * sizeof(vprop_t); + total_send += indices[i].size(); + } + indices_send.reserve(total_send); + for(auto & index_set : indices) { + indices_send.insert(indices_send.end(), index_set.begin(), + index_set.end()); + } + std::vector sizes_recv(indices.size()); + std::vector displs_recv(indices.size()); + std::vector sizes_send_data(indices.size()); + std::vector displs_send_data(indices.size()); + dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), + DART_TYPE_BYTE, graph.team().dart_id()); + int total_recv = 0; + for(int i = 0; i < sizes_recv.size(); ++i) { + sizes_send_data[i] = sizes_recv[i] * sizeof(vprop_t); + displs_send_data[i] = total_recv * sizeof(vprop_t); + displs_recv[i] = total_recv; + total_recv += sizes_recv[i]; + } + std::vector indices_recv(total_recv); + dart_alltoallv(indices_send.data(), sizes_send.data(), displs_send.data(), + DART_TYPE_INT, indices_recv.data(), sizes_recv.data(), + displs_recv.data(), graph.team().dart_id()); + + std::vector data_send; + data_send.reserve(total_recv); + std::vector data_recv(total_send); + + trace.exit_state("send indices"); + // exchange data + trace.enter_state("get components"); + for(auto & index : indices_recv) { + data_send.push_back(graph.vertex_attributes(index - start)); + } + trace.exit_state("get components"); + trace.enter_state("send components"); + dart_alltoallv(data_send.data(), sizes_send_data.data(), + displs_send_data.data(), DART_TYPE_BYTE, data_recv.data(), + sizes_recv_data.data(), displs_recv_data.data(), + graph.team().dart_id()); + trace.exit_state("send components"); + trace.enter_state("create map"); + std::unordered_map output; + output.reserve(total_send); + for(int i = 0; i < data_recv.size(); ++i) { + output[indices_send[i]] = data_recv[i]; + } + trace.exit_state("create map"); + return output; +} + +/* template auto cc_get_components( GraphType & graph, @@ -125,6 +199,7 @@ auto cc_get_components( trace.exit_state("send components"); return components_recv; } +*/ template void cc_set_data( @@ -265,7 +340,19 @@ void connected_components(GraphType & g) { if(gr_all == 0) break; while(1) { int pj = 0; - std::vector components = internal::cc_get_components(g, trace); + matrix_t indices(g.team().size()); + { + std::vector> comp_set(g.team().size()); + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto c = g[it].attributes(); + auto & set = comp_set[c.unit]; + if(set.find(c.comp) == set.end()) { + indices[c.unit].push_back(c.comp); + set.insert(c.comp); + } + } + } + auto components = internal::cc_get_components(indices, start, g, trace); trace.enter_state("set data (pj)"); for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { From 5531da9d14facff8145f78dcad1df096b2dde8ca Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Fri, 23 Feb 2018 13:02:53 +0100 Subject: [PATCH 086/102] optimized cc again --- .../algorithm/graph/ConnectedComponents.h | 45 +++++++++---------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/dash/include/dash/algorithm/graph/ConnectedComponents.h b/dash/include/dash/algorithm/graph/ConnectedComponents.h index f1a62e55a..6bdc5a69b 100644 --- a/dash/include/dash/algorithm/graph/ConnectedComponents.h +++ b/dash/include/dash/algorithm/graph/ConnectedComponents.h @@ -284,26 +284,27 @@ void connected_components(GraphType & g) { dash::barrier(); trace.exit_state("barrier"); + matrix_t indices(g.team().size()); + matrix_t permutations(g.team().size()); + { + trace.enter_state("compute indices"); + int i = 0; + for(auto it = g.out_edges().lbegin(); it != g.out_edges().lend(); ++it) { + auto trg = g[it].target(); + auto lpos = trg.lpos(); + // TODO: use more sophsticated sorting mechanism + indices[lpos.unit].push_back(lpos.index); + permutations[lpos.unit].push_back(i); + ++i; + } + trace.exit_state("compute indices"); + } + while(1) { int gr = 0; { - std::vector data; - { - trace.enter_state("compute indices"); - matrix_t indices(g.team().size()); - matrix_t permutations(g.team().size()); - int i = 0; - for(auto it = g.out_edges().lbegin(); it != g.out_edges().lend(); ++it) { - auto trg = g[it].target(); - auto lpos = trg.lpos(); - // TODO: use more sophsticated sorting mechanism - indices[lpos.unit].push_back(lpos.index); - permutations[lpos.unit].push_back(i); - ++i; - } - trace.exit_state("compute indices"); - data = internal::cc_get_data(indices, permutations, g, trace); - } + std::vector data = internal::cc_get_data(indices, permutations, + g, trace); trace.enter_state("compute pairs"); matrix_pair_t data_pairs(g.team().size()); { @@ -316,12 +317,10 @@ void connected_components(GraphType & g) { auto src_comp = src.attributes(); auto trg_comp = data[i]; if(src_comp.comp < trg_comp.comp) { - auto pair_it = pair_set[trg_comp.unit].find(trg_comp.comp); - if(pair_it == pair_set[trg_comp.unit].end()) { - data_pairs[trg_comp.unit].push_back( - std::make_pair(trg_comp.comp, src_comp) - ); - pair_set[trg_comp.unit].insert(trg_comp.comp); + auto & set = pair_set[trg_comp.unit]; + if(set.find(trg_comp.comp) == set.end()) { + data_pairs[trg_comp.unit].emplace_back(trg_comp.comp, src_comp); + set.insert(trg_comp.comp); } gr = 1; } From 6051bc20368a65b8a0c844fbb68c53e4e24d5e4e Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Fri, 23 Feb 2018 17:25:05 +0100 Subject: [PATCH 087/102] updated mst algorithm --- .../algorithm/graph/MinimumSpanningTree.h | 142 ++++++++++++------ 1 file changed, 94 insertions(+), 48 deletions(-) diff --git a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h index 2bfd80648..03628bffe 100644 --- a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h +++ b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h @@ -100,40 +100,78 @@ auto mst_get_data( return output; } - template auto mst_get_components( + matrix_t & indices, + int start, GraphType & graph, dash::util::Trace & trace -) -> std::vector { - trace.enter_state("send components"); +) -> std::unordered_map { typedef typename GraphType::vertex_properties_type vprop_t; - std::size_t size = graph.local_vertex_size() * sizeof(vprop_t); - std::size_t size_n = graph.local_vertex_size(); - std::vector sizes_recv(graph.team().size()); - std::vector displs_recv(graph.team().size()); - std::size_t total_recv = 0; - dart_allgather(&size, sizes_recv.data(), sizeof(std::size_t), + trace.enter_state("send indices"); + // exchange indices to get + std::vector sizes_send(indices.size()); + std::vector displs_send(indices.size()); + std::vector sizes_recv_data(indices.size()); + std::vector displs_recv_data(indices.size()); + std::vector indices_send; + int total_send = 0; + for(int i = 0; i < indices.size(); ++i) { + sizes_send[i] = indices[i].size(); + displs_send[i] = total_send; + sizes_recv_data[i] = indices[i].size() * sizeof(vprop_t); + displs_recv_data[i] = total_send * sizeof(vprop_t); + total_send += indices[i].size(); + } + indices_send.reserve(total_send); + for(auto & index_set : indices) { + indices_send.insert(indices_send.end(), index_set.begin(), + index_set.end()); + } + std::vector sizes_recv(indices.size()); + std::vector displs_recv(indices.size()); + std::vector sizes_send_data(indices.size()); + std::vector displs_send_data(indices.size()); + dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), DART_TYPE_BYTE, graph.team().dart_id()); - + int total_recv = 0; for(int i = 0; i < sizes_recv.size(); ++i) { + sizes_send_data[i] = sizes_recv[i] * sizeof(vprop_t); + displs_send_data[i] = total_recv * sizeof(vprop_t); displs_recv[i] = total_recv; total_recv += sizes_recv[i]; } + std::vector indices_recv(total_recv); + dart_alltoallv(indices_send.data(), sizes_send.data(), displs_send.data(), + DART_TYPE_INT, indices_recv.data(), sizes_recv.data(), + displs_recv.data(), graph.team().dart_id()); - std::vector components_send; - components_send.reserve(size_n); - for(int i = 0; i < size_n; ++i) { - components_send.push_back(graph.vertex_attributes(i)); - } - - std::vector components_recv(total_recv / sizeof(vprop_t)); - dart_allgatherv(components_send.data(), size, DART_TYPE_BYTE, - components_recv.data(), sizes_recv.data(), displs_recv.data(), - graph.team().dart_id()); + std::vector data_send; + data_send.reserve(total_recv); + std::vector data_recv(total_send); + trace.exit_state("send indices"); + // exchange data + trace.enter_state("get components"); + for(auto & index : indices_recv) { + data_send.push_back(graph.vertex_attributes(index - start)); + } + trace.exit_state("get components"); + trace.enter_state("send components"); + dart_alltoallv(data_send.data(), sizes_send_data.data(), + displs_send_data.data(), DART_TYPE_BYTE, data_recv.data(), + sizes_recv_data.data(), displs_recv_data.data(), + graph.team().dart_id()); trace.exit_state("send components"); - return components_recv; + + trace.enter_state("create map"); + std::unordered_map output; + output.reserve(total_send); + for(int i = 0; i < data_recv.size(); ++i) { + output[indices_send[i]] = data_recv[i]; + } + trace.exit_state("create map"); + return output; } template @@ -233,30 +271,30 @@ void minimum_spanning_tree(GraphType & g) { dash::barrier(); trace.exit_state("barrier"); + matrix_t indices(g.team().size()); + matrix_t permutations(g.team().size()); + { + trace.enter_state("compute indices"); + int i = 0; + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto v = g[it]; + for(auto e_it = v.out_edges().lbegin(); e_it != v.out_edges().lend(); + ++e_it) { + auto trg = g[e_it].target(); + auto lpos = trg.lpos(); + indices[lpos.unit].push_back(lpos.index); + permutations[lpos.unit].push_back(i); + ++i; + } + } + trace.exit_state("compute indices"); + } + while(1) { int gr = 0; { - std::vector data; - { - trace.enter_state("compute indices"); - matrix_t indices(g.team().size()); - matrix_t permutations(g.team().size()); - int i = 0; - for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { - auto v = g[it]; - for(auto e_it = v.out_edges().lbegin(); e_it != v.out_edges().lend(); - ++e_it) { - auto trg = g[e_it].target(); - auto lpos = trg.lpos(); - indices[lpos.unit].push_back(lpos.index); - permutations[lpos.unit].push_back(i); - ++i; - } - } - trace.exit_state("compute indices"); - data = internal::mst_get_data(indices, permutations, g, trace); - } - + std::vector data = internal::mst_get_data(indices, permutations, + g, trace); trace.enter_state("compute pairs"); matrix_min_pairs_t data_pairs(g.team().size()); std::vector> pair_set(g.team().size()); @@ -287,15 +325,11 @@ void minimum_spanning_tree(GraphType & g) { std::make_tuple(src_comp.comp, trg_comp_min, min_weight, myid, ledgepos) ); - set.insert(src_comp.comp); - } - set = pair_set[trg_comp_min.unit]; - if(set.find(trg_comp_min.comp) == set.end()) { data_pairs[trg_comp_min.unit].push_back( std::make_tuple(trg_comp_min.comp, src_comp, min_weight, myid, ledgepos) ); - set.insert(trg_comp_min.comp); + set.insert(src_comp.comp); } gr = 1; } @@ -312,7 +346,19 @@ void minimum_spanning_tree(GraphType & g) { if(gr_all == 0) break; while(1) { int pj = 0; - std::vector components = internal::mst_get_components(g, trace); + matrix_t indices(g.team().size()); + { + std::vector> comp_set(g.team().size()); + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto c = g[it].attributes(); + auto & set = comp_set[c.unit]; + if(set.find(c.comp) == set.end()) { + indices[c.unit].push_back(c.comp); + set.insert(c.comp); + } + } + } + auto components = internal::mst_get_components(indices, start, g, trace); trace.enter_state("set data (pj)"); for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { From cc0cafb3509483510f03f9cffe5cb9d817960fad Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 26 Feb 2018 14:46:03 +0100 Subject: [PATCH 088/102] further optimized algorithms --- dash/include/dash/Graph.h | 4 +- .../algorithm/graph/ConnectedComponents.h | 176 +++++++++++------- .../algorithm/graph/MinimumSpanningTree.h | 137 ++++++++++---- .../dash/memory/GlobHeapContiguousMem.h | 4 + 4 files changed, 214 insertions(+), 107 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 5ff60dad8..2f62b73de 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -768,8 +768,8 @@ class Graph { e.properties = prop; } - vertex_size_type local_vertex_size() { - return _glob_mem_vertex->local_size(); + vertex_size_type vertex_size(team_unit_t unit) { + return _glob_mem_vertex->global_size(unit); } private: diff --git a/dash/include/dash/algorithm/graph/ConnectedComponents.h b/dash/include/dash/algorithm/graph/ConnectedComponents.h index 6bdc5a69b..25d6edfef 100644 --- a/dash/include/dash/algorithm/graph/ConnectedComponents.h +++ b/dash/include/dash/algorithm/graph/ConnectedComponents.h @@ -11,6 +11,8 @@ template using matrix_pair_t = std::vector>>; namespace internal { +// TODO: merge overlapping code of the following functions with the functions +// provided in MinimumSpanningTree.h template auto cc_get_data( @@ -96,23 +98,55 @@ auto cc_get_components( int start, GraphType & graph, dash::util::Trace & trace -) -> std::unordered_map { +) -> std::pair< + std::unordered_map, + std::vector>> +{ typedef typename GraphType::vertex_properties_type vprop_t; trace.enter_state("send indices"); - // exchange indices to get + auto myid = graph.team().myid(); + std::size_t lsize = graph.vertex_size(myid); + std::vector thresholds(indices.size()); + for(int i = 0; i < thresholds.size(); ++i) { + team_unit_t unit { i }; + // TODO: find a threshold that provides an optimal tradeoff for any + // amount of units + thresholds[i] = graph.vertex_size(unit) * 20; + } + + std::vector total_sizes(indices.size()); std::vector sizes_send(indices.size()); + std::vector sizes_send_longdt(indices.size()); std::vector displs_send(indices.size()); std::vector sizes_recv_data(indices.size()); + std::vector cumul_sizes_recv_data(indices.size()); std::vector displs_recv_data(indices.size()); - std::vector indices_send; - int total_send = 0; for(int i = 0; i < indices.size(); ++i) { sizes_send[i] = indices[i].size(); - displs_send[i] = total_send; + sizes_send_longdt[i] = indices[i].size(); sizes_recv_data[i] = indices[i].size() * sizeof(vprop_t); - displs_recv_data[i] = total_send * sizeof(vprop_t); - total_send += indices[i].size(); } + dart_allreduce(sizes_send_longdt.data(), total_sizes.data(), indices.size(), + DART_TYPE_LONGLONG, DART_OP_SUM, graph.team().dart_id()); + int total_send = 0; + int total_recv_data = 0; + for(int i = 0; i < indices.size(); ++i) { + if(total_sizes[i] > thresholds[i]) { + sizes_send[i] = 0; + team_unit_t unit { i }; + sizes_recv_data[i] = graph.vertex_size(unit) * sizeof(vprop_t); + indices[i].clear(); + } else { + sizes_send[i] = indices[i].size(); + sizes_recv_data[i] = indices[i].size() * sizeof(vprop_t); + } + displs_send[i] = total_send; + displs_recv_data[i] = total_recv_data; + total_send += sizes_send[i]; + total_recv_data += sizes_recv_data[i]; + cumul_sizes_recv_data[i] = total_recv_data / sizeof(vprop_t); + } + std::vector indices_send; indices_send.reserve(total_send); for(auto & index_set : indices) { indices_send.insert(indices_send.end(), index_set.begin(), @@ -125,82 +159,68 @@ auto cc_get_components( dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), DART_TYPE_BYTE, graph.team().dart_id()); int total_recv = 0; + int total_send_data = 0; for(int i = 0; i < sizes_recv.size(); ++i) { - sizes_send_data[i] = sizes_recv[i] * sizeof(vprop_t); - displs_send_data[i] = total_recv * sizeof(vprop_t); - displs_recv[i] = total_recv; + if(total_sizes[myid] > thresholds[myid]) { + sizes_send_data[i] = lsize * sizeof(vprop_t); + displs_send_data[i] = 0; + } else { + sizes_send_data[i] = sizes_recv[i] * sizeof(vprop_t); + displs_send_data[i] = total_send_data; + displs_recv[i] = total_recv; + } total_recv += sizes_recv[i]; + total_send_data += sizes_send_data[i]; } std::vector indices_recv(total_recv); - dart_alltoallv(indices_send.data(), sizes_send.data(), displs_send.data(), - DART_TYPE_INT, indices_recv.data(), sizes_recv.data(), - displs_recv.data(), graph.team().dart_id()); - - std::vector data_send; - data_send.reserve(total_recv); - std::vector data_recv(total_send); - + dart_alltoallv(indices_send.data(), sizes_send.data(), displs_send.data(), + DART_TYPE_INT, indices_recv.data(), sizes_recv.data(), + displs_recv.data(), graph.team().dart_id()); trace.exit_state("send indices"); - // exchange data trace.enter_state("get components"); - for(auto & index : indices_recv) { - data_send.push_back(graph.vertex_attributes(index - start)); + std::vector data_send; + std::vector data_recv(total_recv_data / sizeof(vprop_t)); + if(total_sizes[myid] > thresholds[myid]) { + data_send.reserve(lsize); + for(int i = 0; i < lsize; ++i) { + data_send.push_back(graph.vertex_attributes(i)); + } + } else { + data_send.reserve(total_recv); + int i = 0; + for(auto & index : indices_recv) { + data_send.push_back(graph.vertex_attributes(index - start)); + ++i; + } } trace.exit_state("get components"); trace.enter_state("send components"); - dart_alltoallv(data_send.data(), sizes_send_data.data(), - displs_send_data.data(), DART_TYPE_BYTE, data_recv.data(), - sizes_recv_data.data(), displs_recv_data.data(), - graph.team().dart_id()); + dart_alltoallv(data_send.data(), sizes_send_data.data(), + displs_send_data.data(), DART_TYPE_BYTE, data_recv.data(), + sizes_recv_data.data(), displs_recv_data.data(), + graph.team().dart_id()); trace.exit_state("send components"); - trace.enter_state("create map"); - std::unordered_map output; - output.reserve(total_send); + std::unordered_map output_regular; + std::vector> output_contiguous(indices.size()); + output_regular.reserve(total_send); + int current_unit = 0; + int j = 0; for(int i = 0; i < data_recv.size(); ++i) { - output[indices_send[i]] = data_recv[i]; + if(i >= cumul_sizes_recv_data[current_unit]) { + ++current_unit; + } + if(total_sizes[current_unit] > thresholds[current_unit]) { + output_contiguous[current_unit].push_back(data_recv[i]); + } else { + output_regular[indices_send[j]] = data_recv[i]; + ++j; + } } trace.exit_state("create map"); - return output; + return std::make_pair(output_regular, output_contiguous); } -/* -template -auto cc_get_components( - GraphType & graph, - dash::util::Trace & trace -) -> std::vector { - trace.enter_state("send components"); - typedef typename GraphType::vertex_properties_type vprop_t; - std::size_t size = graph.local_vertex_size() * sizeof(vprop_t); - std::size_t size_n = graph.local_vertex_size(); - std::vector sizes_recv(graph.team().size()); - std::vector displs_recv(graph.team().size()); - int total_recv = 0; - dart_allgather(&size, sizes_recv.data(), sizeof(std::size_t), - DART_TYPE_BYTE, graph.team().dart_id()); - - for(int i = 0; i < sizes_recv.size(); ++i) { - displs_recv[i] = total_recv; - total_recv += sizes_recv[i]; - } - - std::vector components_send; - components_send.reserve(size_n); - for(int i = 0; i < size_n; ++i) { - components_send.push_back(graph.vertex_attributes(i)); - } - - std::vector components_recv(total_recv / sizeof(vprop_t)); - dart_allgatherv(components_send.data(), size, DART_TYPE_BYTE, - components_recv.data(), sizes_recv.data(), displs_recv.data(), - graph.team().dart_id()); - - trace.exit_state("send components"); - return components_recv; -} -*/ - template void cc_set_data( matrix_pair_t & data_pairs, @@ -284,6 +304,14 @@ void connected_components(GraphType & g) { dash::barrier(); trace.exit_state("barrier"); + std::vector unit_sizes(g.team().size()); + std::size_t total_size = 0; + for(int i = 0; i < g.team().size(); ++i) { + unit_sizes[i] = total_size; + team_unit_t unit { i }; + total_size += g.vertex_size(unit); + } + matrix_t indices(g.team().size()); matrix_t permutations(g.team().size()); { @@ -356,10 +384,16 @@ void connected_components(GraphType & g) { trace.enter_state("set data (pj)"); for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { auto v = g[it]; - auto comp = v.attributes().comp; - if(comp != 0) { - auto comp_next = components[comp]; - if(comp != comp_next.comp) { + auto comp = v.attributes(); + if(comp.comp != 0) { + vprop_t comp_next; + if(components.second[comp.unit].size() > 0) { + comp_next = + components.second[comp.unit][comp.comp - unit_sizes[comp.unit]]; + } else { + comp_next = components.first[comp.comp]; + } + if(comp.comp != comp_next.comp) { v.set_attributes(comp_next); pj = 1; } diff --git a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h index 03628bffe..8e849ee76 100644 --- a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h +++ b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h @@ -20,6 +20,8 @@ template using matrix_pair_t = std::vector>>; namespace internal { +// TODO: merge overlapping code of the following functions with the functions +// provided in ConnectedComponents.h template auto mst_get_data( @@ -106,23 +108,55 @@ auto mst_get_components( int start, GraphType & graph, dash::util::Trace & trace -) -> std::unordered_map { +) -> std::pair< + std::unordered_map, + std::vector>> +{ typedef typename GraphType::vertex_properties_type vprop_t; trace.enter_state("send indices"); - // exchange indices to get + auto myid = graph.team().myid(); + std::size_t lsize = graph.vertex_size(myid); + std::vector thresholds(indices.size()); + for(int i = 0; i < thresholds.size(); ++i) { + team_unit_t unit { i }; + // TODO: find a threshold that provides an optimal tradeoff for any + // amount of units + thresholds[i] = graph.vertex_size(unit) * 20; + } + + std::vector total_sizes(indices.size()); std::vector sizes_send(indices.size()); + std::vector sizes_send_longdt(indices.size()); std::vector displs_send(indices.size()); std::vector sizes_recv_data(indices.size()); + std::vector cumul_sizes_recv_data(indices.size()); std::vector displs_recv_data(indices.size()); - std::vector indices_send; - int total_send = 0; for(int i = 0; i < indices.size(); ++i) { sizes_send[i] = indices[i].size(); - displs_send[i] = total_send; + sizes_send_longdt[i] = indices[i].size(); sizes_recv_data[i] = indices[i].size() * sizeof(vprop_t); - displs_recv_data[i] = total_send * sizeof(vprop_t); - total_send += indices[i].size(); } + dart_allreduce(sizes_send_longdt.data(), total_sizes.data(), indices.size(), + DART_TYPE_LONGLONG, DART_OP_SUM, graph.team().dart_id()); + int total_send = 0; + int total_recv_data = 0; + for(int i = 0; i < indices.size(); ++i) { + if(total_sizes[i] > thresholds[i]) { + sizes_send[i] = 0; + team_unit_t unit { i }; + sizes_recv_data[i] = graph.vertex_size(unit) * sizeof(vprop_t); + indices[i].clear(); + } else { + sizes_send[i] = indices[i].size(); + sizes_recv_data[i] = indices[i].size() * sizeof(vprop_t); + } + displs_send[i] = total_send; + displs_recv_data[i] = total_recv_data; + total_send += sizes_send[i]; + total_recv_data += sizes_recv_data[i]; + cumul_sizes_recv_data[i] = total_recv_data / sizeof(vprop_t); + } + std::vector indices_send; indices_send.reserve(total_send); for(auto & index_set : indices) { indices_send.insert(indices_send.end(), index_set.begin(), @@ -135,43 +169,64 @@ auto mst_get_components( dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), DART_TYPE_BYTE, graph.team().dart_id()); int total_recv = 0; + int total_send_data = 0; for(int i = 0; i < sizes_recv.size(); ++i) { - sizes_send_data[i] = sizes_recv[i] * sizeof(vprop_t); - displs_send_data[i] = total_recv * sizeof(vprop_t); - displs_recv[i] = total_recv; + if(total_sizes[myid] > thresholds[myid]) { + sizes_send_data[i] = lsize * sizeof(vprop_t); + displs_send_data[i] = 0; + } else { + sizes_send_data[i] = sizes_recv[i] * sizeof(vprop_t); + displs_send_data[i] = total_send_data; + displs_recv[i] = total_recv; + } total_recv += sizes_recv[i]; + total_send_data += sizes_send_data[i]; } std::vector indices_recv(total_recv); - dart_alltoallv(indices_send.data(), sizes_send.data(), displs_send.data(), - DART_TYPE_INT, indices_recv.data(), sizes_recv.data(), - displs_recv.data(), graph.team().dart_id()); - - std::vector data_send; - data_send.reserve(total_recv); - std::vector data_recv(total_send); - + dart_alltoallv(indices_send.data(), sizes_send.data(), displs_send.data(), + DART_TYPE_INT, indices_recv.data(), sizes_recv.data(), + displs_recv.data(), graph.team().dart_id()); trace.exit_state("send indices"); - // exchange data trace.enter_state("get components"); - for(auto & index : indices_recv) { - data_send.push_back(graph.vertex_attributes(index - start)); + std::vector data_send; + std::vector data_recv(total_recv_data / sizeof(vprop_t)); + if(total_sizes[myid] > thresholds[myid]) { + data_send.reserve(lsize); + for(int i = 0; i < lsize; ++i) { + data_send.push_back(graph.vertex_attributes(i)); + } + } else { + data_send.reserve(total_recv); + for(auto & index : indices_recv) { + data_send.push_back(graph.vertex_attributes(index - start)); + } } trace.exit_state("get components"); trace.enter_state("send components"); - dart_alltoallv(data_send.data(), sizes_send_data.data(), - displs_send_data.data(), DART_TYPE_BYTE, data_recv.data(), - sizes_recv_data.data(), displs_recv_data.data(), - graph.team().dart_id()); + dart_alltoallv(data_send.data(), sizes_send_data.data(), + displs_send_data.data(), DART_TYPE_BYTE, data_recv.data(), + sizes_recv_data.data(), displs_recv_data.data(), + graph.team().dart_id()); trace.exit_state("send components"); - trace.enter_state("create map"); - std::unordered_map output; - output.reserve(total_send); + std::unordered_map output_regular; + std::vector> output_contiguous(indices.size()); + output_regular.reserve(total_send); + int current_unit = 0; + int j = 0; for(int i = 0; i < data_recv.size(); ++i) { - output[indices_send[i]] = data_recv[i]; + if(i >= cumul_sizes_recv_data[current_unit]) { + ++current_unit; + } + if(total_sizes[current_unit] > thresholds[current_unit]) { + output_contiguous[current_unit].push_back(data_recv[i]); + } else { + output_regular[indices_send[j]] = data_recv[i]; + ++j; + } } trace.exit_state("create map"); - return output; + return std::make_pair(output_regular, output_contiguous); } template @@ -271,6 +326,14 @@ void minimum_spanning_tree(GraphType & g) { dash::barrier(); trace.exit_state("barrier"); + std::vector unit_sizes(g.team().size()); + std::size_t total_size = 0; + for(int i = 0; i < g.team().size(); ++i) { + unit_sizes[i] = total_size; + team_unit_t unit { i }; + total_size += g.vertex_size(unit); + } + matrix_t indices(g.team().size()); matrix_t permutations(g.team().size()); { @@ -363,10 +426,16 @@ void minimum_spanning_tree(GraphType & g) { trace.enter_state("set data (pj)"); for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { auto v = g[it]; - auto comp = v.attributes().comp; - if(comp != 0) { - auto comp_next = components[comp]; - if(comp > comp_next.comp) { + auto comp = v.attributes(); + if(comp.comp != 0) { + vprop_t comp_next; + if(components.second[comp.unit].size() > 0) { + comp_next = + components.second[comp.unit][comp.comp - unit_sizes[comp.unit]]; + } else { + comp_next = components.first[comp.comp]; + } + if(comp.comp > comp_next.comp) { v.set_attributes(comp_next); pj = 1; } diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h index 6c5468518..e34ac4ddd 100644 --- a/dash/include/dash/memory/GlobHeapContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -426,6 +426,10 @@ class GlobHeapContiguousMem { (*_container)[index] = value; } + size_type global_size(team_unit_t unit) { + return _bucket_cumul_sizes[unit][0]; + } + private: /** From 17e0b48af3caf0b986388817f603f2de4af511dd Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 26 Feb 2018 18:06:35 +0100 Subject: [PATCH 089/102] added meta functions --- dash/include/dash/Graph.h | 41 ++------ dash/include/dash/graph/EdgeIterator.h | 78 ++++++++++++---- dash/include/dash/graph/InEdgeIterator.h | 85 ++++++++++++++--- dash/include/dash/graph/OutEdgeIterator.h | 86 ++++++++++++++--- dash/include/dash/graph/VertexIterator.h | 93 ++++++++++++++++--- .../include/dash/memory/GlobHeapCombinedMem.h | 12 +++ .../dash/memory/GlobHeapContiguousMem.h | 14 ++- 7 files changed, 316 insertions(+), 93 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 2f62b73de..f44fddf22 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -704,6 +704,11 @@ class Graph { _glob_mem_edge->add_globmem(*_glob_mem_in_edge); // Register deallocator at the respective team instance _team->register_deallocator(this, std::bind(&Graph::deallocate, this)); + + _vertices = vertex_it_wrapper(this); + _in_edges = in_edge_it_wrapper(this); + _out_edges = out_edge_it_wrapper(this); + _edges = edge_it_wrapper(this); return true; } @@ -744,34 +749,6 @@ class Graph { return *_team; } - global_vertex_iterator vertex_gptr(local_vertex_iterator it) { - return global_vertex_iterator(_glob_mem_vertex, _myid, it.pos()); - } - - vertex_properties_type & vertex_attributes(vertex_size_type local_vertex) { - return _glob_mem_vertex->get(local_vertex).properties; - } - - void set_vertex_attributes(vertex_size_type local_vertex, - vertex_properties_type prop) { - auto & v = _glob_mem_vertex->get(local_vertex); - v.properties = prop; - } - - edge_properties_type & out_edge_attributes(edge_size_type local_edge) { - return _glob_mem_out_edge->get(local_edge).properties; - } - - void set_out_edge_attributes(edge_size_type local_edge, - edge_properties_type prop) { - auto & e = _glob_mem_out_edge->get(local_edge); - e.properties = prop; - } - - vertex_size_type vertex_size(team_unit_t unit) { - return _glob_mem_vertex->global_size(unit); - } - private: /** @@ -864,13 +841,13 @@ class Graph { /** Edges that have to be added to vertices on another unit in next commit */ edge_list_type _remote_edges; /** wrapper for vertex iterator ranges */ - vertex_it_wrapper _vertices = vertex_it_wrapper(this); + vertex_it_wrapper _vertices; /** wrapper for edge iterator ranges */ - edge_it_wrapper _edges = edge_it_wrapper(this); + edge_it_wrapper _edges; /** wrapper for in-edge iterator ranges */ - in_edge_it_wrapper _in_edges = in_edge_it_wrapper(this); + in_edge_it_wrapper _in_edges; /** wrapper for out-edge iterator ranges */ - out_edge_it_wrapper _out_edges = out_edge_it_wrapper(this); + out_edge_it_wrapper _out_edges; }; diff --git a/dash/include/dash/graph/EdgeIterator.h b/dash/include/dash/graph/EdgeIterator.h index 607f44f9f..0ce497fb4 100644 --- a/dash/include/dash/graph/EdgeIterator.h +++ b/dash/include/dash/graph/EdgeIterator.h @@ -9,25 +9,33 @@ namespace dash { template struct EdgeIteratorWrapper { - typedef Graph graph_type; - typedef typename Graph::global_edge_iterator iterator; - typedef const iterator const_iterator; - typedef typename Graph::local_edge_iterator local_iterator; - typedef const local_iterator const_local_iterator; - typedef typename Graph::edge_properties_type edge_properties_type; + typedef Graph graph_type; + typedef typename graph_type::glob_mem_edge_comb_type glob_mem_type; + typedef typename Graph::global_edge_iterator iterator; + typedef const iterator const_iterator; + typedef typename Graph::local_edge_iterator local_iterator; + typedef const local_iterator const_local_iterator; + typedef typename graph_type::edge_properties_type properties_type; + typedef typename graph_type::edge_size_type size_type; /** * Constructs the wrapper. */ EdgeIteratorWrapper(graph_type * graph) - : _graph(graph) + : _graph(graph), + _gmem(graph->_glob_mem_edge) { } + /* + * Default constructor. + */ + EdgeIteratorWrapper() = default; + /** * Returns global iterator to the beginning of the edge list. */ iterator begin() { - return _graph->_glob_mem_edge->begin(); + return _gmem->begin(); //return iterator(_graph->_glob_mem_out_edge, 0); } @@ -35,7 +43,7 @@ struct EdgeIteratorWrapper { * Returns global iterator to the beginning of the edge list. */ const_iterator begin() const { - return _graph->_glob_mem_edge->begin(); + return _gmem->begin(); //return iterator(_graph->_glob_mem_out_edge, 0); } @@ -43,49 +51,85 @@ struct EdgeIteratorWrapper { * Returns global iterator to the end of the edge list. */ iterator end() { - return _graph->_glob_mem_edge->end(); - //return iterator(_graph->_glob_mem_out_edge, _graph->_glob_mem_out_edge->size()); + return _gmem->end(); } /** * Returns global iterator to the end of the edge list. */ const_iterator end() const { - return _graph->_glob_mem_edge->end(); - //return iterator(_graph->_glob_mem_out_edge, _graph->_glob_mem_out_edge->size()); + return _gmem->end(); } /** * Returns local iterator to the beginning of the edge list. */ local_iterator lbegin() { - return _graph->_glob_mem_edge->lbegin(); + return _gmem->lbegin(); } /** * Returns local iterator to the beginning of the edge list. */ const_local_iterator lbegin() const { - return _graph->_glob_mem_edge->lbegin(); + return _gmem->lbegin(); } /** * Returns local iterator to the end of the edge list. */ local_iterator lend() { - return _graph->_glob_mem_edge->lend(); + return _gmem->lend(); } /** * Returns local iterator to the end of the edge list. */ const_local_iterator lend() const { - return _graph->_glob_mem_edge->lend(); + return _gmem->lend(); + } + + /** + * Returns the amount of edges in the whole graph. + */ + size_type size() { + return _gmem->size(); + } + + /** + * Returns the amount of in-edges the specified unit currently holds in + * global memory space. + */ + size_type size(team_unit_t unit) { + return _gmem->size(unit); + } + + /** + * Returns the amount of edges this unit currently holds in local memory + * space. + */ + size_type lsize() { + return _gmem->lsize(); + } + + /* + * Returns whether theare are edges in global memory space. + */ + bool empty() { + return size() == 0; + } + + /** + * Returns the maximum number of edges the graph can store. + */ + size_type max_size() { + return std::numeric_limits::max(); } private: graph_type * _graph; + glob_mem_type * _gmem; }; diff --git a/dash/include/dash/graph/InEdgeIterator.h b/dash/include/dash/graph/InEdgeIterator.h index f1d9438e8..80cfb7984 100644 --- a/dash/include/dash/graph/InEdgeIterator.h +++ b/dash/include/dash/graph/InEdgeIterator.h @@ -10,24 +10,32 @@ template struct InEdgeIteratorWrapper { typedef Graph graph_type; + typedef typename graph_type::glob_mem_edge_type glob_mem_type; typedef typename Graph::global_in_edge_iterator iterator; typedef const iterator const_iterator; typedef typename Graph::local_in_edge_iterator local_iterator; typedef const local_iterator const_local_iterator; - typedef typename Graph::edge_properties_type edge_properties_type; + typedef typename graph_type::edge_properties_type properties_type; + typedef typename graph_type::edge_size_type size_type; /** * Constructs the wrapper. */ InEdgeIteratorWrapper(graph_type * graph) - : _graph(graph) + : _graph(graph), + _gmem(graph->_glob_mem_in_edge) { } + + /** + * Default constructor. + */ + InEdgeIteratorWrapper() = default; /** * Returns global iterator to the beginning of the edge list. */ iterator begin() { - return _graph->_glob_mem_in_edge->begin(); + return _gmem->begin(); //return iterator(_graph->_glob_mem_in_edge, 0); } @@ -35,7 +43,7 @@ struct InEdgeIteratorWrapper { * Returns global iterator to the beginning of the edge list. */ const_iterator begin() const { - return _graph->_glob_mem_in_edge->begin(); + return _gmem->begin(); //return iterator(_graph->_glob_mem_in_edge, 0); } @@ -43,49 +51,102 @@ struct InEdgeIteratorWrapper { * Returns global iterator to the end of the edge list. */ iterator end() { - return _graph->_glob_mem_in_edge->end(); - //return iterator(_graph->_glob_mem_in_edge, _graph->_glob_mem_in_edge->size()); + return _gmem->end(); } /** * Returns global iterator to the end of the edge list. */ const_iterator end() const { - return _graph->_glob_mem_in_edge->end(); - //return iterator(_graph->_glob_mem_in_edge, _graph->_glob_mem_in_edge->size()); + return _gmem->end(); } /** * Returns local iterator to the beginning of the edge list. */ local_iterator lbegin() { - return _graph->_glob_mem_in_edge->lbegin(); + return _gmem->lbegin(); } /** * Returns local iterator to the beginning of the edge list. */ const_local_iterator lbegin() const { - return _graph->_glob_mem_in_edge->lbegin(); + return _gmem->lbegin(); } /** * Returns local iterator to the end of the edge list. */ local_iterator lend() { - return _graph->_glob_mem_in_edge->lend(); + return _gmem->lend(); } /** * Returns local iterator to the end of the edge list. */ const_local_iterator lend() const { - return _graph->_glob_mem_in_edge->lend(); + return _gmem->lend(); + } + + /** + * Directly gets the attributes of an in-edge identified by a local offset + */ + properties_type & attributes(size_type local_index) { + // TODO: Allow this only when there are no uncommitted changes + return _gmem->get(local_index).properties; + } + + /** + * Directly sets the attributes of an in-edge identified by a local offset + */ + void set_attributes(size_type local_index, properties_type prop) { + // TODO: Allow this only when there are no uncommitted changes + auto & v = _gmem->get(local_index); + v.properties = prop; + } + + /** + * Returns the amount of in-edges in the whole graph. + */ + size_type size() { + return _gmem->size(); + } + + /** + * Returns the amount of in-edges the specified unit currently holds in + * global memory space. + */ + size_type size(team_unit_t unit) { + return _gmem->size(unit); + } + + /** + * Returns the amount of in-edges this unit currently holds in local memory + * space. + */ + size_type lsize() { + return _gmem->lsize(); + } + + /* + * Returns whether theare are in-edges in global memory space. + */ + bool empty() { + return size() == 0; + } + + /** + * Returns the maximum number of in-edges the graph can store. + */ + size_type max_size() { + return std::numeric_limits::max(); } private: graph_type * _graph; + glob_mem_type * _gmem; }; diff --git a/dash/include/dash/graph/OutEdgeIterator.h b/dash/include/dash/graph/OutEdgeIterator.h index 97d6db2ec..e3000ddcf 100644 --- a/dash/include/dash/graph/OutEdgeIterator.h +++ b/dash/include/dash/graph/OutEdgeIterator.h @@ -10,24 +10,31 @@ template struct OutEdgeIteratorWrapper { typedef Graph graph_type; + typedef typename graph_type::glob_mem_edge_type glob_mem_type; typedef typename Graph::global_out_edge_iterator iterator; typedef const iterator const_iterator; typedef typename Graph::local_out_edge_iterator local_iterator; typedef const local_iterator const_local_iterator; - typedef typename Graph::edge_properties_type edge_properties_type; - + typedef typename graph_type::edge_properties_type properties_type; + typedef typename graph_type::edge_size_type size_type; /** * Constructs the wrapper. */ OutEdgeIteratorWrapper(graph_type * graph) - : _graph(graph) + : _graph(graph), + _gmem(graph->_glob_mem_out_edge) { } + /* + * Default constructor. + */ + OutEdgeIteratorWrapper() = default; + /** * Returns global iterator to the beginning of the edge list. */ iterator begin() { - return _graph->_glob_mem_out_edge->begin(); + return _gmem->begin(); //return iterator(_graph->_glob_mem_out_edge, 0); } @@ -35,7 +42,7 @@ struct OutEdgeIteratorWrapper { * Returns global iterator to the beginning of the edge list. */ const_iterator begin() const { - return _graph->_glob_mem_out_edge->begin(); + return _gmem->begin(); //return iterator(_graph->_glob_mem_out_edge, 0); } @@ -43,49 +50,102 @@ struct OutEdgeIteratorWrapper { * Returns global iterator to the end of the edge list. */ iterator end() { - return _graph->_glob_mem_out_edge->end(); - //return iterator(_graph->_glob_mem_out_edge, _graph->_glob_mem_out_edge->size()); + return _gmem->end(); } /** * Returns global iterator to the end of the edge list. */ const_iterator end() const { - return _graph->_glob_mem_out_edge->end(); - //return iterator(_graph->_glob_mem_out_edge, _graph->_glob_mem_out_edge->size()); + return _gmem->end(); } /** * Returns local iterator to the beginning of the edge list. */ local_iterator lbegin() { - return _graph->_glob_mem_out_edge->lbegin(); + return _gmem->lbegin(); } /** * Returns local iterator to the beginning of the edge list. */ const_local_iterator lbegin() const { - return _graph->_glob_mem_out_edge->lbegin(); + return _gmem->lbegin(); } /** * Returns local iterator to the end of the edge list. */ local_iterator lend() { - return _graph->_glob_mem_out_edge->lend(); + return _gmem->lend(); } /** * Returns local iterator to the end of the edge list. */ const_local_iterator lend() const { - return _graph->_glob_mem_out_edge->lend(); + return _gmem->lend(); + } + + /** + * Directly gets the attributes of an out-edge identified by a local offset + */ + properties_type & attributes(size_type local_index) { + // TODO: Allow this only when there are no uncommitted changes + return _gmem->get(local_index).properties; + } + + /** + * Directly sets the attributes of an out-edge identified by a local offset + */ + void set_attributes(size_type local_index, properties_type prop) { + // TODO: Allow this only when there are no uncommitted changes + auto & v = _gmem->get(local_index); + v.properties = prop; + } + + /** + * Returns the amount of out-edges in the whole graph. + */ + size_type size() { + return _gmem->size(); + } + + /** + * Returns the amount of out-edges the specified unit currently holds in + * global memory space. + */ + size_type size(team_unit_t unit) { + return _gmem->size(unit); + } + + /** + * Returns the amount of out-edges this unit currently holds in local memory + * space. + */ + size_type lsize() { + return _gmem->lsize(); + } + + /* + * Returns whether theare are out-edges in global memory space. + */ + bool empty() { + return size() == 0; + } + + /** + * Returns the maximum number of out-edges the graph can store. + */ + size_type max_size() { + return std::numeric_limits::max(); } private: graph_type * _graph; + glob_mem_type * _gmem; }; diff --git a/dash/include/dash/graph/VertexIterator.h b/dash/include/dash/graph/VertexIterator.h index 735dde4a3..c02d75783 100644 --- a/dash/include/dash/graph/VertexIterator.h +++ b/dash/include/dash/graph/VertexIterator.h @@ -9,78 +9,143 @@ namespace dash { template struct VertexIteratorWrapper { - typedef Graph graph_type; - typedef typename Graph::global_vertex_iterator iterator; - typedef const iterator const_iterator; - typedef typename Graph::local_vertex_iterator local_iterator; - typedef const local_iterator const_local_iterator; + typedef Graph graph_type; + typedef typename graph_type::glob_mem_vert_type glob_mem_type; + typedef typename Graph::global_vertex_iterator iterator; + typedef const iterator const_iterator; + typedef typename Graph::local_vertex_iterator local_iterator; + typedef const local_iterator const_local_iterator; + typedef typename graph_type::vertex_properties_type properties_type; + typedef typename graph_type::vertex_size_type size_type; /** * Constructs the wrapper. */ VertexIteratorWrapper(graph_type * graph) - : _graph(graph) + : _graph(graph), + _gmem(graph->_glob_mem_vertex) { } + /* + * Default constructor. + */ + VertexIteratorWrapper() = default; + /** * Returns global iterator to the beginning of the vertex list. */ iterator begin() { - return _graph->_glob_mem_vertex->begin(); + return _gmem->begin(); } /** * Returns global iterator to the beginning of the vertex list. */ const_iterator begin() const { - return _graph->_glob_mem_vertex->begin(); + return _gmem->begin(); } /** * Returns global iterator to the end of the vertex list. */ iterator end() { - return _graph->_glob_mem_vertex->end(); + return _gmem->end(); } /** * Returns global iterator to the end of the vertex list. */ const_iterator end() const { - return _graph->_glob_mem_vertex->end(); + return _gmem->end(); } /** * Returns local iterator to the beginning of the vertex list. */ local_iterator lbegin() { - return _graph->_glob_mem_vertex->lbegin(); + return _gmem->lbegin(); } /** * Returns local iterator to the beginning of the vertex list. */ const_local_iterator lbegin() const { - return _graph->_glob_mem_vertex->lbegin(); + return _gmem->lbegin(); } /** * Returns local iterator to the end of the vertex list. */ local_iterator lend() { - return _graph->_glob_mem_vertex->lend(); + return _gmem->lend(); } /** * Returns local iterator to the end of the vertex list. */ const_local_iterator lend() const { - return _graph->_glob_mem_vertex->lend(); + return _gmem->lend(); + } + + + /** + * Directly gets the attributes of a vertex identified by a local offset + */ + properties_type & attributes(size_type local_index) { + // TODO: Allow this only when there are no uncommitted changes + return _gmem->get(local_index).properties; + } + + /** + * Directly sets the attributes of a vertex identified by a local offset + */ + void set_attributes(size_type local_index, properties_type prop) { + // TODO: Allow this only when there are no uncommitted changes + auto & v = _gmem->get(local_index); + v.properties = prop; + } + + /** + * Returns the amount of vertices in the whole graph. + */ + size_type size() { + return _gmem->size(); + } + + /** + * Returns the amount of vertices the specified unit currently holds in + * global memory space. + */ + size_type size(team_unit_t unit) { + return _gmem->size(unit); + } + + /** + * Returns the amount of vertices this unit currently holds in local memory + * space. + */ + size_type lsize() { + return _gmem->lsize(); + } + + /* + * Returns whether theare are vertices in global memory space. + */ + bool empty() { + return size() == 0; + } + + /** + * Returns the maximum number of vertices the graph can store. + */ + size_type max_size() { + return std::numeric_limits::max(); } private: graph_type * _graph; + glob_mem_type * _gmem; }; diff --git a/dash/include/dash/memory/GlobHeapCombinedMem.h b/dash/include/dash/memory/GlobHeapCombinedMem.h index a8d7e3af4..ff5cb7f73 100644 --- a/dash/include/dash/memory/GlobHeapCombinedMem.h +++ b/dash/include/dash/memory/GlobHeapCombinedMem.h @@ -156,6 +156,18 @@ class GlobHeapCombinedMem { return bucket_size; } + size_type lsize() { + return _local_size; + } + + size_type size(team_unit_t unit) { + return _bucket_cumul_sizes[unit][1]; + } + + size_type size() { + return _size; + } + private: /** diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h index e34ac4ddd..49e9d96ab 100644 --- a/dash/include/dash/memory/GlobHeapContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -418,18 +418,22 @@ class GlobHeapContiguousMem { return (*_container)[index]; } - size_type local_size() { - return _local_size; - } - void set(index_type index, value_type value) { (*_container)[index] = value; } - size_type global_size(team_unit_t unit) { + size_type lsize() { + return _local_size; + } + + size_type size(team_unit_t unit) { return _bucket_cumul_sizes[unit][0]; } + size_type size() { + return _size; + } + private: /** From 0453ca15eb9b6b27e0821ad4774706d61f7ef7b8 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Mon, 26 Feb 2018 18:07:15 +0100 Subject: [PATCH 090/102] updated algorithms --- .../algorithm/graph/ConnectedComponents.h | 35 +++++++++--------- .../algorithm/graph/MinimumSpanningTree.h | 37 +++++++++---------- 2 files changed, 35 insertions(+), 37 deletions(-) diff --git a/dash/include/dash/algorithm/graph/ConnectedComponents.h b/dash/include/dash/algorithm/graph/ConnectedComponents.h index 25d6edfef..c5229b3e6 100644 --- a/dash/include/dash/algorithm/graph/ConnectedComponents.h +++ b/dash/include/dash/algorithm/graph/ConnectedComponents.h @@ -68,7 +68,7 @@ auto cc_get_data( // exchange data trace.enter_state("get components"); for(auto & index : indices_recv) { - data_send.push_back(graph.vertex_attributes(index)); + data_send.push_back(graph.vertices().attributes(index)); } trace.exit_state("get components"); trace.enter_state("send components"); @@ -105,13 +105,13 @@ auto cc_get_components( typedef typename GraphType::vertex_properties_type vprop_t; trace.enter_state("send indices"); auto myid = graph.team().myid(); - std::size_t lsize = graph.vertex_size(myid); + std::size_t lsize = graph.vertices().size(myid); std::vector thresholds(indices.size()); for(int i = 0; i < thresholds.size(); ++i) { team_unit_t unit { i }; // TODO: find a threshold that provides an optimal tradeoff for any // amount of units - thresholds[i] = graph.vertex_size(unit) * 20; + thresholds[i] = graph.vertices().size(unit) * 20; } std::vector total_sizes(indices.size()); @@ -134,7 +134,7 @@ auto cc_get_components( if(total_sizes[i] > thresholds[i]) { sizes_send[i] = 0; team_unit_t unit { i }; - sizes_recv_data[i] = graph.vertex_size(unit) * sizeof(vprop_t); + sizes_recv_data[i] = graph.vertices().size(unit) * sizeof(vprop_t); indices[i].clear(); } else { sizes_send[i] = indices[i].size(); @@ -183,13 +183,13 @@ auto cc_get_components( if(total_sizes[myid] > thresholds[myid]) { data_send.reserve(lsize); for(int i = 0; i < lsize; ++i) { - data_send.push_back(graph.vertex_attributes(i)); + data_send.push_back(graph.vertices().attributes(i)); } } else { data_send.reserve(total_recv); int i = 0; for(auto & index : indices_recv) { - data_send.push_back(graph.vertex_attributes(index - start)); + data_send.push_back(graph.vertices().attributes(index - start)); ++i; } } @@ -261,7 +261,7 @@ void cc_set_data( trace.enter_state("set components"); for(auto & pair : pairs_recv) { int ind = pair.first - start; - graph.set_vertex_attributes(pair.first - start, pair.second); + graph.vertices().set_attributes(pair.first - start, pair.second); } trace.exit_state("set components"); } @@ -289,13 +289,20 @@ void connected_components(GraphType & g) { trace.enter_state("vertex setup"); int i = 0; auto myid = dash::myid(); - auto git = g.vertex_gptr(g.vertices().lbegin()); - auto start = git.pos(); + std::vector unit_sizes(g.team().size()); + std::size_t total_size = 0; + for(int i = 0; i < g.team().size(); ++i) { + unit_sizes[i] = total_size; + team_unit_t unit { i }; + total_size += g.vertices().size(unit); + } + + auto start = unit_sizes[myid]; for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { auto index = it.pos(); auto gindex = index + start; vprop_t prop { gindex, myid }; - g.set_vertex_attributes(index, prop); + g.vertices().set_attributes(index, prop); ++i; } trace.exit_state("vertex setup"); @@ -304,14 +311,6 @@ void connected_components(GraphType & g) { dash::barrier(); trace.exit_state("barrier"); - std::vector unit_sizes(g.team().size()); - std::size_t total_size = 0; - for(int i = 0; i < g.team().size(); ++i) { - unit_sizes[i] = total_size; - team_unit_t unit { i }; - total_size += g.vertex_size(unit); - } - matrix_t indices(g.team().size()); matrix_t permutations(g.team().size()); { diff --git a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h index 8e849ee76..c83a3ee6e 100644 --- a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h +++ b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h @@ -77,7 +77,7 @@ auto mst_get_data( // exchange data trace.enter_state("get components"); for(auto & index : indices_recv) { - data_send.push_back(graph.vertex_attributes(index)); + data_send.push_back(graph.vertices().attributes(index)); } trace.exit_state("get components"); trace.enter_state("send components"); @@ -115,13 +115,13 @@ auto mst_get_components( typedef typename GraphType::vertex_properties_type vprop_t; trace.enter_state("send indices"); auto myid = graph.team().myid(); - std::size_t lsize = graph.vertex_size(myid); + std::size_t lsize = graph.vertices().size(myid); std::vector thresholds(indices.size()); for(int i = 0; i < thresholds.size(); ++i) { team_unit_t unit { i }; // TODO: find a threshold that provides an optimal tradeoff for any // amount of units - thresholds[i] = graph.vertex_size(unit) * 20; + thresholds[i] = graph.vertices().size(unit) * 20; } std::vector total_sizes(indices.size()); @@ -144,7 +144,7 @@ auto mst_get_components( if(total_sizes[i] > thresholds[i]) { sizes_send[i] = 0; team_unit_t unit { i }; - sizes_recv_data[i] = graph.vertex_size(unit) * sizeof(vprop_t); + sizes_recv_data[i] = graph.vertices().size(unit) * sizeof(vprop_t); indices[i].clear(); } else { sizes_send[i] = indices[i].size(); @@ -193,12 +193,12 @@ auto mst_get_components( if(total_sizes[myid] > thresholds[myid]) { data_send.reserve(lsize); for(int i = 0; i < lsize; ++i) { - data_send.push_back(graph.vertex_attributes(i)); + data_send.push_back(graph.vertices().attributes(i)); } } else { data_send.reserve(total_recv); for(auto & index : indices_recv) { - data_send.push_back(graph.vertex_attributes(index - start)); + data_send.push_back(graph.vertices().attributes(index - start)); } } trace.exit_state("get components"); @@ -280,10 +280,10 @@ void mst_set_data_min( trace.exit_state("send pairs"); trace.enter_state("set components"); for(auto & pair : mapping) { - graph.set_vertex_attributes(pair.first - start, std::get<1>(pair.second)); + graph.vertices().set_attributes(pair.first - start, std::get<1>(pair.second)); if(dash::myid() == get<3>(pair.second)) { typename GraphType::edge_properties_type eprop { -1 }; - graph.set_out_edge_attributes(std::get<4>(pair.second), eprop); + graph.out_edges().set_attributes(std::get<4>(pair.second), eprop); } } trace.exit_state("set components"); @@ -311,13 +311,20 @@ void minimum_spanning_tree(GraphType & g) { trace.enter_state("vertex setup"); int i = 0; auto myid = dash::myid(); - auto git = g.vertex_gptr(g.vertices().lbegin()); - auto start = git.pos(); + std::vector unit_sizes(g.team().size()); + std::size_t total_size = 0; + for(int i = 0; i < g.team().size(); ++i) { + unit_sizes[i] = total_size; + team_unit_t unit { i }; + total_size += g.vertices().size(unit); + } + + auto start = unit_sizes[myid]; for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { auto index = it.pos(); auto gindex = index + start; vprop_t prop { gindex, myid }; - g.set_vertex_attributes(index, prop); + g.vertices().set_attributes(index, prop); ++i; } trace.exit_state("vertex setup"); @@ -326,14 +333,6 @@ void minimum_spanning_tree(GraphType & g) { dash::barrier(); trace.exit_state("barrier"); - std::vector unit_sizes(g.team().size()); - std::size_t total_size = 0; - for(int i = 0; i < g.team().size(); ++i) { - unit_sizes[i] = total_size; - team_unit_t unit { i }; - total_size += g.vertex_size(unit); - } - matrix_t indices(g.team().size()); matrix_t permutations(g.team().size()); { From 61376d8a58043764cb375a7890e4cee9615516e0 Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 1 Mar 2018 18:55:15 +0100 Subject: [PATCH 091/102] added external vertex->unit mapping to graph constructor --- dash/include/dash/Graph.h | 26 +++++-- dash/include/dash/graph/internal/Graph.h | 86 ++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 6 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index f44fddf22..41f3a0560 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -160,6 +160,7 @@ class Graph { ) : _team(&team), _myid(team.myid()), + _size(team.size()), _remote_edges(team.size()) { allocate(n_vertices, n_vertex_edges); @@ -175,20 +176,26 @@ class Graph { * * /todo Add */ - template + template< + typename ForwardIterator, + typename VertexMapFunction = BlockedVertexMapper + > Graph( ForwardIterator begin, ForwardIterator end, vertex_size_type n_vertices, - Team & team = dash::Team::All() + Team & team = dash::Team::All(), + VertexMapFunction vertex_owner = BlockedVertexMapper() ) : _team(&team), _myid(team.myid()), + _size(team.size()), _remote_edges(team.size()) { // TODO: find heuristic to allocate a reasonable amount of edge memory // per vertex allocate(n_vertices, 0); + //VertexMapFunction vertex_owner; std::map lvertices; std::map gvertices; @@ -198,7 +205,7 @@ class Graph { for(auto it = begin; it != end; ++it) { auto item = *it; auto v = edgelist_source(item); - if(vertex_owner(v, n_vertices) == _myid) { + if(vertex_owner(v, n_vertices, _size, _myid) == _myid) { auto u = edgelist_target(item); if(lvertices.find(v) == lvertices.end()) { @@ -206,7 +213,7 @@ class Graph { // to be in order lvertices[v] = local_vertex_iterator(); } - auto target_owner = vertex_owner(u, n_vertices); + auto target_owner = vertex_owner(u, n_vertices, _size, _myid); if(target_owner == _myid) { if(lvertices.find(u) == lvertices.end()) { // add dummy first, more vertices are added later and they have @@ -308,11 +315,11 @@ class Graph { for(auto it = begin; it != end; ++it) { auto item = *it; auto v = edgelist_source(item); - if(vertex_owner(v, n_vertices) == _myid) { + if(vertex_owner(v, n_vertices, _size, _myid) == _myid) { auto v_it = lvertices[v]; auto u = edgelist_target(item); - if(vertex_owner(u, n_vertices) == _myid) { + if(vertex_owner(u, n_vertices, _size, _myid) == _myid) { auto u_it = lvertices[u]; edgelist_add_edge(v_it, u_it, item); } else { @@ -782,12 +789,17 @@ class Graph { return glob_mem_edge->push_back(source.offset, edge); } + /* team_unit_t vertex_owner(vertex_size_type v, vertex_size_type n_vertices) { int owner_id = static_cast(v) / (static_cast(n_vertices) / _team->size()); + int size = _team->size(); + int owner_id = static_cast(v) / (n_vertices + * ((static_cast(_myid) + 1) / (size * (size + 1) / 2))); team_unit_t owner { owner_id }; return owner; } + */ // TODO: Generalize following methods for other edge attributes than { int } @@ -834,6 +846,8 @@ class Graph { glob_mem_edge_comb_type * _glob_mem_edge = nullptr; /** Unit ID of the current unit */ team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; + /** Amount of units in the team */ + std::size_t _size; /** Index of the vertex container in _glob_mem_vertex */ vertex_cont_ref_type _vertex_container_ref; /** Amount of edge elements to be pre-allocated for every vertex */ diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index 55e3d87e8..be0691c3c 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -12,6 +12,92 @@ enum GraphDirection { //BidirectionalGraph }; +/** + * Maps vertices to units with equal block sizes. + */ +template +class BlockedVertexMapper { + +public: + + typedef GraphType graph_type; + typedef typename graph_type::vertex_size_type vertex_size_type; + + /** + * Returns the unit a vertex is mapped to. + */ + dash::team_unit_t operator()(vertex_size_type v, vertex_size_type n_vertices, + std::size_t n_units, dash::team_unit_t myid) { + int owner = static_cast(v) / (static_cast(n_vertices) / + n_units); + dash::team_unit_t unit { owner }; + return unit; + } + +}; + +/** + * Maps vertices to units using a logarithmic function. + */ +template +class LogarithmicVertexMapper { + +public: + + typedef GraphType graph_type; + typedef typename graph_type::vertex_size_type vertex_size_type; + + /** + * Constructs the logarithmic vertex mapper. + * Factors are calculated with the following formula: + * + * factor[unit] = log10((unit + start) * scale) + */ + LogarithmicVertexMapper(vertex_size_type n_vertices, std::size_t n_units, + double start = 2, double scale = 1) + : _blocks(n_units) + { + double factor_sum = 0; + std::vector factors(n_units); + for(double i = 0; i < n_units; ++i) { + factors[i] = log10((i + start) * scale); + factor_sum += factors[i]; + } + int total_vertices = 0; + for(int i = 0; i < n_units; ++i) { + if(i == n_units - 1) { + // avoid double errors + _blocks[i] = n_vertices - total_vertices; + } else { + _blocks[i] = n_vertices * (factors[i] / factor_sum); + total_vertices += _blocks[i]; + } + } + } + + /** + * Returns the unit a vertex is mapped to. + */ + dash::team_unit_t operator()(vertex_size_type v, vertex_size_type n_vertices, + std::size_t n_units, dash::team_unit_t myid) { + int owner = n_units - 1; + // TODO: can this be done in O(c)? + for(int i = 0; i < n_units; ++i) { + if(v < _blocks[i]) { + owner = i; + break; + } + } + dash::team_unit_t unit { owner }; + return unit; + } + +private: + + std::vector _blocks; + +}; + /** * Index type for vertices. */ From cf8f89f2b1b5ccb21a8b00e888dfcc380544da3c Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 8 Mar 2018 15:37:05 +0100 Subject: [PATCH 092/102] fixed mst algorithm --- .../algorithm/graph/MinimumSpanningTree.h | 133 +++++++++++++++--- 1 file changed, 115 insertions(+), 18 deletions(-) diff --git a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h index c83a3ee6e..e17be7626 100644 --- a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h +++ b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h @@ -232,12 +232,14 @@ auto mst_get_components( template void mst_set_data_min( matrix_min_pairs_t & data_pairs, + std::vector> & remote_edges, int start, GraphType & graph, dash::util::Trace & trace ) { trace.enter_state("send pairs"); typedef typename GraphType::vertex_properties_type vprop_t; + auto myid = graph.team().myid(); std::vector sizes_send(data_pairs.size()); std::vector displs_send(data_pairs.size()); std::vector> pairs_send; @@ -265,7 +267,7 @@ void mst_set_data_min( dart_alltoallv(pairs_send.data(), sizes_send.data(), displs_send.data(), DART_TYPE_BYTE, pairs_recv.data(), sizes_recv.data(), displs_recv.data(), graph.team().dart_id()); - std::unordered_map> mapping; + std::unordered_map> mapping; for(auto & pair : pairs_recv) { auto it = mapping.find(std::get<0>(pair)); if(it != mapping.end()) { @@ -280,15 +282,83 @@ void mst_set_data_min( trace.exit_state("send pairs"); trace.enter_state("set components"); for(auto & pair : mapping) { - graph.vertices().set_attributes(pair.first - start, std::get<1>(pair.second)); - if(dash::myid() == get<3>(pair.second)) { - typename GraphType::edge_properties_type eprop { -1 }; - graph.out_edges().set_attributes(std::get<4>(pair.second), eprop); + auto trg_comp = std::get<1>(pair.second); + graph.vertices().set_attributes(pair.first - start, trg_comp); + //std::cout << ">" << pair.first << ":" << trg_comp.comp << std::endl; + /* + if(trg_comp.unit == myid) { + auto it = mapping.find(trg_comp.comp); + vprop_t src_comp { pair.first, myid }; + + if(it != mapping.end()) { + auto weight = std::get<2>(it->second); + if(weight > std::get<2>(pair.second)) { + graph.vertices().set_attributes(trg_comp.comp - start, src_comp); + std::cout << "> " << trg_comp.comp << ":" << src_comp.comp << " *" << std::endl; + } + } else { + graph.vertices().set_attributes(trg_comp.comp - start, src_comp); + std::cout << "> " << trg_comp.comp << ":" << src_comp.comp << " *" << std::endl; + } + } + */ + if(myid == get<3>(pair.second)) { + auto edge = std::get<4>(pair.second); + auto prop = graph.out_edges().attributes(edge); + prop.is_min = true; + graph.out_edges().set_attributes(edge, prop); + } else if(std::get<3>(pair.second) != std::get<1>(pair.second).unit) { + // if the edge belongs to a unit that does not hold any of the component + // vertices + remote_edges[std::get<3>(pair.second)].push_back( + std::get<4>(pair.second)); } } trace.exit_state("set components"); } +template +void mst_set_edges( + std::vector> remote_edges, + GraphType & graph, + dash::util::Trace & trace +) { + typedef typename GraphType::edge_properties_type eprop_t; + trace.enter_state("send edges"); + std::vector sizes_send(remote_edges.size()); + std::vector displs_send(remote_edges.size()); + std::vector send; + std::size_t total_send = 0; + for(int i = 0; i < remote_edges.size(); ++i) { + sizes_send[i] = remote_edges[i].size() * sizeof(std::size_t); + displs_send[i] = total_send; + total_send += sizes_send[i]; + } + send.reserve(total_send / sizeof(std::size_t)); + for(auto & set : remote_edges) { + send.insert(send.end(), set.begin(), set.end()); + } + std::vector sizes_recv(remote_edges.size()); + std::vector displs_recv(remote_edges.size()); + dart_alltoall(sizes_send.data(), sizes_recv.data(), 1, + DART_TYPE_INT, graph.team().dart_id()); + std::size_t total_recv = 0; + for(int i = 0; i < sizes_recv.size(); ++i) { + displs_recv[i] = total_recv; + total_recv += sizes_recv[i]; + } + std::vector recv(total_recv / sizeof(std::size_t)); + dart_alltoallv(send.data(), sizes_send.data(), displs_send.data(), + DART_TYPE_BYTE, recv.data(), sizes_recv.data(), + displs_recv.data(), graph.team().dart_id()); + for(auto & edge : recv) { + auto prop = graph.out_edges().attributes(edge); + prop.is_min = true; + graph.out_edges().set_attributes(edge, prop); + } + trace.exit_state("send edges"); +} + } // namespace internal // TODO: component type should be Graph::vertex_size_type @@ -298,6 +368,7 @@ void mst_set_data_min( * - (int) comp * Requires the graph's edges to store the following attributep: * - (int) weight + * - (bool) is_min * * Output: * TODO: store the tree information in the edges OR in a separate data @@ -352,6 +423,7 @@ void minimum_spanning_tree(GraphType & g) { trace.exit_state("compute indices"); } + std::vector> remote_edges(g.team().size()); while(1) { int gr = 0; { @@ -359,7 +431,8 @@ void minimum_spanning_tree(GraphType & g) { g, trace); trace.enter_state("compute pairs"); matrix_min_pairs_t data_pairs(g.team().size()); - std::vector> pair_set(g.team().size()); + std::vector>> pair_map(g.team().size()); int i = 0; for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { auto v = g[it]; @@ -367,6 +440,8 @@ void minimum_spanning_tree(GraphType & g) { int min_weight = std::numeric_limits::max(); vprop_t trg_comp_min; int ledgepos = -1; + int src = 0; + int trg = 0; for(auto e_it = v.out_edges().lbegin(); e_it != v.out_edges().lend(); ++e_it) { auto e = g[e_it]; @@ -377,27 +452,48 @@ void minimum_spanning_tree(GraphType & g) { min_weight = e_weight; trg_comp_min = trg_comp; ledgepos = e_it.pos(); + src = e.source().pos(); + trg = e.target().pos(); } ++i; } if(ledgepos >= 0) { - auto & set = pair_set[src_comp.unit]; - if(set.find(src_comp.comp) == set.end()) { - data_pairs[src_comp.unit].push_back( - std::make_tuple(src_comp.comp, trg_comp_min, min_weight, myid, - ledgepos) - ); - data_pairs[trg_comp_min.unit].push_back( - std::make_tuple(trg_comp_min.comp, src_comp, min_weight, myid, - ledgepos) - ); - set.insert(src_comp.comp); + auto & map = pair_map[src_comp.unit]; + auto m_it = map.find(src_comp.comp); + if(m_it == map.end()) { + map[src_comp.comp] = std::make_tuple(src_comp, trg_comp_min, + min_weight, ledgepos); + } else { + if(std::get<2>(m_it->second) > min_weight) { + m_it->second = std::make_tuple(src_comp, trg_comp_min, min_weight, + ledgepos); + } } gr = 1; } } + for(int i = 0; i < pair_map.size(); ++i) { + for(auto & pair : pair_map[i]) { + data_pairs[i].emplace_back(std::get<0>(pair.second).comp, + std::get<1>(pair.second), std::get<2>(pair.second), myid, + std::get<3>(pair.second)); + //std::cout << i << ":" << std::get<0>(pair.second).comp << "-" << std::get<1>(pair.second).comp << std::endl; + auto trg_unit = std::get<1>(pair.second).unit; + if(std::get<0>(pair.second).unit != trg_unit) { + data_pairs[trg_unit].emplace_back(std::get<1>(pair.second).comp, + std::get<0>(pair.second), std::get<2>(pair.second), myid, + std::get<3>(pair.second)); + } + /* + std::cout << trg_unit << ":" << std::get<1>(pair.second).comp << "-" << std::get<0>(pair.second).comp << " *" << std::endl; + } else { + std::cout << trg_unit << ":" << std::get<1>(pair.second).comp << "-" << std::get<0>(pair.second).comp << std::endl; + } + */ + } + } trace.exit_state("compute pairs"); - internal::mst_set_data_min(data_pairs, start, g, trace); + internal::mst_set_data_min(data_pairs, remote_edges, start, g, trace); } int gr_all = 0; @@ -449,6 +545,7 @@ void minimum_spanning_tree(GraphType & g) { if(pj_all == 0) break; } } + internal::mst_set_edges(remote_edges, g, trace); dash::barrier(); From 4a7845904b397881163ead7071184417c729232e Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 8 Mar 2018 15:38:08 +0100 Subject: [PATCH 093/102] fixed logarithmic vertex mapper --- dash/include/dash/graph/internal/Graph.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index be0691c3c..10fa0dbdb 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -67,10 +67,10 @@ class LogarithmicVertexMapper { for(int i = 0; i < n_units; ++i) { if(i == n_units - 1) { // avoid double errors - _blocks[i] = n_vertices - total_vertices; + _blocks[i] = n_vertices; } else { - _blocks[i] = n_vertices * (factors[i] / factor_sum); - total_vertices += _blocks[i]; + total_vertices += n_vertices * (factors[i] / factor_sum); + _blocks[i] = total_vertices; } } } @@ -94,7 +94,7 @@ class LogarithmicVertexMapper { private: - std::vector _blocks; + std::vector _blocks; }; From 4e832398916b6f38df6e6119a61a565726542aed Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 8 Mar 2018 15:40:01 +0100 Subject: [PATCH 094/102] fixed algorithm tests --- .../test/algorithm/ConnectedComponentsTest.cc | 61 +++++++++++--- .../test/algorithm/MinimumSpanningTreeTest.cc | 81 +++++++++++++------ 2 files changed, 104 insertions(+), 38 deletions(-) diff --git a/dash/test/algorithm/ConnectedComponentsTest.cc b/dash/test/algorithm/ConnectedComponentsTest.cc index 53e9a9eb2..c61441ecd 100644 --- a/dash/test/algorithm/ConnectedComponentsTest.cc +++ b/dash/test/algorithm/ConnectedComponentsTest.cc @@ -634,18 +634,6 @@ TEST_F(ConnectedComponentsTest, AlgorithmRun) {371, 431}, {431, 564}, {564, 779}, {779, 835}, {835, 877}, {877, 986} }; - DASH_LOG_DEBUG("ConnectedComponentsTest.AlgorithmRun", - "construction started"); - graph_t g(edge_list.begin(), edge_list.end(), 1000); - DASH_LOG_DEBUG("ConnectedComponentsTest.AlgorithmRun", - "construction finished"); - - dash::barrier(); - - DASH_LOG_DEBUG("ConnectedComponentsTest.AlgorithmRun", "algorithm started"); - dash::connected_components(g); - DASH_LOG_DEBUG("ConnectedComponentsTest.AlgorithmRun", "algorithm finished"); - std::vector results = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -690,6 +678,20 @@ TEST_F(ConnectedComponentsTest, AlgorithmRun) 0, 0, 0 }; + DASH_LOG_DEBUG("ConnectedComponentsTest.Blocked.AlgorithmRun", + "construction started"); + graph_t g(edge_list.begin(), edge_list.end(), 1000); + DASH_LOG_DEBUG("ConnectedComponentsTest.Blocked.AlgorithmRun", + "construction finished"); + + dash::barrier(); + + DASH_LOG_DEBUG("ConnectedComponentsTest.Blocked.AlgorithmRun", + "algorithm started"); + dash::connected_components(g); + DASH_LOG_DEBUG("ConnectedComponentsTest.Blocked.AlgorithmRun", + "algorithm finished"); + int wrong_components = 0; if(dash::myid() == 0) { int index = 0; @@ -705,7 +707,42 @@ TEST_F(ConnectedComponentsTest, AlgorithmRun) // each component set according to results array EXPECT_EQ_U(0, wrong_components); } + + auto & team = dash::Team::All(); + dash::LogarithmicVertexMapper mapper(1000, team.size()); + DASH_LOG_DEBUG("ConnectedComponentsTest.Logarithmic.AlgorithmRun", + "construction started"); + // TODO: creating 2 graphs in their own scopes results in a segfault + // probably a problem with the destructor + // happens only in GTest though + graph_t g2(edge_list.begin(), edge_list.end(), 1000, team, mapper); + DASH_LOG_DEBUG("ConnectedComponentsTest.Logarithmic.AlgorithmRun", + "construction finished"); + + dash::barrier(); + + DASH_LOG_DEBUG("ConnectedComponentsTest.Logarithmic.AlgorithmRun", + "algorithm started"); + dash::connected_components(g2); + DASH_LOG_DEBUG("ConnectedComponentsTest.Logarithmic.AlgorithmRun", + "algorithm finished"); + + wrong_components = 0; + if(dash::myid() == 0) { + int index = 0; + for(auto it = g2.vertices().begin(); it != g2.vertices().end(); ++it) { + auto v = g2[it]; + if(v.attributes().comp != results[index]) { + ++wrong_components; + break; + } + ++index; + } + // each component set according to results array + EXPECT_EQ_U(0, wrong_components); + } dash::barrier(); + } diff --git a/dash/test/algorithm/MinimumSpanningTreeTest.cc b/dash/test/algorithm/MinimumSpanningTreeTest.cc index 6e5d6afa0..cfa4199ac 100644 --- a/dash/test/algorithm/MinimumSpanningTreeTest.cc +++ b/dash/test/algorithm/MinimumSpanningTreeTest.cc @@ -9,6 +9,7 @@ struct vprop { struct eprop { int weight; + bool is_min; }; typedef dash::Graph graph_t; @@ -34,45 +35,73 @@ TEST_F(MinimumSpanningTreeTest, AlgorithmRun) 10}, {{15, 5}, 7}, {{15, 10}, 8} }; - DASH_LOG_DEBUG("MinimumSpanningTreeTest.AlgorithmRun", + std::size_t TREE_SIZE = 39; + + DASH_LOG_DEBUG("MinimumSpanningTreeTest.Blocked.AlgorithmRun", "construction started"); graph_t g(edge_list.begin(), edge_list.end(), 20); - DASH_LOG_DEBUG("MinimumSpanningTreeTest.AlgorithmRun", + DASH_LOG_DEBUG("MinimumSpanningTreeTest.Blocked.AlgorithmRun", "construction finished"); - DASH_LOG_DEBUG("MinimumSpanningTreeTest.AlgorithmRun", "algorithm started"); + DASH_LOG_DEBUG("MinimumSpanningTreeTest.Blocked.AlgorithmRun", + "algorithm started"); dash::minimum_spanning_tree(g); - DASH_LOG_DEBUG("MinimumSpanningTreeTest.AlgorithmRun", "algorithm finished"); - - std::vector> results = { - {0, 10}, {0, 15}, {1, 17}, {1, 13}, {1, 10}, {2, 10}, {2, 3}, {4, 3}, - {5, 0}, {6, 12}, {7, 5}, {8, 0}, {9, 2}, {11,13}, {12, 1}, {14, 13}, - {15, 19}, {16, 13}, {18, 8} - }; + DASH_LOG_DEBUG("MinimumSpanningTreeTest.Blocked.AlgorithmRun", + "algorithm finished"); - int wrong_edges = 0; + std::unordered_set results; + std::size_t total_weight = 0; if(dash::myid() == 0) { - int i = 0; for(auto it = g.out_edges().begin(); it != g.out_edges().end(); ++it) { auto e = g[it]; - if(e.attributes().weight == -1) { - bool found = false; - for(auto & result : results ) { - // undirected graph: check edges in both directions - if((result.first == e.source().pos() - && result.second == e.target().pos()) - || (result.first == e.target().pos() - && result.second == e.source().pos())) { - found = true; - } + if(e.attributes().is_min) { + auto u = e.source().pos(); + auto v = e.target().pos(); + if(v < u) std::swap(u, v); + auto hash = std::hash()(u) ^ std::hash()(v); + if(results.find(hash) == results.end()) { + results.insert(hash); + total_weight += e.attributes().weight; } - if(!found) { - ++wrong_edges; + } + } + EXPECT_EQ_U(TREE_SIZE, total_weight); + } + + auto & team = dash::Team::All(); + dash::LogarithmicVertexMapper mapper(20, team.size()); + DASH_LOG_DEBUG("MinimumSpanningTreeTest.Logarithmic.AlgorithmRun", + "construction started"); + // TODO: creating 2 graphs in their own scopes results in a segfault + // probably a problem with the destructor + // happens only in GTest though + graph_t g2(edge_list.begin(), edge_list.end(), 20, team, mapper); + DASH_LOG_DEBUG("MinimumSpanningTreeTest.Logarithmic.AlgorithmRun", + "construction finished"); + + DASH_LOG_DEBUG("MinimumSpanningTreeTest.Logarithmic.AlgorithmRun", + "algorithm started"); + dash::minimum_spanning_tree(g2); + DASH_LOG_DEBUG("MinimumSpanningTreeTest.Logarithmic.AlgorithmRun", + "algorithm finished"); + + results.clear(); + total_weight = 0; + if(dash::myid() == 0) { + for(auto it = g2.out_edges().begin(); it != g2.out_edges().end(); ++it) { + auto e = g2[it]; + if(e.attributes().is_min) { + auto u = e.source().pos(); + auto v = e.target().pos(); + if(v < u) std::swap(u, v); + auto hash = std::hash()(u) ^ std::hash()(v); + if(results.find(hash) == results.end()) { + results.insert(hash); + total_weight += e.attributes().weight; } } - ++i; } - EXPECT_EQ_U(0, wrong_edges); + EXPECT_EQ_U(TREE_SIZE, total_weight); } dash::barrier(); From f2d31ff9615c1312ef3b10cbbac87b9a484c636f Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Thu, 8 Mar 2018 17:07:12 +0100 Subject: [PATCH 095/102] fixed mst algorithm again --- .../algorithm/graph/MinimumSpanningTree.h | 37 +++---------------- 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h index e17be7626..3e2c79214 100644 --- a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h +++ b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h @@ -284,24 +284,6 @@ void mst_set_data_min( for(auto & pair : mapping) { auto trg_comp = std::get<1>(pair.second); graph.vertices().set_attributes(pair.first - start, trg_comp); - //std::cout << ">" << pair.first << ":" << trg_comp.comp << std::endl; - /* - if(trg_comp.unit == myid) { - auto it = mapping.find(trg_comp.comp); - vprop_t src_comp { pair.first, myid }; - - if(it != mapping.end()) { - auto weight = std::get<2>(it->second); - if(weight > std::get<2>(pair.second)) { - graph.vertices().set_attributes(trg_comp.comp - start, src_comp); - std::cout << "> " << trg_comp.comp << ":" << src_comp.comp << " *" << std::endl; - } - } else { - graph.vertices().set_attributes(trg_comp.comp - start, src_comp); - std::cout << "> " << trg_comp.comp << ":" << src_comp.comp << " *" << std::endl; - } - } - */ if(myid == get<3>(pair.second)) { auto edge = std::get<4>(pair.second); auto prop = graph.out_edges().attributes(edge); @@ -330,26 +312,26 @@ void mst_set_edges( std::vector send; std::size_t total_send = 0; for(int i = 0; i < remote_edges.size(); ++i) { - sizes_send[i] = remote_edges[i].size() * sizeof(std::size_t); + sizes_send[i] = remote_edges[i].size(); displs_send[i] = total_send; total_send += sizes_send[i]; } - send.reserve(total_send / sizeof(std::size_t)); + send.reserve(total_send); for(auto & set : remote_edges) { send.insert(send.end(), set.begin(), set.end()); } std::vector sizes_recv(remote_edges.size()); std::vector displs_recv(remote_edges.size()); - dart_alltoall(sizes_send.data(), sizes_recv.data(), 1, - DART_TYPE_INT, graph.team().dart_id()); + dart_alltoall(sizes_send.data(), sizes_recv.data(), sizeof(std::size_t), + DART_TYPE_BYTE, graph.team().dart_id()); std::size_t total_recv = 0; for(int i = 0; i < sizes_recv.size(); ++i) { displs_recv[i] = total_recv; total_recv += sizes_recv[i]; } - std::vector recv(total_recv / sizeof(std::size_t)); + std::vector recv(total_recv); dart_alltoallv(send.data(), sizes_send.data(), displs_send.data(), - DART_TYPE_BYTE, recv.data(), sizes_recv.data(), + DART_TYPE_INT, recv.data(), sizes_recv.data(), displs_recv.data(), graph.team().dart_id()); for(auto & edge : recv) { auto prop = graph.out_edges().attributes(edge); @@ -477,19 +459,12 @@ void minimum_spanning_tree(GraphType & g) { data_pairs[i].emplace_back(std::get<0>(pair.second).comp, std::get<1>(pair.second), std::get<2>(pair.second), myid, std::get<3>(pair.second)); - //std::cout << i << ":" << std::get<0>(pair.second).comp << "-" << std::get<1>(pair.second).comp << std::endl; auto trg_unit = std::get<1>(pair.second).unit; if(std::get<0>(pair.second).unit != trg_unit) { data_pairs[trg_unit].emplace_back(std::get<1>(pair.second).comp, std::get<0>(pair.second), std::get<2>(pair.second), myid, std::get<3>(pair.second)); } - /* - std::cout << trg_unit << ":" << std::get<1>(pair.second).comp << "-" << std::get<0>(pair.second).comp << " *" << std::endl; - } else { - std::cout << trg_unit << ":" << std::get<1>(pair.second).comp << "-" << std::get<0>(pair.second).comp << std::endl; - } - */ } } trace.exit_state("compute pairs"); From e9de737ef48d6280b290d39d971e3ece9ff7b39f Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Fri, 9 Mar 2018 13:40:01 +0100 Subject: [PATCH 096/102] added size method to wrapper function objects --- dash/include/dash/graph/internal/Graph.h | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index 10fa0dbdb..103b2f355 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -23,6 +23,18 @@ class BlockedVertexMapper { typedef GraphType graph_type; typedef typename graph_type::vertex_size_type vertex_size_type; + /** + * Default constructor. + */ + BlockedVertexMapper() = default; + + /** + * Constructs the blocked vertex mapper. + */ + BlockedVertexMapper(vertex_size_type n_vertices, std::size_t n_units) + : _blocksize(n_vertices / n_units) + { } + /** * Returns the unit a vertex is mapped to. */ @@ -34,6 +46,17 @@ class BlockedVertexMapper { return unit; } + /** + * Returns the blocksize of a given unit. + */ + std::size_t size(team_unit_t unit) { + return _blocksize; + } + +private: + + std::size_t _blocksize; + }; /** @@ -72,6 +95,7 @@ class LogarithmicVertexMapper { total_vertices += n_vertices * (factors[i] / factor_sum); _blocks[i] = total_vertices; } + if(dash::myid() == 0) std::cout << n_vertices * (factors[i] / factor_sum) << std::endl; } } @@ -92,6 +116,16 @@ class LogarithmicVertexMapper { return unit; } + /** + * Returns the blocksize of a given unit. + */ + std::size_t size(team_unit_t unit) { + if(unit == 0) { + return _blocks[0]; + } + return _blocks[unit] - _blocks[unit - 1]; + } + private: std::vector _blocks; From 97d12dbdc37b7aa2759c282811fe56968995dc4a Mon Sep 17 00:00:00 2001 From: Stefan Effenberger Date: Sun, 18 Mar 2018 12:58:08 +0100 Subject: [PATCH 097/102] removed unnecessary code --- dash/include/dash/graph/internal/Graph.h | 1 - dash/include/dash/memory/GlobHeapContiguousMem.h | 5 ----- 2 files changed, 6 deletions(-) diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index 103b2f355..cac1da6c1 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -95,7 +95,6 @@ class LogarithmicVertexMapper { total_vertices += n_vertices * (factors[i] / factor_sum); _blocks[i] = total_vertices; } - if(dash::myid() == 0) std::cout << n_vertices * (factors[i] / factor_sum) << std::endl; } } diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h index 49e9d96ab..7dacd52c8 100644 --- a/dash/include/dash/memory/GlobHeapContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -219,11 +219,6 @@ class GlobHeapContiguousMem { ++count; } _bucket_cumul_sizes[_myid][0] = _local_size; - // 2 local buckets per global bucket - count /= 2; - for(int i = count; i < *max_buckets; ++i) { - add_container(0); - } // detach old container location from global memory space, if it has // been attached before From 79bbac0d967a26fc3b7875be62ef2ef8dc1c2e2d Mon Sep 17 00:00:00 2001 From: Tobias Fuchs Date: Sat, 9 Mar 2019 00:17:42 +0100 Subject: [PATCH 098/102] Updated dart_alltoallv feat-graph (missing in dev) --- dart-impl/mpi/src/dart_communication.c | 25 ++++++++++++++++++++----- dash/test/container/GraphTest.cc | 17 +++++++++-------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/dart-impl/mpi/src/dart_communication.c b/dart-impl/mpi/src/dart_communication.c index e533c4eae..6c0efe5ed 100644 --- a/dart-impl/mpi/src/dart_communication.c +++ b/dart-impl/mpi/src/dart_communication.c @@ -2168,6 +2168,10 @@ dart_ret_t dart_allgatherv( return DART_OK; } +#if 0 +/* + * Implementation from branch feat-graph, to be discussed + */ dart_ret_t dart_alltoall( const void * sendbuf, void * recvbuf, @@ -2219,7 +2223,12 @@ dart_ret_t dart_alltoall( teamid, nelem); return DART_OK; } +#endif +#if 0 +/* + * Implementation from branch feat-graph, to be discussed + */ dart_ret_t dart_alltoallv( const void * sendbuf, const size_t * nsendcounts, @@ -2230,14 +2239,15 @@ dart_ret_t dart_alltoallv( const size_t * recvdispls, dart_team_t teamid) { - MPI_Datatype mpi_dtype = dart__mpi__datatype_struct(dtype)->basic.mpi_type; + MPI_Datatype mpi_dtype = dart__mpi__datatype_struct(dtype)->contiguous.mpi_type; MPI_Comm comm; int comm_size; DART_LOG_TRACE("dart_alltoallv() team:%d nsendelem:%"PRIu64"", - teamid, nsendelem); + teamid, nsendcounts); if (teamid == DART_UNDEFINED_TEAM_ID) { - DART_LOG_ERROR("dart_alltoallv ! failed: team may not be DART_UNDEFINED_TEAM_ID"); + DART_LOG_ERROR( + "dart_alltoallv ! failed: team may not be DART_UNDEFINED_TEAM_ID"); return DART_ERR_INVAL; } @@ -2257,13 +2267,17 @@ dart_ret_t dart_alltoallv( int *irecvdispls = malloc(sizeof(int) * comm_size); for (int i = 0; i < comm_size; i++) { if (nsendcounts[i] > INT_MAX || senddispls[i] > INT_MAX) { - DART_LOG_ERROR("dart_alltoallv ! failed: nsendcounts[%i] > INT_MAX || senddispls[%i] > INT_MAX", i, i); + DART_LOG_ERROR( + "dart_alltoallv ! failed: " + "nsendcounts[%i] > INT_MAX || senddispls[%i] > INT_MAX", i, i); free(insendcounts); free(isenddispls); return DART_ERR_INVAL; } if (nrecvcounts[i] > INT_MAX || recvdispls[i] > INT_MAX) { - DART_LOG_ERROR("dart_alltoallv ! failed: nrecvcounts[%i] > INT_MAX || recvdispls[%i] > INT_MAX", i, i); + DART_LOG_ERROR( + "dart_alltoallv ! failed: " + "nrecvcounts[%i] > INT_MAX || recvdispls[%i] > INT_MAX", i, i); free(inrecvcounts); free(irecvdispls); return DART_ERR_INVAL; @@ -2298,6 +2312,7 @@ dart_ret_t dart_alltoallv( DART_LOG_TRACE("dart_alltoallv > team:%d", teamid); return DART_OK; } +#endif dart_ret_t dart_allreduce( const void * sendbuf, diff --git a/dash/test/container/GraphTest.cc b/dash/test/container/GraphTest.cc index 61b6df150..f7c448842 100644 --- a/dash/test/container/GraphTest.cc +++ b/dash/test/container/GraphTest.cc @@ -637,24 +637,25 @@ TEST_F(GraphTest, Construction) DASH_LOG_DEBUG("GraphTest.Construction", "construction finished"); int unrecognized_edges = 0; - if(dash::myid() == 0) { - for(auto it = g.vertices().begin(); it != g.vertices().end(); ++it) { + if (dash::myid() == 0) { + for (auto it = g.vertices().begin(); it != g.vertices().end(); ++it) { auto v = g[it]; - for(auto e_it = v.out_edges().begin(); e_it != v.out_edges().end(); + for (auto e_it = v.out_edges().begin(); e_it != v.out_edges().end(); ++e_it) { auto e = g[e_it]; - //TODO: use map for faster runtime + //TODO: use std::map bool unrecognized = true; - for(auto el_it = edge_list.begin(); el_it != edge_list.end(); - ++el_it) { - if(el_it->first == e.source().pos() + for (auto el_it = edge_list.begin(); + el_it != edge_list.end(); + ++el_it) { + if ( el_it->first == e.source().pos() && el_it->second == e.target().pos()) { edge_list.erase(el_it); unrecognized = false; break; } } - if(unrecognized) { + if(unrecognized) { ++unrecognized_edges; } } From 3e6ce99c82b0846f006e2a6380c5d578bfc634ae Mon Sep 17 00:00:00 2001 From: Tobias Fuchs Date: Tue, 12 Mar 2019 20:34:46 +0100 Subject: [PATCH 099/102] added dart_alltoallv, added graph examples, fixing compiler errors --- dart-impl/mpi/src/dart_communication.c | 2 - dash/examples/ex.02.graph-deref/main.cpp | 121 +++++++++ dash/examples/ex.02.graph/main.cpp | 173 +++--------- dash/examples/ex.02.graph/rmatrandom.h | 247 ++++++++++++++++++ .../dash/memory/GlobHeapContiguousLocalPtr.h | 2 +- .../dash/memory/GlobHeapContiguousMem.h | 15 +- dash/test/container/GraphTest.cc | 6 +- 7 files changed, 424 insertions(+), 142 deletions(-) create mode 100644 dash/examples/ex.02.graph-deref/main.cpp create mode 100644 dash/examples/ex.02.graph/rmatrandom.h diff --git a/dart-impl/mpi/src/dart_communication.c b/dart-impl/mpi/src/dart_communication.c index 6c0efe5ed..52963303b 100644 --- a/dart-impl/mpi/src/dart_communication.c +++ b/dart-impl/mpi/src/dart_communication.c @@ -2225,7 +2225,6 @@ dart_ret_t dart_alltoall( } #endif -#if 0 /* * Implementation from branch feat-graph, to be discussed */ @@ -2312,7 +2311,6 @@ dart_ret_t dart_alltoallv( DART_LOG_TRACE("dart_alltoallv > team:%d", teamid); return DART_OK; } -#endif dart_ret_t dart_allreduce( const void * sendbuf, diff --git a/dash/examples/ex.02.graph-deref/main.cpp b/dash/examples/ex.02.graph-deref/main.cpp new file mode 100644 index 000000000..e44512464 --- /dev/null +++ b/dash/examples/ex.02.graph-deref/main.cpp @@ -0,0 +1,121 @@ +#include +#include +#include +#include +// #include "rmatrandom.h" + +struct vprop { + int comp; +}; + +struct eprop { + int comp; +}; + +typedef dash::Graph graph_t; + +int main(int argc, char* argv[]) { + dash::init(&argc, &argv); + + int n_unit_edges = 1; + int n_vertices_full = 448; + int n_edges_full = n_vertices_full * n_unit_edges; + // not exactly n_vertices_full are generated due to rounding + int n_vertices_start = n_vertices_full / dash::size(); + int n_size_rounds = 5; + int n_rounds = 1; + for(int rounds = 0; rounds < n_size_rounds; ++rounds) { + for(int i = 0; i < n_rounds; ++i) { + int scale = rounds + 1; + int n_vertices = n_vertices_start * scale; + graph_t g(n_vertices, n_unit_edges); + + for(int j = 0; j < n_vertices; ++j) { + g.add_vertex(); + } + + g.commit(); + + int n_edges = n_vertices * n_unit_edges; + auto src = g.vertices().lbegin(); + auto trg = src + 1; + auto end = g.vertices().lend(); + for(int j = 0; j < n_edges; ++j) { + g.add_edge(src, trg); + ++src; + ++trg; + if(trg == end) { + src = g.vertices().lbegin(); + trg = src + 1; + } + } + + g.commit(); + + std::clock_t g_begin_time = clock(); + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + auto v = g[it]; + v.attributes(); + } + std::clock_t g_end_time = clock(); + double time = double(g_end_time - g_begin_time) / CLOCKS_PER_SEC; + + double all_time; + dart_reduce(&time, &all_time, 1, DART_TYPE_DOUBLE, DART_OP_SUM, 0, + g.team().dart_id()); + + if(dash::myid() == 0) { + std::cout << "[round " << i << "] " << n_vertices_full * scale << + " vertices per node dereferenced (local): " << all_time << + std::endl; + } + + if(dash::myid() == 0) { + std::clock_t g_begin_time = clock(); + for(auto it = g.vertices().begin(); it != g.vertices().end(); ++it) { + auto v = g[it]; + v.attributes(); + } + std::clock_t g_end_time = clock(); + std::cout << "[round " << i << "] " << n_vertices_full * scale << + " vertices per node dereferenced (global): " << + double(g_end_time - g_begin_time) / CLOCKS_PER_SEC << std::endl; + } + + g_begin_time = clock(); + for(auto it = g.out_edges().lbegin(); it != g.out_edges().lend(); ++it) { + auto e = g[it]; + e.attributes(); + } + g_end_time = clock(); + time = double(g_end_time - g_begin_time) / CLOCKS_PER_SEC; + + dart_reduce(&time, &all_time, 1, DART_TYPE_DOUBLE, DART_OP_SUM, 0, + g.team().dart_id()); + + if(dash::myid() == 0) { + std::cout << "[round " << i << "] " << n_edges_full * scale << + " edges per node dereferenced (local): " << all_time << std::endl; + } + + if(dash::myid() == 0) { + std::clock_t g_begin_time = clock(); + for(auto it = g.out_edges().begin(); it != g.out_edges().end(); ++it) { + auto e = g[it]; + e.attributes(); + } + std::clock_t g_end_time = clock(); + std::cout << "[round " << i << "] " << n_edges_full * scale << + " edges per node dereferenced (global): " << + double(g_end_time - g_begin_time) / CLOCKS_PER_SEC << std::endl; + } + dash::barrier(); + } + if(dash::myid() == 0) std::cout << "-----------------" << std::endl; + } + + dash::finalize(); + return 0; +} + + diff --git a/dash/examples/ex.02.graph/main.cpp b/dash/examples/ex.02.graph/main.cpp index e05deaab7..78853da1a 100644 --- a/dash/examples/ex.02.graph/main.cpp +++ b/dash/examples/ex.02.graph/main.cpp @@ -1,145 +1,58 @@ -#include #include +#include +#include +#include -struct vprop { - int id; - int level; -}; +#include "rmatrandom.h" -struct eprop { - int id; +struct vprop { + int comp; + int unit; }; -typedef dash::Graph graph_t; -typedef graph_t::vertex_index_type vertex_index_t; -typedef graph_t::vertex_type vertex_t; -typedef graph_t::edge_type edge_t; - -void create_graph(graph_t &g) { - // add 4 v to every unit - for (int i = (dash::myid() * 4) + 1; i <= (dash::myid() * 4) + 4; ++i) { - vprop prop{i, -1}; - g.add_vertex(prop); - } - - std::vector v; - v.emplace_back(dash::team_unit_t(0), 0); - v.emplace_back(dash::team_unit_t(0), 1); - v.emplace_back(dash::team_unit_t(0), 2); - v.emplace_back(dash::team_unit_t(0), 3); - v.emplace_back(dash::team_unit_t(1), 0); - v.emplace_back(dash::team_unit_t(1), 1); - v.emplace_back(dash::team_unit_t(1), 2); - v.emplace_back(dash::team_unit_t(1), 3); - - if (dash::myid() == 0) { - g.add_edge(v[0], v[1]); - g.add_edge(v[0], v[4]); - g.add_edge(v[1], v[5]); - g.add_edge(v[1], v[6]); - g.add_edge(v[2], v[6]); - g.add_edge(v[3], v[7]); - } - if (dash::myid() == 1) { - g.add_edge(v[4], v[5]); - g.add_edge(v[5], v[6]); - g.add_edge(v[6], v[7]); - } - - g.barrier(); -} +typedef dash::Graph graph_t; + +int main(int argc, char* argv[]) { + if(argc == 3) { + dash::init(&argc, &argv); + int n_vertices = 1000 * atoi(argv[1]); + int n_edges = 4000 * atoi(argv[2]); + auto & team = dash::Team::All(); + dash::LogarithmicVertexMapper mapper(n_vertices, team.size()); + for(int i = 0; i < 5; ++i) { + RMATRandomGenerator begin(n_vertices, n_edges, team.size(), + team.myid(), mapper, 0.25, 0.25, 0.25, 0.25); + RMATRandomGenerator end(0, 0, team.size(), team.myid(), mapper, + 0, 0, 0, 0); + + std::clock_t g_begin_time = clock(); + graph_t g(begin, end, n_vertices, team, mapper); + std::clock_t g_end_time = clock(); + if(dash::myid() == 0) { + std::cout << "[round " << i + 1 << "] construction: " << + double(g_end_time - g_begin_time) / CLOCKS_PER_SEC << std::endl; + } -void breadth_first_search(graph_t &g, vertex_index_t source) { - dash::Team &team = g.team(); + dash::barrier(); + std::clock_t begin_time = clock(); - std::vector frontier; - std::vector> neighbours(team.size()); + //dash::util::TraceStore::on(); + //dash::util::TraceStore::clear(); - if (source.unit == team.myid()) { - frontier.push_back(source); - } + dash::connected_components(g); - int level = 0; - while (true) { - for (auto f_it = frontier.begin(); f_it != frontier.end(); ++f_it) { - vertex_index_t v_index = *f_it; - vertex_t &v = g.vertices[v_index]; - if (v.properties.level == -1) { - for (auto e_it = g.out_edges.vbegin(v); e_it != g.out_edges.vend(v); - ++e_it) { - edge_t e = *e_it; - auto target = e.target(); - neighbours[target.unit.id].push_back(target); - } - v.properties.level = level; - } - } - std::cout << neighbours[0].size() << " " << neighbours[1].size() - << std::endl; + //dash::barrier(); + //dash::util::TraceStore::off(); + //dash::util::TraceStore::write(std::cout); - std::vector neighbour_data; - std::vector neighbour_count(team.size()); - std::vector neighbour_displs(team.size()); - for (int i = 0; i < neighbours.size(); ++i) { - for (auto &neighbour : neighbours[i]) { - neighbour_data.push_back(neighbour); - neighbour_count[i] += sizeof(vertex_index_t); - } - for (int j = i + 1; j < neighbour_displs.size(); ++j) { - neighbour_displs[j] += neighbour_count[i]; + std::clock_t end_time = clock(); + if(dash::myid() == 0) { + std::cout << "[round " << i + 1 << "] algorithm: " << + double(end_time - begin_time) / CLOCKS_PER_SEC << std::endl; } - neighbours[i].clear(); - } - std::vector neighbour_recv_count(team.size()); - dart_alltoall(neighbour_count.data(), neighbour_recv_count.data(), - sizeof(std::size_t), DART_TYPE_BYTE, team.dart_id()); - int total = 0; - std::vector neighbour_recv_displs(team.size()); - for (int i = 0; i < neighbour_recv_count.size(); ++i) { - total += neighbour_recv_count[i]; - for (int j = i + 1; j < neighbour_recv_displs.size(); ++j) { - neighbour_recv_displs[j] += neighbour_recv_count[i]; - } - } - int global_count = 0; - dart_allreduce(&total, &global_count, 1, DART_TYPE_INT, - DART_OP_SUM, team.dart_id()); - // frontier is empty on all nodes -> terminate algorithm - if (global_count == 0) { - break; } - std::vector new_frontier(total / sizeof(vertex_index_t)); - dart_alltoallv(neighbour_data.data(), neighbour_count.data(), - neighbour_displs.data(), DART_TYPE_BYTE, new_frontier.data(), - neighbour_recv_count.data(), neighbour_recv_displs.data(), - team.dart_id()); - - frontier = new_frontier; - - ++level; + dash::finalize(); } + return 0; } -int main(int argc, char *argv[]) { - dash::init(&argc, &argv); - graph_t g(8); - - create_graph(g); - breadth_first_search(g, vertex_index_t(dash::team_unit_t(0), 0)); - if (dash::myid() == 1) { - std::cout << g.num_vertices() << std::endl; - for (auto it = g.vertices.begin(); it != g.vertices.end(); ++it) { - vertex_t v = *it; - std::cout << "vertex " << v.properties.id << ": level " - << v.properties.level << std::endl; - } - /* - for(auto it = g.out_edges.begin(); it != g.out_edges.end(); ++it) { - edge_t e = *it; - std::cout << "edge: " << e.source().offset << " -> " << e.target().offset - << std::endl; - } - */ - } - dash::barrier(); -} diff --git a/dash/examples/ex.02.graph/rmatrandom.h b/dash/examples/ex.02.graph/rmatrandom.h new file mode 100644 index 000000000..5c8329598 --- /dev/null +++ b/dash/examples/ex.02.graph/rmatrandom.h @@ -0,0 +1,247 @@ +#ifndef DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED +#define DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED + +#include +#include +#include +#include + +class xorshift { + +public: + + typedef unsigned long result_type; + + template + xorshift(SeedGenerator & s) + : x(s()), + y(s()), + z(s()) + { } + + result_type operator()() { + result_type t; + x ^= x << 16; + x ^= x >> 5; + x ^= x << 1; + + t = x; + x = y; + y = z; + z = t ^ x ^ y; + + return z; + } + + // std::uniform distribution too slow + double dist() { + return static_cast((*this)()) / max(); + } + + result_type min() { + return 0; + } + + result_type max() { + return std::numeric_limits::max() - 1; + } + +private: + + result_type x, y, z; + +}; + +template +class RMATRandomGenerator { + +public: + typedef GraphType graph_type; + typedef RMATRandomGenerator self_t; + typedef typename graph_type::vertex_size_type vertex_size_type; + typedef typename graph_type::edge_size_type edge_size_type; + typedef std::pair value_type; + typedef std::uniform_int_distribution dist_int_type; + +private: + +public: + + /** + * Begin iterator + */ + template + RMATRandomGenerator( + vertex_size_type n, + edge_size_type m, + int n_units, + dash::team_unit_t myid, + VertexMapperFunction owner, + double a, + double b, + double c, + double d + ) { + std::random_device rd; + xorshift gen(rd); + + // the intel compiler on SUPERMUC falls into a "internal error loop" with + // the end constructor. therefore, this constructor is misused as end + // constructor + if(n == 0) { + _done = true; + } else { + // generate 50% of the edges with RMAT + m /= 2; + edge_size_type m_unit_random = m / n_units; + int SCALE = int(floor(log(double(n))/log(2.))); + + std::map edge_map; + + // generate whole graph on each unit, but only use edges belonging to this + // unit + edge_size_type generated = 0; + edge_size_type local_edges = 0; + do { + edge_size_type rejected = 0; + do { + vertex_size_type u, v; + std::tie(u, v) = generate_rmat_edge(gen, n, SCALE, a, b, c, d); + + if (owner(u, n, n_units, myid) == myid) { + // reject loop edges and multi-edges + if (u != v + && edge_map.find(std::make_pair(u, v)) == edge_map.end()) { + edge_map[std::make_pair(u, v)] = true; + ++local_edges; + } else { + ++rejected; + } + } + ++generated; + } while (generated < m); + // generate more edges based on the amount of edges rejected on each unit + int rejected_all; + dart_allreduce(&rejected, &rejected_all, 1, DART_TYPE_INT, DART_OP_SUM, + dash::Team::All().dart_id()); + generated -= rejected_all; + } while (generated < m); + + // reserve space for generated rmat edges and for coming random edges + _values.reserve(local_edges + m_unit_random); + typename std::map::reverse_iterator em_end = + edge_map.rend(); + for (typename std::map::reverse_iterator em_i = + edge_map.rbegin(); em_i != em_end; ++em_i) { + _values.push_back(em_i->first); + } + + // generate 50% random edges + // source edge has to belong to this unit + int start = 0; + int end = 0; + if(myid == 0) { + end = owner.size(myid) - 1; + } else { + for(int i = 0; i < myid; ++i) { + dash::team_unit_t unit { i }; + start += owner.size(unit); + } + end = start + owner.size(myid) - 1; + } + dist_int_type dist_u(start, end); + dist_int_type dist_v(0, n - 1); + for(int i = 0; i < m_unit_random; ++i) { + vertex_size_type u = dist_u(gen); + vertex_size_type v = dist_v(gen); + _values.push_back(std::make_pair(u, v)); + } + + _current = _values.size() - 1; + } + } + + /** + * End iterator + * + * This constructor somehow results in an "internal error loop" with the + * intel compiler of the SUPERMUC system. + */ + RMATRandomGenerator() + : _done(true) + { } + + self_t & operator++() { + if(_current > 0) { + --_current; + } else { + _done = true; + } + + return *this; + } + + const value_type & operator*() const { + return _values[_current]; + } + + const value_type * operator->() const { + return &_values[_current]; + } + + + + bool operator!=(const self_t & other) { + return !(_done && other._done); + } + +private: + + value_type generate_rmat_edge( + xorshift & gen, + vertex_size_type n, + unsigned int SCALE, + double a, + double b, + double c, + double d) { + vertex_size_type u = 0; + vertex_size_type v = 0; + vertex_size_type step = n/2; + + for (unsigned int j = 0; j < SCALE; ++j) { + double p = gen.dist(); + if (p < a) + ; + else if (p >= a && p < a + b) + v += step; + else if (p >= a + b && p < a + b + c) + u += step; + else { + u += step; + v += step; + } + + step /= 2; + + a *= 0.9 + 0.2 * gen.dist(); + b *= 0.9 + 0.2 * gen.dist(); + c *= 0.9 + 0.2 * gen.dist(); + d *= 0.9 + 0.2 * gen.dist(); + + double S = a + b + c + d; + + a /= S; b /= S; c /= S; + d = 1. - a - b - c; + } + + return std::make_pair(u, v); + } + + std::vector _values; + int _current; + bool _done = false; + +}; + +#endif // DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED diff --git a/dash/include/dash/memory/GlobHeapContiguousLocalPtr.h b/dash/include/dash/memory/GlobHeapContiguousLocalPtr.h index aaf85f235..06e6dbcb4 100644 --- a/dash/include/dash/memory/GlobHeapContiguousLocalPtr.h +++ b/dash/include/dash/memory/GlobHeapContiguousLocalPtr.h @@ -1,4 +1,4 @@ -#ifndef DASH__MEMORY__GLOB_HEAP_CONTIUGOUS_LOCAL_PTR_H__INCLUDED +#ifndef DASH__MEMORY__GLOB_HEAP_CONTIGUOUS_LOCAL_PTR_H__INCLUDED #define DASH__MEMORY__GLOB_HEAP_CONTIGUOUS_LOCAL_PTR_H__INCLUDED #include diff --git a/dash/include/dash/memory/GlobHeapContiguousMem.h b/dash/include/dash/memory/GlobHeapContiguousMem.h index 7dacd52c8..f8ede2a85 100644 --- a/dash/include/dash/memory/GlobHeapContiguousMem.h +++ b/dash/include/dash/memory/GlobHeapContiguousMem.h @@ -44,7 +44,7 @@ struct container_data { //TODO: adapt to GlobMem concept /** - * Global memory space for multiple, dynaimcally allocated contiguous memory + * Global memory space for multiple, dynamically allocated contiguous memory * regions. */ template @@ -100,7 +100,6 @@ class GlobHeapContiguousMem { } } - /** * Adds a new bucket into memory space. */ @@ -176,13 +175,13 @@ class GlobHeapContiguousMem { container_type * new_container = new container_type(); new_container->reserve(_local_size); - int count = 0; - int elements = 0; - int bucket_num = 0; - size_type bucket_cumul = 0; + int count = 0; + int elements = 0; + int bucket_num = 0; + size_type bucket_cumul = 0; auto unattached_container_it = _unattached_containers.begin(); + int elements_before = 0; bucket_type * last_bucket; - int elements_before = 0; for(auto & bucket : _buckets) { elements_before = elements; // move data to new range @@ -277,7 +276,7 @@ class GlobHeapContiguousMem { // update global iterators _begin = global_iterator(this, 0); - _end = global_iterator(this, _size); + _end = global_iterator(this, _size); } /** diff --git a/dash/test/container/GraphTest.cc b/dash/test/container/GraphTest.cc index f7c448842..3f2578977 100644 --- a/dash/test/container/GraphTest.cc +++ b/dash/test/container/GraphTest.cc @@ -1,3 +1,6 @@ + +#include + #include "GraphTest.h" #include @@ -5,7 +8,8 @@ struct vprop { int comp; }; -typedef dash::Graph graph_t; +typedef dash::Graph + graph_t; TEST_F(GraphTest, Construction) { From ec7e50f650ecf0c87786e12b8281e3ee5a79aef6 Mon Sep 17 00:00:00 2001 From: Tobias Fuchs Date: Tue, 12 Mar 2019 20:44:05 +0100 Subject: [PATCH 100/102] Fixed compiler errors in Graph --- dash/include/dash/Graph.h | 37 ++++++++++++++---------- dash/include/dash/graph/internal/Graph.h | 27 ++++++++++++----- 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/dash/include/dash/Graph.h b/dash/include/dash/Graph.h index 41f3a0560..2d117e649 100644 --- a/dash/include/dash/Graph.h +++ b/dash/include/dash/Graph.h @@ -43,7 +43,6 @@ namespace dash { * */ - /** * Distributed, dynamic graph container for sparse graphs. */ @@ -51,23 +50,29 @@ template< GraphDirection Direction = DirectedGraph, typename VertexProperties = EmptyProperties, // user-defined struct typename EdgeProperties = EmptyProperties, // user-defined struct - typename VertexSizeType = int, - typename EdgeSizeType = int, - template typename EdgeContainer = std::vector, - template typename VertexContainer = std::vector + typename VertexSizeType = int, + typename EdgeSizeType = int, +//template typename EdgeContainer = std::vector, +//template typename VertexContainer = std::vector + template class _EdgeContainer = std::vector, + template class _VertexContainer = std::vector > class Graph { public: - typedef Graph self_t; + _EdgeContainer, _VertexContainer> self_t; + typedef Vertex vertex_type; typedef Edge edge_type; - typedef VertexContainer vertex_container_type; - typedef EdgeContainer edge_container_type; + + typedef std::vector vertex_container_type; + typedef std::vector edge_container_type; + +//typedef VertexContainer vertex_container_type; +//typedef EdgeContainer edge_container_type; private: @@ -174,7 +179,6 @@ class Graph { * Partitions vertices based on their id: * vertex_id / (n / num_units) = owner * - * /todo Add */ template< typename ForwardIterator, @@ -474,6 +478,7 @@ class Graph { */ local_vertex_iterator add_vertex() { VertexProperties prop; + // : Reference to temporary! return add_vertex(prop); } @@ -571,7 +576,7 @@ class Graph { * actually been added. */ std::pair add_edge( - const local_vertex_iterator & source, + const local_vertex_iterator & source, const global_vertex_iterator & target ) { EdgeProperties prop; @@ -835,15 +840,15 @@ class Graph { private: /** the team containing all units using the container */ - Team * _team = nullptr; + Team * _team = nullptr; /** Global memory allocation and access to vertices */ - glob_mem_vert_type * _glob_mem_vertex = nullptr; + glob_mem_vert_type * _glob_mem_vertex = nullptr; /** Global memory allocation and access to inbound edges */ - glob_mem_edge_type * _glob_mem_in_edge = nullptr; + glob_mem_edge_type * _glob_mem_in_edge = nullptr; /** Global memory allocation and access to outbound edges */ - glob_mem_edge_type * _glob_mem_out_edge = nullptr; + glob_mem_edge_type * _glob_mem_out_edge = nullptr; /** Access to inbound and outbound edges */ - glob_mem_edge_comb_type * _glob_mem_edge = nullptr; + glob_mem_edge_comb_type * _glob_mem_edge = nullptr; /** Unit ID of the current unit */ team_unit_t _myid{DART_UNDEFINED_UNIT_ID}; /** Amount of units in the team */ diff --git a/dash/include/dash/graph/internal/Graph.h b/dash/include/dash/graph/internal/Graph.h index cac1da6c1..52ac414df 100644 --- a/dash/include/dash/graph/internal/Graph.h +++ b/dash/include/dash/graph/internal/Graph.h @@ -76,9 +76,12 @@ class LogarithmicVertexMapper { * * factor[unit] = log10((unit + start) * scale) */ - LogarithmicVertexMapper(vertex_size_type n_vertices, std::size_t n_units, - double start = 2, double scale = 1) - : _blocks(n_units) + LogarithmicVertexMapper( + vertex_size_type n_vertices, + std::size_t n_units, + double start = 2, + double scale = 1) + : _blocks(n_units) { double factor_sum = 0; std::vector factors(n_units); @@ -101,8 +104,12 @@ class LogarithmicVertexMapper { /** * Returns the unit a vertex is mapped to. */ - dash::team_unit_t operator()(vertex_size_type v, vertex_size_type n_vertices, - std::size_t n_units, dash::team_unit_t myid) { + dash::team_unit_t operator()( + vertex_size_type v, + vertex_size_type n_vertices, + std::size_t n_units, + dash::team_unit_t myid) + { int owner = n_units - 1; // TODO: can this be done in O(c)? for(int i = 0; i < n_units; ++i) { @@ -126,7 +133,6 @@ class LogarithmicVertexMapper { } private: - std::vector _blocks; }; @@ -137,10 +143,16 @@ class LogarithmicVertexMapper { template struct VertexIndex { + typedef VertexIndex self_t; + /** * Default constructor. */ - VertexIndex() = default; + VertexIndex() = default; + ~VertexIndex() = default; + VertexIndex(const self_t &) = default; + VertexIndex(self_t &&) = default; + self_t & operator=(const self_t &) = default; /** * Index Constructor. @@ -154,7 +166,6 @@ struct VertexIndex { team_unit_t unit; /** The offset of the vertex in local memory space of unit */ IndexType offset; - }; /** From 8eb4a0ddcfeeea1ac74406d93b795be089661669 Mon Sep 17 00:00:00 2001 From: Tobias Fuchs Date: Tue, 12 Mar 2019 21:02:47 +0100 Subject: [PATCH 101/102] Added graph microbench examples --- dash/examples/ex.02.graph-iter/main.cpp | 142 ++++++++++++ dash/examples/ex.02.graph-iter/rmatrandom.h | 210 ++++++++++++++++++ dash/examples/ex.02.graph/rmatrandom.h | 159 +++++-------- dash/examples/ex.02.graph_insert/main.cpp | 77 +++++++ dash/examples/ex.02.graph_insert/rmatrandom.h | 210 ++++++++++++++++++ 5 files changed, 700 insertions(+), 98 deletions(-) create mode 100644 dash/examples/ex.02.graph-iter/main.cpp create mode 100644 dash/examples/ex.02.graph-iter/rmatrandom.h create mode 100644 dash/examples/ex.02.graph_insert/main.cpp create mode 100644 dash/examples/ex.02.graph_insert/rmatrandom.h diff --git a/dash/examples/ex.02.graph-iter/main.cpp b/dash/examples/ex.02.graph-iter/main.cpp new file mode 100644 index 000000000..1a195ae8f --- /dev/null +++ b/dash/examples/ex.02.graph-iter/main.cpp @@ -0,0 +1,142 @@ +#include +#include +#include +#include + +#include "rmatrandom.h" + +struct vprop { + int comp; +}; + +struct eprop { + int comp; +}; + +typedef dash::Graph graph_t; + +int main(int argc, char* argv[]) { + dash::init(&argc, &argv); + + int n_unit_edges = 1; + int n_vertices_full = 100000; + int n_edges_full = n_vertices_full * n_unit_edges; + // not exactly n_vertices_full are generated due to rounding + int n_vertices_start = n_vertices_full / dash::size(); + int n_size_rounds = 3; + int n_rounds = 5; + int n_iter_rounds = 10; + for(int rounds = 0; rounds < n_size_rounds; ++rounds) { + for(int i = 0; i < n_rounds; ++i) { + int scale = pow(10, rounds); + int n_vertices = n_vertices_start * scale; + graph_t g(n_vertices, n_unit_edges); + + for(int j = 0; j < n_vertices; ++j) { + g.add_vertex(); + } + + std::clock_t g_begin_time = clock(); + g.commit(); + std::clock_t g_end_time = clock(); + if(dash::myid() == 0) { + std::cout << "[round " << i << "] " << n_vertices_full * scale << + " vertices per node iterated (commit): " << + double(g_end_time - g_begin_time) / CLOCKS_PER_SEC << std::endl; + } + + int n_edges = n_vertices * n_unit_edges; + auto src = g.vertices().lbegin(); + auto trg = src + 1; + auto end = g.vertices().lend(); + for(int j = 0; j < n_edges; ++j) { + g.add_edge(src, trg); + ++src; + ++trg; + if(trg == end) { + src = g.vertices().lbegin(); + trg = src + 1; + } + } + + g_begin_time = clock(); + g.commit(); + g_end_time = clock(); + if(dash::myid() == 0) { + std::cout << "[round " << i << "] " << n_edges_full * scale << + " edges per node iterated (commit): " << + double(g_end_time - g_begin_time) / CLOCKS_PER_SEC << std::endl; + } + + g_begin_time = clock(); + for(int j = 0; j < n_iter_rounds; ++j) { + for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) { + g[it]; + } + } + g_end_time = clock(); + double time = double(g_end_time - g_begin_time) / CLOCKS_PER_SEC; + double all_time = 0; + + dart_reduce(&time, &all_time, 1, DART_TYPE_DOUBLE, DART_OP_SUM, 0, + g.team().dart_id()); + + if(dash::myid() == 0) { + std::cout << "[round " << i << "] " << n_vertices_full * scale << + " vertices per node iterated (local): " << all_time << std::endl; + } + + if(dash::myid() == 0) { + std::clock_t g_begin_time = clock(); + for(int j = 0; j < n_iter_rounds; ++j) { + for(auto it = g.vertices().begin(); it != g.vertices().end(); ++it) { + g[it]; + } + } + std::clock_t g_end_time = clock(); + std::cout << "[round " << i << "] " << n_vertices_full * scale << + " vertices per node iterated (global): " << + double(g_end_time - g_begin_time) / CLOCKS_PER_SEC << std::endl; + } + + g_begin_time = clock(); + for(int j = 0; j < n_iter_rounds; ++j) { + for(auto it = g.out_edges().lbegin(); it != g.out_edges().lend(); ++it) { + g[it]; + } + } + g_end_time = clock(); + time = double(g_end_time - g_begin_time) / CLOCKS_PER_SEC; + + dart_reduce(&time, &all_time, 1, DART_TYPE_DOUBLE, DART_OP_SUM, 0, + g.team().dart_id()); + + if(dash::myid() == 0) { + std::cout << "[round " << i << "] " << n_edges_full * scale << + " edges per node iterated (local): " << all_time << std::endl; + } + + if(dash::myid() == 0) { + int count = 0; + std::clock_t g_begin_time = clock(); + for(int j = 0; j < n_iter_rounds; ++j) { + for(auto it = g.out_edges().begin(); it != g.out_edges().end(); ++it) { + g[it]; + ++count; + } + } + std::clock_t g_end_time = clock(); + std::cout << "[round " << i << "] " << n_edges_full * scale << + " edges per node iterated (global): " << + double(g_end_time - g_begin_time) / CLOCKS_PER_SEC << std::endl; + } + dash::barrier(); + } + if(dash::myid() == 0) std::cout << "-----------------" << std::endl; + } + + dash::finalize(); + return 0; +} + + diff --git a/dash/examples/ex.02.graph-iter/rmatrandom.h b/dash/examples/ex.02.graph-iter/rmatrandom.h new file mode 100644 index 000000000..ef30897da --- /dev/null +++ b/dash/examples/ex.02.graph-iter/rmatrandom.h @@ -0,0 +1,210 @@ +#ifndef DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED +#define DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED + +#include +#include +#include +#include + +class xorshift { + +public: + typedef unsigned long result_type; + + template + xorshift(SeedGenerator &s) : x(s()), y(s()), z(s()) {} + + result_type operator()() { + result_type t; + x ^= x << 16; + x ^= x >> 5; + x ^= x << 1; + + t = x; + x = y; + y = z; + z = t ^ x ^ y; + + return z; + } + + // std::uniform distribution too slow + double dist() { return static_cast((*this)()) / max(); } + + result_type min() { return 0; } + + result_type max() { return std::numeric_limits::max() - 1; } + +private: + result_type x, y, z; +}; + +template class RMATRandomGenerator { + +public: + typedef GraphType graph_type; + typedef RMATRandomGenerator self_t; + typedef typename graph_type::vertex_size_type vertex_size_type; + typedef typename graph_type::edge_size_type edge_size_type; + typedef std::pair value_type; + typedef std::uniform_int_distribution dist_int_type; + +private: +public: + /** + * Begin iterator + */ + template + RMATRandomGenerator(vertex_size_type n, edge_size_type m, int n_units, + dash::team_unit_t myid, VertexMapperFunction owner, + double a, double b, double c, double d) { + std::random_device rd; + xorshift gen(rd); + + // the intel compiler on SUPERMUC falls into a "internal error loop" with + // the end constructor. therefore, this constructor is misused as end + // constructor + if (n == 0) { + _done = true; + } else { + // generate 50% of the edges with RMAT + m /= 2; + edge_size_type m_unit_random = m / n_units; + int SCALE = int(floor(log(double(n)) / log(2.))); + + std::map edge_map; + + // generate whole graph on each unit, but only use edges belonging to this + // unit + edge_size_type generated = 0; + edge_size_type local_edges = 0; + do { + edge_size_type rejected = 0; + do { + vertex_size_type u, v; + std::tie(u, v) = generate_rmat_edge(gen, n, SCALE, a, b, c, d); + + if (owner(u, n, n_units, myid) == myid) { + // reject loop edges and multi-edges + if (u != v && + edge_map.find(std::make_pair(u, v)) == edge_map.end()) { + edge_map[std::make_pair(u, v)] = true; + ++local_edges; + } else { + ++rejected; + } + } + ++generated; + } while (generated < m); + // generate more edges based on the amount of edges rejected on each + // unit + int rejected_all; + dart_allreduce(&rejected, &rejected_all, 1, DART_TYPE_INT, DART_OP_SUM, + dash::Team::All().dart_id()); + generated -= rejected_all; + } while (generated < m); + + // reserve space for generated rmat edges and for coming random edges + _values.reserve(local_edges + m_unit_random); + typename std::map::reverse_iterator em_end = + edge_map.rend(); + for (typename std::map::reverse_iterator em_i = + edge_map.rbegin(); + em_i != em_end; ++em_i) { + _values.push_back(em_i->first); + } + + // generate 50% random edges + // source edge has to belong to this unit + int start = 0; + int end = 0; + if (myid == 0) { + end = owner.size(myid) - 1; + } else { + for (int i = 0; i < myid; ++i) { + dash::team_unit_t unit{i}; + start += owner.size(unit); + } + end = start + owner.size(myid) - 1; + } + dist_int_type dist_u(start, end); + dist_int_type dist_v(0, n - 1); + for (int i = 0; i < m_unit_random; ++i) { + vertex_size_type u = dist_u(gen); + vertex_size_type v = dist_v(gen); + _values.push_back(std::make_pair(u, v)); + } + + _current = _values.size() - 1; + } + } + + /** + * End iterator + * + * This constructor somehow results in an "internal error loop" with the + * intel compiler of the SUPERMUC system. + */ + RMATRandomGenerator() : _done(true) {} + + self_t &operator++() { + if (_current > 0) { + --_current; + } else { + _done = true; + } + + return *this; + } + + const value_type &operator*() const { return _values[_current]; } + + const value_type *operator->() const { return &_values[_current]; } + + bool operator!=(const self_t &other) { return !(_done && other._done); } + +private: + value_type generate_rmat_edge(xorshift &gen, vertex_size_type n, + unsigned int SCALE, double a, double b, + double c, double d) { + vertex_size_type u = 0; + vertex_size_type v = 0; + vertex_size_type step = n / 2; + + for (unsigned int j = 0; j < SCALE; ++j) { + double p = gen.dist(); + if (p < a) + ; + else if (p >= a && p < a + b) + v += step; + else if (p >= a + b && p < a + b + c) + u += step; + else { + u += step; + v += step; + } + + step /= 2; + + a *= 0.9 + 0.2 * gen.dist(); + b *= 0.9 + 0.2 * gen.dist(); + c *= 0.9 + 0.2 * gen.dist(); + d *= 0.9 + 0.2 * gen.dist(); + + double S = a + b + c + d; + + a /= S; + b /= S; + c /= S; + d = 1. - a - b - c; + } + + return std::make_pair(u, v); + } + + std::vector _values; + int _current; + bool _done = false; +}; + +#endif // DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED diff --git a/dash/examples/ex.02.graph/rmatrandom.h b/dash/examples/ex.02.graph/rmatrandom.h index 5c8329598..ef30897da 100644 --- a/dash/examples/ex.02.graph/rmatrandom.h +++ b/dash/examples/ex.02.graph/rmatrandom.h @@ -1,100 +1,76 @@ #ifndef DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED #define DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED -#include #include -#include +#include #include +#include class xorshift { public: - typedef unsigned long result_type; - - template - xorshift(SeedGenerator & s) - : x(s()), - y(s()), - z(s()) - { } + + template + xorshift(SeedGenerator &s) : x(s()), y(s()), z(s()) {} result_type operator()() { - result_type t; - x ^= x << 16; - x ^= x >> 5; - x ^= x << 1; + result_type t; + x ^= x << 16; + x ^= x >> 5; + x ^= x << 1; - t = x; - x = y; - y = z; - z = t ^ x ^ y; + t = x; + x = y; + y = z; + z = t ^ x ^ y; - return z; + return z; } // std::uniform distribution too slow - double dist() { - return static_cast((*this)()) / max(); - } + double dist() { return static_cast((*this)()) / max(); } - result_type min() { - return 0; - } + result_type min() { return 0; } - result_type max() { - return std::numeric_limits::max() - 1; - } + result_type max() { return std::numeric_limits::max() - 1; } private: - result_type x, y, z; - }; -template -class RMATRandomGenerator { +template class RMATRandomGenerator { public: - typedef GraphType graph_type; - typedef RMATRandomGenerator self_t; - typedef typename graph_type::vertex_size_type vertex_size_type; - typedef typename graph_type::edge_size_type edge_size_type; + typedef GraphType graph_type; + typedef RMATRandomGenerator self_t; + typedef typename graph_type::vertex_size_type vertex_size_type; + typedef typename graph_type::edge_size_type edge_size_type; typedef std::pair value_type; - typedef std::uniform_int_distribution dist_int_type; + typedef std::uniform_int_distribution dist_int_type; private: - public: - /** * Begin iterator */ - template - RMATRandomGenerator( - vertex_size_type n, - edge_size_type m, - int n_units, - dash::team_unit_t myid, - VertexMapperFunction owner, - double a, - double b, - double c, - double d - ) { + template + RMATRandomGenerator(vertex_size_type n, edge_size_type m, int n_units, + dash::team_unit_t myid, VertexMapperFunction owner, + double a, double b, double c, double d) { std::random_device rd; - xorshift gen(rd); + xorshift gen(rd); // the intel compiler on SUPERMUC falls into a "internal error loop" with // the end constructor. therefore, this constructor is misused as end // constructor - if(n == 0) { + if (n == 0) { _done = true; } else { // generate 50% of the edges with RMAT m /= 2; edge_size_type m_unit_random = m / n_units; - int SCALE = int(floor(log(double(n))/log(2.))); + int SCALE = int(floor(log(double(n)) / log(2.))); std::map edge_map; @@ -110,8 +86,8 @@ class RMATRandomGenerator { if (owner(u, n, n_units, myid) == myid) { // reject loop edges and multi-edges - if (u != v - && edge_map.find(std::make_pair(u, v)) == edge_map.end()) { + if (u != v && + edge_map.find(std::make_pair(u, v)) == edge_map.end()) { edge_map[std::make_pair(u, v)] = true; ++local_edges; } else { @@ -120,19 +96,21 @@ class RMATRandomGenerator { } ++generated; } while (generated < m); - // generate more edges based on the amount of edges rejected on each unit + // generate more edges based on the amount of edges rejected on each + // unit int rejected_all; dart_allreduce(&rejected, &rejected_all, 1, DART_TYPE_INT, DART_OP_SUM, - dash::Team::All().dart_id()); + dash::Team::All().dart_id()); generated -= rejected_all; } while (generated < m); // reserve space for generated rmat edges and for coming random edges _values.reserve(local_edges + m_unit_random); - typename std::map::reverse_iterator em_end = - edge_map.rend(); - for (typename std::map::reverse_iterator em_i = - edge_map.rbegin(); em_i != em_end; ++em_i) { + typename std::map::reverse_iterator em_end = + edge_map.rend(); + for (typename std::map::reverse_iterator em_i = + edge_map.rbegin(); + em_i != em_end; ++em_i) { _values.push_back(em_i->first); } @@ -140,18 +118,18 @@ class RMATRandomGenerator { // source edge has to belong to this unit int start = 0; int end = 0; - if(myid == 0) { + if (myid == 0) { end = owner.size(myid) - 1; } else { - for(int i = 0; i < myid; ++i) { - dash::team_unit_t unit { i }; + for (int i = 0; i < myid; ++i) { + dash::team_unit_t unit{i}; start += owner.size(unit); } end = start + owner.size(myid) - 1; } dist_int_type dist_u(start, end); dist_int_type dist_v(0, n - 1); - for(int i = 0; i < m_unit_random; ++i) { + for (int i = 0; i < m_unit_random; ++i) { vertex_size_type u = dist_u(gen); vertex_size_type v = dist_v(gen); _values.push_back(std::make_pair(u, v)); @@ -163,16 +141,14 @@ class RMATRandomGenerator { /** * End iterator - * - * This constructor somehow results in an "internal error loop" with the + * + * This constructor somehow results in an "internal error loop" with the * intel compiler of the SUPERMUC system. */ - RMATRandomGenerator() - : _done(true) - { } + RMATRandomGenerator() : _done(true) {} - self_t & operator++() { - if(_current > 0) { + self_t &operator++() { + if (_current > 0) { --_current; } else { _done = true; @@ -181,33 +157,19 @@ class RMATRandomGenerator { return *this; } - const value_type & operator*() const { - return _values[_current]; - } + const value_type &operator*() const { return _values[_current]; } - const value_type * operator->() const { - return &_values[_current]; - } + const value_type *operator->() const { return &_values[_current]; } - - - bool operator!=(const self_t & other) { - return !(_done && other._done); - } + bool operator!=(const self_t &other) { return !(_done && other._done); } private: - - value_type generate_rmat_edge( - xorshift & gen, - vertex_size_type n, - unsigned int SCALE, - double a, - double b, - double c, - double d) { + value_type generate_rmat_edge(xorshift &gen, vertex_size_type n, + unsigned int SCALE, double a, double b, + double c, double d) { vertex_size_type u = 0; vertex_size_type v = 0; - vertex_size_type step = n/2; + vertex_size_type step = n / 2; for (unsigned int j = 0; j < SCALE; ++j) { double p = gen.dist(); @@ -231,7 +193,9 @@ class RMATRandomGenerator { double S = a + b + c + d; - a /= S; b /= S; c /= S; + a /= S; + b /= S; + c /= S; d = 1. - a - b - c; } @@ -239,9 +203,8 @@ class RMATRandomGenerator { } std::vector _values; - int _current; - bool _done = false; - + int _current; + bool _done = false; }; #endif // DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED diff --git a/dash/examples/ex.02.graph_insert/main.cpp b/dash/examples/ex.02.graph_insert/main.cpp new file mode 100644 index 000000000..04bf70c13 --- /dev/null +++ b/dash/examples/ex.02.graph_insert/main.cpp @@ -0,0 +1,77 @@ +#include +#include +#include +#include + +#include "rmatrandom.h" + +struct vprop { + int comp; +}; + +struct eprop { + int comp; +}; + +typedef dash::Graph graph_t; + +int main(int argc, char* argv[]) { + dash::init(&argc, &argv); + + int n_vertices_start = 1000; + if(dash::myid() != 0) { + n_vertices_start = 0; + } + int n_unit_edges = 1; + int n_rounds = 1; + int n_size_rounds = 4; + for(int rounds = 3; rounds < n_size_rounds; ++rounds) { + for(int i = 0; i < n_rounds; ++i) { + int n_vertices = n_vertices_start * pow(10, rounds); + graph_t g(n_vertices, n_unit_edges); + + std::clock_t g_begin_time = clock(); + for(int j = 0; j < n_vertices; ++j) { + g.add_vertex(); + } + std::clock_t g_end_time = clock(); + if(dash::myid() == 0) { + std::cout << "[round " << i << "] " << n_vertices << + " vertices added: " << double(g_end_time - g_begin_time) + / CLOCKS_PER_SEC << std::endl; + } + + g.commit(); +/* + int n_edges = n_vertices * n_unit_edges; + if(dash::myid() != 0) { + n_edges = 0; + } + auto src = g.vertices().lbegin(); + auto trg = src + 1; + auto end = g.vertices().lend(); + g_begin_time = clock(); + for(int j = 0; j < n_edges; ++j) { + g.add_edge(src, trg); + ++src; + ++trg; + if(trg == end) { + src = g.vertices().lbegin(); + trg = src + 1; + } + } + g_end_time = clock(); + if(dash::myid() == 0) { + std::cout << "[round " << i << "] " << n_edges << " edges added: " << + double(g_end_time - g_begin_time) / CLOCKS_PER_SEC << std::endl; + } +*/ + } + dash::barrier(); + if(dash::myid() == 0) std::cout << "----------------------" << std::endl; + } + + dash::finalize(); + return 0; +} + diff --git a/dash/examples/ex.02.graph_insert/rmatrandom.h b/dash/examples/ex.02.graph_insert/rmatrandom.h new file mode 100644 index 000000000..ef30897da --- /dev/null +++ b/dash/examples/ex.02.graph_insert/rmatrandom.h @@ -0,0 +1,210 @@ +#ifndef DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED +#define DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED + +#include +#include +#include +#include + +class xorshift { + +public: + typedef unsigned long result_type; + + template + xorshift(SeedGenerator &s) : x(s()), y(s()), z(s()) {} + + result_type operator()() { + result_type t; + x ^= x << 16; + x ^= x >> 5; + x ^= x << 1; + + t = x; + x = y; + y = z; + z = t ^ x ^ y; + + return z; + } + + // std::uniform distribution too slow + double dist() { return static_cast((*this)()) / max(); } + + result_type min() { return 0; } + + result_type max() { return std::numeric_limits::max() - 1; } + +private: + result_type x, y, z; +}; + +template class RMATRandomGenerator { + +public: + typedef GraphType graph_type; + typedef RMATRandomGenerator self_t; + typedef typename graph_type::vertex_size_type vertex_size_type; + typedef typename graph_type::edge_size_type edge_size_type; + typedef std::pair value_type; + typedef std::uniform_int_distribution dist_int_type; + +private: +public: + /** + * Begin iterator + */ + template + RMATRandomGenerator(vertex_size_type n, edge_size_type m, int n_units, + dash::team_unit_t myid, VertexMapperFunction owner, + double a, double b, double c, double d) { + std::random_device rd; + xorshift gen(rd); + + // the intel compiler on SUPERMUC falls into a "internal error loop" with + // the end constructor. therefore, this constructor is misused as end + // constructor + if (n == 0) { + _done = true; + } else { + // generate 50% of the edges with RMAT + m /= 2; + edge_size_type m_unit_random = m / n_units; + int SCALE = int(floor(log(double(n)) / log(2.))); + + std::map edge_map; + + // generate whole graph on each unit, but only use edges belonging to this + // unit + edge_size_type generated = 0; + edge_size_type local_edges = 0; + do { + edge_size_type rejected = 0; + do { + vertex_size_type u, v; + std::tie(u, v) = generate_rmat_edge(gen, n, SCALE, a, b, c, d); + + if (owner(u, n, n_units, myid) == myid) { + // reject loop edges and multi-edges + if (u != v && + edge_map.find(std::make_pair(u, v)) == edge_map.end()) { + edge_map[std::make_pair(u, v)] = true; + ++local_edges; + } else { + ++rejected; + } + } + ++generated; + } while (generated < m); + // generate more edges based on the amount of edges rejected on each + // unit + int rejected_all; + dart_allreduce(&rejected, &rejected_all, 1, DART_TYPE_INT, DART_OP_SUM, + dash::Team::All().dart_id()); + generated -= rejected_all; + } while (generated < m); + + // reserve space for generated rmat edges and for coming random edges + _values.reserve(local_edges + m_unit_random); + typename std::map::reverse_iterator em_end = + edge_map.rend(); + for (typename std::map::reverse_iterator em_i = + edge_map.rbegin(); + em_i != em_end; ++em_i) { + _values.push_back(em_i->first); + } + + // generate 50% random edges + // source edge has to belong to this unit + int start = 0; + int end = 0; + if (myid == 0) { + end = owner.size(myid) - 1; + } else { + for (int i = 0; i < myid; ++i) { + dash::team_unit_t unit{i}; + start += owner.size(unit); + } + end = start + owner.size(myid) - 1; + } + dist_int_type dist_u(start, end); + dist_int_type dist_v(0, n - 1); + for (int i = 0; i < m_unit_random; ++i) { + vertex_size_type u = dist_u(gen); + vertex_size_type v = dist_v(gen); + _values.push_back(std::make_pair(u, v)); + } + + _current = _values.size() - 1; + } + } + + /** + * End iterator + * + * This constructor somehow results in an "internal error loop" with the + * intel compiler of the SUPERMUC system. + */ + RMATRandomGenerator() : _done(true) {} + + self_t &operator++() { + if (_current > 0) { + --_current; + } else { + _done = true; + } + + return *this; + } + + const value_type &operator*() const { return _values[_current]; } + + const value_type *operator->() const { return &_values[_current]; } + + bool operator!=(const self_t &other) { return !(_done && other._done); } + +private: + value_type generate_rmat_edge(xorshift &gen, vertex_size_type n, + unsigned int SCALE, double a, double b, + double c, double d) { + vertex_size_type u = 0; + vertex_size_type v = 0; + vertex_size_type step = n / 2; + + for (unsigned int j = 0; j < SCALE; ++j) { + double p = gen.dist(); + if (p < a) + ; + else if (p >= a && p < a + b) + v += step; + else if (p >= a + b && p < a + b + c) + u += step; + else { + u += step; + v += step; + } + + step /= 2; + + a *= 0.9 + 0.2 * gen.dist(); + b *= 0.9 + 0.2 * gen.dist(); + c *= 0.9 + 0.2 * gen.dist(); + d *= 0.9 + 0.2 * gen.dist(); + + double S = a + b + c + d; + + a /= S; + b /= S; + c /= S; + d = 1. - a - b - c; + } + + return std::make_pair(u, v); + } + + std::vector _values; + int _current; + bool _done = false; +}; + +#endif // DASH__EXAMPLES__EX02_GRAPH__RMATRANDOM_H__INCLUDED From 1a86e365fa204f7080f8c06152c922aed6757838 Mon Sep 17 00:00:00 2001 From: Tobias Fuchs Date: Tue, 12 Mar 2019 21:35:49 +0100 Subject: [PATCH 102/102] Fixed clang compiler errors, added acknowledgements --- dash/examples/ex.02.graph/main.cpp | 4 ++-- .../include/dash/algorithm/graph/ConnectedComponents.h | 9 +++++++-- .../include/dash/algorithm/graph/MinimumSpanningTree.h | 5 +++++ dash/test/algorithm/ConnectedComponentsTest.cc | 4 ++-- dash/test/algorithm/MinimumSpanningTreeTest.cc | 10 ++++++++-- 5 files changed, 24 insertions(+), 8 deletions(-) diff --git a/dash/examples/ex.02.graph/main.cpp b/dash/examples/ex.02.graph/main.cpp index 78853da1a..6fe08f57f 100644 --- a/dash/examples/ex.02.graph/main.cpp +++ b/dash/examples/ex.02.graph/main.cpp @@ -6,8 +6,8 @@ #include "rmatrandom.h" struct vprop { - int comp; - int unit; + dash::default_index_t comp; + dash::global_unit_t unit; }; typedef dash::Graph graph_t; diff --git a/dash/include/dash/algorithm/graph/ConnectedComponents.h b/dash/include/dash/algorithm/graph/ConnectedComponents.h index c5229b3e6..348025498 100644 --- a/dash/include/dash/algorithm/graph/ConnectedComponents.h +++ b/dash/include/dash/algorithm/graph/ConnectedComponents.h @@ -1,3 +1,8 @@ +/** + * Author: Steffan Effenberger (github: @stiefn) + * via https://github.com/stiefn/dyn-data-structures-thesis + * + */ #ifndef DASH__ALGORITHM____GRAPH__CONNECTED_COMPONENTS_H__ #define DASH__ALGORITHM____GRAPH__CONNECTED_COMPONENTS_H__ @@ -6,7 +11,7 @@ namespace dash { -typedef std::vector> matrix_t; +typedef std::vector> matrix_t; template using matrix_pair_t = std::vector>>; @@ -282,7 +287,7 @@ void cc_set_data( */ template void connected_components(GraphType & g) { - typedef typename GraphType::vertex_properties_type vprop_t; + typedef typename GraphType::vertex_properties_type vprop_t; dash::util::Trace trace("ConnectedComponents"); // set component to global index in iteration space // TODO: find faster method for this diff --git a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h index 3e2c79214..31d78fc08 100644 --- a/dash/include/dash/algorithm/graph/MinimumSpanningTree.h +++ b/dash/include/dash/algorithm/graph/MinimumSpanningTree.h @@ -1,3 +1,8 @@ +/** + * Author: Steffan Effenberger (github: @stiefn) + * via https://github.com/stiefn/dyn-data-structures-thesis + * + */ #ifndef DASH__ALGORITHM____GRAPH__MINIMUM_SPANNING_TREE_H__ #define DASH__ALGORITHM____GRAPH__MINIMUM_SPANNING_TREE_H__ diff --git a/dash/test/algorithm/ConnectedComponentsTest.cc b/dash/test/algorithm/ConnectedComponentsTest.cc index c61441ecd..5ca0382df 100644 --- a/dash/test/algorithm/ConnectedComponentsTest.cc +++ b/dash/test/algorithm/ConnectedComponentsTest.cc @@ -3,8 +3,8 @@ #include struct vprop { - int comp; - int unit; + dash::default_index_t comp; + dash::global_unit_t unit; }; typedef dash::Graph graph_t; diff --git a/dash/test/algorithm/MinimumSpanningTreeTest.cc b/dash/test/algorithm/MinimumSpanningTreeTest.cc index cfa4199ac..ef49d79e2 100644 --- a/dash/test/algorithm/MinimumSpanningTreeTest.cc +++ b/dash/test/algorithm/MinimumSpanningTreeTest.cc @@ -1,10 +1,16 @@ +/** + * Author: Steffan Effenberger (github: @stiefn) + * via https://github.com/stiefn/dyn-data-structures-thesis + * + */ + #include "MinimumSpanningTreeTest.h" #include #include struct vprop { - int comp; - short unit; + dash::default_index_t comp; + dash::global_unit_t unit; }; struct eprop {