Cuda coll

configure.ac

@@ -1411,6 +1411,10 @@ m4_ifdef([project_oshmem],
 
 opal_show_subtitle "Final output"
 
+if test "$OPAL_cuda_support" != "0"; then
+  AC_CONFIG_FILES([opal/datatype/cuda/Makefile])
+fi
+
 AC_CONFIG_FILES([
     Makefile
 

ompi/mca/coll/adapt/Makefile.am

@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2014      The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+dist_ompidata_DATA = help-mpi-coll-adapt.txt
+
+sources = \
+	coll_adapt_component.c \
+	coll_adapt_module.c \
+    coll_adapt_bcast.c \
+	coll_adapt_ibcast.c \
+	coll_adapt_reduce.c \
+	coll_adapt_ireduce.c \
+	coll_adapt_allreduce.c \
+	coll_adapt_iallreduce.c \
+	coll_adapt_alltoall.c \
+	coll_adapt_ialltoall.c \
+	coll_adapt_alltoallv.c \
+	coll_adapt_ialltoallv.c \
+    coll_adapt.h \
+	coll_adapt_algorithms.h \
+	coll_adapt_context.h \
+	coll_adapt_context.c \
+	coll_adapt_inbuf.c \
+	coll_adapt_inbuf.h \
+	coll_adapt_item.c \
+	coll_adapt_item.h
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+component_noinst =
+component_install =
+if MCA_BUILD_ompi_coll_adapt_DSO
+component_install += mca_coll_adapt.la
+else
+component_noinst += libmca_coll_adapt.la
+endif
+
+mcacomponentdir = $(ompilibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_coll_adapt_la_SOURCES = $(sources)
+mca_coll_adapt_la_LDFLAGS = -module -avoid-version
+mca_coll_adapt_la_LIBADD =
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_coll_adapt_la_SOURCES =$(sources)
+libmca_coll_adapt_la_LDFLAGS = -module -avoid-version

ompi/mca/coll/adapt/coll_adapt.h

@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) 2014      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/** @file */
+
+#ifndef MCA_COLL_ADAPT_EXPORT_H
+#define MCA_COLL_ADAPT_EXPORT_H
+
+#include "ompi_config.h"
+
+#include "mpi.h"
+#include "opal/mca/mca.h"
+#include "opal/datatype/opal_convertor.h"
+#include "ompi/mca/coll/coll.h"
+
+BEGIN_C_DECLS
+
+typedef struct mca_coll_adapt_module_t mca_coll_adapt_module_t;
+
+/** 
+ * Structure to hold the adapt coll component.  First it holds the
+ * base coll component, and then holds a bunch of
+ * adapt-coll-component-specific stuff (e.g., current MCA param
+ * values). 
+ */
+typedef struct mca_coll_adapt_component_t {
+    /** Base coll component */
+    mca_coll_base_component_2_0_0_t super;
+
+    /** MCA parameter: Priority of this component */
+    int adapt_priority;
+
+    /** MCA parameter: Length of a cache line or page (in bytes) */
+    int adapt_control_size;
+
+    /** MCA parameter: Number of "in use" flags in each
+        communicator's area in the data mpool */
+    int adapt_comm_num_in_use_flags;
+
+    /** MCA parameter: Number of segments for each communicator in
+        the data mpool */
+    int adapt_comm_num_segments;
+
+    /** MCA parameter: Fragment size for data */
+    int adapt_fragment_size;
+
+    /** MCA parameter: Degree of tree for tree-based collectives */
+    int adapt_tree_degree;
+
+    /** MCA parameter: Number of processes to use in the
+        calculation of the "info" MCA parameter */
+    int adapt_info_comm_size;
+
+    /******* end of MCA params ********/
+
+    /** How many fragment segments are protected by a single
+        in-use flags.  This is solely so that we can only perform
+        the division once and then just use the value without
+        having to re-calculate. */
+    int adapt_segs_per_inuse_flag;
+} mca_coll_adapt_component_t;
+
+/**
+ * Structure for the sm coll module to hang off the communicator.
+ * Contains communicator-specific information, including pointers
+ * into the per-communicator shmem data data segment for this
+ * comm's sm collective operations area.
+ */
+typedef struct mca_coll_adapt_comm_t {
+    /* Meta data that we get back from the common mmap allocation
+       function */
+    mca_coll_adapt_module_t *adapt_bootstrap_meta;
+
+    /** Pointer to my barrier control pages (odd index pages are
+        "in", even index pages are "out") */
+    uint32_t *mcb_barrier_control_me;
+
+    /** Pointer to my parent's barrier control pages (will be NULL
+        for communicator rank 0; odd index pages are "in", even
+        index pages are "out") */
+    uint32_t *mcb_barrier_control_parent;
+
+    /** Pointers to my childrens' barrier control pages (they're
+        contiguous in memory, so we only point to the base -- the
+        number of children is in my entry in the mcb_tree); will
+        be NULL if this process has no children (odd index pages
+        are "in", even index pages are "out") */
+    uint32_t *mcb_barrier_control_children;
+
+    /** Number of barriers that we have executed (i.e., which set
+        of barrier buffers to use). */
+    int mcb_barrier_count;
+
+    /** Operation number (i.e., which segment number to use) */
+    uint32_t mcb_operation_count;
+} mca_coll_adapt_comm_t;
+
+/** Coll sm module */
+struct mca_coll_adapt_module_t {
+    /** Base module */
+	mca_coll_base_module_t super;
+
+    /* Whether this module has been lazily initialized or not yet */
+    bool enabled;
+
+    /* Data that hangs off the communicator */
+	mca_coll_adapt_comm_t *adapt_comm_data;
+
+        /* Underlying reduce function and module */
+	mca_coll_base_module_reduce_fn_t previous_reduce;
+	mca_coll_base_module_t *previous_reduce_module;
+};
+OBJ_CLASS_DECLARATION(mca_coll_adapt_module_t);
+
+/**
+ * Global component instance
+ */
+OMPI_MODULE_DECLSPEC extern mca_coll_adapt_component_t mca_coll_adapt_component;
+
+/*
+ * coll module functions
+ */
+int mca_coll_adapt_init_query(bool enable_progress_threads,
+                              bool enable_mpi_threads);
+
+mca_coll_base_module_t *
+mca_coll_adapt_comm_query(struct ompi_communicator_t *comm, int *priority);
+
+/* Lazily enable a module (since it involves expensive/slow mmap
+   allocation, etc.) */
+int ompi_coll_adapt_lazy_enable(mca_coll_base_module_t *module,
+                                struct ompi_communicator_t *comm);
+
+
+int mca_coll_adapt_ft_event(int state);
+
+#endif /* MCA_COLL_ADAPT_EXPORT_H */

ompi/mca/coll/adapt/coll_adapt_algorithms.h

@@ -0,0 +1,144 @@
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/coll_base_topo.h"  //ompi_coll_tree_t
+
+int mca_coll_adapt_bcast(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_bcast_binomial(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_bcast_in_order_binomial(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_bcast_binary(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_bcast_pipeline(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_bcast_chain(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_bcast_linear(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_bcast_topoaware_linear(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_bcast_topoaware_chain(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_bcast_two_trees_binary(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_bcast_two_trees_binomial(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_bcast_two_chains(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_bcast_generic(void *buff, int count, struct ompi_datatype_t *dtype, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module, ompi_coll_tree_t *tree);
+
+int mca_coll_adapt_bcast_two_trees_generic(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module, ompi_coll_tree_t** trees);
+
+int mca_coll_adapt_ibcast(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_ibcast_binomial(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_ibcast_in_order_binomial(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_ibcast_bininary(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_ibcast_pipeline(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_ibcast_chain(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_ibcast_linear(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_ibcast_topoaware_linear(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_ibcast_topoaware_chain(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+
+int mca_coll_adapt_ibcast_two_trees_binary(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_ibcast_two_trees_binomial(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_ibcast_two_chains(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_ibcast_generic(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module, ompi_coll_tree_t* tree);
+
+int mca_coll_adapt_ibcast_two_trees_generic(void *buff, int count, struct ompi_datatype_t *datatype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module, ompi_coll_tree_t** trees);
+
+int mca_coll_adapt_reduce(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_reduce_binomial(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_reduce_in_order_binomial(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_reduce_binary(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_reduce_pipeline(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_reduce_chain(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_reduce_linear(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_reduce_topoaware_linear(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_reduce_topoaware_chain(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+
+int mca_coll_adapt_reduce_generic(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, mca_coll_base_module_t *module, ompi_coll_tree_t* tree);
+
+int mca_coll_adapt_ireduce(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_ireduce_binomial(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_ireduce_in_order_binomial(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_ireduce_binary(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_ireduce_pipeline(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+
+int mca_coll_adapt_ireduce_chain(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_ireduce_linear(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_ireduce_generic(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module, ompi_coll_tree_t* tree);
+
+int mca_coll_adapt_allreduce_intra_nonoverlapping(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_allreduce_intra_recursivedoubling(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_allreduce(const void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_iallreduce(void *sbuf, void *rbuf, int count, struct ompi_datatype_t *dtype, struct ompi_op_t *op, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_alltoallv(const void *sbuf, const int *scounts, const int *sdisps, struct ompi_datatype_t *sdtype, void* rbuf, const int *rcounts, const int *rdisps, struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm, mca_coll_base_module_t *module);
+
+int mca_coll_adapt_ialltoallv(const void *sbuf, const int *scounts, const int *sdisps, struct ompi_datatype_t *sdtype, void* rbuf, const int *rcounts, const int *rdisps, struct ompi_datatype_t *rdtype, struct ompi_communicator_t *comm, ompi_request_t ** request, mca_coll_base_module_t *module);
+
+//get thread id for test
+static inline uint64_t gettid(void) {
+    pthread_t ptid = pthread_self();
+    uint64_t threadId = 0;
+    int min;
+    if (sizeof(threadId) < sizeof(ptid)) {
+        min = sizeof(threadId);
+    }
+    else
+        min = sizeof(ptid);
+    memcpy(&threadId, &ptid, min);
+    return threadId;
+}
+
+//print tree for test
+static inline void print_tree(ompi_coll_tree_t* tree, int rank) {
+    int i;
+    printf("[%d, prev = %d, next_size = %d]:", rank, tree->tree_prev, tree->tree_nextsize);
+    for( i = 0; i < tree->tree_nextsize; i++ ){
+        printf(" %d", tree->tree_next[i]);
+    }
+    if (rank == tree->tree_root) {
+        printf(" root = %d", tree->tree_root);
+    }
+    printf("\n");
+}
+
+static inline int adapt_request_free(ompi_request_t** request)
+{
+    (*request)->req_state = OMPI_REQUEST_INVALID;
+    OBJ_RELEASE(*request);
+    *request = MPI_REQUEST_NULL;
+    return OMPI_SUCCESS;
+}