diff --git a/CMakeExt/GoogleTest.cmake b/CMakeExt/GoogleTest.cmake index 1e0bfde24..c843248ff 100644 --- a/CMakeExt/GoogleTest.cmake +++ b/CMakeExt/GoogleTest.cmake @@ -52,7 +52,7 @@ else() # Wrap download, configure and build steps in a script to log output LOG_DOWNLOAD ON LOG_CONFIGURE ON - LOG_BUILD ON + LOG_BUILD OFF ) set(GTEST_LOCATION "${GTEST_PREFIX}/src/GTestExternal-build/googlemock/gtest") set(GTEST_INCLUDES "${GTEST_PREFIX}/src/GTestExternal/googletest/include") diff --git a/CMakeExt/dyloc.cmake b/CMakeExt/dyloc.cmake new file mode 100644 index 000000000..752c86804 --- /dev/null +++ b/CMakeExt/dyloc.cmake @@ -0,0 +1,99 @@ + +include(ExternalProject) + +# Discussion of integration alternatives: +# https://stackoverflow.com/questions/9689183/cmake-googletest + + +set(DYLOC_SOURCE_PATH + "$ENV{DYLOC_SOURCE_PATH}" CACHE PATH + "Base path to the dyloc source distribution") + +if(ENABLE_DYLOC) + set(DART_IMPLEMENTATIONS ${DART_IMPLEMENTATIONS} PARENT_SCOPE) + set(ENVIRONMENT_TYPE ${ENVIRONMENT_TYPE} PARENT_SCOPE) + set(ENABLE_LOGGING ${ENABLE_LOGGING} PARENT_SCOPE) + + set(DYLOC_PREFIX "${CMAKE_BINARY_DIR}/dyloc") + set(DYLOC_INSTALL_PREFIX "${DYLOC_PREFIX}/install") + + message(STATUS "Building dyloc in ${DYLOC_PREFIX}") + + list( + APPEND DYLOC_CMAKE_ARGS + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DENVIRONMENT_TYPE=${ENVIRONMENT_TYPE} + -DCMAKE_INSTALL_PREFIX=${DYLOC_INSTALL_PREFIX} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DBOOST_INCLUDEDIR=${BOOST_INCLUDE_DIRS} + -DBOOST_LIBRARYDIR=${BOOST_LIBRARIES} + -DBUILD_TESTS=OFF + -DENABLE_LOGGING=${ENABLE_LOGGING} + -DENABLE_PAPI=${ENABLE_PAPI} + -DENABLE_LIBNUMA=${ENABLE_LIBNUMA} + -DDART_IMPLEMENTATIONS=${DART_IMPLEMENTATIONS} + -DDART_PREFIX=${CMAKE_BINARY_DIR} + -DDART_INCLUDE_DIRS=${CMAKE_SOURCE_DIR}/dart-if/include + ) + if (DYLOC_SOURCE_PATH) + ExternalProject_Add( + dylocExternal + URL ${DYLOC_SOURCE_PATH} + PREFIX "${DYLOC_PREFIX}" + CMAKE_ARGS ${DYLOC_CMAKE_ARGS} + INSTALL_DIR ${DYLOC_INSTALL_PREFIX} + LOG_INSTALL ON + ) + else() + ExternalProject_Add( + dylocExternal + GIT_REPOSITORY https://github.com/dash-project/dyloc.git + GIT_TAG master + TIMEOUT 10 + PREFIX "${DYLOC_PREFIX}" + CMAKE_ARGS ${DYLOC_CMAKE_ARGS} + INSTALL_DIR ${DYLOC_INSTALL_PREFIX} + LOG_INSTALL ON + ) + endif() + + set(DYLOC_BUILD_DIR "${DYLOC_PREFIX}/src/dylocExternal-build/dyloc") + set(DYLOC_BASE "${DYLOC_INSTALL_PREFIX}") + set(DYLOC_INCLUDES "${DYLOC_BASE}/include") + set(DYLOC_LIBRARY "${DYLOC_BASE}/lib/${LIBPREFIX}dyloc${LIBSUFFIX}") + set(DYLOC_COMMON_LIBRARY "${DYLOC_BASE}/lib/${LIBPREFIX}dyloc-common${LIBSUFFIX}") + set(DYLOCXX_LIBRARY "${DYLOC_BASE}/lib/${LIBPREFIX}dylocxx${LIBSUFFIX}") + + message(STATUS "dyloc source path: " ${DYLOC_SOURCE_PATH}) + message(STATUS "dyloc include path: " ${DYLOC_INCLUDES}) + + add_dependencies(dylocExternal dart-mpi) + add_dependencies(dylocExternal dart-base) + + add_library(dyloc-common IMPORTED STATIC GLOBAL) + set_target_properties( + dyloc-common + PROPERTIES + IMPORTED_LOCATION "${DYLOC_LIBRARY}" + IMPORTED_LINK_INTERFACE_LIBRARIES "${CMAKE_THREAD_LIBS_INIT}") + add_dependencies(dyloc-common dylocExternal) + + add_library(dylocxx IMPORTED STATIC GLOBAL) + set_target_properties( + dylocxx + PROPERTIES + IMPORTED_LOCATION "${DYLOC_LIBRARY}" + IMPORTED_LINK_INTERFACE_LIBRARIES "${CMAKE_THREAD_LIBS_INIT}") + add_dependencies(dylocxx dylocExternal) + + add_library(dyloc IMPORTED STATIC GLOBAL) + set_target_properties( + dyloc + PROPERTIES + IMPORTED_LOCATION "${DYLOC_LIBRARY}" + IMPORTED_LINK_INTERFACE_LIBRARIES "${CMAKE_THREAD_LIBS_INIT}") + add_dependencies(dyloc dylocExternal) + +endif() + diff --git a/CMakeExt/Hwloc.cmake b/CMakeExt/hwloc.cmake similarity index 100% rename from CMakeExt/Hwloc.cmake rename to CMakeExt/hwloc.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 567407ae7..82ab7ca23 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,6 +79,8 @@ option(ENABLE_DEFAULT_INDEX_TYPE_LONG "Specify whether to use int64_t as default index type" on) option(ENABLE_LIBNUMA "Specify whether libnuma features are enabled" on) +option(ENABLE_DYLOC + "Specify whether dyloc features are enabled" on) option(ENABLE_HWLOC "Specify whether hwloc features are enabled" on) option(ENABLE_HWLOC_PCI @@ -128,7 +130,7 @@ endif() # has been loaded: include(${CMAKE_SOURCE_DIR}/CMakeExt/MPI.cmake) include(${CMAKE_SOURCE_DIR}/CMakeExt/PAPI.cmake) -include(${CMAKE_SOURCE_DIR}/CMakeExt/Hwloc.cmake) +include(${CMAKE_SOURCE_DIR}/CMakeExt/hwloc.cmake) include(${CMAKE_SOURCE_DIR}/CMakeExt/Likwid.cmake) include(${CMAKE_SOURCE_DIR}/CMakeExt/NUMA.cmake) include(${CMAKE_SOURCE_DIR}/CMakeExt/IPM.cmake) @@ -224,6 +226,7 @@ add_subdirectory(dart-if) # DART base: add_subdirectory(dart-impl) +# DASH: add_subdirectory(dash) ## Documentation @@ -274,6 +277,8 @@ message(INFO "Default index type long: (ENABLE_DEFAULT_INDEX_TYPE_LONG) " ${ENABLE_DEFAULT_INDEX_TYPE_LONG}) message(INFO "libnuma support: (ENABLE_LIBNUMA) " ${ENABLE_LIBNUMA}) +message(INFO "dyloc support: (ENABLE_DYLOC) " + ${ENABLE_DYLOC}) message(INFO "hwloc support: (ENABLE_HWLOC) " ${ENABLE_HWLOC}) message(INFO "hwloc PCI support: (ENABLE_HWLOC_PCI) " @@ -327,6 +332,11 @@ if (ENABLE_BLAS) else() message(NOTE "BLAS disabled") endif() +if (ENABLE_DYLOC) + message(INFO "dyloc enabled") +else() + message(NOTE "dyloc disabled") +endif() if (ENABLE_HWLOC) if (HWLOC_FOUND) message(INFO "hwloc enabled") diff --git a/dart-if/include/dash/dart/if/dart_locality.h b/dart-if/include/dash/dart/if/dart_locality.h deleted file mode 100644 index d2083550d..000000000 --- a/dart-if/include/dash/dart/if/dart_locality.h +++ /dev/null @@ -1,241 +0,0 @@ -/** - * \file dart_locality.h - * \defgroup DartLocality Locality- and topolgy discovery - * \ingroup DartInterface - * - * A set of routines to query and remodel the locality domain hierarchy and the logical arrangement of teams. - * - */ - -#ifndef DART__LOCALITY_H_ -#define DART__LOCALITY_H_ - -#include -#include - - -/** - * \defgroup DartLocality Locality- and topolgy discovery - * \ingroup DartInterface - */ -#ifdef __cplusplus -extern "C" { -#endif - -/** \cond DART_HIDDEN_SYMBOLS */ -#define DART_INTERFACE_ON -/** \endcond */ - -/** - * Initialize information of the specified team. - * - * \threadsafe_none - * \ingroup DartLocality - */ -dart_ret_t dart_team_locality_init( - dart_team_t team) DART_NOTHROW; - -/** - * Initialize information of the specified team. - * - * \threadsafe_none - * \ingroup DartLocality - */ -dart_ret_t dart_team_locality_finalize( - dart_team_t team) DART_NOTHROW; - -/** - * Locality information of the team domain with the specified id tag. - * - * \threadsafe - * \ingroup DartLocality - */ -dart_ret_t dart_domain_team_locality( - dart_team_t team, - const char * domain_tag, - dart_domain_locality_t ** team_domain_out) DART_NOTHROW; - -/** - * Default constructor. - * Create an empty locality domain object. - * - * \threadsafe - * \ingroup DartLocality - */ -dart_ret_t dart_domain_create( - dart_domain_locality_t ** domain_out) DART_NOTHROW; - -/** - * Copy-constructor. - * Create a new locality domain object as a deep copy of a specified - * locality domain. - * - * \threadsafe - * \ingroup DartLocality - */ -dart_ret_t dart_domain_clone( - const dart_domain_locality_t * domain_in, - dart_domain_locality_t ** domain_out) DART_NOTHROW; - -/** - * Destructor. - * Delete a locality domain object. - * - * \threadsafe - * \ingroup DartLocality - */ -dart_ret_t dart_domain_destroy( - dart_domain_locality_t * domain) DART_NOTHROW; - -/** - * Assignment operator. - * Overwrites domain object \c domain_lhs with a deep copy of domain object - * \c domain_rhs. - * - * \threadsafe - * \ingroup DartLocality - */ -dart_ret_t dart_domain_assign( - dart_domain_locality_t * domain_lhs, - const dart_domain_locality_t * domain_rhs) DART_NOTHROW; - -/** - * Locality information of the subdomain with the specified id tag. - * - * \threadsafe - * \ingroup DartLocality - */ -dart_ret_t dart_domain_find( - const dart_domain_locality_t * domain_in, - const char * domain_tag, - dart_domain_locality_t ** subdomain_out) DART_NOTHROW; - -/** - * Remove domains in locality domain hierarchy that do not match the - * specified domain tags and are not a parent of a matched domain. - * - * \threadsafe - * \ingroup DartLocality - */ -dart_ret_t dart_domain_select( - dart_domain_locality_t * domain_in, - int num_subdomain_tags, - const char ** subdomain_tags) DART_NOTHROW; - -/** - * Remove domains in locality domain hierarchy matching the specified domain - * tags. - * - * \threadsafe - * \ingroup DartLocality - */ -dart_ret_t dart_domain_exclude( - dart_domain_locality_t * domain_in, - int num_subdomain_tags, - const char ** subdomain_tags) DART_NOTHROW; - -/** - * Insert locality domain into subdomains of a domain at the specified - * relative index. - * - * Tags of inserted subdomains are updated according to the parent domain. - * Units mapped to inserted subdomains are added to ancestor domains - * recursively. Units mapped to inserted subdomains must not be mapped - * to the target domain hierarchy already. - * - * \threadsafe - * \ingroup DartLocality - */ -dart_ret_t dart_domain_add_subdomain( - dart_domain_locality_t * domain, - dart_domain_locality_t * subdomain, - int subdomain_rel_id) DART_NOTHROW; - -/** - * Move locality domain in the locality hierarchy. - * - * The specified domain is added to the children of another domain in the - * same locality hierarchy at the specified relative index. - * - * Tags of inserted subdomains are updated according to the parent domain. - * Units mapped to inserted subdomains are added to ancestor domains - * recursively. Units mapped to inserted subdomains must not be mapped - * to the target domain hierarchy already. - * - * \threadsafe - * \ingroup DartLocality - */ -dart_ret_t dart_domain_move_subdomain( - dart_domain_locality_t * domain, - dart_domain_locality_t * new_parent_domain, - int new_domain_rel_id) DART_NOTHROW; - -/** - * Split locality domain hierarchy at given domain tag into \c num_parts - * groups at specified scope. - * - * \threadsafe - * \ingroup DartLocality - */ -dart_ret_t dart_domain_split_scope( - const dart_domain_locality_t * domain_in, - dart_locality_scope_t scope, - int num_parts, - dart_domain_locality_t * split_domain_out) DART_NOTHROW; - -/** - * Domain tags of domains at the specified locality scope. - * - * \threadsafe - * \ingroup DartLocality - */ -dart_ret_t dart_domain_scope_tags( - const dart_domain_locality_t * domain_in, - dart_locality_scope_t scope, - int * num_domains_out, - char *** domain_tags_out) DART_NOTHROW; - -/** - * Locality domains at the specified locality scope. - * - * \threadsafe - * \ingroup DartLocality - */ -dart_ret_t dart_domain_scope_domains( - const dart_domain_locality_t * domain_in, - dart_locality_scope_t scope, - int * num_domains_out, - dart_domain_locality_t *** domains_out) DART_NOTHROW; - -/** - * Adds entries to locality hierarchy to group locality domains. - * - * \threadsafe - * \ingroup DartLocality - */ -dart_ret_t dart_domain_group( - dart_domain_locality_t * domain_in, - int num_group_subdomains, - const char ** group_subdomain_tags, - char * group_domain_tag_out) DART_NOTHROW; - -/** - * Locality information of the unit with the specified team-relative id. - * - * \threadsafe - * \ingroup DartLocality - */ -dart_ret_t dart_unit_locality( - dart_team_t team, - dart_team_unit_t unit, - dart_unit_locality_t ** loc) DART_NOTHROW; - -/** \cond DART_HIDDEN_SYMBOLS */ -#define DART_INTERFACE_OFF -/** \endcond */ - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* DART__LOCALITY_H_ */ diff --git a/dart-if/include/dash/dart/if/dart_team_group.h b/dart-if/include/dash/dart/if/dart_team_group.h index 02ca53f2a..9138f14a1 100644 --- a/dart-if/include/dash/dart/if/dart_team_group.h +++ b/dart-if/include/dash/dart/if/dart_team_group.h @@ -358,7 +358,8 @@ dart_ret_t dart_team_get_group( * * (2,3 are unique with respect to the parent team (1)). * - * \param teamid The parent team to use whose units participate in the collective operation. + * \param teamid The parent team to use whose units participate in the + * collective operation. * \param group The group object to build the new team from. * \param[out] newteam Will contain the new team ID upon successful return. * diff --git a/dart-impl/base/CMakeLists.txt b/dart-impl/base/CMakeLists.txt index 87c43a23b..e66081fdc 100644 --- a/dart-impl/base/CMakeLists.txt +++ b/dart-impl/base/CMakeLists.txt @@ -120,6 +120,11 @@ set_target_properties( COMPILE_FLAGS ${ADDITIONAL_COMPILE_FLAGS} ) +set_target_properties( + ${DASH_DART_BASE_LIBRARY} + PROPERTIES POSITION_INDEPENDENT_CODE TRUE +) + ## Installation DeployLibrary(${DASH_DART_BASE_LIBRARY}) diff --git a/dart-impl/base/include/dash/dart/base/hwinfo.h b/dart-impl/base/include/dash/dart/base/hwinfo.h deleted file mode 100644 index c760fecec..000000000 --- a/dart-impl/base/include/dash/dart/base/hwinfo.h +++ /dev/null @@ -1,28 +0,0 @@ -/** - * \file dash/dart/base/hwinfo.h - */ -#ifndef DART__BASE__HWINFO_H__ -#define DART__BASE__HWINFO_H__ - -#include - -/** - * Initializes hwinfo object with uninitialized defaults. - */ -dart_ret_t dart_hwinfo_init( - dart_hwinfo_t * hwinfo); - -/** - * Resolves the current unit's hardware locality information. - * - * Locality information is obtained from a series of specializes libraries - * such as PAPI, hwloc, likwid, libnuma and platform-dependent system - * functions, depending on libraries enabled in the DART build configuration. - * - * If a locality property cannot be reliably resolved or deduced, the - * respective entry is set to \c -1 or an empty string. - */ -dart_ret_t dart_hwinfo( - dart_hwinfo_t * hwinfo); - -#endif /* DART__BASE__HWINFO_H__ */ diff --git a/dart-impl/base/include/dash/dart/base/locality.h b/dart-impl/base/include/dash/dart/base/locality.h deleted file mode 100644 index 1a4a5f012..000000000 --- a/dart-impl/base/include/dash/dart/base/locality.h +++ /dev/null @@ -1,200 +0,0 @@ -/** - * \file dash/dart/base/locality.h - * - * Fundamental algorithms on DART locality types independent from a concrete - * implementation of the DART interface. - */ -#ifndef DART__BASE__LOCALITY_H__ -#define DART__BASE__LOCALITY_H__ - -#include - -#include -#include - - -/* ======================================================================== * - * Init / Finalize * - * ======================================================================== */ - -dart_ret_t dart__base__locality__init(); - -dart_ret_t dart__base__locality__finalize(); - -/* ======================================================================== * - * Create / Delete * - * ======================================================================== */ - -dart_ret_t dart__base__locality__create( - dart_team_t team); - -dart_ret_t dart__base__locality__delete( - dart_team_t team); - -/* ======================================================================== * - * Domain Locality * - * ======================================================================== */ - -static inline -dart_ret_t dart__base__locality__create_domain( - dart_domain_locality_t ** domain_out) -{ - *domain_out = malloc(sizeof(dart_domain_locality_t)); - dart_ret_t ret = dart__base__locality__domain__init(*domain_out); - if (DART_OK != ret) { - free(*domain_out); - *domain_out = NULL; - } - return ret; -} - -static inline -dart_ret_t dart__base__locality__clone_domain( - const dart_domain_locality_t * domain_in, - dart_domain_locality_t ** domain_out) -{ - DART_LOG_TRACE("dart__base__locality__clone_domain() " - "domain %s (%p)", - domain_in->domain_tag, domain_in); - - *domain_out = NULL; - dart_ret_t ret = dart__base__locality__create_domain(domain_out); - if (DART_OK != ret) { return ret; } - DART_LOG_TRACE("dart__base__locality__clone_domain: copy"); - ret = dart__base__locality__domain__copy(domain_in, *domain_out); - - DART_LOG_TRACE("dart__base__locality__clone_domain > " - "domain %s (%p) -> (%p)", - domain_in->domain_tag, domain_in, *domain_out); - return ret; -} - -static inline -dart_ret_t dart__base__locality__assign_domain( - dart_domain_locality_t * domain_lhs, - const dart_domain_locality_t * domain_rhs) -{ - DART_LOG_TRACE("dart__base__locality__assign_domain() " - "lhs (%p) <- rhs (%p): %s", - domain_lhs, domain_rhs, domain_rhs->domain_tag); - - dart_ret_t ret = dart__base__locality__domain__destruct(domain_lhs); - if (DART_OK != ret) { return ret; } - ret = dart__base__locality__domain__copy(domain_rhs, domain_lhs); - - DART_LOG_TRACE("dart__base__locality__assign_domain > " - "lhs (%p) <- rhs (%p): %s", - domain_lhs, domain_rhs, domain_rhs->domain_tag); - return ret; -} - -static inline -dart_ret_t dart__base__locality__destruct_domain( - dart_domain_locality_t * domain) -{ - DART_LOG_TRACE("dart__base__locality__destruct_domain() " - "domain %s (%p)", - domain->domain_tag, domain); - dart_ret_t ret = dart__base__locality__domain__destruct(domain); - if (ret != DART_OK) { return ret; } - free(domain); - DART_LOG_TRACE("dart__base__locality__destruct_domain > ~(%p)", - domain); - return DART_OK; -} - -static inline -dart_ret_t dart__base__locality__select_subdomains( - dart_domain_locality_t * domain, - const char ** subdomain_tags, - int num_subdomain_tags) -{ - static const int remove_matches = 0; - return dart__base__locality__domain__filter_subdomains( - domain, subdomain_tags, num_subdomain_tags, remove_matches); -} - -static inline -dart_ret_t dart__base__locality__exclude_subdomains( - dart_domain_locality_t * domain, - const char ** subdomain_tags, - int num_subdomain_tags) -{ - static const int remove_matches = 1; - return dart__base__locality__domain__filter_subdomains( - domain, subdomain_tags, num_subdomain_tags, remove_matches); -} - -static inline -dart_ret_t dart__base__locality__add_subdomain( - dart_domain_locality_t * domain, - dart_domain_locality_t * subdomain, - int subdomain_rel_id) -{ - return dart__base__locality__domain__add_subdomain( - domain, subdomain, subdomain_rel_id); -} - -static inline -dart_ret_t dart__base__locality__remove_subdomain( - dart_domain_locality_t * domain, - int subdomain_rel_id) -{ - return dart__base__locality__domain__remove_subdomain( - domain, subdomain_rel_id); -} - -static inline -dart_ret_t dart__base__locality__move_subdomain( - dart_domain_locality_t * subdomain, - dart_domain_locality_t * new_parent_domain, - int new_subdomain_rel_id) -{ - return dart__base__locality__domain__move_subdomain( - subdomain, new_parent_domain, new_subdomain_rel_id); -} - -dart_ret_t dart__base__locality__team_domain( - dart_team_t team, - dart_domain_locality_t ** domain_out); - -dart_ret_t dart__base__locality__domain( - const dart_domain_locality_t * domain_in, - const char * domain_tag, - dart_domain_locality_t ** domain_out); - -dart_ret_t dart__base__locality__domain_split_tags( - const dart_domain_locality_t * domain_in, - dart_locality_scope_t scope, - int num_parts, - int ** group_sizes_out, - char **** group_domain_tags_out); - -dart_ret_t dart__base__locality__domain_group( - dart_domain_locality_t * domain, - int group_size, - const char ** group_subdomain_tags, - char * group_domain_tag_out); - -dart_ret_t dart__base__locality__scope_domains( - const dart_domain_locality_t * domain_in, - dart_locality_scope_t scope, - int * num_domains_out, - dart_domain_locality_t *** domains_out); - -dart_ret_t dart__base__locality__scope_domain_tags( - const dart_domain_locality_t * domain_in, - dart_locality_scope_t scope, - int * num_domains_out, - char *** domain_tags_out); - -/* ======================================================================== * - * Unit Locality * - * ======================================================================== */ - -dart_ret_t dart__base__locality__unit( - dart_team_t team, - dart_team_unit_t unit, - dart_unit_locality_t ** locality); - -#endif /* DART__BASE__LOCALITY_H__ */ diff --git a/dart-impl/base/src/hwinfo.c b/dart-impl/base/src/hwinfo.c deleted file mode 100644 index 03e2bc845..000000000 --- a/dart-impl/base/src/hwinfo.c +++ /dev/null @@ -1,436 +0,0 @@ -#include -#ifdef DART__PLATFORM__LINUX -/* _GNU_SOURCE required for sched_getcpu() */ -# define _GNU_SOURCE -# include -#endif -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -#include -#include - -#include -#include -#include -#include -#include -#include - -#ifdef DART_ENABLE_LIKWID -# include -#endif - -#ifdef DART_ENABLE_HWLOC -# include -# include -#endif - -#ifdef DART_ENABLE_PAPI -# include -#endif - -#ifdef DART_ENABLE_NUMA -# include -# include -#endif - -static const int BYTES_PER_MB = (1024 * 1024); - -/* NOTE: The dart_hwinfo function must only return reliable information, - * typically obtained from system functions or libraries such as - * hwloc, PAPI, likwid etc. - * Hardware locality attributes that cannot be obtained or deduced - * are initialized with -1. - * Assumptions and approximations based on the common case - * are specific to the use case and must be decided in client code. - * Otherwise, users of this function would consider a possibly - * incorrect hwinfo attribute setting as actual system information. - * - * For example, assuming - * - * (numa node memory) = (system memory) / (num numa) - * - * to derive default values for memory capacity is viable in most - * cases but harmful for compute nodes with accelerator components - * where this would lead to drastic host/target load imbalance. - */ - -dart_ret_t dart_hwinfo_init( - dart_hwinfo_t * hw) -{ - hw->num_numa = -1; - hw->numa_id = -1; - hw->num_cores = -1; - hw->core_id = -1; - hw->cpu_id = -1; - hw->min_cpu_mhz = -1; - hw->max_cpu_mhz = -1; - hw->min_threads = -1; - hw->max_threads = -1; - hw->cache_ids[0] = -1; - hw->cache_ids[1] = -1; - hw->cache_ids[2] = -1; - hw->cache_sizes[0] = -1; - hw->cache_sizes[1] = -1; - hw->cache_sizes[2] = -1; - hw->cache_line_sizes[0] = -1; - hw->cache_line_sizes[1] = -1; - hw->cache_line_sizes[2] = -1; - hw->max_shmem_mbps = -1; - hw->system_memory_bytes = -1; - hw->numa_memory_bytes = -1; - hw->num_scopes = -1; - - dart_locality_scope_pos_t undef_scope; - undef_scope.scope = DART_LOCALITY_SCOPE_UNDEFINED; - undef_scope.index = -1; - for (int s = 0; s < DART_LOCALITY_MAX_DOMAIN_SCOPES; s++) { - hw->scopes[s] = undef_scope; - } - - return DART_OK; -} - -dart_ret_t dart_hwinfo( - dart_hwinfo_t * hwinfo) -{ - DART_LOG_DEBUG("dart_hwinfo()"); - - dart_hwinfo_t hw; - dart_hwinfo_init(&hw); - - char * max_shmem_mbps_str = getenv("DASH_MAX_SHMEM_MBPS"); - if (NULL != max_shmem_mbps_str) { - hw.max_shmem_mbps = (int)(atoi(max_shmem_mbps_str)); - DART_LOG_TRACE("dart_hwinfo: DASH_MAX_SHMEM_MBPS set: %d", - hw.max_shmem_mbps); - } else { - DART_LOG_TRACE("dart_hwinfo: DASH_MAX_SHMEM_MBPS not set"); - } - if (hw.max_shmem_mbps <= 0) { - /* TODO: Intermediate workaround for load balancing, use -1 - * instead: */ - hw.max_shmem_mbps = 1235; - } - - gethostname(hw.host, DART_LOCALITY_HOST_MAX_SIZE); - -#ifdef DART_ENABLE_LIKWID - DART_LOG_TRACE("dart_hwinfo: using likwid"); - /* - * see likwid API documentation: - * https://rrze-hpc.github.io/likwid/Doxygen/C-likwidAPI-code.html - */ - int likwid_ret = topology_init(); - if (likwid_ret < 0) { - DART_LOG_ERROR("dart_hwinfo: " - "likwid: topology_init failed, returned %d", likwid_ret); - } else { - CpuInfo_t info = get_cpuInfo(); - CpuTopology_t topo = get_cpuTopology(); - if (hw.min_cpu_mhz < 0 || hw.max_cpu_mhz < 0) { - hw.min_cpu_mhz = info->clock; - hw.max_cpu_mhz = info->clock; - } - if (hw.num_numa < 0) { - hw.num_numa = hw.num_sockets; - } - if (hw.num_cores < 0) { - hw.num_cores = topo->numCoresPerSocket * hw.num_sockets; - } - topology_finalize(); - DART_LOG_TRACE("dart_hwinfo: likwid: " - "num_sockets: %d num_numa: %d num_cores: %d", - hw.num_sockets, hw.num_numa, hw.num_cores); - } -#endif /* DART_ENABLE_LIKWID */ - -#ifdef DART_ENABLE_HWLOC - DART_LOG_TRACE("dart_hwinfo: using hwloc"); - - hwloc_topology_t topology; - hwloc_topology_init(&topology); - hwloc_topology_set_flags(topology, -#if HWLOC_API_VERSION < 0x00020000 - HWLOC_TOPOLOGY_FLAG_IO_DEVICES - | HWLOC_TOPOLOGY_FLAG_IO_BRIDGES - /* | HWLOC_TOPOLOGY_FLAG_WHOLE_IO */ -#else - HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM -#endif - ); - hwloc_topology_load(topology); - - /* hwloc can resolve the physical index (os_index) of the active unit, - * not the logical index. - * Queries in the topology hierarchy require the logical index, however. - * As a workaround, units scan the topology for the logical index of the - * CPU object that has a matching physical index. - */ - - /* Get PU of active thread: */ - hwloc_cpuset_t cpuset = hwloc_bitmap_alloc(); - int flags = 0; // HWLOC_CPUBIND_PROCESS; - int cpu_os_id = -1; - int ret = hwloc_get_last_cpu_location(topology, cpuset, flags); - if (!ret) { - cpu_os_id = hwloc_bitmap_first(cpuset); - } - hwloc_bitmap_free(cpuset); - - hwloc_obj_t cpu_obj; - for (cpu_obj = - hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, 0); - cpu_obj; - cpu_obj = cpu_obj->next_cousin) { - if ((int)cpu_obj->os_index == cpu_os_id) { - hw.cpu_id = cpu_obj->logical_index; - break; - } - } - - DART_LOG_TRACE("dart_hwinfo: hwloc : cpu_id phys:%d logical:%d", - cpu_os_id, hw.cpu_id); - - /* PU (cpu_id) to CORE (core_id) object: */ - hwloc_obj_t core_obj; - for (core_obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, hw.cpu_id); - core_obj; - core_obj = core_obj->parent) { - if (core_obj->type == HWLOC_OBJ_CORE) { break; } - } - if (core_obj) { - hw.core_id = core_obj->logical_index; - - DART_LOG_TRACE("dart_hwinfo: hwloc : core logical index: %d", - hw.core_id); - - hw.num_scopes = 0; - int cache_level = 0; - /* Use CORE object to resolve caches: */ - hwloc_obj_t obj; - for (obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_CORE, hw.core_id); - obj; - obj = obj->parent) { - - if (obj->type == HWLOC_OBJ_MACHINE) { break; } - hw.scopes[hw.num_scopes].scope = - dart__base__hwloc__obj_type_to_dart_scope(obj->type); - hw.scopes[hw.num_scopes].index = obj->logical_index; - DART_LOG_TRACE("dart_hwinfo: hwloc: parent[%d] (scope:%d index:%d)", - hw.num_scopes, - hw.scopes[hw.num_scopes].scope, - hw.scopes[hw.num_scopes].index); - hw.num_scopes++; - -#if HWLOC_API_VERSION < 0x00020000 - if (obj->type == HWLOC_OBJ_CACHE) { - hw.cache_sizes[cache_level] = obj->attr->cache.size; - hw.cache_line_sizes[cache_level] = obj->attr->cache.linesize; - hw.cache_ids[cache_level] = obj->logical_index; - cache_level++; - } -#else - if (obj->type == HWLOC_OBJ_L1CACHE) { - hw.cache_sizes[0] = obj->attr->cache.size; - hw.cache_line_sizes[0] = obj->attr->cache.linesize; - hw.cache_ids[0] = obj->logical_index; - cache_level++; - } - else if (obj->type == HWLOC_OBJ_L2CACHE) { - hw.cache_sizes[1] = obj->attr->cache.size; - hw.cache_line_sizes[1] = obj->attr->cache.linesize; - hw.cache_ids[1] = obj->logical_index; - cache_level++; - } - else if (obj->type == HWLOC_OBJ_L3CACHE) { - hw.cache_sizes[2] = obj->attr->cache.size; - hw.cache_line_sizes[2] = obj->attr->cache.linesize; - hw.cache_ids[2] = obj->logical_index; - cache_level++; - } -#endif - else if (obj->type == HWLOC_OBJ_NODE) { - hw.numa_id = obj->logical_index; - } else if (obj->type == -#if HWLOC_API_VERSION > 0x00011000 - HWLOC_OBJ_PACKAGE -#else - HWLOC_OBJ_SOCKET -#endif - ) { - } - } - } - - if (hw.numa_id < 0) { - hwloc_obj_t numa_obj; - for (numa_obj = - hwloc_get_obj_by_type(topology, HWLOC_OBJ_PU, hw.cpu_id); - numa_obj; - numa_obj = numa_obj->parent) { - if (numa_obj->type == HWLOC_OBJ_NODE) { - hw.numa_id = numa_obj->logical_index; - break; - } - } - } - if (hw.num_numa < 0) { - int n_numa_nodes = hwloc_get_nbobjs_by_type( - topology, DART__HWLOC_OBJ_NUMANODE); - if (n_numa_nodes > 0) { - hw.num_numa = n_numa_nodes; - } - } - if (hw.num_cores < 0) { - int n_cores = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_CORE); - if (n_cores > 0) { - hw.num_cores = n_cores; - } - } - if (hw.min_threads < 0 && hw.max_threads < 0 && - hw.num_cores > 0 && hw.max_threads < 0) { - int n_cpus = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_PU); - hw.min_threads = 1; - hw.max_threads = n_cpus / hw.num_cores; - } - - if(hw.system_memory_bytes < 0) { - hwloc_obj_t obj; - obj = hwloc_get_obj_by_type(topology, HWLOC_OBJ_MACHINE, 0); - hw.system_memory_bytes = obj->memory.total_memory / BYTES_PER_MB; - } - if(hw.numa_memory_bytes < 0) { - hwloc_obj_t obj; - obj = hwloc_get_obj_by_type(topology, DART__HWLOC_OBJ_NUMANODE, 0); - if(obj != NULL) { - hw.numa_memory_bytes = obj->memory.total_memory / BYTES_PER_MB; - } else { - /* No NUMA domain: */ - hw.numa_memory_bytes = hw.system_memory_bytes; - } - } - - hwloc_topology_destroy(topology); - DART_LOG_TRACE("dart_hwinfo: hwloc: " - "num_numa:%d numa_id:%d " - "num_cores:%d core_id:%d cpu_id:%d", - hw.num_numa, hw.numa_id, - hw.num_cores, hw.core_id, hw.cpu_id); -#endif /* DART_ENABLE_HWLOC */ - -#ifdef DART_ENABLE_PAPI - DART_LOG_TRACE("dart_hwinfo: using PAPI"); - - const PAPI_hw_info_t * papi_hwinfo = NULL; - if (dart__base__locality__papi_init(&papi_hwinfo) == DART_OK) { - if (hw.num_numa < 0) { - hw.num_numa = papi_hwinfo->nnodes; - } - if (hw.num_cores < 0) { - int num_sockets = papi_hwinfo->sockets; - int cores_per_socket = papi_hwinfo->cores; - hw.num_cores = num_sockets * cores_per_socket; - } - if (hw.min_cpu_mhz < 0 || hw.max_cpu_mhz < 0) { - hw.min_cpu_mhz = papi_hwinfo->cpu_min_mhz; - hw.max_cpu_mhz = papi_hwinfo->cpu_max_mhz; - } - DART_LOG_TRACE("dart_hwinfo: PAPI: num_numa:%d num_cores:%d", - hw.num_numa, hw.num_cores); - } -#endif /* DART_ENABLE_PAPI */ - -#ifdef DART__PLATFORM__LINUX - if (hw.cpu_id < 0) { - hw.cpu_id = sched_getcpu(); - } -#else - DART_LOG_ERROR("dart_hwinfo: " - "HWLOC or PAPI required if not running on a Linux platform"); - return DART_ERR_OTHER; -#endif - -#ifdef DART__ARCH__IS_MIC - /* - * Hardware information for Intel MIC can be hard-coded as hardware - * specs of MIC model variants are invariant: - */ - DART_LOG_TRACE("dart_hwinfo: MIC architecture"); - - if (hw.num_numa < 0) { hw.num_numa = 1; } - if (hw.num_cores < 0) { hw.num_cores = 60; } - if (hw.min_cpu_mhz < 0 || hw.max_cpu_mhz < 0) { - hw.min_cpu_mhz = 1100; - hw.max_cpu_mhz = 1100; - } - if (hw.min_threads < 0 || hw.max_threads < 0) { - hw.min_threads = 4; - hw.max_threads = 4; - } - if (hw.numa_id < 0) { - hw.numa_id = 0; - } -#endif - -#ifdef DART__PLATFORM__POSIX - if (hw.num_cores < 0) { - /* - * NOTE: includes hyperthreading - */ - int posix_ret = sysconf(_SC_NPROCESSORS_ONLN); - hw.num_cores = (posix_ret > 0) ? posix_ret : hw.num_cores; - DART_LOG_TRACE( - "dart_hwinfo: POSIX: hw.num_cores = %d", - hw.num_cores); - } - - if(hw.system_memory_bytes < 0) { - long pages = sysconf(_SC_AVPHYS_PAGES); - long page_size = sysconf(_SC_PAGE_SIZE); - if (pages > 0 && page_size > 0) { - hw.system_memory_bytes = (int) ((pages * page_size) / BYTES_PER_MB); - } - } - -#endif - -#ifdef DART_ENABLE_NUMA - DART_LOG_TRACE("dart_hwinfo: using numalib"); - if (hw.num_numa < 0) { - hw.num_numa = numa_max_node() + 1; - } - if (hw.numa_id < 0 && hw.cpu_id >= 0) { - hw.numa_id = numa_node_of_cpu(hw.cpu_id); - } -#endif - - if (hw.num_scopes < 1) { - /* No domain hierarchy could be resolved. - * Use flat topology, with all units assigned to domains in CORE scope: */ - hw.num_scopes = 1; - hw.scopes[0].scope = DART_LOCALITY_SCOPE_CORE; - hw.scopes[0].index = (hw.core_id >= 0) ? hw.core_id : hw.cpu_id; - } - - DART_LOG_TRACE("dart_hwinfo: finished: " - "num_numa:%d numa_id:%d cpu_id:%d, num_cores:%d " - "min_threads:%d max_threads:%d", - hw.num_numa, hw.numa_id, hw.cpu_id, hw.num_cores, - hw.min_threads, hw.max_threads); - - *hwinfo = hw; - - DART_LOG_DEBUG("dart_hwinfo >"); - return DART_OK; -} diff --git a/dart-impl/base/src/internal/domain_locality.c b/dart-impl/base/src/internal/domain_locality.c deleted file mode 100644 index 578ce69eb..000000000 --- a/dart-impl/base/src/internal/domain_locality.c +++ /dev/null @@ -1,1176 +0,0 @@ -/** - * \file dash/dart/base/internal/domain_locality.c - * - * Implementation of the \c dart_domain_locality_t object type providing - * primitive operations like construction, assignment, destruction and - * node traversal to be used in locality algorihms. - * - * \see dash/dart/base/locality.h - */ -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -/* ================================================================== * - * Private Functions: Declarations * - * ================================================================== */ - -dart_ret_t dart__base__locality__domain__create_node_subdomains( - dart_domain_locality_t * node_domain, - dart_host_topology_t * host_topology, - dart_unit_mapping_t * unit_mapping); - -dart_ret_t dart__base__locality__domain__create_module_subdomains( - dart_domain_locality_t * module_domain, - dart_host_topology_t * host_topology, - dart_unit_mapping_t * unit_mapping, - int module_scope_level); - -dart_ret_t dart__base__locality__domain__child_rec( - const dart_domain_locality_t * domain, - const char * subdomain_tag, - dart_domain_locality_t ** subdomain_out); - -/* ================================================================== * - * Internal Functions * - * ================================================================== */ - -dart_ret_t dart__base__locality__domain__init( - dart_domain_locality_t * loc) -{ - if (loc == NULL) { - DART_LOG_ERROR("dart__base__locality__domain_locality_init ! null"); - return DART_ERR_INVAL; - } - loc->domain_tag[0] = '\0'; - loc->num_aliases = 0; - loc->aliases = NULL; - loc->host[0] = '\0'; - loc->scope = DART_LOCALITY_SCOPE_UNDEFINED; - loc->global_index = -1; - loc->team = DART_TEAM_NULL; - loc->level = 0; - loc->relative_index = 0; - loc->parent = NULL; - loc->children = NULL; - loc->num_domains = 0; - loc->unit_ids = NULL; - loc->num_units = -1; - loc->num_nodes = -1; - loc->num_cores = -1; - loc->shared_mem_bytes = -1; - - return DART_OK; -} - -dart_ret_t dart__base__locality__domain__destruct( - dart_domain_locality_t * domain) -{ - dart_ret_t ret = DART_OK; - - if (domain == NULL) { - DART_LOG_DEBUG("dart__base__locality__domain__destruct > domain NULL"); - return DART_OK; - } - - if (domain->num_domains > 0 && NULL == domain->children) { - DART_LOG_ERROR("dart__base__locality__domain__destruct ! " - "domain.domains must not be NULL for " - "domain.num_domains = %d in %s", - domain->num_domains, domain->domain_tag); - return DART_ERR_INVAL; - } - else if (domain->num_domains == 0 && NULL != domain->children) { - DART_LOG_ERROR("dart__base__locality__domain__destruct ! " - "domain.domains expected to be NULL for " - "domain.num_domains = %d in %s", - domain->num_domains, domain->domain_tag); - return DART_ERR_INVAL; - } - - /* deallocate child nodes in depth-first recursion: */ - for (int subdom_idx = 0; subdom_idx < domain->num_domains; ++subdom_idx) { - ret = dart__base__locality__domain__destruct( - domain->children[subdom_idx]); - if (DART_OK != ret) { - DART_LOG_ERROR("dart__base__locality__domain__destruct ! " - "failed to destroy domain(%p)->children[%d]", - (void *)domain, subdom_idx); - return ret; - } - } - /* deallocate node itself: */ - if (NULL != domain->children) { - free(domain->children); - domain->children = NULL; - } - if (NULL != domain->unit_ids) { - free(domain->unit_ids); - domain->unit_ids = NULL; - } - if (NULL != domain->aliases) { -// free(domain->aliases); - domain->aliases = NULL; - } - domain->num_domains = 0; - domain->num_units = 0; - domain->num_aliases = 0; - - return DART_OK; -} - -dart_ret_t dart__base__locality__domain__copy( - const dart_domain_locality_t * domain_src, - dart_domain_locality_t * domain_dst) -{ - DART_LOG_TRACE("dart__base__locality__domain__copy() %s (%p -> %p)", - domain_src->domain_tag, domain_src, domain_dst); - dart_ret_t ret = DART_ERR_OTHER; - - dart__base__locality__domain__init(domain_dst); - - // memcpy(domain_dst, domain_src, sizeof(dart_domain_locality_t)); - *domain_dst = *domain_src; - - // TODO: adjust domain aliases in domain_dst to corresponding - // pointers to subdomains of domain_dst (currently point - // to subdomains in domain_src). - domain_dst->num_aliases = 0; - domain_dst->aliases = NULL; - - /* Copy unit ids: - */ - if (domain_src->num_units > 0) { - if (NULL == domain_src->unit_ids) { - DART_LOG_ERROR("dart__base__locality__domain__copy: domain %s " - "has num_units = %d but domain->unit_ids is NULL", - domain_src->domain_tag, domain_src->num_units); - return DART_ERR_OTHER; - } - domain_dst->num_units = domain_src->num_units; - domain_dst->unit_ids = malloc(sizeof(dart_global_unit_t) * - domain_src->num_units); - for (int u = 0; u < domain_src->num_units; u++) { - domain_dst->unit_ids[u] = domain_src->unit_ids[u]; - } - } else { - if (NULL != domain_src->unit_ids) { - DART_LOG_ERROR("dart__base__locality__domain__copy: domain %s " - "has num_units = %d, expected unit_ids == NULL", - domain_src->domain_tag, domain_src->num_units); - return DART_ERR_OTHER; - } - domain_dst->unit_ids = NULL; - } - - /* Copy subdomains: - */ - if (domain_src->num_domains > 0) { - if (NULL == domain_src->children) { - DART_LOG_ERROR("dart__base__locality__domain__copy: domain %s " - "has num_domains = %d, expected domains != NULL", - domain_src->domain_tag, domain_src->num_domains); - return DART_ERR_OTHER; - } - domain_dst->num_domains = domain_src->num_domains; - domain_dst->children = malloc(sizeof(dart_domain_locality_t *) * - domain_src->num_domains); - } else { - if (NULL != domain_src->children) { - DART_LOG_ERROR("dart__base__locality__domain__copy: domain %s " - "has num_domains = %d, expected domains = NULL", - domain_src->domain_tag, domain_src->num_domains); - return DART_ERR_OTHER; - } - domain_dst->children = NULL; - } - - if (domain_src->num_domains > 16) { - DART_LOG_WARN("dart__base__locality__domain__filter_subdomains: " - "domain %s num_domains:%d is unusual", - domain_src->domain_tag, domain_src->num_domains); - } - - /* Recursively copy subdomains: - */ - for (int sd = 0; sd < domain_src->num_domains; sd++) { - domain_dst->children[sd] = malloc(sizeof(dart_domain_locality_t)); - - const dart_domain_locality_t * subdomain_src = - domain_src->children[sd]; - dart_domain_locality_t * subdomain_dst = - domain_dst->children[sd]; - - ret = dart__base__locality__domain__copy( - subdomain_src, - subdomain_dst); - if (ret != DART_OK) { - return ret; - } - domain_dst->children[sd]->parent = domain_dst; - } - - return DART_OK; -} - -dart_ret_t dart__base__locality__domain__update_subdomains( - dart_domain_locality_t * domain) -{ - int is_unit_scope = ((int)domain->scope >= (int)DART_LOCALITY_SCOPE_CORE); - DART_LOG_TRACE("dart__base__locality__domain__update_subdomains() " - "domain: %s, scope: %d, subdomains: %d, units: %d, " - "in unit scope: %s", - domain->domain_tag, domain->scope, - domain->num_domains, domain->num_units, - (is_unit_scope ? "true" : "false")); - - if (!is_unit_scope) { - domain->num_units = 0; - } - for (int sd = 0; sd < domain->num_domains; sd++) { - dart_domain_locality_t * subdomain = domain->children[sd]; - - subdomain->team = domain->team; - subdomain->level = domain->level + 1; - subdomain->relative_index = sd; - subdomain->parent = domain; - - sprintf(subdomain->domain_tag, "%s.%d", domain->domain_tag, sd); - - DART_LOG_TRACE("dart__base__locality__domain__update_subdomains --v"); - /* Recursively update subdomains: */ - dart__base__locality__domain__update_subdomains( - subdomain); - DART_LOG_TRACE("dart__base__locality__domain__update_subdomains --^"); - - if (!is_unit_scope) { - domain->num_units += subdomain->num_units; - } - } - if (domain->num_units == 0 && domain->unit_ids != NULL) { - DART_LOG_DEBUG("dart__base__locality__domain__update_subdomains: " - "free(domain->unit_ids)"); - free(domain->unit_ids); - domain->unit_ids = NULL; - } - if (domain->num_domains == 0 && domain->children != NULL) { - DART_LOG_DEBUG("dart__base__locality__domain__update_subdomains: " - "free(domain->children)"); - for (int sd = 0; sd < domain->num_domains; sd++) { - if (NULL != domain->children[sd]) { - free(domain->children[sd]); - domain->children[sd] = NULL; - } - } - free(domain->children); - domain->children = NULL; - } - if (domain->num_units > 0) { - if (is_unit_scope) { - dart_global_unit_t unit_id = domain->unit_ids[0]; - DART_LOG_DEBUG("dart__base__locality__domain__update_subdomains: " - "free(domain->unit_ids)"); - free(domain->unit_ids); - domain->unit_ids = malloc(sizeof(dart_global_unit_t)); - domain->unit_ids[0] = unit_id; - } else { - domain->unit_ids = malloc(sizeof(dart_global_unit_t) * domain->num_units); - int domain_unit_idx = 0; - for (int sd = 0; sd < domain->num_domains; sd++) { - dart_domain_locality_t * subdomain = domain->children[sd]; - memcpy(domain->unit_ids + domain_unit_idx, - subdomain->unit_ids, - sizeof(dart_global_unit_t) * subdomain->num_units); - domain_unit_idx += subdomain->num_units; - } - } - } else { - free(domain->unit_ids); - domain->unit_ids = NULL; - } - DART_LOG_TRACE("dart__base__locality__domain__update_subdomains > " - "domain: %s, scope: %d, subdomains: %d, units: %d", - domain->domain_tag, domain->scope, - domain->num_domains, domain->num_units); - return DART_OK; -} - -/** - * Find subdomain at arbitrary level below a specified domain. - */ -dart_ret_t dart__base__locality__domain__child( - const dart_domain_locality_t * domain, - const char * subdomain_tag, - dart_domain_locality_t ** subdomain_out) -{ - dart_ret_t ret = dart__base__locality__domain__child_rec( - domain, subdomain_tag, subdomain_out); - - if (DART_ERR_NOTFOUND == ret) { - DART_LOG_TRACE("dart__base__locality__domain__child - " - "no subdomain %s in %s", - subdomain_tag, domain->domain_tag); - } else { - DART_LOG_TRACE("dart__base__locality__domain__child - " - "domain:%s, subdomain_tag:%s -> %s", - (NULL != domain) ? domain->domain_tag : "?", - subdomain_tag, (*subdomain_out)->domain_tag); - } - return ret; -} - -dart_ret_t dart__base__locality__domain__child_rec( - const dart_domain_locality_t * domain, - const char * subdomain_tag, - dart_domain_locality_t ** subdomain_out) -{ - if (strcmp(domain->domain_tag, subdomain_tag) == 0) { - *subdomain_out = (dart_domain_locality_t *)(domain); - return DART_OK; - } - /* - * TODO: Optimize, currently using exhaustive search. - */ - for (int sd = 0; sd < domain->num_domains; ++sd) { - if (dart__base__locality__domain__child_rec( - domain->children[sd], subdomain_tag, subdomain_out) - == DART_OK) { - return DART_OK; - } - } - *subdomain_out = NULL; - return DART_ERR_NOTFOUND; -} - -/** - * Find common parent (lowest common ancestor) of specified domains by - * their common domain tag prefix. - */ -dart_ret_t dart__base__locality__domain__parent( - const dart_domain_locality_t * domain_in, - const char ** subdomain_tags, - int num_subdomain_tags, - dart_domain_locality_t ** domain_out) -{ - *domain_out = NULL; - - /* Parent domain tag of subdomains is common prefix of subdomain tags: - */ - char subdomains_prefix[DART_LOCALITY_DOMAIN_TAG_MAX_SIZE]; - int subdomains_prefix_len = dart__base__strscommonprefix( - subdomain_tags, - num_subdomain_tags, - subdomains_prefix); - /* Remove trailing '.' */ - if (subdomains_prefix[subdomains_prefix_len - 1] == '.') { - subdomains_prefix_len--; - subdomains_prefix[subdomains_prefix_len] = '\0'; - } - if (subdomains_prefix_len == 0) { - *domain_out = (dart_domain_locality_t *)(domain_in); - return DART_OK; - } - - /* Find domain tagged with subdomains prefix: */ - return dart__base__locality__domain__child( - domain_in, subdomains_prefix, domain_out); -} - -/** - * Remove all child nodes from a domain that do not match the specified - * domain tags. - */ -dart_ret_t dart__base__locality__domain__filter_subdomains_if( - dart_domain_locality_t * domain, - dart_domain_predicate_t pred) -{ - dart__unused(domain); - dart__unused(pred); - - return DART_OK; -} - -/** - * Remove all child nodes from a domain that match or do not match the - * specified domain tags. - */ -dart_ret_t dart__base__locality__domain__filter_subdomains( - dart_domain_locality_t * domain, - const char ** subdomain_tags, - int num_subdomain_tags, - int remove_matches) -{ -// TODO DEBUG: check if memory operations on domains are correct in -// this function: -// - dart_ret_t ret = DART_OK; - - int is_unit_scope = ((int)domain->scope >= - (int)DART_LOCALITY_SCOPE_CORE); - int matched = 0; - int unit_idx = 0; - int subdomain_idx = 0; - // int num_numa = 0; - - DART_LOG_TRACE("dart__base__locality__domain__filter_subdomains() " - "domain: %s, level: %d, domains: %d, units: %d", - domain->domain_tag, domain->level, - domain->num_domains, domain->num_units); - -#ifdef DART_ENABLE_LOGGING - for (int gsd = 0; gsd < num_subdomain_tags; gsd++) { - DART_LOG_TRACE("dart__base__locality__domain__filter_subdomains: " - "subdomain_tags[%d]: %s", - gsd, subdomain_tags[gsd]); - } -#endif - - if (is_unit_scope) { - return DART_OK; - } - - for (int sd = 0; sd < domain->num_domains; sd++) { - /* -------------------------------------------------------------- * - * Selection predicate is just this block. * - * Could use functors to allow arbitrary selection functions. * - * */ - char * subdomain_tag = domain->children[sd]->domain_tag; - size_t subdomain_tag_len = strlen(subdomain_tag); - matched = 0; - for (int dt = 0; dt < num_subdomain_tags; dt++) { - size_t filter_tag_len = strlen(subdomain_tags[dt]); - size_t common_prefix_len = dart__base__strcommonprefix( - subdomain_tag, - subdomain_tags[dt], - NULL); - /* When removing matches: - * Match domains with full domain tag filter in prefix, - * e.g. ".0.1" matches ".0.1.0" - * -> minimum match length is length of filter tag - * When selecting matches: - * Match domain tags that are fully included in filter tag - * -> minimum match length is length of subdomain: - */ - size_t min_tag_match_len = (remove_matches == 1) - ? filter_tag_len - : subdomain_tag_len; - - if (common_prefix_len >= min_tag_match_len) { - matched = 1; - break; - } - } - /* * - * -------------------------------------------------------------- */ - - if (matched == remove_matches) { - continue; - } - DART_LOG_TRACE("dart__base__locality__domain__filter_subdomains : " - " --v subdomain[%d] = %s matched", - sd, subdomain_tag); - - if (subdomain_idx != sd) { - domain->children[subdomain_idx] = domain->children[sd]; - domain->children[subdomain_idx]->relative_index = subdomain_idx; - domain->children[sd] = NULL; - } - - ret = dart__base__locality__domain__filter_subdomains( - domain->children[subdomain_idx], - subdomain_tags, - num_subdomain_tags, - remove_matches); - if (ret != DART_OK) { return ret; } - - DART_LOG_TRACE("dart__base__locality__domain__filter_subdomains : " - " --^ subdomain[%d] = %s: domains: %d, units: %d", - sd, - domain->children[subdomain_idx]->domain_tag, - domain->children[subdomain_idx]->num_domains, - domain->children[subdomain_idx]->num_units); - - /* Collect units and domains bottom-up after maximum recursion - * depth has been reached: - */ - if (domain->children[subdomain_idx]->num_units > 0) { - DART_ASSERT(domain->num_units > unit_idx); - memcpy(domain->unit_ids + unit_idx, - domain->children[subdomain_idx]->unit_ids, - domain->children[subdomain_idx]->num_units * - sizeof(dart_global_unit_t)); - unit_idx += domain->children[subdomain_idx]->num_units; - } - - // num_numa += domain->children[subdomain_idx]->hwinfo.num_numa; - subdomain_idx++; - } - - /* - * Bottom-up accumulation of units and domains: - */ - DART_LOG_TRACE("dart__base__locality__domain__filter_subdomains : " - "--> collected in %s: domains: %d, units: %d", - domain->domain_tag, subdomain_idx, unit_idx); - - // domain->hwinfo.num_numa = num_numa; - - if (NULL != domain->unit_ids) { - if (domain->num_units != unit_idx) { - if (unit_idx == 0) { - free(domain->unit_ids); - domain->unit_ids = NULL; - } else { - domain->unit_ids = - realloc(domain->unit_ids, unit_idx * sizeof(dart_global_unit_t)); - DART_ASSERT(domain->unit_ids != NULL); - } - domain->num_units = unit_idx; - } - } - - if (NULL != domain->children) { - if (domain->num_domains != subdomain_idx) { - if (subdomain_idx > domain->num_domains) { - DART_LOG_WARN("dart__base__locality__domain__filter_subdomains: " - "number of subdomains increased from %d to %d in %s", - domain->num_domains, subdomain_idx, - domain->domain_tag); - // Filtering should never increase number of subdomains: - DART_ASSERT(subdomain_idx <= domain->num_domains); - for (int sd = domain->num_domains; sd < subdomain_idx; sd++) { - domain->children[sd] = malloc(sizeof(dart_domain_locality_t)); - } - } else { - /* delete subdomains at rel. index > new number of subdomains: - */ - for (int sd = subdomain_idx; sd < domain->num_domains; sd++) { - if (NULL != domain->children[sd]) { - dart__base__locality__domain__destruct( - domain->children[sd]); - free(domain->children[sd]); - domain->children[sd] = NULL; - } - } - } - if (subdomain_idx == 0) { - free(domain->children); - domain->children = NULL; - } else { - domain->children = realloc(domain->children, - subdomain_idx * - sizeof(dart_domain_locality_t *)); - DART_ASSERT_MSG( - domain->children != NULL, - "dart__base__locality__domain__filter_subdomains: realloc failed"); - } - domain->num_domains = subdomain_idx; - } - } - DART_LOG_TRACE("dart__base__locality__domain__filter_subdomains >"); - return DART_OK; -} - -dart_ret_t dart__base__locality__domain__add_subdomain( - dart_domain_locality_t * domain, - dart_domain_locality_t * subdomain, - int subdomain_rel_id) -{ - /* inserts pointer to subdomain and updates number of subdomains but - * does not recalculate domain attributes. - */ - domain->num_domains++; - domain->children = realloc(domain->children, - domain->num_domains * - sizeof(dart_domain_locality_t *)); - DART_ASSERT_MSG( - domain->children != NULL, - "dart__base__locality__domain__add_subdomain: realloc failed"); - - if (subdomain_rel_id < 0) { - /* append at end of subdomains: */ - domain->children[domain->num_domains - 1] = subdomain; - } else { - /* move all subdomains after insertion index: */ - for (int sd_move = subdomain_rel_id; - sd_move < domain->num_domains-1; sd_move++) { - domain->children[sd_move + 1] = domain->children[sd_move]; - } - domain->children[subdomain_rel_id] = subdomain; - } - return DART_OK; -} - -dart_ret_t dart__base__locality__domain__remove_subdomain( - dart_domain_locality_t * domain, - int subdomain_rel_id) -{ - // TODO: also remove the subdomain from all aliases lists - - /* reduces number of subdomains but does not recalculate domain - * attributes and does not destruct the removed subdomain. - */ - - /* move all subdomains after removed index: */ - for (int sd_move = subdomain_rel_id; - sd_move < domain->num_domains-1; sd_move++) { - domain->children[sd_move] = domain->children[sd_move + 1]; - } - domain->num_domains--; - - if (domain->num_domains <= 0) { - free(domain->children); - domain->children = NULL; - domain->num_domains = 0; - } else { - domain->children = realloc(domain->children, - domain->num_domains * - sizeof(dart_domain_locality_t *)); - DART_ASSERT_MSG( - domain->children != NULL, - "dart__base__locality__domain__add_subdomain: realloc failed"); - } - return DART_OK; -} - -dart_ret_t dart__base__locality__domain__move_subdomain( - dart_domain_locality_t * subdomain, - dart_domain_locality_t * new_parent_domain, - int new_subdomain_rel_id) -{ - /* Moves pointer to subdomain and updates number of subdomains in source- - * and target parent domain but does not recalculate domain attributes. - */ - dart_ret_t ret; - - ret = dart__base__locality__domain__add_subdomain( - new_parent_domain, subdomain, new_subdomain_rel_id); - if (DART_OK != ret) { return ret; } - - ret = dart__base__locality__domain__remove_subdomain( - subdomain->parent, subdomain->relative_index); - if (DART_OK != ret) { return ret; } - - return DART_OK; -} - -dart_ret_t dart__base__locality__domain__create_subdomains( - dart_domain_locality_t * global_domain, - dart_host_topology_t * host_topology, - dart_unit_mapping_t * unit_mapping) -{ - /* Iterate node domains in given root domain: */ - int num_nodes; - DART_ASSERT_RETURNS( - dart__base__host_topology__num_nodes( - host_topology, &num_nodes), - DART_OK); - DART_LOG_TRACE("dart__base__locality__domain__create_node_subdomains: " - "num_nodes:%d", num_nodes); - - /* Child domains of root are at node level: */ - global_domain->num_cores = 0; - global_domain->num_domains = num_nodes; - global_domain->num_aliases = 0; - global_domain->scope = DART_LOCALITY_SCOPE_GLOBAL; - global_domain->level = 0; - global_domain->shared_mem_bytes = 0; - global_domain->global_index = 0; - global_domain->relative_index = 0; - global_domain->children = malloc(num_nodes * - sizeof(dart_domain_locality_t *)); - strcpy(global_domain->domain_tag, "."); - - for (int n = 0; n < num_nodes; n++) { - global_domain->children[n] = malloc(sizeof(dart_domain_locality_t)); - - dart_domain_locality_t * node_domain = global_domain->children[n]; - dart__base__locality__domain__init(node_domain); - - node_domain->scope = DART_LOCALITY_SCOPE_NODE; - node_domain->level = global_domain->level + 1; - node_domain->shared_mem_bytes = -1; - node_domain->global_index = n; - node_domain->relative_index = n; - node_domain->parent = global_domain; - node_domain->team = global_domain->team; - node_domain->num_units = 0; - sprintf(node_domain->domain_tag, ".%d", node_domain->relative_index); - - const char * node_hostname; - DART_ASSERT_RETURNS( - dart__base__host_topology__node( - host_topology, n, &node_hostname), - DART_OK); - strncpy(node_domain->host, node_hostname, DART_LOCALITY_HOST_MAX_SIZE); - - DART_ASSERT_RETURNS( - dart__base__locality__domain__create_node_subdomains( - node_domain, host_topology, unit_mapping), - DART_OK); - - for (int sd = 0; sd < node_domain->num_domains; sd++) { - dart_domain_locality_t * node_subdomain = node_domain->children[sd]; - int node_num_units_prev = node_domain->num_units; - node_domain->num_units += node_subdomain->num_units; - - if (node_domain->num_units > node_num_units_prev) { - node_domain->unit_ids = realloc(node_domain->unit_ids, - node_domain->num_units * - sizeof(dart_global_unit_t)); - - DART_ASSERT_MSG( - node_domain->unit_ids != NULL, - "dart__base__locality__domain__create_subdomains: realloc failed"); - - memcpy(node_domain->unit_ids + node_num_units_prev, - node_subdomain->unit_ids, - node_subdomain->num_units * sizeof(dart_global_unit_t)); - } - } - - /* Bottom-up recursion operations: */ - global_domain->num_cores += node_domain->num_cores; - } - return DART_OK; -} - -dart_ret_t dart__base__locality__domain__create_node_subdomains( - dart_domain_locality_t * node_domain, - dart_host_topology_t * host_topology, - dart_unit_mapping_t * unit_mapping) -{ - /* Expects node_domain to be initialized. */ - - DART_LOG_TRACE( - "dart__base__locality__domain__create_node_subdomains() " - "node_domain { host:%s, domain_tag:%s, num_units:%d }", - node_domain->host, node_domain->domain_tag, - node_domain->num_units); - /* - * TODO: Should only be performed by node leader units and the - * result broadcasted to units located on the node. - */ - - int num_modules; - DART_ASSERT_RETURNS( - dart__base__host_topology__num_node_modules( - host_topology, node_domain->host, &num_modules), - DART_OK); - DART_LOG_TRACE( - "dart__base__locality__domain__create_node_subdomains: " - "node_hostname:%s num_modules:%d", - node_domain->host, num_modules); - - node_domain->num_aliases = 0; - node_domain->num_domains = num_modules; - node_domain->children = malloc(num_modules * - sizeof(dart_domain_locality_t *)); - - int sum_module_cores = 0; - for (int m = 0; m < num_modules; m++) { - node_domain->children[m] = malloc(sizeof(dart_domain_locality_t)); - dart_domain_locality_t * module_domain = node_domain->children[m]; - dart__base__locality__domain__init(module_domain); - - module_domain->scope = DART_LOCALITY_SCOPE_MODULE; - module_domain->level = node_domain->level + 1; - module_domain->shared_mem_bytes = -1; - module_domain->global_index = m; - module_domain->relative_index = m; - module_domain->parent = node_domain; - module_domain->team = node_domain->team; - sprintf(module_domain->domain_tag, "%s.%d", - node_domain->domain_tag, - module_domain->relative_index); - - const char * module_hostname; - DART_ASSERT_RETURNS( - dart__base__host_topology__node_module( - host_topology, node_domain->host, m, &module_hostname), - DART_OK); - DART_LOG_TRACE( - "dart__base__locality__domain__create_node_subdomains: " - "module_index:%d module_hostname:%s", - m, module_hostname); - strncpy(module_domain->host, module_hostname, - DART_LOCALITY_HOST_MAX_SIZE); - - const dart_global_unit_t * module_unit_ids; - const int * module_numa_ids; - int module_num_numa; - DART_ASSERT_RETURNS( - dart__base__host_topology__host_domain( - host_topology, module_domain->host, - &module_unit_ids, &module_domain->num_units, - &module_numa_ids, &module_num_numa), - DART_OK); - - module_domain->unit_ids = - malloc(module_domain->num_units * sizeof(dart_global_unit_t)); - memcpy(module_domain->unit_ids, module_unit_ids, - module_domain->num_units * sizeof(dart_global_unit_t)); - - DART_ASSERT_RETURNS( - dart__base__locality__domain__create_module_subdomains( - module_domain, host_topology, unit_mapping, 0), - DART_OK); - - /* Bottom-up recursion operations here */ - - sum_module_cores += module_domain->num_cores; - } - node_domain->num_cores = sum_module_cores; - - DART_LOG_TRACE( - "dart__base__locality__domain__create_node_subdomains >"); - return DART_OK; -} - -dart_ret_t dart__base__locality__domain__create_module_subdomains( - dart_domain_locality_t * module_domain, - dart_host_topology_t * host_topology, - dart_unit_mapping_t * unit_mapping, - int module_scope_level) -{ - DART_LOG_TRACE( - "dart__base__locality__domain__create_module_subdomains() " - "module_scope_level:%d module_domain { " - "host:%s, domain_tag:%s, num_units:%d, global_index:%d }", - module_scope_level, - module_domain->host, module_domain->domain_tag, - module_domain->num_units, module_domain->global_index); - - DART_LOG_TRACE_UNITID_ARRAY( - "dart__base__locality__domain__create_module_subdomains", "%d", - module_domain->unit_ids, module_domain->num_units); - /* - * NOTE: Locality scopes may be heterogeneous but are expected - * to be homogeneous within a module domain. - * For example, this would be a valid use case: - * - * module[0] { unit[0]: [CORE,CACHE,PACKAGE,NUMA], - * unit[1]: [CORE,CACHE,PACKAGE,NUMA] } - * module[1] { unit[2]: [CORE,CACHE,CACHE,CACHE,NUMA], - * unit[2]: [CORE,CACHE,CACHE,CACHE,NUMA] } - */ - if (module_domain->num_units < 1) { - module_domain->num_units = 0; - module_domain->unit_ids = NULL; - DART_LOG_TRACE( - "dart__base__locality__domain__create_module_subdomains > " - "no units"); - return DART_OK; - } - - /* Collect scope lists of all units at the given module, converting: - * - * u[0].scopes: [CORE:0, CACHE:4, CACHE:0, NUMA:0] - * u[1].scopes: [CORE:1, CACHE:5, CACHE:0, NUMA:0] - * u[2].scopes: [CORE:2, CACHE:6, CACHE:1, NUMA:0] - * u[3].scopes: [CORE:3, CACHE:7, CACHE:1, NUMA:0] - * - * to transposed structure: - * - * level[0]: { scope:NUMA, gids: [ 0 ], - * sub_gids:[[ 0 , 1 ]] } - * - * level[1]: { scope:CACHE, gids: [ 0, 1 ], - * sub_gids:[[4 , 5],[6 , 7]] } - * - * level[2]: { scope:CACHE, gids: [ 4, 5, 6, 7 ], - * sub_gids:[[0],[1],[2],[3]] } - * - * such that subdomains of a domain with global index G are referenced - * in sub_gids[G]. - */ - - /* Obtain array of distinct global indices from unit scopes at current - * level: - */ - - dart_locality_scope_t module_scopes[DART_LOCALITY_MAX_DOMAIN_SCOPES]; - - int num_scopes = 0; - dart_unit_locality_t * module_leader_unit_loc; - dart_team_unit_t unit_id; - DART_ASSERT_RETURNS( - dart_team_unit_g2l(module_domain->team, module_domain->unit_ids[0], - &unit_id), - DART_OK); - DART_LOG_TRACE( - "dart__base__locality__domain__create_module_subdomains: " - "module_domain->unit_ids[0]: %d -> local: %d", - module_domain->unit_ids[0].id, unit_id.id); - - DART_ASSERT_RETURNS( - dart__base__unit_locality__at( - unit_mapping, unit_id, &module_leader_unit_loc), - DART_OK); - num_scopes = module_leader_unit_loc->hwinfo.num_scopes; - - DART_ASSERT(num_scopes > 0); - - if (module_scope_level == 0) { - /* At module level, get number of cores from leader unit: */ - int num_module_cores = module_leader_unit_loc->hwinfo.num_cores; - module_domain->num_cores = num_module_cores; - } else { - // TODO: Clarify - } - - for (int s = 0; s < num_scopes; s++) { - module_scopes[s] = module_leader_unit_loc->hwinfo.scopes[s].scope; - } - DART_LOG_TRACE_ARRAY( - "dart__base__locality__domain__create_module_subdomains", "%d", - module_scopes, num_scopes); - - int subdomain_gid_idx = num_scopes - (module_scope_level + 1); - -#if defined(DART_ENABLE_LOGGING) - dart_locality_scope_t subdomain_scope = module_scopes[subdomain_gid_idx]; - DART_LOG_TRACE("dart__base__locality__domain__create_module_subdomains: " - "-- module_scope_level:%d subdomain_gid_idx:%d " - "-> subdomain_scope:%d", - module_scope_level, subdomain_gid_idx, subdomain_scope); -#endif - - /* Array of the global indices of the current module subdomains. - * Maximum number of global indices, including duplicates, is number - * of units: - */ - int module_subdomain_gids[module_domain->num_units]; - - /* Copy global indices from scopes list of all units U at the module - * domain to module_subdomain_gids[U]: - */ - int gid_idx = 0; - for (int u_idx = 0; u_idx < module_domain->num_units; u_idx++) { - dart_global_unit_t unit_gid = module_domain->unit_ids[u_idx]; - dart_team_unit_t unit_lid; - DART_ASSERT_RETURNS( - dart_team_unit_g2l(module_domain->team, unit_gid, &unit_lid), - DART_OK); - - dart_unit_locality_t * module_unit_loc; - DART_ASSERT_RETURNS( - dart__base__unit_locality__at( - unit_mapping, unit_lid, &module_unit_loc), - DART_OK); - dart_hwinfo_t * unit_hwinfo = &module_unit_loc->hwinfo; - - int unit_level_gid = unit_hwinfo->scopes[subdomain_gid_idx+1].index; - int unit_sub_gid = -1; - if (subdomain_gid_idx >= 0) { - unit_sub_gid = unit_hwinfo->scopes[subdomain_gid_idx].index; - } - DART_LOG_TRACE( - "dart__base__locality__domain__create_module_subdomains: ---- " - "module_domain.unit_ids[%d] => unit:%d sub_gid:%d level_gid:%d " - "module_domain.global_index:%d", - u_idx, unit_lid.id, unit_sub_gid, unit_level_gid, - module_domain->global_index); - /* Ignore units that are not contained in current module domain: */ - if (module_scope_level == 0 || - unit_level_gid == module_domain->global_index) { - module_subdomain_gids[gid_idx] = unit_sub_gid; - DART_LOG_TRACE( - "dart__base__locality__domain__create_module_subdomains: ---- " - "level:%d unit:%d module_subdomain_gids[%d]:%d", - module_scope_level, u_idx, gid_idx, module_subdomain_gids[gid_idx]); - gid_idx++; - } - } - int num_module_units = gid_idx; - /* Sorts values in array module_subdomain_gids such that the first - * num_subdomain elements contain its distinct values: */ - int num_subdomains = dart__base__intsunique( - module_subdomain_gids, num_module_units); - - module_domain->num_domains = num_subdomains; - module_domain->children = malloc(module_domain->num_domains * - sizeof(dart_domain_locality_t *)); - - DART_LOG_TRACE_ARRAY( - "dart__base__locality__domain__create_module_subdomains", "%d", - module_subdomain_gids, num_subdomains); - - DART_LOG_TRACE("dart__base__locality__domain__create_module_subdomains: " - "module_domain.num_domains:%d, module_domain.num_units:%d " - " == %d ?", - module_domain->num_domains, module_domain->num_units, - num_module_units); - - for (int sd = 0; sd < module_domain->num_domains; sd++) { - - DART_LOG_TRACE("dart__base__locality__domain__create_module_subdomains: " - "module subdomain index:%d / num_domains:%d", - sd, module_domain->num_domains); - - module_domain->children[sd] = malloc(sizeof(dart_domain_locality_t)); - dart_domain_locality_t * subdomain = module_domain->children[sd]; - dart__base__locality__domain__init(subdomain); - - subdomain->level = module_domain->level + 1; - subdomain->scope = module_scopes[subdomain_gid_idx]; - subdomain->relative_index = sd; - subdomain->global_index = module_subdomain_gids[sd]; - subdomain->parent = module_domain; - subdomain->team = module_domain->team; - - /* Set module subdomain tag: */ - sprintf(subdomain->domain_tag, "%s.%d", - module_domain->domain_tag, - subdomain->relative_index); - - DART_LOG_TRACE("dart__base__locality__domain__create_module_subdomains: " - "-- module->domains[%d].domain_tag:%s", - sd, subdomain->domain_tag); - - /* Set module subdomain hostname: */ - const char * module_subdomain_host; - if (dart__base__host_topology__node_module( - host_topology, module_domain->host, sd, - &module_subdomain_host) - == DART_OK) { - strncpy(subdomain->host, module_subdomain_host, - DART_LOCALITY_HOST_MAX_SIZE); - } else { - strncpy(subdomain->host, module_domain->host, - DART_LOCALITY_HOST_MAX_SIZE); - } - DART_LOG_TRACE("dart__base__locality__domain__create_module_subdomains: " - "-- module->children[%d].host:%s", - sd, subdomain->host); - - /* Set module subdomain units. Filter units in module domain by - * global index of the subdomain of this iteration: */ - subdomain->num_units = 0; - subdomain->unit_ids = malloc(module_domain->num_units * - sizeof(dart_global_unit_t)); - for (int u_idx = 0; u_idx < module_domain->num_units; u_idx++) { - dart_global_unit_t unit_gid = module_domain->unit_ids[u_idx]; - dart_team_unit_t unit_lid; - DART_ASSERT_RETURNS( - dart_team_unit_g2l(module_domain->team, unit_gid, &unit_lid), - DART_OK); - - dart_unit_locality_t * unit_loc; - DART_ASSERT_RETURNS( - dart__base__unit_locality__at( - unit_mapping, unit_lid, &unit_loc), - DART_OK); - DART_LOG_TRACE( - "dart__base__locality__domain__create_module_subdomains: ---- " - "module_unit[%d](= unit:%d).scopes[%d].index:%d =?= " - "subdomain.global_index:%d", - u_idx, unit_lid.id, subdomain_gid_idx, - unit_loc->hwinfo.scopes[subdomain_gid_idx].index, - subdomain->global_index); - - if (unit_loc->hwinfo.scopes[subdomain_gid_idx].index == - subdomain->global_index) { - subdomain->unit_ids[subdomain->num_units] = unit_gid; - subdomain->num_units++; - } - } - DART_LOG_TRACE("dart__base__locality__domain__create_module_subdomains: " - "-- module->children[%d].num_units:%d", - sd, subdomain->num_units); - - if (subdomain->num_units > 0) { - subdomain->unit_ids = realloc(subdomain->unit_ids, - subdomain->num_units * sizeof(dart_global_unit_t)); - DART_ASSERT(NULL != subdomain->unit_ids); - } else { - free(subdomain->unit_ids); - subdomain->num_units = 0; - subdomain->unit_ids = NULL; - } - - /* Number of units in subdomain is set at this point. - * Below module level, a module subdomain's number of affine - * cores is: - */ - int balanced_cores_per_subdomain = module_domain->num_cores / - module_domain->num_units; - subdomain->num_cores = balanced_cores_per_subdomain * - subdomain->num_units; - - if (subdomain->num_cores < 1) { - subdomain->num_cores = 1; - } - - if (subdomain_gid_idx <= 0) { - - /* Reached CORE scope: */ - DART_LOG_TRACE( - "dart__base__locality__domain__create_module_subdomains: " - "reached CORE scope (num_units:%d)", - subdomain->num_units); - - /* TODO: Travis might map multiple MPI processes to the same - * physical core. - */ - for (int u_idx = 0; u_idx < subdomain->num_units; u_idx++) { - dart_global_unit_t unit_gid = subdomain->unit_ids[u_idx]; - dart_team_unit_t unit_lid; - DART_ASSERT_RETURNS( - dart_team_unit_g2l(subdomain->team, unit_gid, &unit_lid), - DART_OK); - - DART_LOG_TRACE( - "dart__base__locality__domain__create_module_subdomains: " - "reached CORE scope (num_units:%d), setting domain tag for " - "unit_lid:%d to %s", - subdomain->num_units, unit_lid.id, subdomain->domain_tag); - - dart_unit_locality_t * unit_loc; - DART_ASSERT_RETURNS( - dart__base__unit_locality__at( - unit_mapping, unit_lid, &unit_loc), - DART_OK); - - strncpy(unit_loc->domain_tag, subdomain->domain_tag, - DART_LOCALITY_DOMAIN_TAG_MAX_SIZE); - unit_loc->hwinfo.num_cores = subdomain->num_cores / - subdomain->num_units; - if (unit_loc->hwinfo.num_cores < 1) { - unit_loc->hwinfo.num_cores = 1; - } - } - } else { - /* Recurse to next scope level in the module domain: */ - DART_ASSERT_RETURNS( - dart__base__locality__domain__create_module_subdomains( - subdomain, host_topology, unit_mapping, - module_scope_level+1), - DART_OK); - } - } - - DART_LOG_TRACE("dart__base__locality__domain__create_module_subdomains >"); - return DART_OK; -} - diff --git a/dart-impl/base/src/internal/host_topology.c b/dart-impl/base/src/internal/host_topology.c deleted file mode 100644 index cde15ddf7..000000000 --- a/dart-impl/base/src/internal/host_topology.c +++ /dev/null @@ -1,927 +0,0 @@ -/** - * \file dash/dart/base/internal/host_topology.c - * - */ - -#include -#include - -#include -#include -#include - -#include -#include -#include - -#include -#include -#include - -#ifdef DART_ENABLE_HWLOC -# include -# include -#endif - -/* ===================================================================== * - * Private Functions * - * ===================================================================== */ - -static int cmpstr_(const void * p1, const void * p2) { - return strcmp(* (char * const *) p1, * (char * const *) p2); -} - -static dart_ret_t dart__base__host_topology__module_locations( - dart_module_location_t ** module_locations, - int * num_modules) -{ - *module_locations = NULL; - *num_modules = 0; - -#if defined(DART_ENABLE_HWLOC) && defined(DART_ENABLE_HWLOC_PCI) - DART_LOG_TRACE("dart__base__host_topology__module_locations: using hwloc"); - - hwloc_topology_t topology; - hwloc_topology_init(&topology); - hwloc_topology_set_flags(topology, -#if HWLOC_API_VERSION < 0x00020000 - HWLOC_TOPOLOGY_FLAG_IO_DEVICES - | HWLOC_TOPOLOGY_FLAG_IO_BRIDGES - /* | HWLOC_TOPOLOGY_FLAG_WHOLE_IO */ -#else - HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM -#endif - ); - hwloc_topology_load(topology); - DART_LOG_TRACE("dart__base__host_topology__module_locations: " - "hwloc: indexing PCI devices"); - /* Alternative: HWLOC_TYPE_DEPTH_PCI_DEVICE */ - int n_pcidev = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_PCI_DEVICE); - - DART_LOG_TRACE("dart__base__host_topology__module_locations: " - "hwloc: %d PCI devices found", n_pcidev); - for (int pcidev_idx = 0; pcidev_idx < n_pcidev; pcidev_idx++) { - hwloc_obj_t coproc_obj = - hwloc_get_obj_by_type(topology, HWLOC_OBJ_PCI_DEVICE, pcidev_idx); - if (NULL != coproc_obj) { - DART_LOG_TRACE("dart__base__host_topology__module_locations: " - "hwloc: PCI device: (name:%s arity:%d)", - coproc_obj->name, coproc_obj->arity); - if (NULL != coproc_obj->name && - NULL != strstr(coproc_obj->name, "Xeon Phi")) { - DART_LOG_TRACE("dart__base__host_topology__module_locations: " - "hwloc: Xeon Phi device"); - if (coproc_obj->arity > 0) { - for (int pd_i = 0; pd_i < (int)coproc_obj->arity; pd_i++) { - hwloc_obj_t coproc_child_obj = coproc_obj->children[pd_i]; - DART_LOG_TRACE("dart__base__host_topology__module_locations: " - "hwloc: Xeon Phi child node: (name:%s arity:%d)", - coproc_child_obj->name, coproc_child_obj->arity); - (*num_modules)++; - *module_locations = realloc(*module_locations, - *num_modules * - sizeof(dart_module_location_t)); - dart_module_location_t * module_loc = - module_locations[(*num_modules)-1]; - - char * hostname = module_loc->host; - char * mic_hostname = module_loc->module; - char * mic_dev_name = coproc_child_obj->name; - - gethostname(hostname, DART_LOCALITY_HOST_MAX_SIZE); - - int n_chars_written = snprintf( - mic_hostname, DART_LOCALITY_HOST_MAX_SIZE, - "%s-%s", hostname, mic_dev_name); - if (n_chars_written < 0 || - n_chars_written >= DART_LOCALITY_HOST_MAX_SIZE) { - DART_LOG_ERROR("dart__base__host_topology__module_locations: " - "MIC host name '%s-%s could not be assigned", - hostname, mic_dev_name); - } - - DART_LOG_TRACE("dart__base__host_topology__module_locations: " - "hwloc: Xeon Phi module hostname: %s " - "node hostname: %s", - module_loc->module, module_loc->host); - - /* Get host of MIC device: */ - hwloc_obj_t mic_host_obj = - hwloc_get_non_io_ancestor_obj(topology, coproc_obj); - if (mic_host_obj != NULL) { - - module_loc->pos.scope = - dart__base__hwloc__obj_type_to_dart_scope(mic_host_obj->type); - module_loc->pos.index = mic_host_obj->logical_index; - DART_LOG_TRACE("dart__base__host_topology__module_locations: " - "hwloc: Xeon Phi scope pos: " - "(type:%d -> scope:%d idx:%d)", - mic_host_obj->type, - module_loc->pos.scope, - module_loc->pos.index); - } - } - } - } - } - } - hwloc_topology_destroy(topology); - DART_LOG_TRACE("dart__base__host_topology__module_locations > " - "num_modules:%d", *num_modules); -#endif /* ifdef DART_ENABLE_HWLOC */ - return DART_OK; -} - -static dart_ret_t dart__base__host_topology__update_module_locations( - dart_unit_mapping_t * unit_mapping, - dart_host_topology_t * topo) -{ - int num_hosts = topo->num_hosts; - dart_team_t team = unit_mapping->team; - - /* - * Initiate all-to-all exchange of module locations like Xeon Phi - * hostnames and their assiocated NUMA domain in their parent node. - * - * Select one leader unit per node for communication: - */ - - dart_unit_locality_t * my_uloc; - - /* - * unit ID of leader unit (relative to the team specified in unit_mapping) - * of the active unit's local compute node. - * Example: - * - * node 0: node 1: - * unit ids: { 0, 1, 2 } unit ids: { 3, 4, 5 } - * - * leader unit at units 0,1,2: 0 - * leader unit at units 3,4,5: 3 - */ - dart_team_unit_t local_leader_unit_id = DART_UNDEFINED_TEAM_UNIT_ID; - dart_team_unit_t my_id; - dart_group_t leader_group; - dart_group_t local_group; - dart_team_t leader_team; /* team of all node leaders */ - - DART_ASSERT_RETURNS( - dart_team_myid(team, &my_id), - DART_OK); - DART_ASSERT_RETURNS( - dart__base__unit_locality__at( - unit_mapping, my_id, &my_uloc), - DART_OK); - DART_ASSERT_RETURNS( - dart_group_create(&leader_group), - DART_OK); - DART_ASSERT_RETURNS( - dart_group_create(&local_group), - DART_OK); - - dart_unit_locality_t * unit_locality; - dart__base__unit_locality__at( - unit_mapping, my_id, &unit_locality); - - char * local_hostname = unit_locality->hwinfo.host; - DART_LOG_TRACE("dart__base__host_topology__init: " - "local_hostname:%s", local_hostname); - - - /* Compose leader group and local group: */ - for (int h = 0; h < num_hosts; h++) { - /* Get unit ids at local unit's host */ - dart_host_units_t * host_units = &topo->host_units[h]; - dart_host_domain_t * host_domain = &topo->host_domains[h]; - /* Select first unit id at local host as leader: */ - dart_global_unit_t leader_unit_id = host_units->units[0]; - DART_ASSERT_RETURNS( - dart_group_addmember(leader_group, leader_unit_id), - DART_OK); - DART_LOG_TRACE("dart__base__host_topology__init: " - "num. units on host %s: %d", - topo->host_names[h], host_units->num_units); - DART_LOG_TRACE("dart__base__host_topology__init: " - "leader unit on host %s: %d", - topo->host_names[h], leader_unit_id.id); - DART_ASSERT( - strncmp(topo->host_names[h], host_domain->host, - DART_LOCALITY_HOST_MAX_SIZE) == 0); - if (strncmp(host_domain->host, local_hostname, - DART_LOCALITY_HOST_MAX_SIZE) == 0) { - /* set local leader: */ - DART_ASSERT_RETURNS( - dart_team_unit_g2l(team, leader_unit_id, - &local_leader_unit_id), - DART_OK); - /* collect units in local group: */ - for (int u_idx = 0; u_idx < host_units->num_units; u_idx++) { - DART_LOG_TRACE("dart__base__host_topology__init: " - "add unit %d to local group", - host_units->units[u_idx].id); - DART_ASSERT_RETURNS( - dart_group_addmember(local_group, host_units->units[u_idx]), - DART_OK); - } - } - } - - DART_LOG_TRACE("dart__base__host_topology__init: " - "myid:%d (in team %d) local_leader_unit_id:%d", - my_id.id, team, local_leader_unit_id.id); - - size_t num_leaders; - DART_ASSERT_RETURNS( - dart_group_size(leader_group, &num_leaders), - DART_OK); - DART_LOG_TRACE("dart__base__host_topology__init: num_leaders:%zu", - num_leaders); - - if (num_leaders > 1) { - DART_LOG_TRACE("dart__base__host_topology__init: create leader team"); - DART_ASSERT_RETURNS( - dart_team_create(team, leader_group, &leader_team), - DART_OK); - DART_LOG_TRACE("dart__base__host_topology__init: leader team: %d", - leader_team); - } else { - leader_team = team; - } - - DART_ASSERT_RETURNS( - dart_group_destroy(&leader_group), - DART_OK); - - if (my_id.id == local_leader_unit_id.id) { - dart_module_location_t * module_locations = NULL; - dart_team_unit_t my_leader_id; - DART_ASSERT_RETURNS( - dart_team_myid(leader_team, &my_leader_id), - DART_OK); - DART_LOG_TRACE("dart__base__host_topology__init: " - "num_leaders:%zu my_leader_id:%d (in team %d)", - num_leaders, my_leader_id.id, leader_team); - - /* local module locations to send: */ - int max_node_modules = 2; - int num_local_modules = 0; - dart_module_location_t * local_module_locations; - DART_ASSERT_RETURNS( - dart__base__host_topology__module_locations( - &local_module_locations, &num_local_modules), - DART_OK); - /* Number of bytes to receive from each leader unit in allgatherv: */ - size_t * recvcounts = calloc(num_leaders, sizeof(size_t)); - /* Displacement at which to place data received from each leader: */ - size_t * displs = calloc(num_leaders, sizeof(size_t)); - recvcounts[my_leader_id.id] = num_local_modules * - sizeof(dart_module_location_t); - /* - * Allgather of leaders is only required if there is more than - * one leader unit. - */ - if (num_leaders > 1) { - /* all module locations to receive: */ - module_locations = malloc(sizeof(dart_module_location_t) * - max_node_modules * num_leaders); - DART_ASSERT_RETURNS( - dart_allgather( - NULL, - recvcounts, - 1, - DART_TYPE_SIZET, - leader_team), - DART_OK); - - displs[0] = 0; - for (size_t lu = 1; lu < num_leaders; lu++) { - DART_LOG_TRACE("dart__base__host_topology__init: " - "allgather: leader unit %zu sent %zu", - lu, recvcounts[lu]); - displs[lu] = displs[lu-1] + recvcounts[lu]; - } - - DART_ASSERT_RETURNS( - dart_allgatherv( - local_module_locations, - recvcounts[my_leader_id.id], - DART_TYPE_BYTE, - module_locations, - recvcounts, - displs, - leader_team), - DART_OK); - - DART_LOG_DEBUG("dart__base__host_topology__init: " - "free(local_module_locations)"); - free(local_module_locations); - } else { - module_locations = local_module_locations; - } - - topo->num_nodes = topo->num_hosts; - topo->num_host_levels = 0; - for (size_t lu = 0; lu < num_leaders; lu++) { - /* Number of modules received from leader unit lu: */ - size_t lu_num_modules = recvcounts[lu] / - sizeof(dart_module_location_t); - for (size_t m = 0; m < lu_num_modules; m++) { - int m_displ = displs[lu] / sizeof(dart_module_location_t); - dart_module_location_t * module_loc = - &module_locations[m_displ + m]; -#ifdef DART_ENABLE_LOGGING - dart_team_unit_t luid = {lu}; - dart_global_unit_t gu; - DART_ASSERT_RETURNS( - dart_team_unit_l2g(leader_team, luid, &gu), - DART_OK); - DART_LOG_TRACE("dart__base__host_topology__init: " - "leader unit id: %d (global unit id: %d) " - "module_location { " - "host:%s module:%s scope:%d rel.idx:%d } " - "num_hosts:%d", - luid.id, gu.id, module_loc->host, module_loc->module, - module_loc->pos.scope, module_loc->pos.index, - num_hosts); -#endif // DART_ENABLE_LOGGING - for (int h = 0; h < num_hosts; ++h) { - dart_host_domain_t * host_domain = &topo->host_domains[h]; - if (strncmp(host_domain->host, module_loc->module, - DART_LOCALITY_HOST_MAX_SIZE) - == 0) { - DART_LOG_TRACE("dart__base__host_topology__init: " - "setting parent of %s to %s", - host_domain->host, module_loc->host); - /* Classify host as module: */ - strncpy(host_domain->parent, module_loc->host, - DART_LOCALITY_HOST_MAX_SIZE); - host_domain->scope_pos = module_loc->pos; - host_domain->level = 1; - if (topo->num_host_levels < host_domain->level) { - topo->num_host_levels = host_domain->level; - } - break; - } - } - } - } - if (num_leaders > 1) { - dart_barrier(leader_team); - } - - DART_LOG_DEBUG("dart__base__host_topology__init: " - "free(module_locations)"); - free(module_locations); - - DART_LOG_DEBUG("dart__base__host_topology__init: " - "free(displs)"); - free(displs); - - DART_LOG_DEBUG("dart__base__host_topology__init: " - "free(recvcounts)"); - free(recvcounts); - - if (num_leaders > 1) { - DART_LOG_TRACE("dart__base__host_topology__init: finalize leader team"); - DART_ASSERT_RETURNS( - dart_team_destroy(&leader_team), - DART_OK); - } - } - dart_barrier(team); - - /* - * Broadcast updated host topology data from leader to all units at - * local node: - */ - if (DART_UNDEFINED_UNIT_ID != local_leader_unit_id.id) { - dart_team_t local_team; - dart_team_unit_t host_topo_bcast_root = local_leader_unit_id; - dart_team_t host_topo_bcast_team = team; - if (num_hosts > 1) { - DART_LOG_TRACE("dart__base__host_topology__init: create local team"); - DART_ASSERT_RETURNS( - dart_team_create(team, local_group, &local_team), - DART_OK); - /* Leader unit ID local team is always 0: */ - host_topo_bcast_team = local_team; - host_topo_bcast_root.id = 0; - } - - DART_LOG_TRACE("dart__base__host_topology__init: " - "broadcasting module locations from leader unit %d " - "to units in team %d", - local_leader_unit_id.id, host_topo_bcast_team); - - DART_ASSERT_RETURNS( - dart_bcast( - topo->host_domains, - sizeof(dart_host_domain_t) * num_hosts, - DART_TYPE_BYTE, - host_topo_bcast_root, - host_topo_bcast_team), - DART_OK); - - if (num_hosts > 1) { - DART_LOG_TRACE("dart__base__host_topology__init: finalize local team"); - DART_ASSERT_RETURNS( - dart_team_destroy(&local_team), - DART_OK); - } - - DART_LOG_TRACE("dart__base__host_topology__init: updated host topology:"); - topo->num_nodes = num_hosts; - for (int h = 0; h < num_hosts; ++h) { - /* Get unit ids at local unit's host */ - dart_host_domain_t * hdom = &topo->host_domains[h]; - if (hdom->level > 0) { - topo->num_nodes--; - } - DART_LOG_TRACE("dart__base__host_topology__init: " - "host[%d]: (host:%s parent:%s level:%d, scope_pos:" - "(scope:%d rel.idx:%d))", - h, hdom->host, hdom->parent, hdom->level, - hdom->scope_pos.scope, hdom->scope_pos.index); - } - } - - - DART_ASSERT_RETURNS( - dart_group_destroy(&local_group), - DART_OK); - -#if 1 - /* Classify hostnames into categories 'node' and 'module'. - * Typically, modules have the hostname of their nodes as prefix in their - * hostname, e.g.: - * - * computer-node-124 <-- node, heterogenous - * |- compute_node-124-sys <-- module, homogenous - * |- compute-node-124-mic0 <-- module, homogenous - * '- compute-node-124-mic1 <-- module, homogenous - * - * Find shortest strings in array of distinct host names: - */ - int hostname_min_len = INT_MAX; - int hostname_max_len = 0; - for (int n = 0; n < num_hosts; ++n) { - topo->host_domains[n].level = 0; - topo->host_domains[n].parent[0] = '\0'; - int hostname_len = strlen(topo->host_names[n]); - if (hostname_len < hostname_min_len) { - hostname_min_len = hostname_len; - } - if (hostname_len > hostname_max_len) { - hostname_max_len = hostname_len; - } - } - DART_LOG_TRACE("dart__base__host_topology__init: " - "host name length min: %d, max: %d", - hostname_min_len, hostname_max_len); - - topo->num_host_levels = 0; - topo->num_nodes = num_hosts; - if (hostname_min_len != hostname_max_len) { - topo->num_nodes = 0; - int num_modules = 0; - /* Match short hostnames as prefix of every other hostname: */ - for (int top = 0; top < num_hosts; ++top) { - if (strlen(topo->host_names[top]) == (size_t)hostname_min_len) { - ++topo->num_nodes; - /* Host name is node, find its modules in all other hostnames: */ - char * short_name = topo->host_names[top]; - DART_LOG_TRACE("dart__base__host_topology__init: node: %s", - short_name); - for (int sub = 0; sub < num_hosts; ++sub) { - char * other_name = topo->host_names[sub]; - /* Other hostname is longer and has short host name in prefix: */ - if (strlen(other_name) > (size_t)hostname_min_len && - strncmp(short_name, other_name, hostname_min_len) == 0) { - DART_LOG_TRACE("dart__base__host_topology__init: " - "module: %s, parent node: %s", - other_name, short_name); - num_modules++; - /* Increment topology level of other host: */ - int node_level = topo->host_domains[top].level + 1; - if (node_level > topo->num_host_levels) { - topo->num_host_levels = node_level; - } - topo->host_domains[sub].level = node_level; - /* Set short hostname as parent: */ - strncpy(topo->host_domains[sub].parent, short_name, - DART_LOCALITY_HOST_MAX_SIZE); - } - } - } - } - if (num_hosts > topo->num_nodes + num_modules) { - /* some hosts are modules of node that is not in host names: */ - topo->num_nodes += num_hosts - (topo->num_nodes + num_modules); - } - DART_LOG_TRACE("dart__base__host_topology__init: " - "hosts: %d nodes: %d modules: %d", - topo->num_hosts, topo->num_nodes, num_modules); - } -#endif - DART_LOG_TRACE("dart__base__host_topology__init >"); - return DART_OK; -} - -/* ===================================================================== * - * Internal Functions * - * ===================================================================== */ - -dart_ret_t dart__base__host_topology__create( - dart_unit_mapping_t * unit_mapping, - dart_host_topology_t ** host_topology) -{ - *host_topology = NULL; - dart_team_t team = unit_mapping->team; - size_t num_units; - - DART_LOG_TRACE("dart__base__host_topology__create: team:%d", team); - - DART_ASSERT_RETURNS(dart_team_size(team, &num_units), DART_OK); - DART_ASSERT_MSG(num_units == unit_mapping->num_units, - "Number of units in mapping differs from team size"); - - /* Copy host names of all units into array: - */ - const int max_host_len = DART_LOCALITY_HOST_MAX_SIZE; - DART_LOG_TRACE("dart__base__locality__create: copying host names"); - char ** hostnames = malloc(sizeof(char *) * num_units); - for (size_t u = 0; u < num_units; ++u) { - hostnames[u] = malloc(sizeof(char) * max_host_len); - dart_unit_locality_t * ul; - dart_team_unit_t luid = {u}; - DART_ASSERT_RETURNS( - dart__base__unit_locality__at(unit_mapping, luid, &ul), - DART_OK); - strncpy(hostnames[u], ul->hwinfo.host, max_host_len); - } - - dart_host_topology_t * topo = malloc(sizeof(dart_host_topology_t)); - - /* - * Find unique host names in array 'hostnames' and count units per - * host in same pass: - */ - DART_LOG_TRACE("dart__base__host_topology__init: " - "filtering host names of %ld units", num_units); - qsort(hostnames, num_units, sizeof(char*), cmpstr_); - size_t last_host_idx = 0; - /* Maximum number of units mapped to a single host: */ - int max_host_units = 0; - /* Number of units mapped to current host: */ - int num_host_units = 0; - for (size_t u = 0; u < num_units; ++u) { - ++num_host_units; - if (u == last_host_idx) { continue; } - /* copies next differing host name to the left, like: - * - * [ a a a a b b b c c c ] last_host_index++ = 1 - * .-----' - * v - * -> [ a b a a b b b c c c ] last_host_index++ = 2 - * .---------' - * v - * -> [ a b c a b b b c c c ] last_host_index++ = 3 - * ... - */ - if (strcmp(hostnames[u], hostnames[last_host_idx]) != 0) { - ++last_host_idx; - strncpy(hostnames[last_host_idx], hostnames[u], max_host_len); - if (num_host_units > max_host_units) { - max_host_units = num_host_units; - } - num_host_units = 0; - } - } - if (max_host_units == 0) { - /* All units mapped to same host: */ - max_host_units = num_host_units; - } - - DART_ASSERT_MSG(max_host_units > 0, - "Resolved max. units per host is <= 0"); - - /* All entries after index last_host_ids are duplicates now: */ - int num_hosts = last_host_idx + 1; - DART_LOG_TRACE("dart__base__host_topology__init: number of hosts: %d", - num_hosts); - DART_LOG_TRACE("dart__base__host_topology__init: max. number of units " - "mapped to a hosts: %d", max_host_units); - - /* Map units to hosts: */ - topo->host_domains = malloc(num_hosts * sizeof(dart_host_domain_t)); - topo->host_units = malloc(num_hosts * sizeof(dart_host_units_t)); - - for (int h = 0; h < num_hosts; ++h) { - dart_host_domain_t * host_domain = &topo->host_domains[h]; - dart_host_units_t * host_units = &topo->host_units[h]; - /* Histogram of NUMA ids: */ - int numa_id_hist[DART_LOCALITY_MAX_NUMA_ID] = { 0 }; - /* Allocate array with capacity of maximum units on a single host: */ - host_units->units = malloc(sizeof(dart_global_unit_t) - * max_host_units); - host_units->num_units = 0; - host_domain->host[0] = '\0'; - host_domain->parent[0] = '\0'; - host_domain->num_numa = 0; - host_domain->level = 0; - host_domain->scope_pos.scope = DART_LOCALITY_SCOPE_NODE; - host_domain->scope_pos.index = 0; - - memset(host_domain->numa_ids, 0, - sizeof(int) * DART_LOCALITY_MAX_NUMA_ID); - strncpy(host_domain->host, hostnames[h], max_host_len); - - DART_LOG_TRACE("dart__base__host_topology__init: mapping units to %s", - hostnames[h]); - /* Iterate over all units: */ - for (size_t u = 0; u < num_units; ++u) { - dart_unit_locality_t * ul; - dart_team_unit_t luid = {u}; - DART_ASSERT_RETURNS( - dart__base__unit_locality__at(unit_mapping, luid, &ul), - DART_OK); - if (strncmp(ul->hwinfo.host, hostnames[h], max_host_len) == 0) { - /* Unit is local to host at index h: */ - dart_global_unit_t guid; - DART_ASSERT_RETURNS( - dart_team_unit_l2g(team, ul->unit, &guid), - DART_OK); - host_units->units[host_units->num_units] = guid; - host_units->num_units++; - - int unit_numa_id = ul->hwinfo.numa_id; - - DART_LOG_TRACE("dart__base__host_topology__init: " - "mapping unit %ld to host '%s', NUMA id: %d", - u, hostnames[h], unit_numa_id); - if (unit_numa_id >= 0) { - if (numa_id_hist[unit_numa_id] == 0) { - host_domain->numa_ids[host_domain->num_numa] = unit_numa_id; - host_domain->num_numa++; - } - numa_id_hist[unit_numa_id]++; - } - } - } - DART_LOG_TRACE("dart__base__host_topology__init: " - "found %d NUMA domains on host %s", - host_domain->num_numa, hostnames[h]); - for (int n = 0; n < host_domain->num_numa; ++n) { - DART_LOG_TRACE("dart__base__host_topology__init: numa_id[%d]:%d", - n, host_domain->numa_ids[n]); - } - - /* Shrink unit array to required capacity: */ - if (host_units->num_units < max_host_units) { - DART_LOG_TRACE("dart__base__host_topology__init: shrinking node unit " - "array from %d to %d elements", - max_host_units, host_units->num_units); - // Either realloc(addr != 0, n >= 0) -> free or realloc - // or realloc(addr = 0, n > 0) -> malloc - DART_ASSERT(host_units->units != NULL || - host_units->num_units > 0); - // Note: realloc with zero-size is argued unsafe in certain scenarios: - // https://www.securecoding.cert.org/confluence/display/c/\ - // MEM04-C.+Beware+of+zero-length+allocations - if (host_units->num_units > 0) { - host_units->units = realloc(host_units->units, - host_units->num_units * - sizeof(dart_global_unit_t)); - DART_ASSERT(host_units->units != NULL); - } else { - free(host_units->units); - host_units->units = NULL; - } - } - } - - topo->num_host_levels = 0; - topo->num_nodes = num_hosts; - topo->num_hosts = num_hosts; - topo->num_units = num_units; - topo->host_names = hostnames; - - DART_ASSERT_RETURNS( - dart__base__host_topology__update_module_locations( - unit_mapping, topo), - DART_OK); - -#if 0 - topo->num_hosts = num_hosts; -//topo->host_names = (char **)(realloc(hostnames, num_hosts * sizeof(char*))); - topo->host_names = hostnames; - DART_ASSERT(topo->host_names != NULL); -#endif - - *host_topology = topo; - return DART_OK; -} - -dart_ret_t dart__base__host_topology__destruct( - dart_host_topology_t * topo) -{ - DART_LOG_DEBUG("dart__base__host_topology__destruct()"); - if (NULL != topo->host_domains) { - DART_LOG_DEBUG("dart__base__host_topology__init: " - "free(topo->host_domains)"); - free(topo->host_domains); - topo->host_domains = NULL; - } - if (NULL != topo->host_names) { - for (size_t h = 0; h < topo->num_units; ++h) { - if (NULL != topo->host_names[h]) { - DART_LOG_DEBUG("dart__base__host_topology__init: " - "free(topo->host_names[%zu])", h); - free(topo->host_names[h]); - topo->host_names[h] = NULL; - } - } - DART_LOG_DEBUG("dart__base__host_topology__init: " - "free(topo->host_names)"); - free(topo->host_names); - topo->host_names = NULL; - } - if (NULL != topo->host_units) { - if (NULL != topo->host_units->units) { - DART_LOG_DEBUG("dart__base__host_topology__init: " - "free(topo->host_units->units)"); - free(topo->host_units->units); - topo->host_units->units = NULL; - } - DART_LOG_DEBUG("dart__base__host_topology__init: " - "free(topo->host_units)"); - free(topo->host_units); - topo->host_units = NULL; - } - DART_LOG_DEBUG("dart__base__host_topology__destruct >"); - return DART_OK; -} - - -dart_ret_t dart__base__host_topology__num_nodes( - dart_host_topology_t * topo, - int * num_nodes) -{ - *num_nodes = topo->num_nodes; - return DART_OK; -} - -dart_ret_t dart__base__host_topology__node( - dart_host_topology_t * topo, - int node_index, - const char ** node_hostname) -{ - *node_hostname = NULL; - int n_index = 0; - for (int h = 0; h < topo->num_hosts; ++h) { - if (topo->host_domains[h].level == 0) { - if (n_index == node_index) { - *node_hostname = topo->host_names[h]; - return DART_OK; - } - n_index++; - } - } - DART_LOG_ERROR("dart__base__host_topology__node: " - "failed to load node at index:%d, num.hosts:%d num.nodes:%d", - node_index, topo->num_hosts, topo->num_nodes); - return DART_ERR_NOTFOUND; -} - -dart_ret_t dart__base__host_topology__num_node_modules( - dart_host_topology_t * topo, - const char * node_hostname, - int * num_modules) -{ - *num_modules = 0; - for (int h = 0; h < topo->num_hosts; ++h) { - /* also includes node itself */ - char * m_hostname = topo->host_domains[h].host; - if (strncmp(node_hostname, m_hostname, strlen(node_hostname)) == 0) { - *num_modules += 1; - } - } - return DART_OK; -} - -dart_ret_t dart__base__host_topology__node_module( - dart_host_topology_t * topo, - const char * node_hostname, - int module_index, - const char ** module_hostname) -{ - int m_index = 0; - for (int h = 0; h < topo->num_hosts; ++h) { - char * m_hostname = topo->host_names[h]; - /* also includes node itself */ - if (strncmp(node_hostname, m_hostname, strlen(node_hostname)) == 0) { - if (m_index == module_index) { - *module_hostname = m_hostname; - return DART_OK; - } - m_index++; - } - } - return DART_ERR_NOTFOUND; -} - -dart_ret_t dart__base__host_topology__node_units( - dart_host_topology_t * topo, - const char * hostname, - dart_global_unit_t ** units, - int * num_units) -{ - DART_LOG_TRACE("dart__base__host_topolgoy__node_units() host: %s", - hostname); - *num_units = 0; - *units = NULL; - int host_found = 0; - /* - * Also includes units in sub-modules, e.g. a query for host name - * "some-node" would also include units from "sub-node-*": - */ - - /* First pass: Find total number of units: */ - for (int h = 0; h < topo->num_hosts; ++h) { - dart_host_domain_t * host_domain = &topo->host_domains[h]; - dart_host_units_t * host_units = &topo->host_units[h]; - if (strncmp(host_domain->host, hostname, strlen(hostname)) - == 0) { - *num_units += host_units->num_units; - host_found = 1; - } - } - if (!host_found) { - DART_LOG_ERROR("dart__base__host_topology__node_units ! " - "no entry for host '%s')", hostname); - return DART_ERR_NOTFOUND; - } - /* Second pass: Copy unit ids: */ - dart_global_unit_t * node_unit_ids = malloc(*num_units * - sizeof(dart_global_unit_t)); - int node_unit_idx = 0; - for (int h = 0; h < topo->num_hosts; ++h) { - dart_host_domain_t * host_domain = &topo->host_domains[h]; - dart_host_units_t * host_units = &topo->host_units[h]; - if (strncmp(host_domain->host, hostname, strlen(hostname)) - == 0) { - for (int nu = 0; nu < host_units->num_units; ++nu) { - node_unit_ids[node_unit_idx + nu] = host_units->units[nu]; - } - node_unit_idx += host_units->num_units; - } - } - *units = node_unit_ids; - - DART_LOG_TRACE_UNITID_ARRAY( - "dart__base__host_topology__node_units >", "%d", *units, *num_units); - return DART_OK; -} - -dart_ret_t dart__base__host_topology__host_domain( - dart_host_topology_t * topo, - const char * hostname, - const dart_global_unit_t ** units, - int * num_units, - const int ** numa_ids, - int * num_numa_domains) -{ - DART_LOG_TRACE("dart__base__host_topolgoy__module_units() host: %s", - hostname); - *num_units = 0; - *num_numa_domains = 0; - *units = NULL; - int host_found = 0; - /* - * Does not include units in sub-modules, e.g. a query for host name - * "some-node" would not include units from "sub-node-*": - */ - for (int h = 0; h < topo->num_hosts; ++h) { - dart_host_domain_t * host_domain = &topo->host_domains[h]; - dart_host_units_t * host_units = &topo->host_units[h]; - if (strncmp(host_domain->host, hostname, DART_LOCALITY_HOST_MAX_SIZE) - == 0) { - *num_numa_domains = host_domain->num_numa; - *numa_ids = host_domain->numa_ids; - *num_units = host_units->num_units; - *units = host_units->units; - host_found = 1; - break; - } - } - if (!host_found) { - DART_LOG_ERROR("dart__base__host_topology__module_units ! " - "no entry for host '%s')", hostname); - return DART_ERR_NOTFOUND; - } - - DART_LOG_TRACE_UNITID_ARRAY( - "dart__base__host_topology__module_units >", "%d", *units, *num_units); - return DART_OK; -} - diff --git a/dart-impl/base/src/internal/unit_locality.c b/dart-impl/base/src/internal/unit_locality.c deleted file mode 100644 index c8e8df484..000000000 --- a/dart-impl/base/src/internal/unit_locality.c +++ /dev/null @@ -1,281 +0,0 @@ -/** - * \file dash/dart/base/internal/unit_locality.c - */ - -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#ifdef DART_ENABLE_LIKWID -# include -#endif - -#ifdef DART_ENABLE_HWLOC -# include -# include -#endif - -#ifdef DART_ENABLE_PAPI -# include -#endif - -#ifdef DART_ENABLE_NUMA -# include -# include -#endif - -#include -#include -#include - -#include -#include - -/** - * Functions for exchanging and collecting locality information of units - * in a team. - */ - - -/* ======================================================================== * - * Private Functions * - * ======================================================================== */ - -dart_ret_t dart__base__unit_locality__init( - dart_unit_locality_t * loc); - -dart_ret_t dart__base__unit_locality__local_unit_new( - dart_team_t team, - dart_unit_locality_t * loc); - -/* ======================================================================== * - * Init / Finalize * - * ======================================================================== */ - -/** - * Exchange and collect all locality information of all units in the - * specified team in an array of \c dart_unit_mapping_t objects. - * - * Note that locality information does not contain the units' locality - * domain tags. - * - * \note - * This is a collective N-to-N (allgather) operation. - */ -dart_ret_t dart__base__unit_locality__create( - dart_team_t team, - dart_unit_mapping_t ** unit_mapping) -{ - dart_ret_t ret; - dart_team_unit_t myid = DART_UNDEFINED_TEAM_UNIT_ID; - size_t nunits = 0; - *unit_mapping = NULL; - DART_LOG_DEBUG("dart__base__unit_locality__create()"); - - DART_ASSERT_RETURNS(dart_team_myid(team, &myid), DART_OK); - DART_ASSERT_RETURNS(dart_team_size(team, &nunits), DART_OK); - - size_t nbytes = sizeof(dart_unit_locality_t); - - /* get local unit's locality information: */ - dart_unit_locality_t * uloc = malloc(sizeof(dart_unit_locality_t)); - ret = dart__base__unit_locality__local_unit_new(team, uloc); - if (ret != DART_OK) { - DART_LOG_ERROR("dart__base__unit_locality__create ! " - "dart__base__unit_locality__local_unit_new failed: %d", - ret); - free(uloc); - return ret; - } - DART_LOG_TRACE("dart__base__unit_locality__create: unit %d of %ld: " - "sending %ld bytes: " - "host:'%s' core_id:%d numa_id:%d nthreads:%d", - myid.id, nunits, nbytes, - uloc->hwinfo.host, - uloc->hwinfo.cpu_id, uloc->hwinfo.numa_id, - uloc->hwinfo.max_threads); - - dart_unit_mapping_t * mapping = malloc(sizeof(dart_unit_mapping_t)); - mapping->num_units = nunits; - mapping->team = team; - mapping->unit_localities = malloc(nunits * - sizeof(dart_unit_locality_t)); - dart_barrier(team); - - /* all-to-all exchange of locality data across all units: - * (send, recv, nbytes, team) */ - DART_LOG_DEBUG("dart__base__unit_locality__create: dart_allgather"); - ret = dart_allgather(uloc, - mapping->unit_localities, - nbytes, - DART_TYPE_BYTE, - team); - - dart_barrier(team); - free(uloc); - - if (ret != DART_OK) { - DART_LOG_ERROR("dart__base__unit_locality__create ! " - "dart_allgather failed: %d", ret); - free(mapping); - return ret; - } -#ifdef DART_ENABLE_LOGGING - for (size_t u = 0; u < nunits; ++u) { - dart_unit_locality_t * ulm_u = &mapping->unit_localities[u]; - DART_LOG_TRACE("dart__base__unit_locality__create: unit[%d]: " - "unit:%d host:'%s' " - "num_cores:%d core_id:%d cpu_id:%d " - "num_numa:%d numa_id:%d " - "nthreads:%d", - (int)(u), ulm_u->unit.id, - ulm_u->hwinfo.host, - ulm_u->hwinfo.num_cores, ulm_u->hwinfo.core_id, - ulm_u->hwinfo.cpu_id, - ulm_u->hwinfo.num_numa, ulm_u->hwinfo.numa_id, - ulm_u->hwinfo.max_threads); - } -#endif - - *unit_mapping = mapping; - - DART_LOG_DEBUG("dart__base__unit_locality__create >"); - return DART_OK; -} - -dart_ret_t dart__base__unit_locality__destruct( - dart_unit_mapping_t * unit_mapping) -{ - DART_LOG_DEBUG("dart__base__unit_locality__destruct() team: %d", - unit_mapping->team); - - if (NULL != unit_mapping) { - if (NULL != unit_mapping->unit_localities) { - free(unit_mapping->unit_localities); - unit_mapping->unit_localities = NULL; - } - free(unit_mapping); - } - - DART_LOG_DEBUG("dart__base__unit_locality__destruct >"); - return DART_OK; -} - -/* ======================================================================== * - * Lookup * - * ======================================================================== */ - -/** - * Get the specified unit's locality information from a set of unit - * mappings. - */ -dart_ret_t dart__base__unit_locality__at( - dart_unit_mapping_t * unit_mapping, - dart_team_unit_t unit, - dart_unit_locality_t ** loc) -{ - if ((size_t)(unit.id) >= unit_mapping->num_units) { - DART_LOG_ERROR("dart__base__unit_locality__get ! " - "unit id %d out of bounds, team size: %zu", - unit.id, unit_mapping->num_units); - return DART_ERR_INVAL; - } - *loc = &(unit_mapping->unit_localities[unit.id]); - return DART_OK; -} - -/* ======================================================================== * - * Private Functions * - * ======================================================================== */ - -/** - * Initialize unit locality information from HW locality. - */ -dart_ret_t dart__base__unit_locality__local_unit_new( - dart_team_t team, - dart_unit_locality_t * uloc) -{ - DART_LOG_DEBUG("dart__base__unit_locality__local_unit_new() loc(%p)", - (void *)uloc); - if (uloc == NULL) { - DART_LOG_ERROR("dart__base__unit_locality__local_unit_new ! null"); - return DART_ERR_INVAL; - } - dart_team_unit_t myid; - - DART_ASSERT_RETURNS( - dart__base__unit_locality__init(uloc), - DART_OK); - DART_ASSERT_RETURNS( - dart_team_myid(team, &myid), - DART_OK); - - dart_hwinfo_t hwinfo; - DART_ASSERT_RETURNS(dart_hwinfo(&hwinfo), DART_OK); - -#if 1 || __TODO__CLARIFY__ - /* Is this call required? */ - dart_domain_locality_t * tloc; - DART_ASSERT_RETURNS( - dart_domain_team_locality(team, ".", &tloc), - DART_OK); -#endif - - uloc->unit = myid; - uloc->team = team; - uloc->hwinfo = hwinfo; - -#if defined(DART__BASE__LOCALITY__SIMULATE_MICS) - /* Assigns every second unit to a MIC host name. - * Useful for simulating a heterogeneous node-level topology - * for debugging. - */ - if (myid % 3 == 1) { - strncat(uloc->hwinfo.host, "-mic0", 5); - } -#endif - - DART_LOG_DEBUG("dart__base__unit_locality__local_unit_new > loc(%p)", - (void *)uloc); - return DART_OK; -} - -/** - * Default constructor of dart_unit_locality_t. - */ -dart_ret_t dart__base__unit_locality__init( - dart_unit_locality_t * loc) -{ - DART_LOG_TRACE("dart__base__unit_locality__init() loc: %p", - (void *)loc); - if (loc == NULL) { - DART_LOG_ERROR("dart__base__unit_locality__init ! null"); - return DART_ERR_INVAL; - } - - loc->unit = DART_UNDEFINED_TEAM_UNIT_ID; - loc->team = DART_UNDEFINED_TEAM_ID; - - dart_hwinfo_init(&loc->hwinfo); - - /* - DART_ASSERT_RETURNS( - dart__base__locality__domain__init(&loc->domain), - DART_OK); - */ - loc->domain_tag[0] = '\0'; - - DART_LOG_TRACE("dart__base__unit_locality__init >"); - return DART_OK; -} - diff --git a/dart-impl/base/src/locality.c b/dart-impl/base/src/locality.c deleted file mode 100644 index db1a94fe3..000000000 --- a/dart-impl/base/src/locality.c +++ /dev/null @@ -1,1202 +0,0 @@ -/** - * \file dash/dart/base/locality.c - */ - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - -#include -#include -#include -#include - -/* ====================================================================== * - * Private Data * - * ====================================================================== */ - -#define DART__BASE__LOCALITY__MAX_TEAM_DOMAINS 32 - -static dart_host_topology_t * -dart__base__locality__host_topology_[DART__BASE__LOCALITY__MAX_TEAM_DOMAINS]; - -static dart_unit_mapping_t * -dart__base__locality__unit_mapping_[DART__BASE__LOCALITY__MAX_TEAM_DOMAINS]; - -static dart_domain_locality_t * -dart__base__locality__global_domain_[DART__BASE__LOCALITY__MAX_TEAM_DOMAINS]; - -/* ====================================================================== * - * Private Functions * - * ====================================================================== */ - -static int cmpstr_(const void * p1, const void * p2) -{ - return strcmp(* (char * const *) p1, * (char * const *) p2); -} - -dart_ret_t dart__base__locality__scope_domains_rec( - const dart_domain_locality_t * domain, - dart_locality_scope_t scope, - int * num_domains_out, - dart_domain_locality_t *** domains_out); - -dart_ret_t dart__base__locality__group_subdomains( - dart_domain_locality_t * domain, - const char ** group_subdomain_tags, - int num_group_subdomain_tags, - char * group_domain_tag_out); - -/* ====================================================================== * - * Init / Finalize * - * ====================================================================== */ - -dart_ret_t dart__base__locality__init() -{ - for (int td = 0; td < DART__BASE__LOCALITY__MAX_TEAM_DOMAINS; ++td) { - dart__base__locality__global_domain_[td] = NULL; - dart__base__locality__host_topology_[td] = NULL; - dart__base__locality__unit_mapping_[td] = NULL; - } - return dart__base__locality__create(DART_TEAM_ALL); -} - -dart_ret_t dart__base__locality__finalize() -{ - for (dart_team_t t = 0; t < DART__BASE__LOCALITY__MAX_TEAM_DOMAINS; ++t) { - dart__base__locality__delete(t); - } - - dart_barrier(DART_TEAM_ALL); - return DART_OK; -} - -/* ====================================================================== * - * Create / Delete * - * ====================================================================== */ - -/** - * Exchange and collect locality information of all units in the specified - * team. - * - * The team's unit locality information is stored in private array - * \c dart__base__locality__unit_mapping_[team] with a capacity for - * \c DART__BASE__LOCALITY__MAX_TEAM_DOMAINS teams. - * - * Outline of the locality initialization procedure: - * - * 1. All units collect their local hardware locality information - * -> dart_hwinfo_t - * - * 2. All-to-all exchange of hardware locality data - * -> dart_unit_mapping_t { unit, team, hwinfo, domain } - * - * 3. Construct host topology from unit mapping data - * -> dart_host_topology_t - * - * 4. Initialize locality domain hierarchy from unit mapping data and - * host topology - * -> dart_domain_locality_t - */ -dart_ret_t dart__base__locality__create( - dart_team_t team) -{ - DART_LOG_DEBUG("dart__base__locality__create() team(%d)", team); - - /* - * TODO: Clarify if returning would be sufficient instead of failing - * assertion. - */ - DART_ASSERT_MSG( - (NULL == dart__base__locality__unit_mapping_[team] && - NULL == dart__base__locality__global_domain_[team] && - NULL == dart__base__locality__host_topology_[team]), - "dash__base__locality__create(): " - "locality data of team is already initialized"); - - dart_domain_locality_t * team_global_domain = - malloc(sizeof(dart_domain_locality_t)); - dart__base__locality__global_domain_[team] = - team_global_domain; - - /* Initialize the global domain as the root entry in the locality - * hierarchy: - */ - team_global_domain->scope = DART_LOCALITY_SCOPE_GLOBAL; - team_global_domain->level = 0; - team_global_domain->relative_index = 0; - team_global_domain->team = team; - team_global_domain->parent = NULL; - team_global_domain->num_domains = 0; - team_global_domain->children = NULL; - team_global_domain->num_units = 0; - team_global_domain->host[0] = '\0'; - team_global_domain->domain_tag[0] = '.'; - team_global_domain->domain_tag[1] = '\0'; - - size_t num_units = 0; - DART_ASSERT_RETURNS(dart_team_size(team, &num_units), DART_OK); - team_global_domain->num_units = num_units; - team_global_domain->unit_ids = malloc(num_units * - sizeof(dart_global_unit_t)); - for (size_t u = 0; u < num_units; ++u) { - dart_team_unit_t luid = { u }; - DART_ASSERT_RETURNS( - dart_team_unit_l2g(team, luid, &team_global_domain->unit_ids[u]), - DART_OK); - } - - /* Exchange unit locality information between all units: - */ - dart_unit_mapping_t * unit_mapping; - DART_ASSERT_RETURNS( - dart__base__unit_locality__create(team, &unit_mapping), - DART_OK); - dart__base__locality__unit_mapping_[team] = unit_mapping; - - /* Resolve host topology from the unit's host names: - */ - dart_host_topology_t * topo; - DART_ASSERT_RETURNS( - dart__base__host_topology__create(unit_mapping, &topo), - DART_OK); - dart__base__locality__host_topology_[team] = topo; - size_t num_nodes = topo->num_nodes; - DART_LOG_TRACE("dart__base__locality__create: nodes: %ld", num_nodes); - - team_global_domain->num_nodes = num_nodes; - -#ifdef DART_ENABLE_LOGGING - for (int h = 0; h < topo->num_hosts; ++h) { - dart_host_units_t * node_units = &topo->host_units[h]; - dart_host_domain_t * node_domain = &topo->host_domains[h]; - char * hostname = topo->host_names[h]; - DART_LOG_TRACE("dart__base__locality__create: " - "host %s: units:%d level:%d parent:%s", hostname, - node_units->num_units, - node_domain->level, node_domain->parent); - for (int u = 0; u < node_units->num_units; ++u) { - DART_LOG_TRACE("dart__base__locality__create: %s unit[%d]: %d", - hostname, u, node_units->units[u].id); - } - } -#endif - - DART_LOG_DEBUG("dart__base__locality__create: " - "constructing domain hierarchy"); - /* Recursively create locality information of the global domain's - * sub-domains: - */ - DART_ASSERT_RETURNS( - dart__base__locality__domain__create_subdomains( - dart__base__locality__global_domain_[team], - dart__base__locality__host_topology_[team], - dart__base__locality__unit_mapping_[team]), - DART_OK); - - DART_LOG_DEBUG("dart__base__locality__create >"); - return DART_OK; -} - -dart_ret_t dart__base__locality__delete( - dart_team_t team) -{ - dart_ret_t ret = DART_OK; - - DART_LOG_DEBUG("dart__base__locality__delete() team(%d)", team); - - if (NULL != dart__base__locality__global_domain_[team]) { - ret = dart__base__locality__domain__destruct( - dart__base__locality__global_domain_[team]); - if (ret != DART_OK) { - DART_LOG_ERROR("dart__base__locality__delete ! " - "dart__base__locality__domain_delete failed: %d", ret); - return ret; - } - DART_LOG_DEBUG("dart__base__locality__delete: " - "free(dart__base__locality__global_domain_[%d])", team); - free(dart__base__locality__global_domain_[team]); - dart__base__locality__global_domain_[team] = NULL; - } - - if (NULL != dart__base__locality__host_topology_[team]) { - ret = dart__base__host_topology__destruct( - dart__base__locality__host_topology_[team]); - if (ret != DART_OK) { - DART_LOG_ERROR("dart__base__locality__delete ! " - "dart__base__host_topology__destruct failed: %d", ret); - return ret; - } - DART_LOG_DEBUG("dart__base__locality__delete: " - "free(dart__base__locality__host_topology_[%d])", team); - free(dart__base__locality__host_topology_[team]); - dart__base__locality__host_topology_[team] = NULL; - } - - if (NULL != dart__base__locality__unit_mapping_[team]) { - ret = dart__base__unit_locality__destruct( - dart__base__locality__unit_mapping_[team]); - if (ret != DART_OK) { - DART_LOG_ERROR("dart__base__locality__delete ! " - "dart__base__unit_locality__destruct failed: %d", ret); - return ret; - } - DART_LOG_DEBUG("dart__base__locality__delete: " - "free(dart__base__locality__unit_mapping_[%d])", team); - dart__base__locality__unit_mapping_[team] = NULL; - } - - DART_LOG_DEBUG("dart__base__locality__delete > team(%d)", team); - return DART_OK; -} - -/* ====================================================================== * - * Domain Locality * - * ====================================================================== */ - -dart_ret_t dart__base__locality__team_domain( - dart_team_t team, - dart_domain_locality_t ** domain_out) -{ - DART_LOG_DEBUG("dart__base__locality__team_domain() team(%d)", team); - dart_ret_t ret = DART_ERR_NOTFOUND; - - *domain_out = NULL; - dart_domain_locality_t * domain = - dart__base__locality__global_domain_[team]; - - ret = dart__base__locality__domain(domain, ".", domain_out); - - DART_LOG_DEBUG("dart__base__locality__team_domain > " - "team(%d) -> domain(%p)", team, (void *)(*domain_out)); - return ret; -} - -dart_ret_t dart__base__locality__domain( - const dart_domain_locality_t * domain_in, - const char * domain_tag, - dart_domain_locality_t ** domain_out) -{ - return dart__base__locality__domain__child( - domain_in, domain_tag, domain_out); -} - - -dart_ret_t dart__base__locality__scope_domains( - const dart_domain_locality_t * domain_in, - dart_locality_scope_t scope, - int * num_domains_out, - dart_domain_locality_t *** domains_out) -{ - DART_LOG_TRACE("dart__base__locality__scope_domains() domain:%s scope:%d", - domain_in->domain_tag, (int)scope); - - *num_domains_out = 0; - *domains_out = NULL; - - dart_ret_t ret = dart__base__locality__scope_domains_rec( - domain_in, scope, num_domains_out, domains_out); - if (DART_OK == ret && *num_domains_out <= 0) { - DART_LOG_DEBUG("dart__base__locality__scope_domains ! " - "no domains found"); - return DART_ERR_NOTFOUND; - } -#ifdef DART_ENABLE_LOGGING - for (int sd = 0; sd < *num_domains_out; sd++) { - DART_LOG_TRACE("dart__base__locality__scope_domains: " - "scope_domain[%d]: %s", - sd, (*domains_out)[sd]->domain_tag); - } -#endif - DART_LOG_TRACE("dart__base__locality__scope_domains >"); - return ret; -} - -dart_ret_t dart__base__locality__scope_domain_tags( - const dart_domain_locality_t * domain_in, - dart_locality_scope_t scope, - int * num_domains_out, - char *** domain_tags_out) -{ - *num_domains_out = 0; - *domain_tags_out = NULL; - - dart_domain_locality_t ** dart_scope_domains; - dart_ret_t ret = dart__base__locality__scope_domains( - domain_in, - scope, - num_domains_out, - &dart_scope_domains); - - if (ret != DART_OK) { - free(dart_scope_domains); - return ret; - } - if (*num_domains_out <= 0) { - DART_LOG_ERROR("dart__base__locality__scope_domain_tags ! " - "num_domains result at scope %d is %d <= 0", - scope, *num_domains_out); - return DART_ERR_INVAL; - } - - *domain_tags_out = (char **)(malloc(sizeof(char *) * (*num_domains_out))); - for (int sd = 0; sd < *num_domains_out; sd++) { - (*domain_tags_out)[sd] - = malloc(sizeof(char) * - (strlen(dart_scope_domains[sd]->domain_tag) + 1)); - - strcpy((*domain_tags_out)[sd], - dart_scope_domains[sd]->domain_tag); - } - - free(dart_scope_domains); - - if (*domain_tags_out == NULL) { - DART_LOG_ERROR("dart__base__locality__scope_domain_tags ! " - "domain_tags result is undefined"); - return DART_ERR_OTHER; - } - - return DART_OK; -} - -dart_ret_t dart__base__locality__domain_split_tags( - const dart_domain_locality_t * domain_in, - dart_locality_scope_t scope, - int num_parts, - int ** group_sizes_out, - char **** group_domain_tags_out) -{ - /* For 4 domains in the specified scope, a split into 2 parts results - * in a domain hierarchy like: - * - * group_domain_tags[g][d] -> { - * 0: [ domain_0, domain_1 ], - * 1: [ domain_2, domain_3 ], ... - * } - * - */ - - DART_LOG_TRACE("dart__base__locality__domain_split_tags() " - "team(%d) domain(%s) scope(%d) parts(%d)", - domain_in->team, domain_in->domain_tag, scope, num_parts); - - /* Subdomains of global domain. - * Domains of split parts, grouping domains at split scope. */ - char *** group_domain_tags = malloc(num_parts * sizeof(char **)); - int * group_sizes = malloc(num_parts * sizeof(int)); - - /* Get total number and tags of domains in split scope: */ - int num_domains; - char ** domain_tags = NULL; - DART_ASSERT_RETURNS( - dart__base__locality__scope_domain_tags( - domain_in, scope, &num_domains, &domain_tags), - DART_OK); - - if (domain_tags == NULL) { - DART_LOG_ERROR("dart__base__locality__domain_split_tags ! " - "domain_tags is undefined"); - free(group_domain_tags); - free(group_sizes); - return DART_ERR_OTHER; - } - if (num_domains <= 0) { - DART_LOG_ERROR("dart__base__locality__domain_split_tags ! " - "num_domains at scope %d is %d <= 0", - scope, num_domains); - free(group_domain_tags); - free(group_sizes); - return DART_ERR_INVAL; - } - - DART_LOG_TRACE("dart__base__locality__domain_split_tags: " - "number of domains in scope %d: %d", scope, num_domains); - - /* Group domains in split scope into specified number of parts: */ - int max_group_domains = (num_domains + (num_parts-1)) / num_parts; - int group_first_domain_idx = 0; - - DART_LOG_TRACE("dart__base__locality__domain_split_tags: " - "max. domains per group: %d", max_group_domains); - /* - * TODO: Preliminary implementation, should balance number of units in - * groups. - */ - for (int g = 0; g < num_parts; ++g) { - int num_group_subdomains = max_group_domains; - if ((g+1) * max_group_domains > num_domains) { - num_group_subdomains = num_domains - (g * max_group_domains); - } - DART_LOG_TRACE("dart__base__locality__domain_split_tags: " - "domains in group %d: %d", g, num_group_subdomains); - - group_sizes[g] = num_group_subdomains; - group_domain_tags[g] = NULL; - if (num_group_subdomains > 0) { - group_domain_tags[g] = malloc(sizeof(char *) * num_group_subdomains); - } - for (int d_rel = 0; d_rel < num_group_subdomains; ++d_rel) { - int d_abs = group_first_domain_idx + d_rel; - int tag_len = strlen(domain_tags[d_abs]); - group_domain_tags[g][d_rel] = malloc(sizeof(char) * (tag_len + 1)); - strncpy(group_domain_tags[g][d_rel], domain_tags[d_abs], - tag_len); - group_domain_tags[g][d_rel][tag_len] = '\0'; - } - /* Create domain of group: */ - group_first_domain_idx += num_group_subdomains; - } - - *group_sizes_out = group_sizes; - *group_domain_tags_out = group_domain_tags; - - free(domain_tags); - - DART_LOG_TRACE("dart__base__locality__domain_split_tags >"); - return DART_OK; -} - -dart_ret_t dart__base__locality__domain_group( - dart_domain_locality_t * domain, - int group_size, - const char ** group_subdomain_tags, - char * group_domain_tag_out) -{ - DART_LOG_TRACE("dart__base__locality__domain_group() " - "domain_in: (%s: scope:%d @ level:%d) group size: %d", - domain->domain_tag, domain->scope, domain->level, - group_size); -#ifdef DART_ENABLE_LOGGING - for (int sd = 0; sd < group_size; sd++) { - DART_LOG_TRACE("dart__base__locality__domain_group: " - "group_subdomain_tags[%d]: %p = %s", - sd, (void *)(group_subdomain_tags[sd]), - group_subdomain_tags[sd]); - } -#endif - - if (group_size < 1) { - return DART_ERR_INVAL; - } - - dart_ret_t ret = DART_OK; - - DART_LOG_TRACE("dart__base__locality__domain_group: " - "group size: %d", group_size); - - group_domain_tag_out[0] = '\0'; - - /* The group's parent domain: */ - dart_domain_locality_t * group_parent_domain; - ret = dart__base__locality__domain__parent( - domain, - group_subdomain_tags, group_size, - &group_parent_domain); - - if (ret != DART_OK) { - return ret; - } - - DART_LOG_TRACE("dart__base__locality__domain_group: " - "group parent: %s", - group_parent_domain->domain_tag); - - /* Find parents of specified subdomains that are an immediate child node - * of the input domain. - */ - int immediate_subdomains_group = 1; - int num_group_parent_domain_tag_parts = - dart__base__strcnt(group_parent_domain->domain_tag, '.'); - for (int sd = 0; sd < group_size; sd++) { - const char * group_domain_tag = group_subdomain_tags[sd]; - DART_LOG_TRACE("dart__base__locality__domain_group: " - " group_subdomain_tags[%d]: %s", - sd, group_domain_tag); - if (dart__base__strcnt(group_domain_tag, '.') != - num_group_parent_domain_tag_parts + 1) { - immediate_subdomains_group = 0; - break; - } - } - if (immediate_subdomains_group) { - DART_LOG_TRACE("dart__base__locality__domain_group: " - "group of immediate child domains"); - /* Subdomains in group are immediate child nodes of group parent - * domain: - */ - ret = dart__base__locality__group_subdomains( - group_parent_domain, - group_subdomain_tags, group_size, - group_domain_tag_out); - if (ret != DART_OK) { - return ret; - } - } else { - DART_LOG_TRACE("dart__base__locality__domain_group: " - "group of indirect child domains"); - - /* Subdomains in group are in indirect child nodes of group parent - * domain. - * Find immediate child nodes that are parents of group subdomains. - * Example: - * - * parent: .0 - * group domains: { .0.1.2, .0.1.3, .0.2.0 } - * - * --> { .0.1, .0.1, .0.2 } - * - * --> groups: { .0.1, .0.2 } - */ - char ** immediate_subdomain_tags = malloc(sizeof(char *) * - group_size); - char * group_parent_domain_tag = group_parent_domain->domain_tag; - int group_parent_domain_tag_len = strlen(group_parent_domain_tag); - DART_LOG_TRACE("dart__base__locality__domain_group: parent: %s", - group_parent_domain_tag); - for (int sd = 0; sd < group_size; sd++) { - /* Resolve relative index of subdomain: */ - immediate_subdomain_tags[sd] = - calloc(DART_LOCALITY_DOMAIN_TAG_MAX_SIZE, sizeof(char)); - - int group_subdomain_tag_len = strlen(group_subdomain_tags[sd]); - - if (group_subdomain_tag_len <= group_parent_domain_tag_len) { - /* Indicates invalid parameters, usually caused by multiple units - * mapped to the same domain to be grouped. - */ - DART_LOG_ERROR("dart__base__locality__domain_group ! " - "group subdomain %s with invalid parent domain %s", - group_subdomain_tags[sd], group_parent_domain_tag); - - for (int sd_p = 0; sd_p <= sd; sd_p++) { - free(immediate_subdomain_tags[sd_p]); - } - free(immediate_subdomain_tags); - - return DART_ERR_INVAL; - } - - /* Position of second dot separator in tag of grouped domain after the - * end of the parent domain tag, for example: - * parent: .0.1 - * grouped domain: .0.1.4.0 - * dot_pos: 6 ------------' - */ - char * dot_pos = strchr(group_subdomain_tags[sd] + - group_parent_domain_tag_len + 1, '.'); - int immediate_subdomain_tag_len; - if (dot_pos == NULL) { - /* subdomain is immediate child of parent: */ - immediate_subdomain_tag_len = group_subdomain_tag_len; - } else { - /* subdomain is indirect child of parent: */ - immediate_subdomain_tag_len = dot_pos - - group_subdomain_tags[sd]; - } - strncpy(immediate_subdomain_tags[sd], group_subdomain_tags[sd], - immediate_subdomain_tag_len); - immediate_subdomain_tags[sd][immediate_subdomain_tag_len] = '\0'; - } - int num_group_subdomains = dart__base__strsunique( - immediate_subdomain_tags, - group_size); - DART_LOG_TRACE("dart__base__locality__domain_group: " - "num_group_subdomains: %d", num_group_subdomains); -#ifdef DART_ENABLE_LOGGING - for (int gsd = 0; gsd < num_group_subdomains; gsd++) { - DART_LOG_TRACE("dart__base__locality__domain_group: " - "group.subdomain[%d]: %s", - gsd, immediate_subdomain_tags[gsd]); - } -#endif - DART_LOG_TRACE("dart__base__locality__domain_group: " - "group parent domain arity: %d", - group_parent_domain->num_domains); - - /* Create group domain: - */ - dart_domain_locality_t * group_domain = - malloc(sizeof(dart_domain_locality_t)); - - dart__base__locality__domain__init(group_domain); - - group_domain->team = group_parent_domain->team; - group_domain->scope = DART_LOCALITY_SCOPE_GROUP; - group_domain->level = group_parent_domain->level; - group_domain->parent = group_parent_domain; - group_domain->relative_index = group_parent_domain->num_domains; - group_domain->num_nodes = group_parent_domain->num_nodes; - group_domain->num_aliases = 1; - group_domain->aliases = malloc(sizeof(dart_domain_locality_t *)); - group_domain->aliases[0] = group_parent_domain; - group_domain->num_units = 0; - group_domain->unit_ids = NULL; - group_domain->num_domains = num_group_subdomains; - group_domain->children = calloc(num_group_subdomains, - sizeof(dart_domain_locality_t *)); - - DART_LOG_TRACE("dart__base__locality__domain_group: tag group domain"); - strncpy(group_domain->domain_tag, group_parent_domain_tag, - group_parent_domain_tag_len); - group_domain->domain_tag[group_parent_domain_tag_len] = '\0'; - - /* TODO: Check if this implementation is correct. - Incrementing an existing domain's relative index could result - in naming collisions as the relative index of the subdomain - can differ from the last place in their domain tag. - - Possible solutions: - - - parse last index in domain tag to int and increment (obviously - not a formally sound solution) - - proof that invariant rel.index = last index in tag can be - maintained and enforce it in locality functions - */ - int group_domain_tag_len = - sprintf(group_domain->domain_tag + group_parent_domain_tag_len, - ".%d", group_domain->relative_index) + - group_parent_domain_tag_len; - group_domain->domain_tag[group_domain_tag_len] = '\0'; - - DART_LOG_TRACE("dart__base__locality__domain_group: " - "group domain tag: %s", group_domain->domain_tag); - - strcpy(group_domain_tag_out, group_domain->domain_tag); - - /* Initialize group subdomains: - */ - DART_LOG_TRACE("dart__base__locality__domain_group: " - "initialize %d subdomains of group (%s)", - num_group_subdomains, group_domain->domain_tag); - - for (int gsd = 0; gsd < num_group_subdomains; gsd++) { - dart_domain_locality_t * group_subdomain_in; - - /* Copy - * group_subdomain_in = - * domain.domains[domain_tag = group[g].immediate_subdomains[gsd]] - * to - * group[g].domains[gsd]: - */ - DART_LOG_TRACE("dart__base__locality__domain_group: " - "load domain.domains[domain_tag = " - "(group.immediate_subdomain_tags[%d] = %s])", - gsd, immediate_subdomain_tags[gsd]); - - /* Query instance of the group domain's immediate subdomain: - * - */ - ret = dart__base__locality__domain( - domain, immediate_subdomain_tags[gsd], - &group_subdomain_in); - if (ret != DART_OK) { break; } - - DART_LOG_TRACE("dart__base__locality__domain_group: " - "copy domain.domains[domain_tag = %s] to " - "group.domains[%d]", - immediate_subdomain_tags[gsd], gsd); - - group_domain->children[gsd] = malloc(sizeof(dart_domain_locality_t)); - ret = dart__base__locality__domain__copy( - group_subdomain_in, - group_domain->children[gsd]); - if (ret != DART_OK) { break; } - - group_domain->children[gsd]->parent = group_domain; - group_domain->num_units += group_domain->children[gsd]->num_units; - group_domain->num_cores += group_domain->children[gsd]->num_cores; - } /* for group_domain.domains */ - - if (ret != DART_OK) { return ret; } - - group_domain->unit_ids = malloc(sizeof(dart_global_unit_t) * - group_domain->num_units); - - /* Remove entries from group domain that are not part of the group: - */ - DART_LOG_TRACE("dart__base__locality__domain_group: " - "select %d subdomains in group = %s", - group_size, group_domain->domain_tag); - - // TODO DEBUG: check if removed subdomains are correctly destroyed - // - ret = dart__base__locality__select_subdomains( - group_domain, - group_subdomain_tags, - group_size); - if (ret != DART_OK) { return ret; } - - DART_LOG_TRACE("dart__base__locality__domain_group: " - "update group (%s) after adding subdomains", - group_domain->domain_tag); - ret = dart__base__locality__domain__update_subdomains( - group_domain); - if (ret != DART_OK) { return ret; } - - /* Add group domain to lowest common ancestor of grouped domains: - * - * Note: Required to append group domain at the end of the group - * parent's subdomain list to ensure that tags of domains not - * included in group remain valid. - */ - dart__base__locality__add_subdomain( - group_parent_domain, group_domain, -1); - - /* Remove grouped domains from parent's subdomains: - */ - for (int sd = 0; sd < group_parent_domain->num_domains; sd++) { - dart_domain_locality_t * group_parent_subdomain = - group_parent_domain->children[sd]; - /* Whether this sibling of the group domain contains subdomains - * affected by the group: - */ - int contains_grouped_domains = 0; - for (int gd = 0; gd < group_size; ++gd) { - if (strcmp(group_parent_subdomain->domain_tag, - immediate_subdomain_tags[gd]) == 0) { - contains_grouped_domains = 1; - break; - } - } - if (contains_grouped_domains) { - DART_LOG_TRACE("dart__base__locality__domain_group: " - "remove grouped domains from %s", - group_parent_subdomain->domain_tag); - - ret = dart__base__locality__exclude_subdomains( - group_parent_subdomain, - group_subdomain_tags, - group_size); - if (ret != DART_OK) { break; } - } - } - - ret = dart__base__locality__domain__update_subdomains( - group_parent_domain); - - /* Cleanup temporaries: - * - */ - for (int sd = 0; sd < group_size; sd++) { - DART_LOG_TRACE("dart__base__locality__domain_group: " - "free(immediate_subdomain_tags[%d])", sd); - free(immediate_subdomain_tags[sd]); - } - DART_LOG_TRACE("dart__base__locality__domain_group: " - "free(immediate_subdomain_tags)"); - free(immediate_subdomain_tags); - - if (ret != DART_OK) { return ret; } - -#if 0 - group_parent_domain->num_domains++; -#endif - } - if (ret != DART_OK) { return ret; } - - DART_LOG_TRACE("dart__base__locality__domain_group >"); - return DART_OK; -} - -/* ====================================================================== * - * Unit Locality * - * ====================================================================== */ - -dart_ret_t dart__base__locality__unit( - dart_team_t team, - dart_team_unit_t unit, - dart_unit_locality_t ** locality) -{ - DART_LOG_DEBUG("dart__base__locality__unit() team(%d) unit(%d)", - team, unit.id); - *locality = NULL; - - dart_unit_locality_t * uloc; - dart_ret_t ret = dart__base__unit_locality__at( - dart__base__locality__unit_mapping_[team], unit, - &uloc); - if (ret != DART_OK) { - DART_LOG_ERROR("dart_unit_locality: " - "dart__base__locality__unit(team:%d unit:%d) " - "failed (%d)", team, unit.id, ret); - return ret; - } - *locality = uloc; - - DART_LOG_DEBUG("dart__base__locality__unit > team(%d) unit(%d)", - team, unit.id); - return DART_OK; -} - -/* ====================================================================== * - * Private Function Definitions * - * ====================================================================== */ - -/** - * Move subset of a domain's immediate child nodes into a group subdomain. - */ -dart_ret_t dart__base__locality__group_subdomains( - dart_domain_locality_t * domain, - /** Domain tags of child nodes to move into group subdomain. */ - const char ** group_subdomain_tags, - /** Number of child nodes to move into group subdomain. */ - int num_group_subdomain_tags, - /** Domain tag of the created group subdomain. */ - char * group_domain_tag_out) -{ - DART_LOG_TRACE("dart__base__locality__group_subdomains() " - "group parent domain: %s num domains: %d " - "num_group_subdomain_tags: %d", - domain->domain_tag, domain->num_domains, - num_group_subdomain_tags); - - group_domain_tag_out[0] = '\0'; - - if (domain->num_domains < 1) { - DART_LOG_ERROR("dart__base__locality__group_subdomains ! " - "no subdomains, cannot create group"); - return DART_ERR_NOTFOUND; - } - - int num_grouped = num_group_subdomain_tags; - int num_ungrouped = domain->num_domains - num_grouped; - int num_subdomains_new = num_ungrouped + 1; - - if (num_grouped <= 0) { - DART_LOG_ERROR("dart__base__locality__group_subdomains ! " - "requested to group %d <= 0 domains", - num_grouped); - return DART_ERR_INVAL; - } - - /* The domain tag of the group to be added must be a successor of the last - * subdomain's (the group domain's last sibling) tag to avoid collisions. - * Relative index of last subdomain can differ from its last domain tag - * segment, so we need to read and increment the suffix of its domain tag - * to obtain the group's domain tag. */ -#if 0 - int domain_last_rel_idx = domain->children[domain->num_domains - 1] - .relative_index; -#endif - char * domain_last_tag_suffix_pos = - strrchr(domain->children[domain->num_domains - 1]->domain_tag, '.'); - int domain_last_tag_suffix = atoi(domain_last_tag_suffix_pos + 1); - - /* Child nodes are ordered by domain tag. - * Create sorted copy of subdomain tags to partition child nodes in a - * single pass: - */ - char ** group_subdomain_tags_sorted = - malloc(num_group_subdomain_tags * sizeof(char *)); - for (int sdt = 0; sdt < num_group_subdomain_tags; ++sdt) { - group_subdomain_tags_sorted[sdt] = - malloc((strlen(group_subdomain_tags[sdt]) + 1) * sizeof(char)); - strcpy(group_subdomain_tags_sorted[sdt], group_subdomain_tags[sdt]); - } - qsort(group_subdomain_tags_sorted, num_group_subdomain_tags, - sizeof(char*), cmpstr_); - - int num_existing_domain_groups = 0; - for (int sd = 0; sd < domain->num_domains; sd++) { - if (domain->children[sd]->scope == DART_LOCALITY_SCOPE_GROUP) { - num_existing_domain_groups++; - } - } - num_ungrouped -= num_existing_domain_groups; - - /* Partition child nodes of domain into grouped and ungrouped - * subdomains: - */ - dart_domain_locality_t * group_domains = NULL; - dart_domain_locality_t * ungrouped_domains = NULL; - if (num_existing_domain_groups > 0) { - group_domains = - malloc(sizeof(dart_domain_locality_t) * num_existing_domain_groups); - } - if (num_ungrouped > 0) { - ungrouped_domains = - malloc(sizeof(dart_domain_locality_t) * num_ungrouped); - } - dart_domain_locality_t * grouped_domains = - malloc(sizeof(dart_domain_locality_t) * num_grouped); - - /* Copy child nodes into partitions: - */ - int sdt = 0; - int group_idx = 0; - int grouped_idx = 0; - int ungrouped_idx = 0; - int group_domain_rel_idx = num_ungrouped + num_existing_domain_groups; - - for (int sd = 0; sd < domain->num_domains; sd++) { - dart_domain_locality_t * subdom = domain->children[sd]; - dart_domain_locality_t * domain_copy = NULL; - - if (subdom->scope == DART_LOCALITY_SCOPE_GROUP) { - DART_ASSERT_MSG( - group_domains != NULL, "No subdomains at group locality scope"); - domain_copy = &group_domains[group_idx]; - group_idx++; - } - else if (sdt < num_group_subdomain_tags && - strcmp(subdom->domain_tag, group_subdomain_tags_sorted[sdt]) - == 0) { - domain_copy = &grouped_domains[grouped_idx]; - grouped_idx++; - sdt++; - } else { - DART_ASSERT_MSG( - ungrouped_domains != NULL, - "No ungrouped subdomains at group locality scope"); - domain_copy = &ungrouped_domains[ungrouped_idx]; - ungrouped_idx++; - } - DART_ASSERT(domain_copy != NULL); - memcpy(domain_copy, subdom, sizeof(dart_domain_locality_t)); - } - - for (int sdt = 0; sdt < num_group_subdomain_tags; ++sdt) { - DART_LOG_TRACE("dart__base__locality__domain_group: " - "free(group_subdomain_tags[%d])", sdt); - free(group_subdomain_tags_sorted[sdt]); - } - DART_LOG_TRACE("dart__base__locality__domain_group: " - "free(group_subdomain_tags_sorted)"); - free(group_subdomain_tags_sorted); - - /* Check that all subdomains have been found: */ - if (sdt != num_group_subdomain_tags) { - DART_LOG_ERROR("dart__base__locality__group_subdomains ! " - "only found %d of %d requested subdomains", - sdt, num_group_subdomain_tags); - free(ungrouped_domains); - free(group_domains); - free(grouped_domains); - return DART_ERR_NOTFOUND; - } - - /* Append group domain to subdomains: - */ - domain->children = - realloc(domain->children, - sizeof(dart_domain_locality_t *) * num_subdomains_new); - DART_ASSERT(domain->children != NULL); - - domain->num_domains = num_subdomains_new; - - /* Initialize group domain and set it as the input domain's last child - * node: - */ - dart_domain_locality_t * group_domain = - domain->children[group_domain_rel_idx]; - - if (group_domain == NULL) { - DART_LOG_ERROR("dart__base__locality__group_subdomains: " - "Group domain at relative index %d not defined", - group_domain_rel_idx); - free(ungrouped_domains); - return DART_ERR_NOTFOUND; - } - - if (group_domain == NULL) { - DART_LOG_ERROR("dart__base__locality__group_subdomains: " - "Group domain at relative index %d not defined", - group_domain_rel_idx); - return DART_ERR_NOTFOUND; - } - - dart__base__locality__domain__init(group_domain); - - /* TODO: hwinfo of group domain is not initialized. - */ - group_domain->parent = domain; - group_domain->relative_index = group_domain_rel_idx; - group_domain->level = domain->level + 1; - group_domain->scope = DART_LOCALITY_SCOPE_GROUP; - group_domain->num_domains = num_grouped; - group_domain->children = malloc(sizeof(dart_domain_locality_t *) * - num_grouped); - - int tag_len = sprintf(group_domain->domain_tag, "%s", - domain->domain_tag); - int tag_last_idx = domain_last_tag_suffix + 1; - sprintf(group_domain->domain_tag + tag_len, ".%d", tag_last_idx); - DART_LOG_TRACE("dart__base__locality__group_subdomains: " - "group_domain.tag: %s relative index: %d grouped: %d " - "ungrouped: %d", - group_domain->domain_tag, group_domain->relative_index, - num_grouped, num_ungrouped); - - strcpy(group_domain_tag_out, group_domain->domain_tag); - - /* - * Set grouped partition of subdomains as child nodes of group domain: - */ - group_domain->num_units = 0; - group_domain->num_nodes = 0; - for (int gd = 0; gd < num_grouped; gd++) { - group_domain->children[gd] = &grouped_domains[gd]; - } - dart__base__locality__domain__update_subdomains( - group_domain); - - /* - * Collect unit ids of group domain: - */ - group_domain->unit_ids = malloc(sizeof(dart_global_unit_t) * - group_domain->num_units); - int group_domain_unit_idx = 0; - for (int gd = 0; gd < num_grouped; gd++) { - dart_domain_locality_t * group_subdomain = group_domain->children[gd]; - memcpy(group_domain->unit_ids + group_domain_unit_idx, - group_subdomain->unit_ids, - sizeof(dart_global_unit_t) * group_subdomain->num_units); - group_domain_unit_idx += group_subdomain->num_units; - } - - for (int sd = 0; sd < num_ungrouped; sd++) { - DART_ASSERT( - domain->children != NULL); - DART_ASSERT_MSG( - ungrouped_domains != NULL, - "No ungrouped subdomains at group locality scope"); - DART_LOG_TRACE( - "dart__base__locality__group_subdomains: ==> children[%d] u: %s", - sd, domain->children[sd]->domain_tag); - memcpy(domain->children[sd], - &ungrouped_domains[sd], - sizeof(dart_domain_locality_t)); - - /* Pointers are invalidated by realloc, update parent pointers of - * subdomains: */ - domain->children[sd]->parent = domain; - domain->children[sd]->relative_index = sd; - } - - for (int g = 0; g < num_existing_domain_groups; g++) { - int abs_g = g + num_ungrouped; - - DART_LOG_TRACE( - "dart__base__locality__group_subdomains: ==> domains[%d] g: %s", - abs_g, domain->children[abs_g]->domain_tag); - - DART_ASSERT_MSG( - group_domains != NULL, "No subdomains at group locality scope"); - memcpy(domain->children[abs_g], &group_domains[g], - sizeof(dart_domain_locality_t)); - - /* Pointers are invalidated by realloc, update parent pointers of - * subdomains: */ - domain->children[abs_g]->parent = domain; - domain->children[abs_g]->relative_index = abs_g; - } - DART_LOG_TRACE( - "dart__base__locality__group_subdomains: ==> domains[%d] *: %s", - group_domain->relative_index, group_domain->domain_tag); - -#ifdef DART_ENABLE_LOGGING - int g_idx = 0; - for (int sd = 0; sd < domain->num_domains; sd++) { - dart_domain_locality_t * subdom = domain->children[sd]; - DART_LOG_TRACE( - "dart__base__locality__group_subdomains: --> children[%d:%d]: " - "tag:'%s' scope:%d subdomains:%d ADDR[%p]", - sd, subdom->relative_index, subdom->domain_tag, - subdom->scope, subdom->num_domains, (void *)subdom); - if (subdom->scope == DART_LOCALITY_SCOPE_GROUP) { - for (int gsd = 0; gsd < subdom->num_domains; gsd++) { - dart_domain_locality_t * group_subdom = subdom->children[gsd]; - dart__unused(group_subdom); - DART_LOG_TRACE( - "dart__base__locality__group_subdomains: --> groups[%d:%d]." - "domains[%d]: tag:'%s' scope: %d subdomains:%d", - g_idx, group_subdom->relative_index, gsd, - group_subdom->domain_tag, - group_subdom->scope, group_subdom->num_domains); - } - g_idx++; - } - } -#endif - - DART_LOG_TRACE("dart__base__locality__domain_group: " - "free(grouped_domains)"); - free(grouped_domains); - - DART_LOG_TRACE("dart__base__locality__domain_group: " - "free(ungrouped_domains)"); - free(ungrouped_domains); - - DART_LOG_TRACE("dart__base__locality__group_subdomains >"); - return DART_OK; -} - -dart_ret_t dart__base__locality__scope_domains_rec( - const dart_domain_locality_t * domain, - dart_locality_scope_t scope, - int * num_domains_out, - dart_domain_locality_t *** domains_out) -{ - dart_ret_t ret; - DART_LOG_TRACE("dart__base__locality__scope_domains_rec() domain.level:%d", - domain->level); - - if (domain->scope == scope) { - DART_LOG_TRACE("dart__base__locality__scope_domains_rec: " - "domain %s matched", domain->domain_tag); - int dom_idx = *num_domains_out; - *num_domains_out += 1; - dart_domain_locality_t ** domains_temp = - (dart_domain_locality_t **)(realloc(*domains_out, - sizeof(dart_domain_locality_t *) * - (*num_domains_out))); - if (NULL != domains_temp) { - *domains_out = domains_temp; - (*domains_out)[dom_idx] = (dart_domain_locality_t *)(domain); - DART_LOG_TRACE("dart__base__locality__scope_domains_rec: " - "domain %d: %s", - dom_idx, (*domains_out)[dom_idx]->domain_tag); - } else { - DART_LOG_ERROR("dart__base__locality__scope_domains_rec ! " - "realloc failed"); - free(*domains_out); - *domains_out = NULL; - return DART_ERR_OTHER; - } - } else { - for (int d = 0; d < domain->num_domains; ++d) { - ret = dart__base__locality__scope_domains_rec( - domain->children[d], - scope, - num_domains_out, - domains_out); - /* not returning DART_ERR_NOTFOUND if no match in child domain - * as heterogeneous subdomains (child domains in different scopes) - * are to be supported. - */ - if (ret != DART_OK) { - return ret; - } - } - } - DART_LOG_TRACE("dart__base__locality__scope_domains_rec >"); - return DART_OK; -} - diff --git a/dart-impl/mpi/CMakeLists.txt b/dart-impl/mpi/CMakeLists.txt index 9c5884a9b..5f03eb4e5 100644 --- a/dart-impl/mpi/CMakeLists.txt +++ b/dart-impl/mpi/CMakeLists.txt @@ -204,6 +204,11 @@ target_link_libraries( ${ADDITIONAL_LIBRARIES} ) +set_target_properties( + ${DASH_DART_IMPL_MPI_LIBRARY} + PROPERTIES POSITION_INDEPENDENT_CODE TRUE +) + if(MPI_LINK_FLAGS) set_target_properties( ${DASH_DART_IMPL_MPI_LIBRARY} PROPERTIES @@ -246,9 +251,9 @@ install(TARGETS ${DASH_DART_IMPL_MPI_LIBRARY} if(${CMAKE_VERSION} VERSION_GREATER 3.0.0) # exports install(EXPORT "${DASH_DART_IMPL_MPI_LIBRARY}-targets" - DESTINATION cmake) + DESTINATION share/cmake) # install custom config install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${DASH_DART_IMPL_MPI_LIBRARY}-config.cmake" - DESTINATION cmake) + DESTINATION share/cmake) endif() diff --git a/dart-impl/mpi/include/dash/dart/mpi/dart_locality_priv.h b/dart-impl/mpi/include/dash/dart/mpi/dart_locality_priv.h index 52cd8af79..5681d0b57 100644 --- a/dart-impl/mpi/include/dash/dart/mpi/dart_locality_priv.h +++ b/dart-impl/mpi/include/dash/dart/mpi/dart_locality_priv.h @@ -7,6 +7,8 @@ #ifndef DART__MPI__DART_LOCALITY_PRIV_H__ #define DART__MPI__DART_LOCALITY_PRIV_H__ +#if 0 + #include #include @@ -15,4 +17,6 @@ dart_ret_t dart__mpi__locality_init() DART_INTERNAL; dart_ret_t dart__mpi__locality_finalize() DART_INTERNAL; +#endif + #endif /* DART__MPI__DART_LOCALITY_PRIV_H__ */ diff --git a/dart-impl/mpi/src/dart_initialization.c b/dart-impl/mpi/src/dart_initialization.c index ab80d8261..e44a0bcc9 100644 --- a/dart-impl/mpi/src/dart_initialization.c +++ b/dart-impl/mpi/src/dart_initialization.c @@ -17,7 +17,6 @@ #include #include #include -#include #include #define DART_LOCAL_ALLOC_SIZE (1024*1024*16) @@ -190,10 +189,6 @@ dart_ret_t do_init() _dart_initialized = 1; - dart__mpi__locality_init(); - - _dart_initialized = 2; - DART_LOG_DEBUG("dart_init > initialization finished"); return DART_OK; } @@ -277,8 +272,6 @@ dart_ret_t dart_exit() dart_global_unit_t unitid; dart_myid(&unitid); - dart__mpi__locality_finalize(); - _dart_initialized = 0; DART_LOG_DEBUG("%2d: dart_exit()", unitid.id); diff --git a/dart-impl/mpi/src/dart_locality.c b/dart-impl/mpi/src/dart_locality.c deleted file mode 100644 index 140d2e1b8..000000000 --- a/dart-impl/mpi/src/dart_locality.c +++ /dev/null @@ -1,304 +0,0 @@ -/** - * \file dart_locality.c - * - */ -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include -#include -#include -#include - -/* ==================================================================== * - * Domain Locality * - * ==================================================================== */ - -dart_ret_t dart_team_locality_init( - dart_team_t team) -{ - return dart__base__locality__create(team); -} - -dart_ret_t dart_team_locality_finalize( - dart_team_t team) -{ - return dart__base__locality__delete(team); -} - -dart_ret_t dart_domain_team_locality( - dart_team_t team, - const char * domain_tag, - dart_domain_locality_t ** team_domain_out) -{ - DART_LOG_DEBUG("dart_domain_team_locality() team(%d) domain(%s)", - team, domain_tag); - dart_ret_t ret; - - *team_domain_out = NULL; - - dart_domain_locality_t * team_domain = NULL; - ret = dart__base__locality__team_domain(team, &team_domain); - if (ret != DART_OK) { - DART_LOG_ERROR("dart_domain_team_locality: " - "dart__base__locality__team_domain failed (%d)", ret); - return ret; - } - DART_ASSERT(team_domain != NULL); - - *team_domain_out = team_domain; - - if (strcmp(domain_tag, team_domain->domain_tag) != 0) { - dart_domain_locality_t * team_subdomain; - ret = dart__base__locality__domain( - team_domain, domain_tag, &team_subdomain); - if (ret != DART_OK) { - DART_LOG_ERROR("dart_domain_team_locality: " - "dart__base__locality__domain failed " - "for domain tag '%s' -> (%d)", domain_tag, ret); - *team_domain_out = NULL; - return ret; - } - *team_domain_out = team_subdomain; - } - - DART_ASSERT(*team_domain_out != NULL); - - DART_LOG_DEBUG("dart_domain_team_locality > team(%d) domain(%s) -> %p", - team, domain_tag, (void *)(*team_domain_out)); - return DART_OK; -} - -dart_ret_t dart_domain_create( - dart_domain_locality_t ** domain_out) -{ - return dart__base__locality__create_domain(domain_out); -} - -dart_ret_t dart_domain_clone( - const dart_domain_locality_t * domain_in, - dart_domain_locality_t ** domain_out) -{ - return dart__base__locality__clone_domain(domain_in, domain_out); -} - -dart_ret_t dart_domain_destroy( - dart_domain_locality_t * domain) -{ - return dart__base__locality__destruct_domain(domain); -} - -dart_ret_t dart_domain_assign( - dart_domain_locality_t * domain_lhs, - const dart_domain_locality_t * domain_rhs) -{ - return dart__base__locality__assign_domain(domain_lhs, domain_rhs); -} - -dart_ret_t dart_domain_find( - const dart_domain_locality_t * domain_in, - const char * domain_tag, - dart_domain_locality_t ** subdomain_out) -{ - DART_LOG_DEBUG("dart_domain_find() domain_in(%p) domain_tag(%s)", - (void*)domain_in, domain_tag); - dart_ret_t ret = dart__base__locality__domain( - domain_in, domain_tag, subdomain_out); - DART_LOG_DEBUG("dart_domain_find > %d", ret); - return ret; -} - -dart_ret_t dart_domain_select( - dart_domain_locality_t * domain_in, - int num_subdomain_tags, - const char ** subdomain_tags) -{ - return dart__base__locality__select_subdomains( - domain_in, subdomain_tags, num_subdomain_tags); -} - -dart_ret_t dart_domain_exclude( - dart_domain_locality_t * domain_in, - int num_subdomain_tags, - const char ** subdomain_tags) -{ - return dart__base__locality__exclude_subdomains( - domain_in, subdomain_tags, num_subdomain_tags); -} - -dart_ret_t dart_domain_add_subdomain( - dart_domain_locality_t * domain, - dart_domain_locality_t * subdomain, - int subdomain_rel_id) -{ - return dart__base__locality__add_subdomain( - domain, subdomain, subdomain_rel_id); -} - -dart_ret_t dart_domain_remove_subdomain( - dart_domain_locality_t * domain, - int subdomain_rel_id) -{ - return dart__base__locality__remove_subdomain( - domain, subdomain_rel_id); -} - -dart_ret_t dart_domain_move_subdomain( - dart_domain_locality_t * domain, - dart_domain_locality_t * new_parent_domain, - int new_domain_rel_id) -{ - return dart__base__locality__move_subdomain( - domain, new_parent_domain, new_domain_rel_id); -} - -dart_ret_t dart_domain_split_scope( - const dart_domain_locality_t * domain_in, - dart_locality_scope_t scope, - int num_parts, - dart_domain_locality_t * domains_out) -{ - DART_LOG_DEBUG("dart_domain_split_scope() team(%d) domain(%s) " - "into %d parts at scope %d", - domain_in->team, domain_in->domain_tag, num_parts, - scope); - - int * group_sizes = NULL; - char *** group_domain_tags = NULL; - - /* Get domain tags for a split, grouped by locality scope. - * For 4 domains in the specified scope, a split into 2 parts results - * in a grouping of domain tags like: - * - * group_domain_tags = { - * { split_domain_0, split_domain_1 }, - * { split_domain_2, split_domain_3 } - * } - */ - DART_ASSERT_RETURNS( - dart__base__locality__domain_split_tags( - domain_in, scope, num_parts, &group_sizes, &group_domain_tags), - DART_OK); - - /* Use grouping of domain tags to create new locality domain - * hierarchy: - */ - for (int p = 0; p < num_parts; p++) { - DART_LOG_DEBUG("dart_domain_split_scope: split %d / %d", - p + 1, num_parts); - -#ifdef DART_ENABLE_LOGGING - DART_LOG_TRACE("dart_domain_split_scope: groups[%d] size: %d", - p, group_sizes[p]); - for (int g = 0; g < group_sizes[p]; g++) { - DART_LOG_TRACE("dart_domain_split: |- tags[%d]: %s", - g, group_domain_tags[p][g]); - } -#endif - - /* Deep copy of grouped domain so we do not have to recalculate - * groups for every split group : */ - DART_LOG_TRACE("dart_domain_split_scope: copying input domain"); - DART_ASSERT_RETURNS( - dart__base__locality__domain__init( - domains_out + p), - DART_OK); - DART_ASSERT_RETURNS( - dart__base__locality__assign_domain( - domains_out + p, - domain_in), - DART_OK); - - /* Drop domains that are not in split group: */ - DART_LOG_TRACE("dart_domain_split_scope: selecting subdomains"); - DART_ASSERT_RETURNS( - dart__base__locality__select_subdomains( - domains_out + p, - (const char **)(group_domain_tags[p]), - group_sizes[p]), - DART_OK); - } - - DART_LOG_DEBUG("dart_domain_split_scope >"); - return DART_OK; -} - -dart_ret_t dart_domain_scope_tags( - const dart_domain_locality_t * domain_in, - dart_locality_scope_t scope, - int * num_domains_out, - char *** domain_tags_out) -{ - *num_domains_out = 0; - *domain_tags_out = NULL; - - return dart__base__locality__scope_domain_tags( - domain_in, - scope, - num_domains_out, - domain_tags_out); -} - -dart_ret_t dart_domain_scope_domains( - const dart_domain_locality_t * domain_in, - dart_locality_scope_t scope, - int * num_domains_out, - dart_domain_locality_t *** domains_out) -{ - *num_domains_out = 0; - *domains_out = NULL; - - return dart__base__locality__scope_domains( - domain_in, - scope, - num_domains_out, - domains_out); -} - -dart_ret_t dart_domain_group( - dart_domain_locality_t * domain_in, - int num_group_subdomains, - const char ** group_subdomain_tags, - char * group_domain_tag_out) -{ - return dart__base__locality__domain_group( - domain_in, - num_group_subdomains, - group_subdomain_tags, - group_domain_tag_out); -} - -/* ==================================================================== * - * Unit Locality * - * ==================================================================== */ - -dart_ret_t dart_unit_locality( - dart_team_t team, - dart_team_unit_t unit, - dart_unit_locality_t ** locality) -{ - DART_LOG_DEBUG("dart_unit_locality() team(%d) unit(%d)", team, unit.id); - - dart_ret_t ret = dart__base__locality__unit(team, unit, locality); - if (ret != DART_OK) { - DART_LOG_ERROR("dart_unit_locality: " - "dart__base__unit_locality__get(unit:%d) failed (%d)", - unit.id, ret); - *locality = NULL; - return ret; - } - - DART_LOG_DEBUG("dart_unit_locality > team(%d) unit(%d) -> %p", - team, unit.id, (void*)(*locality)); - return DART_OK; -} - diff --git a/dart-impl/mpi/src/dart_locality_priv.c b/dart-impl/mpi/src/dart_locality_priv.c deleted file mode 100644 index 6b63a2933..000000000 --- a/dart-impl/mpi/src/dart_locality_priv.c +++ /dev/null @@ -1,49 +0,0 @@ -/** - * \file dash/dart/mpi/dart_locality_priv.c - * - */ - -#include - -#include -#include -#include -#include - -#include -#include - - -dart_ret_t dart__mpi__locality_init() -{ - DART_LOG_DEBUG("dart__mpi__locality_init()"); - dart_ret_t ret; - - ret = dart__base__locality__init(); - if (ret != DART_OK) { - DART_LOG_ERROR("dart__mpi__locality_init ! " - "dart__base__locality__init failed: %d", ret); - return ret; - } - DART_LOG_DEBUG("dart__mpi__locality_init >"); - return DART_OK; -} - -dart_ret_t dart__mpi__locality_finalize() -{ - DART_LOG_DEBUG("dart__mpi__locality_finalize()"); - dart_ret_t ret; - - ret = dart__base__locality__finalize(); - - dart_barrier(DART_TEAM_ALL); - - if (ret != DART_OK) { - DART_LOG_ERROR("dart__mpi__locality_finalize ! " - "dart__base__locality__finalize failed: %d", ret); - return ret; - } - DART_LOG_DEBUG("dart__mpi__locality_finalize >"); - return DART_OK; -} - diff --git a/dart-impl/mpi/src/dart_team_group.c b/dart-impl/mpi/src/dart_team_group.c index d489750c4..a4cc490fc 100644 --- a/dart-impl/mpi/src/dart_team_group.c +++ b/dart-impl/mpi/src/dart_team_group.c @@ -9,12 +9,10 @@ #include #include #include -#include #include #include #include -#include #include #include @@ -333,6 +331,7 @@ dart_ret_t dart_group_split( return DART_OK; } +#if 0 dart_ret_t dart_group_locality_split( const dart_group_t group, dart_domain_locality_t * domain, @@ -520,7 +519,7 @@ dart_ret_t dart_group_locality_split( DART_LOG_TRACE("dart_group_locality_split >"); return DART_OK; } - +#endif dart_ret_t dart_group_ismember( const dart_group_t g, diff --git a/dash/CMakeLists.txt b/dash/CMakeLists.txt index 28d07de2f..abb5e79dc 100644 --- a/dash/CMakeLists.txt +++ b/dash/CMakeLists.txt @@ -41,6 +41,8 @@ set(ENABLE_PAPI ${ENABLE_PAPI} PARENT_SCOPE) set(ENABLE_COMPTIME_RED ${ENABLE_COMPTIME_RED} PARENT_SCOPE) +set(ENABLE_DYLOC ${ENABLE_DYLOC} + PARENT_SCOPE) # Source- and header files to be compiled (OBJ): @@ -70,6 +72,16 @@ include_directories( set (ADDITIONAL_COMPILE_FLAGS ${CXX_OMP_FLAG} ${ADDITIONAL_COMPILE_FLAGS}) set (ADDITIONAL_COMPILE_FLAGS ${ADDITIONAL_COMPILE_FLAGS} -DDASH) +if (ENABLE_DYLOC) + include(${CMAKE_SOURCE_DIR}/CMakeExt/dyloc.cmake) + set (ADDITIONAL_COMPILE_FLAGS + ${ADDITIONAL_COMPILE_FLAGS} -DDASH_ENABLE_DYLOC) + set (ADDITIONAL_LIBRARIES ${ADDITIONAL_LIBRARIES} + dyloc-common dylocxx) +# set (ADDITIONAL_INCLUDES ${ADDITIONAL_INCLUDES} +# ${DYLOC_INCLUDES}) +endif() + if (ENABLE_DEFAULT_INDEX_TYPE_LONG) set (ADDITIONAL_COMPILE_FLAGS ${ADDITIONAL_COMPILE_FLAGS} -DDASH_ENABLE_DEFAULT_INDEX_TYPE_LONG) @@ -287,6 +299,12 @@ foreach (dart_variant ${DART_IMPLEMENTATIONS_LIST}) ${ADDITIONAL_LINK_FLAGS} ${ADDITIONAL_LIBRARIES} ) + + set_target_properties( + ${DASH_LIBRARY} + PROPERTIES POSITION_INDEPENDENT_CODE TRUE + ) + if (${dart_variant} STREQUAL "mpi") include_directories( ${MPI_INCLUDE_PATH}) @@ -314,6 +332,7 @@ foreach (dart_variant ${DART_IMPLEMENTATIONS_LIST}) ) include_directories( ${ADDITIONAL_INCLUDES} + ${DYLOC_INCLUDES} ) DeployLibrary(${DASH_LIBRARY}) @@ -334,10 +353,10 @@ foreach (dart_variant ${DART_IMPLEMENTATIONS_LIST}) DESTINATION lib EXPORT "${DASH_LIBRARY}-targets") - # exports + # Exports install( EXPORT "${DASH_LIBRARY}-targets" - DESTINATION cmake) + DESTINATION share/cmake) if(${CMAKE_VERSION} VERSION_GREATER 3.0.0 ) configure_package_config_file( @@ -423,17 +442,18 @@ endforeach(dart_variant ${DART_IMPLEMENTATIONS_LIST}) ## Tests # if (BUILD_TESTS) -include(${CMAKE_SOURCE_DIR}/CMakeExt/GoogleTest.cmake) + include(${CMAKE_SOURCE_DIR}/CMakeExt/GoogleTest.cmake) if (BUILD_COVERAGE_TESTS) include(${CMAKE_SOURCE_DIR}/CMakeExt/CodeCoverage.cmake) endif() - foreach(dart_variant ${DART_IMPLEMENTATIONS_LIST}) + foreach(dart_variant ${DART_IMPLEMENTATIONS_LIST}) set(DASH_LIBRARY "dash-${dart_variant}") set(DART_LIBRARY "dart-${dart_variant}") set(DASH_TEST "dash-test-${dart_variant}") include_directories( ${GTEST_INCLUDES} + ${DYLOC_INCLUDES} ${CMAKE_SOURCE_DIR}/dash/include ${ADDITIONAL_INCLUDES} ) diff --git a/dash/dash-config.cmake.in b/dash/dash-config.cmake.in index f71da7c5c..e9f081a60 100644 --- a/dash/dash-config.cmake.in +++ b/dash/dash-config.cmake.in @@ -17,6 +17,6 @@ set(DASH_LIBRARIES ${DASH_LIBRARY} "dart-@dart_variant@") set(DASH_CXX_FLAGS "@VARIANT_ADDITIONAL_COMPILE_FLAGS@ @CMAKE_CXX_FLAGS_RELEASE@") set(DASH_INSTALL_PREFIX "${PACKAGE_PREFIX_DIR}") -find_package(DART-@DART_VARIANT@ REQUIRED HINTS "${DASH_INSTALL_PREFIX}/cmake") +find_package(DART-@DART_VARIANT@ REQUIRED HINTS "${DASH_INSTALL_PREFIX}/share/cmake") -include("${DASH_INSTALL_PREFIX}/cmake/${DASH_LIBRARY}-targets.cmake") +include("${DASH_INSTALL_PREFIX}/share/cmake/${DASH_LIBRARY}-targets.cmake") diff --git a/dash/include/dash/Team.h b/dash/include/dash/Team.h index 6e00ad975..04860a39b 100644 --- a/dash/include/dash/Team.h +++ b/dash/include/dash/Team.h @@ -328,7 +328,7 @@ class Team /// where the team is split. dash::util::Locality::Scope scope, /// Number of parts to split this team's units into - unsigned num_parts); + unsigned num_parts = 0); /** * Split this Team's units into child Team instances at the specified @@ -341,7 +341,7 @@ class Team /// where the team is split. dart_locality_scope_t scope, /// Number of parts to split this team's units into - unsigned num_parts) + unsigned num_parts = 0) { return locality_split( static_cast( @@ -563,9 +563,9 @@ class Team { DASH_LOG_DEBUG("Team.register_team", "team id:", team->_dartid); - DASH_ASSERT_RETURNS( - dart_team_locality_init(team->_dartid), - DART_OK); + // DASH_ASSERT_RETURNS( + // dart_team_locality_init(team->_dartid), + // DART_OK); dash::Team::_teams.insert( std::make_pair(team->_dartid, team)); } @@ -574,14 +574,11 @@ class Team { DASH_LOG_DEBUG("Team.unregister_team", "team id:", team->_dartid); -// if (team->_dartid != DART_TEAM_NULL) - { - DASH_ASSERT_RETURNS( - dart_team_locality_finalize(team->_dartid), - DART_OK); - dash::Team::_teams.erase( - team->_dartid); - } + // DASH_ASSERT_RETURNS( + // dart_team_locality_finalize(team->_dartid), + // DART_OK); + dash::Team::_teams.erase( + team->_dartid); } private: diff --git a/dash/include/dash/util/Locality.h b/dash/include/dash/util/Locality.h index 211d8fac7..8b0f666a7 100644 --- a/dash/include/dash/util/Locality.h +++ b/dash/include/dash/util/Locality.h @@ -2,11 +2,9 @@ #define DASH__UTIL__LOCALITY_H__ #include - #include -#include -#include +#include #include #include @@ -36,19 +34,19 @@ class Locality typedef enum { - Undefined = DART_LOCALITY_SCOPE_UNDEFINED, - Global = DART_LOCALITY_SCOPE_GLOBAL, - Group = DART_LOCALITY_SCOPE_GROUP, - Network = DART_LOCALITY_SCOPE_NETWORK, - Node = DART_LOCALITY_SCOPE_NODE, - Module = DART_LOCALITY_SCOPE_MODULE, - NUMA = DART_LOCALITY_SCOPE_NUMA, - Unit = DART_LOCALITY_SCOPE_UNIT, - Package = DART_LOCALITY_SCOPE_PACKAGE, - Uncore = DART_LOCALITY_SCOPE_UNCORE, - Cache = DART_LOCALITY_SCOPE_CACHE, - Core = DART_LOCALITY_SCOPE_CORE, - CPU = DART_LOCALITY_SCOPE_CPU + Undefined = DYLOC_LOCALITY_SCOPE_UNDEFINED, + Global = DYLOC_LOCALITY_SCOPE_GLOBAL, + Group = DYLOC_LOCALITY_SCOPE_GROUP, + Network = DYLOC_LOCALITY_SCOPE_NETWORK, + Node = DYLOC_LOCALITY_SCOPE_NODE, + Module = DYLOC_LOCALITY_SCOPE_MODULE, + NUMA = DYLOC_LOCALITY_SCOPE_NUMA, + Unit = DYLOC_LOCALITY_SCOPE_UNIT, + Package = DYLOC_LOCALITY_SCOPE_PACKAGE, + Uncore = DYLOC_LOCALITY_SCOPE_UNCORE, + Cache = DYLOC_LOCALITY_SCOPE_CACHE, + Core = DYLOC_LOCALITY_SCOPE_CORE, + CPU = DYLOC_LOCALITY_SCOPE_CPU } Scope; diff --git a/dash/include/dash/util/LocalityDomain.h b/dash/include/dash/util/LocalityDomain.h index d0ea5e89d..278f72e26 100644 --- a/dash/include/dash/util/LocalityDomain.h +++ b/dash/include/dash/util/LocalityDomain.h @@ -4,12 +4,13 @@ #include #include -#include #include #include +#include + #include #include #include @@ -129,7 +130,7 @@ namespace internal { } /** - * Wrapper of a single \c dart_domain_locality_t object. + * Wrapper of a single \c dyloc::locality_domain object. * * Usage examples: * @@ -164,11 +165,11 @@ class LocalityDomain private: LocalityDomain( const self_t & parent, - dart_domain_locality_t * domain); + dyloc::locality_domain * domain); private: - /// Underlying \c dart_domain_locality_t object. - dart_domain_locality_t * _domain = nullptr; + /// Underlying \c dyloc::locality_domain object. + dyloc::locality_domain * _domain = nullptr; /// Copy of _domain->domain_tag to avoid string copying. std::string _domain_tag = "."; /// Cache of lazy-loaded subdomains, mapped by subdomain relative index. @@ -198,10 +199,10 @@ class LocalityDomain LocalityDomain() = default; explicit LocalityDomain( - const dart_domain_locality_t & domain); + const dyloc::locality_domain & domain); explicit LocalityDomain( - dart_domain_locality_t * domain); + dyloc::locality_domain * domain); ~LocalityDomain(); @@ -322,7 +323,8 @@ class LocalityDomain std::vector scope_domains; int num_scope_domains; - dart_domain_locality_t ** dart_scope_domains; +#if 0 + dyloc::locality_domain ** dart_scope_domains; dart_ret_t ret = dart_domain_scope_domains( _domain, static_cast(scope), @@ -342,6 +344,17 @@ class LocalityDomain } free(dart_scope_domains); } +#else + auto & topo = dyloc::team_topology(); + auto scope_dtags = topo.scope_domain_tags( + static_cast(scope)); + for (const auto & scope_dtag : scope_dtags) { + DASH_LOG_TRACE("LocalityDomain.scope_domains", + "scope domain tag:", scope_dtag); + const auto & scope_domain = topo[scope_dtag]; + scope_domains.emplace_back(scope_domain); + } +#endif return scope_domains; } @@ -378,7 +391,7 @@ class LocalityDomain return _domain_tag; } - inline const dart_domain_locality_t & dart_type() const + inline const dyloc::locality_domain & dart_type() const { DASH_ASSERT(_domain != nullptr); return *_domain; @@ -390,11 +403,13 @@ class LocalityDomain return _domain->host; } +#if 0 inline int shared_mem_bytes() const { DASH_ASSERT(_domain != nullptr); - return _domain->shared_mem_bytes; + return _domain->hwinfo.shared_mem_bytes; } +#endif inline iterator begin() { @@ -488,7 +503,7 @@ class LocalityDomain private: inline void init( - dart_domain_locality_t * domain); + dyloc::locality_domain * domain); inline void collect_groups( const std::vector & group_domain_tags) diff --git a/dash/include/dash/util/TeamLocality.h b/dash/include/dash/util/TeamLocality.h index f12a1f469..818289494 100644 --- a/dash/include/dash/util/TeamLocality.h +++ b/dash/include/dash/util/TeamLocality.h @@ -6,7 +6,6 @@ #include #include -#include #include diff --git a/dash/include/dash/util/UnitLocality.h b/dash/include/dash/util/UnitLocality.h index e13f0906a..eb62f614b 100644 --- a/dash/include/dash/util/UnitLocality.h +++ b/dash/include/dash/util/UnitLocality.h @@ -8,7 +8,8 @@ #include #include -#include + +#include #include #include @@ -27,8 +28,7 @@ namespace util { /** * Wrapper of a single \c dart_unit_locality_t object. */ -class UnitLocality -{ +class UnitLocality { private: typedef UnitLocality self_t; diff --git a/dash/src/Team.cc b/dash/src/Team.cc index 80e827935..c50eab8cd 100644 --- a/dash/src/Team.cc +++ b/dash/src/Team.cc @@ -3,6 +3,8 @@ #include +#include + #include #include #include @@ -146,26 +148,27 @@ Team::locality_split( return *result; } + // Number of child teams actually created: + size_t num_split = 0; + if (num_parts == 0) { DASH_THROW( dash::exception::InvalidArgument, "Number of parts to split team must be greater than 0"); } - dart_locality_scope_t dart_scope = static_cast( - static_cast(scope)); - - dart_group_t group; std::vector sub_group_v(num_parts); dart_group_t * sub_groups = sub_group_v.data(); - - size_t num_split = 0; - for (unsigned i = 0; i < num_parts; i++) { DASH_ASSERT_RETURNS( dart_group_create(&sub_groups[i]), DART_OK); } +#if 0 + dart_locality_scope_t dart_scope = static_cast( + static_cast(scope)); + + dart_group_t group; dart_domain_locality_t * domain; DASH_ASSERT_RETURNS( @@ -178,7 +181,34 @@ Team::locality_split( dart_group_locality_split( group, domain, dart_scope, num_parts, &num_split, sub_groups), DART_OK); +#else dart_team_t oldteam = _dartid; + + auto & topo = dyloc::team_topology(_dartid); + auto scope_dtags = topo.scope_domain_tags( + static_cast(scope)); + + if (num_parts == 0) { + num_parts = scope_dtags.size(); + } + + if (num_parts == scope_dtags.size()) { + int group_idx = 0; + for (const auto & scope_dtag : scope_dtags) { + DYLOC_LOG_DEBUG_VAR("Team.locality_split", scope_dtag); + const auto & scope_domain = topo[scope_dtag]; + for (auto scope_domain_unit_id : scope_domain.unit_ids) { + dart_group_addmember(sub_groups[group_idx], scope_domain_unit_id); + } + } + } else { + DASH_THROW( + dash::exception::NotImplemented, + "Team.locality_split does not support nparts != ndomains"); + } + num_split = scope_dtags.size(); +#endif + #if DASH_ENABLE_TRACE_LOGGING for(unsigned i = 0; i < num_parts; i++) { size_t sub_group_size = 0; diff --git a/dash/src/util/Locality.cc b/dash/src/util/Locality.cc index a2a364239..3911c1536 100644 --- a/dash/src/util/Locality.cc +++ b/dash/src/util/Locality.cc @@ -1,10 +1,11 @@ #include +#include + #include #include #include -#include #include #include