codes-org · xwang149 · Apr 11, 2023 · Apr 11, 2023 · Apr 11, 2023 · Apr 18, 2023
diff --git a/Makefile.am b/Makefile.am
@@ -60,10 +60,19 @@ src_libcodes_la_SOURCES += src/workload/methods/codes-recorder-io-wrkld.c
 endif
 
 if USE_ONLINE
-AM_CPPFLAGS += ${ARGOBOTS_CFLAGS} ${SWM_CFLAGS} -DUSE_ONLINE=1
-LDADD += ${SWM_LIBS} ${ARGOBOTS_LIBS}
+AM_CPPFLAGS += ${ARGOBOTS_CFLAGS} -DUSE_ONLINE=1
+LDADD += ${ARGOBOTS_LIBS}
+if USE_SWM
+AM_CPPFLAGS +=  ${SWM_CFLAGS} -DUSE_SWM=1
+LDADD += ${SWM_LIBS}
 src_libcodes_la_SOURCES += src/workload/methods/codes-online-comm-wrkld.C
 endif
+if USE_UNION
+src_libcodes_la_SOURCES += src/workload/methods/codes-conc-online-comm-wrkld.C
+AM_CPPFLAGS += ${UNION_CFLAGS} ${SWM_CFLAGS} -DUSE_UNION=1
+LDADD += ${UNION_LIBS} ${SWM_LIBS}
+endif
+endif
 
 if USE_DUMPI
 AM_CPPFLAGS += ${DUMPI_CFLAGS} -DUSE_DUMPI=1

diff --git a/codes/model-net.h b/codes/model-net.h
@@ -143,7 +143,7 @@ typedef struct model_net_request {
     int      self_event_size;
     char     category[CATEGORY_NAME_MAX];
 
-    //for counting msg app id
+    //Xin: passing app id to routers
     int     app_id;
 
 } model_net_request;

diff --git a/codes/net/dragonfly-custom.h b/codes/net/dragonfly-custom.h
@@ -87,6 +87,14 @@ struct terminal_custom_message
    tw_stime msg_start_time;
    tw_stime saved_busy_time_ross;
    tw_stime saved_fin_chunks_ross;
+
+   //Yao: for counting msg app id
+   int app_id;
+   tw_stime last_received_time;
+   tw_stime last_sent_time;
+   //Xin: for busy time recording
+   tw_stime last_bufupdate_time;
+
 };
 
 #ifdef __cplusplus

diff --git a/codes/net/dragonfly-dally.h b/codes/net/dragonfly-dally.h
@@ -118,6 +118,14 @@ struct terminal_dally_message
    tw_stime msg_start_time;
    tw_stime saved_busy_time_ross;
    tw_stime saved_fin_chunks_ross;
+
+   //Yao: for counting msg app id
+   tw_stime last_received_time;
+   tw_stime last_sent_time;
+
+   //Xin: for busy time recording
+   tw_stime last_bufupdate_time;
+
 };
 
 #ifdef __cplusplus

diff --git a/codes_config.h.in b/codes_config.h.in
@@ -1,18 +1,173 @@
+/* codes_config.h.in.  Generated from configure.ac by autoheader.  */
 
+/* define if the Boost library is available */
+#undef HAVE_BOOST
 
+/* define if the compiler supports basic C++11 syntax */
+#undef HAVE_CXX11
 
-// ross
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#undef HAVE_DLFCN_H
 
-// dumpi
+/* Define to 1 if you have the <execinfo.h> header file. */
+#undef HAVE_EXECINFO_H
 
-// swm
-#define SWM_DATAROOTDIR "${SWM_DATAROOTDIR}"
+/* Define to 1 if you have the <inttypes.h> header file. */
+#undef HAVE_INTTYPES_H
 
+/* Define to 1 if you have the `m' library (-lm). */
+#undef HAVE_LIBM
 
-// damaris
+/* Define to 1 if you have the `pthread' library (-lpthread). */
+#undef HAVE_LIBPTHREAD
 
+/* Define to 1 if you have the `undumpi' library (-lundumpi). */
+#undef HAVE_LIBUNDUMPI
 
-// darshan
+/* Define to 1 if you have the <malloc.h> header file. */
+#undef HAVE_MALLOC_H
 
+/* Define to 1 if you have the <memory.h> header file. */
+#undef HAVE_MEMORY_H
 
-// cortex
+/* Define to 1 if you have the `memset' function. */
+#undef HAVE_MEMSET
+
+/* Define to 1 if you have the <pthread.h> header file. */
+#undef HAVE_PTHREAD_H
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#undef HAVE_STDINT_H
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#undef HAVE_STDLIB_H
+
+/* Define to 1 if you have the <strings.h> header file. */
+#undef HAVE_STRINGS_H
+
+/* Define to 1 if you have the <string.h> header file. */
+#undef HAVE_STRING_H
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#undef HAVE_SYS_STAT_H
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#undef HAVE_SYS_TYPES_H
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#undef HAVE_UNISTD_H
+
+/* If old-style pure reentrant parser syntax is supported by ${YACC} */
+#undef HAVE_YACC_OLD_PURE
+
+/* If old-style push parser syntax is supported by ${YACC} */
+#undef HAVE_YACC_OLD_PUSH
+
+/* Define to the sub-directory in which libtool stores uninstalled libraries.
+   */
+#undef LT_OBJDIR
+
+/* Define to 1 if your C compiler doesn't accept -c and -o together. */
+#undef NO_MINUS_C_MINUS_O
+
+/* Name of package */
+#undef PACKAGE
+
+/* Define to the address where bug reports for this package should be sent. */
+#undef PACKAGE_BUGREPORT
+
+/* Define to the full name of this package. */
+#undef PACKAGE_NAME
+
+/* Define to the full name and version of this package. */
+#undef PACKAGE_STRING
+
+/* Define to the one symbol short name of this package. */
+#undef PACKAGE_TARNAME
+
+/* Define to the home page for this package. */
+#undef PACKAGE_URL
+
+/* Define to the version of this package. */
+#undef PACKAGE_VERSION
+
+/* Define to 1 if you have the ANSI C header files. */
+#undef STDC_HEADERS
+
+/* if using json data files */
+#undef SWM_DATAROOTDIR
+
+/* if using json data files */
+#undef UNION_DATADIR
+
+/* If enabling valgrind-clean build */
+#undef VALGRIND
+
+/* Version number of package */
+#undef VERSION
+
+/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a
+   `char[]'. */
+#undef YYTEXT_POINTER
+
+/* Define for Solaris 2.5.1 so the uint32_t typedef from <sys/synch.h>,
+   <pthread.h>, or <semaphore.h> is not used. If the typedef were allowed, the
+   #define below would cause a syntax error. */
+#undef _UINT32_T
+
+/* Define for Solaris 2.5.1 so the uint64_t typedef from <sys/synch.h>,
+   <pthread.h>, or <semaphore.h> is not used. If the typedef were allowed, the
+   #define below would cause a syntax error. */
+#undef _UINT64_T
+
+/* Define for Solaris 2.5.1 so the uint8_t typedef from <sys/synch.h>,
+   <pthread.h>, or <semaphore.h> is not used. If the typedef were allowed, the
+   #define below would cause a syntax error. */
+#undef _UINT8_T
+
+/* Define to empty if `const' does not conform to ANSI C. */
+#undef const
+
+/* Define to `__inline__' or `__inline' if that's what the C compiler
+   calls it, or to nothing if 'inline' is not supported under any name.  */
+#ifndef __cplusplus
+#undef inline
+#endif
+
+/* Define to the type of a signed integer type of width exactly 16 bits if
+   such a type exists and the standard includes do not define it. */
+#undef int16_t
+
+/* Define to the type of a signed integer type of width exactly 32 bits if
+   such a type exists and the standard includes do not define it. */
+#undef int32_t
+
+/* Define to the type of a signed integer type of width exactly 64 bits if
+   such a type exists and the standard includes do not define it. */
+#undef int64_t
+
+/* Define to the type of a signed integer type of width exactly 8 bits if such
+   a type exists and the standard includes do not define it. */
+#undef int8_t
+
+/* Define to `unsigned int' if <sys/types.h> does not define. */
+#undef size_t
+
+/* Define to `int' if <sys/types.h> does not define. */
+#undef ssize_t
+
+/* Define to the type of an unsigned integer type of width exactly 16 bits if
+   such a type exists and the standard includes do not define it. */
+#undef uint16_t
+
+/* Define to the type of an unsigned integer type of width exactly 32 bits if
+   such a type exists and the standard includes do not define it. */
+#undef uint32_t
+
+/* Define to the type of an unsigned integer type of width exactly 64 bits if
+   such a type exists and the standard includes do not define it. */
+#undef uint64_t
+
+/* Define to the type of an unsigned integer type of width exactly 8 bits if
+   such a type exists and the standard includes do not define it. */
+#undef uint8_t
diff --git a/configure.ac b/configure.ac
@@ -113,27 +113,55 @@ if test "x${use_darshan}" = xyes ; then
 fi
 AM_CONDITIONAL(USE_DARSHAN, [test "x${use_darshan}" = xyes])
 
-
 # check for Argobots
 AC_ARG_WITH([online],[AS_HELP_STRING([--with-online@<:@=DIR@:>@],
-                        [Build with the online workloads and argobots support])],
-                      [use_online=yes],[use_online=no])
-if test "x${use_online}" != "xno" ; then
+                        [Build with the online workloads and argobots support])])
+if test "x${with_online}" != "x" ; then
     AM_CONDITIONAL(USE_ONLINE, true)
     AX_BOOST_BASE([1.66])
     AX_CXX_COMPILE_STDCXX(11, noext, mandatory)
     PKG_CHECK_MODULES_STATIC([ARGOBOTS], [argobots], [],
                       [AC_MSG_ERROR([Could not find working argobots installation via pkg-config])])
+else
+    AM_CONDITIONAL(USE_ONLINE, false)
+fi
+
+#check for SWM
+AC_ARG_WITH([swm],[AS_HELP_STRING([--with-swm@<:@=DIR@:>@],
+                        [location of SWM installation])])
+if test "x${with_swm}" != "x" ; then
+    AM_CONDITIONAL(USE_SWM, true)
     PKG_CHECK_MODULES_STATIC([SWM], [swm], [],
                       [AC_MSG_ERROR([Could not find working swm installation via pkg-config])])
     PKG_CHECK_VAR([SWM_DATAROOTDIR], [swm], [datarootdir], [],
               [AC_MSG_ERROR[Could not find shared directory in SWM]])
     AC_DEFINE_UNQUOTED([SWM_DATAROOTDIR], ["$SWM_DATAROOTDIR"], [if using json
-                        data files])
+                    data files])
 else
-    AM_CONDITIONAL(USE_ONLINE, false)
+  AM_CONDITIONAL(USE_SWM, false)
 fi
 
+#check for UNION 
+AC_ARG_WITH([union],[AS_HELP_STRING([--with-union@<:@=DIR@:>@],
+                        [location of Union installation])])
+if test "x${with_union}" != "x" ; then
+    AM_CONDITIONAL(USE_UNION, true)
+    PKG_CHECK_MODULES_STATIC([UNION], [union], [],
+                      [AC_MSG_ERROR([Could not find working Union installation via pkg-config])])
+    PKG_CHECK_VAR([UNION_DATADIR], [union], [datarootdir], [],
+              [AC_MSG_ERROR[Could not find shared directory in UNION]])
+    AC_DEFINE_UNQUOTED([UNION_DATADIR], ["$UNION_DATADIR"], [if using json data files])
+    PKG_CHECK_MODULES_STATIC([SWM], [swm], [],
+                      [AC_MSG_ERROR([Could not find working swm installation via pkg-config])])
+    PKG_CHECK_VAR([SWM_DATAROOTDIR], [swm], [datarootdir], [],
+              [AC_MSG_ERROR[Could not find shared directory in SWM]])
+    AC_DEFINE_UNQUOTED([SWM_DATAROOTDIR], ["$SWM_DATAROOTDIR"], [if using json
+                    data files])
+else
+    AM_CONDITIONAL(USE_UNION, false)
+fi
+
+
 # check for Recorder
 AM_CONDITIONAL(USE_RECORDER, true)
 RECORDER_CPPFLAGS="-DUSE_RECORDER=1"

diff --git a/doc/workload/union_online_workload.txt b/doc/workload/union_online_workload.txt
@@ -0,0 +1,68 @@
+This document serves the following purposes:
+* CODES updates to accommodate Union online simulations
+* Installation tutorial
+* Completed Experiments
+* Known issues
+
+= CODES updates
+
+The code modifications are started with comment text "Xin:"
+
+== Header file
+
+Added parameters for collecting router traffic data, including:
+* codes/model-net.h
+* codes/net/dragonfly-custom.h
+* codes/net/dragonfly-dally.h
+
+== Makefile
+
+Added checking for Union installation in the autoconf configure script configure.ac
+Added src/workload/methods/codes-conc-online-comm-wrkld.C to code base if compile with Union in Makefile.am
+
+== Union online workload
+
+We add a pluggable workload module "src/workload/methods/codes-conc-online-comm-wrkld.C" into CODES workload generator to hold the actual implementation of Union communication events, such that the messages from Union skeletons can be emitted as simulation events in CODES.
+
+== Router status collection for dragonfly custom and dragonfly dally
+
+Added supportive functions for collecting traffic data on router port on the following network models:
+* dragonfly custom at src/networks/model-net/dragonfly-custom.C
+* dragonfly dally at src/networks/model-net/dragonfly-dally.C
+
+== Updates in MPI replay
+
+Added Union online workload type in MPI workload replay at src/network-workloads/model-net-mpi-replay.c
+
+== Configurations
+
+We added the following items in the CODES configuration file for collecting router traffic information during simulation.
+
+* counting_bool - flag to enable/disable the collection of trouter traffic
+* counting_start - the start time in microsecond for collecting traffic data
+* counting_interval - the time window size in microsecond for collection traffic data
+* counting_windows - the number of time windows for collecting traffic data
+* num_apps - the number of applications in the simulation workload
+* offset - supportive parameter for getting the application id of each packet 
+
+An example configuration can be found at: https://github.com/SPEAR-IIT/Union/blob/master/test/df1d-72-adp.conf
+
+= Installation tutorial
+
+Please follow the Readme at: https://github.com/SPEAR-IIT/Union to install Union and run test simulation of Union online workloads.
+
+= Completed Experiments 
+
+We have completed the following experiments with Union online workload simulation:
+* simulate Conceptual skeletons alone
+* simulate Conceptual and SWM skeletons simultaneously
+* simulate Conceptual and SWM skeletons simultaneously with different synthetic traffic patterns
+
+The above experiments have been done on both dragonfly custom and dragonfly dally network models, with sequential mode and optimistic mode.
+
+= Known Issues
+
+Currently the rendezvous protocol in MPI replay cannot work with Union online workloads.
+The reverse function router_buf_update_rc() does not take care of the cross window reverses for aggregated busytime on port.
+
+