Memory Spaces

.gitignore

@@ -7,3 +7,4 @@ main_spec.out
 main_spec.pdf
 main_spec.toc
 *~
+

content/backmatter.tex

@@ -665,6 +665,24 @@ \subsection{Table~\ref{p2psynctypes}: point-to-point synchronization types}
 
 \chapter{Changes to this Document}\label{sec:changelog}
 
+\section{Version 1.7}
+\label{changelog:v1.7}
+
+Major changes in \openshmem[1.7] include the addition of spaces, which enable
+applications to allocate symmetric memory in heterogeneous memory domains.
+
+The following list describes the specific changes in \openshmem[1.7]:
+\begin{enumerate}
+%
+\item Added spaces, which enable applications to allocate symmetric memory in
+    heterogeneous memory domains. \ChangelogRef{subsec:space, subsec:shmem_space_calloc}%
+%
+\item Added space-specific allocation routines: \FUNC{shmem\_space\_malloc},
+    \FUNC{shmem\_space\_calloc}, and \FUNC{shmem\_space\_free}.
+\ChangelogRef{subsec:shmem_space_malloc, subsec:shmem_space_calloc, subsec:shmem_space_free}%
+%
+\end{enumerate}
+
 \section{Version 1.6}
 \label{changelog:v1.6}
 Major changes in \openshmem[1.6] include the addition of inclusive and

content/environment_variables.tex

@@ -14,7 +14,7 @@
 
 \medskip{}
 
-\begin{longtable}{|p{0.260\textwidth}|p{0.145\textwidth}|p{0.50\textwidth}|}
+\begin{longtable}{|p{0.370\textwidth}|p{0.145\textwidth}|p{0.390\textwidth}|}
 \hline
 \textbf{Variable} & \textbf{Value} & \textbf{Description}
 \tabularnewline\hline
@@ -31,29 +31,45 @@
 %%
 \EnvVarDecl{SHMEM\_SYMMETRIC\_SIZE}
     & Non-negative integer or floating point value with an optional character
-    suffix
-    & Specifies the size (in bytes) of the symmetric heap memory per \ac{PE}.
-    The resulting size is implementation-defined and must be at least as large as
-    the integer ceiling of the product of the numeric prefix and the scaling
-    factor. The allowed character suffixes for the scaling factor are as
-    follows:
+      suffix
+    & Specifies the size (in bytes) of the symmetric heap memory per \ac{PE}
+      for \LibHandleRef{SHMEM\_SPACE\_DEFAULT}. The resulting size is
+      implementation-defined and must be at least as large as the integer
+      ceiling of the product of the numeric prefix and the scaling factor.
+      The allowed character suffixes for the scaling factor are:
       \begin{itemize}
         \item k or K multiplies by \(2^{10}\)  (kibibytes)
         \item m or M multiplies by \(2^{20}\)  (mebibytes)
         \item g or G multiplies by \(2^{30}\)  (gibibytes)
         \item t or T multiplies by \(2^{40}\)  (tebibytes)
       \end{itemize}
-      For example, string \enquote{20m} is equivalent to the integer value
-      20971520, or 20 mebibytes. Similarly the string \enquote{3.1M} is
-      equivalent to the integer value 3250586.
-      Only one multiplier is recognized and any characters following the
-      multiplier are ignored, so \enquote{20kk} will not produce the same
-      result as \enquote{20m}. Usage of string \enquote{.5m} will yield the
-      same result as the string \enquote{0.5m}.
-
-      An invalid value for \ENVVAR{SHMEM\_SYMMETRIC\_SIZE} is an error, which the
-      \openshmem library shall report by either returning a nonzero value from
-      \FUNC{shmem\_init\_thread} or causing program termination.
+      An invalid value is an error, which the \openshmem library shall report
+      by either returning a nonzero value from \FUNC{shmem\_init\_thread} or
+      causing program termination.
+    \tabularnewline\hline
+%%
+\EnvVarDecl{SHMEM\_<DEVICE>\_SYMMETRIC\_SIZE}
+    & Non-negative integer or floating point value with an optional character
+      suffix
+    & Specifies the size (in bytes) of the symmetric heap memory per \ac{PE}
+      for \LibHandleRef{SHMEM\_SPACE\_<DEVICE>}. The resulting size is
+      implementation-defined and must be at least as large as the integer
+      ceiling of the product of the numeric prefix and the scaling factor.
+      The allowed character suffixes are as in \ENVVAR{SHMEM\_SYMMETRIC\_SIZE}.
+      Device tokens are implementation-defined.
+    \tabularnewline\hline
+%%
+\EnvVarDecl{SHMEM\_ENABLE\_<DEVICE>}
+    & Any
+    & Enables the symmetric heap for \LibHandleRef{SHMEM\_SPACE\_<DEVICE>}.
+      When not set or empty for a given \textit{<DEVICE>}, that device's
+      symmetric heap is disabled. Device tokens are implementation-defined.
+    \tabularnewline\hline
+%%
+\EnvVarDecl{SHMEM\_DEFAULT\_SPACE}
+    & \textit{<DEVICE>}
+    & Controls which space \LibHandleRef{SHMEM\_SPACE\_DEFAULT} aliases to.
+      If not set, the implementation chooses the default space.
     \tabularnewline\hline
 %%
 \EnvVarDecl{SHMEM\_DEBUG}
@@ -62,4 +78,4 @@
     \tabularnewline\hline
 \end{longtable}
 
-\medskip{}
+\medskip{}

content/library_constants.tex

@@ -121,6 +121,15 @@
 \ref{subsec:shmmallochint} for more detail about its use.
 \tabularnewline \hline
 %%
+\LibConstDecl{SHMEM\_SPACE\_INVALID} &
+A value corresponding to an invalid space.
+This value can be used to initialize or update space handles to indicate
+that they do not reference a valid space.
+When managed in this way, applications can use an equality comparison
+to test whether a given space handle references a valid space.
+See Section~\ref{subsec:space} for more detail about its use.
+\tabularnewline \hline
+%%
 \begin{DeprecateBlock}
   \LibConstDecl{SHMEM\_SYNC\_VALUE}
   \LibConstDecl{\_SHMEM\_SYNC\_VALUE}

content/library_handles.tex

@@ -33,6 +33,14 @@
 See Section~\ref{subsec:team} for more detail about its use.
 \tabularnewline \hline
 %%
+\LibHandleDecl{SHMEM\_TEAM\_<DEVICE>} &
+Handle of type \CTYPE{shmem\_team\_t} that corresponds to the
+team of \acp{PE} that can access the symmetric heap for \textit{<DEVICE>}.
+It is available when \LibHandleRef{SHMEM\_SPACE\_<DEVICE>} is enabled and accessible.
+Device tokens are implementation-defined.
+See Section~\ref{subsec:space} for more detail about its use.
+\tabularnewline \hline
+%%
 \LibHandleDecl{SHMEM\_CTX\_DEFAULT} &
 Handle of type \CTYPE{shmem\_ctx\_t} that corresponds to the
 default communication context.  All point-to-point communication operations
@@ -41,4 +49,20 @@
 See Section~\ref{sec:ctx} for more detail about its use.
 \tabularnewline \hline
 %%
+\LibHandleDecl{SHMEM\_SPACE\_DEFAULT} &
+A memory space handle representing the default symmetric heap. This
+space aliases to some \LibHandleRef{SHMEM\_SPACE\_<DEVICE>} based
+on implementation choice or user configuration. This space is
+used for memory allocated via \FUNC{shmem\_malloc} and \FUNC{shmem\_calloc}.
+See Section~\ref{subsec:space} for more detail about its use.
+\tabularnewline \hline
+%%
+\LibHandleDecl{SHMEM\_SPACE\_<DEVICE>} &
+Handle of type \CTYPE{shmem\_space\_t} that corresponds to a symmetric
+heap in the memory domain associated with \textit{<DEVICE>}. If the space
+for \textit{<DEVICE>} is not available (e.g., disabled or not supported), this handle
+is set to \LibConstRef{SHMEM\_SPACE\_INVALID}. Device tokens are implementation-defined.
+See Section~\ref{subsec:space} for more detail about its use.
+\tabularnewline \hline
+%%
 \end{longtable}

content/memmgmt_intro.tex

@@ -1,23 +1,43 @@
-\openshmem provides a set of \acp{API} for managing the symmetric heap. The
+\openshmem provides a set of \acp{API} for managing symmetric heaps. The
 \acp{API} allow one to dynamically allocate, deallocate, reallocate, and align
-symmetric data objects in the symmetric heap.
+symmetric data objects in these heaps.
 
-The symmetric memory allocation routines differ from the private heap
+In addition to the traditional (legacy) symmetric heap used by
+\FUNC{shmem\_malloc}, \FUNC{shmem\_calloc}, \FUNC{shmem\_free}, and
+\FUNC{shmem\_realloc}, an implementation may provide multiple symmetric heaps
+associated with memory \emph{spaces} (see Section~\ref{subsec:space}). The
+legacy routines operate on the default symmetric heap, which aliases
+\LibHandleRef{SHMEM\_SPACE\_DEFAULT}, while the space-specific routines
+\FUNC{shmem\_space\_malloc}, \FUNC{shmem\_space\_calloc}, and
+\FUNC{shmem\_space\_free} operate on the symmetric heap associated with a
+given \CTYPE{shmem\_space\_t} handle.
+
+The legacy symmetric memory allocation routines differ from the private heap
 allocation routines in that they must be called by all \acp{PE} in
 the world team.  When specified, each of these routines includes at
 least one call to a procedure that is semantically equivalent to
 \FUNC{shmem\_barrier\_all}.  This ensures that all \acp{PE}
 participate in the memory management, and that the memory on other
 \acp{PE} can be used as soon as the local \ac{PE} returns.  The
-implicit barriers performed by these routines quiet the default
+implicit barriers performed by these legacy routines quiet the default
 context.  It is the user's responsibility to ensure that no
 communication operations involving the given memory block are pending
 on other contexts prior to calling the \FUNC{shmem\_free} and
 \FUNC{shmem\_realloc} routines.
 
-The total size of the symmetric heap is determined at job startup.  One can
-specify the size of the heap using the \ENVVAR{SHMEM\_SYMMETRIC\_SIZE} environment
-variable (where available).
+The space-specific allocation routines are collective on the team
+associated with the space handle and, where applicable, include a call
+that is semantically equivalent to \FUNC{shmem\_team\_sync} on exit.
+
+The total size of each symmetric heap is determined at job startup. The size of
+the default symmetric heap can be controlled with the
+\ENVVAR{SHMEM\_SYMMETRIC\_SIZE} environment variable (where available). The
+sizes of device-specific spaces can be controlled with
+\ENVVAR{SHMEM\_<DEVICE>\_SYMMETRIC\_SIZE}; when
+\LibHandleRef{SHMEM\_SPACE\_DEFAULT} aliases to
+\LibHandleRef{SHMEM\_SPACE\_<DEVICE>}, the device-specific variable takes
+precedence. Availability of device spaces may be controlled with
+\ENVVAR{SHMEM\_ENABLE\_<DEVICE>}.
 
 \begin{DeprecateBlock}
   As of \openshmem[1.2] the use of \FUNC{shmalloc}, \FUNC{shmemalign},

content/programming_model_overview.tex

@@ -72,6 +72,16 @@
     numbering through team-based contexts.
 \end{enumerate}
 
+\item \textbf{Space Management}
+\begin{enumerate}
+    \item \OPR{Spaces}: Spaces are containers for symmetric memory in specific memory
+    domains. They provide the storage for \openshmem symmetric data objects and
+    define accessibility and operational properties of those objects. Allocations
+    within a space are collective and symmetric across the \acp{PE} for which the
+    space is available. Collective and communication operations operate on data
+    resident in one or more spaces as permitted by the implementation
+\end{enumerate}
+
 \item \textbf{\acf{RMA}}
 \begin{enumerate}
     \item \PUT: The local \ac{PE} specifies the \source{} data object, private

content/shmem_space_calloc.tex

@@ -0,0 +1,46 @@
+\apisummary{
+  Collectively allocate a zeroed block of symmetric memory in a specific space.
+}
+
+\begin{apidefinition}
+
+\begin{Csynopsis}
+void *@\FuncDecl{shmem\_space\_calloc}@(shmem_space_t space, size_t count, size_t size);
+\end{Csynopsis}
+
+\begin{apiarguments}
+  \apiargument{IN}{space}{The space within which to allocate.}
+  \apiargument{IN}{count}{The number of elements to allocate.}
+  \apiargument{IN}{size}{The size in bytes of each element to allocate.}
+\end{apiarguments}
+
+\apidescription{
+  The \FUNC{shmem\_space\_calloc} routine is a collective operation on the
+  team associated with \VAR{space}. It allocates a region of
+  remotely-accessible symmetric memory for an array of \VAR{count} objects of
+  \VAR{size} bytes each from the symmetric heap identified by \VAR{space}, and
+  returns the symmetric address of the lowest byte of the allocated symmetric
+  memory. The space is initialized to all bits zero.
+
+  If the allocation succeeds, the pointer returned shall be suitably aligned so
+  that it may be assigned to a pointer to any type of object. If the allocation
+  does not succeed, or either \VAR{count} or \VAR{size} is \CONST{0}, the
+  return value is a null pointer.
+
+  The values of the \VAR{space}, \VAR{count}, and \VAR{size} arguments must be
+  identical on all \acp{PE}; otherwise, the behavior is undefined.
+
+  When \VAR{count} or \VAR{size} is \CONST{0}, the \FUNC{shmem\_space\_calloc}
+  routine returns without performing a barrier; otherwise,
+  \FUNC{shmem\_space\_calloc} calls a procedure that is semantically
+  equivalent to \FUNC{shmem\_team\_sync} on exit.
+}
+
+\apireturnvalues{
+  The \FUNC{shmem\_space\_calloc} routine returns the symmetric address of the
+  allocated space; otherwise, it returns a null pointer.
+}
+
+\end{apidefinition}
+
+

content/shmem_space_free.tex

@@ -0,0 +1,34 @@
+\apisummary{
+  Collectively deallocate symmetric memory in a specific space.
+}
+
+\begin{apidefinition}
+
+\begin{Csynopsis}
+void @\FuncDecl{shmem\_space\_free}@(shmem_space_t space, void *ptr);
+\end{Csynopsis}
+
+\begin{apiarguments}
+  \apiargument{IN}{space}{Space handle from which ptr was allocated.}
+  \apiargument{IN}{ptr}{Symmetric address of an object in the given space.}
+\end{apiarguments}
+
+\apidescription{
+  The \FUNC{shmem\_space\_free} routine is a collective operation on the
+  team associated with \VAR{space} that causes the block to which \VAR{ptr}
+  points to be deallocated, that is, made available for further
+  allocation.  If \VAR{ptr} is a null pointer, no action is performed;
+  otherwise, \FUNC{shmem\_space\_free} calls a function equivalent to barrier on entry.
+  It is the user's responsibility to ensure that no communication
+  operations involving the given memory block are pending on other
+  communication contexts prior to calling \FUNC{shmem\_space\_free}.
+
+  The values of the \VAR{space} and \VAR{ptr} arguments must be identical on all
+  \acp{PE}; otherwise, the behavior is undefined.
+}
+
+\apireturnvalues{
+  None.
+}
+
+\end{apidefinition}

content/shmem_space_malloc.tex

@@ -0,0 +1,41 @@
+\apisummary{
+  Collectively allocate symmetric memory in a specific space.
+}
+
+\begin{apidefinition}
+
+\begin{Csynopsis}
+void *@\FuncDecl{shmem\_space\_malloc}@(shmem_space_t space, size_t size);
+\end{Csynopsis}
+
+\begin{apiarguments}
+  \apiargument{IN}{space}{The space within which to allocate.}
+  \apiargument{IN}{size}{The size, in bytes, of a block to be
+    allocated from the space.}
+\end{apiarguments}
+
+
+\apidescription{
+  The \FUNC{shmem\_space\_malloc} routine is a collective operation on the
+  team associated with \VAR{space} and returns the symmetric address of a
+  block of at least \VAR{size} bytes, which shall be suitably aligned
+  so that it may be assigned to a pointer to any type of object.
+  This block is allocated from the specific symmetric heap identified
+  by \VAR{space}.
+  When \VAR{size} is zero, the \FUNC{shmem\_space\_malloc} routine performs
+  no action and returns a null pointer; otherwise,
+  \FUNC{shmem\_space\_malloc} calls a procedure that is semantically equivalent 
+  to \FUNC{shmem\_team\_sync} on exit. This ensures
+  that all \acp{PE} with access to the space participate
+  in the memory allocation, and that the object on other \acp{PE} can be used as soon as the local
+  \ac{PE} returns. 
+  The values of the \VAR{space} and \VAR{size} arguments must be identical on all
+  \acp{PE}; otherwise, the behavior is undefined.
+}
+
+\apireturnvalues{
+  The \FUNC{shmem\_space\_malloc} routine returns the symmetric address of
+  the allocated space; otherwise, it returns a null pointer.
+}
+
+\end{apidefinition}