From 4d8b25641af78dde4ca8630105f05d7b4db0c84b Mon Sep 17 00:00:00 2001 From: "Lawrence C. Stewart" Date: Mon, 11 Aug 2025 11:22:54 -0400 Subject: [PATCH 01/12] add shmem_space_malloc and shmem_space_free --- content/shmem_space_free.tex | 33 ++++++++++++++++++++++++++ content/shmem_space_malloc.tex | 42 ++++++++++++++++++++++++++++++++++ main_spec.tex | 7 ++++++ 3 files changed, 82 insertions(+) create mode 100644 content/shmem_space_free.tex create mode 100644 content/shmem_space_malloc.tex diff --git a/content/shmem_space_free.tex b/content/shmem_space_free.tex new file mode 100644 index 00000000..7920e047 --- /dev/null +++ b/content/shmem_space_free.tex @@ -0,0 +1,33 @@ +\apisummary{ + Collectively deallocate symmetric memory. +} + +\begin{apidefinition} + +\begin{Csynopsis} +void @\FuncDecl{shmem\_space\_free}@(shmem_space_t space, void *ptr); +\end{Csynopsis} + +\begin{apiarguments} + \apiargument{IN}{ptr}{Symmetric address of an object in the given space.} +\end{apiarguments} + +\apidescription{ + The \FUNC{shmem\_space\_free} routine is a collective operation on the + team associated with \VAR{space} that causes the block to which \VAR{ptr} + points to be deallocated, that is, made available for further + allocation. If \VAR{ptr} is a null pointer, no action is performed; + otherwise, \FUNC{shmem\_space\_free} calls a function equivalent to barrier on entry. + It is the user's responsibility to ensure that no communication + operations involving the given memory block are pending on other + communication contexts prior to calling \FUNC{shmem\_space\_free}. + + The values of the \VAR{space} and \VAR{ptr} arguments must be identical on all + \acp{PE}; otherwise, the behavior is undefined. +} + +\apireturnvalues{ + None. +} + +\end{apidefinition} diff --git a/content/shmem_space_malloc.tex b/content/shmem_space_malloc.tex new file mode 100644 index 00000000..7a587516 --- /dev/null +++ b/content/shmem_space_malloc.tex @@ -0,0 +1,42 @@ +\apisummary{ + Collectively allocate symmetric memory in a specific space. +} + +\begin{apidefinition} + +\begin{Csynopsis} +void *@\FuncDecl{shmem\_space\_malloc}@(shmem_space_t space, size_t size); +\end{Csynopsis} + +\begin{apiarguments} + \apiargument{IN}{space}{The space within which to allocate.} + \apiargument{IN}{size}{The size, in bytes, of a block to be + allocated from the symmetric heap.} +\end{apiarguments} + + +\apidescription{ + The \FUNC{shmem\_space\_malloc} routine is a collective operation on the + \VAR{space} associated team and returns the symmetric address of a + block of at least \VAR{size} bytes, which shall be suitably aligned + so that it may be assigned to a pointer to any type of object. + This block is allocated from the specific symmetric heap identified + by \VAR{space}. + When \VAR{size} is zero, the \FUNC{shmem\_space\_malloc} routine performs + no action and returns a null pointer; otherwise, + \FUNC{shmem\_space\_malloc} calls a procedure that is semantically equivalent + to \FUNC{shmem\_barrier} on exit although only on the set of + \acps{PE} which are part of the \VAR{space} associated team.. This ensures + that all \acp{PE} with access to the space participate + in the memory allocation, and that the memory on other \acp{PE} can be used as soon as the local + \ac{PE} returns. + The value of the \VAR{space} and \VAR{size} arguments must be identical on all + \acp{PE}; otherwise, the behavior is undefined. +} + +\apireturnvalues{ + The \FUNC{shmem\_space\_malloc} routine returns the symmetric address of + the allocated space; otherwise, it returns a null pointer. +} + +\end{apidefinition} diff --git a/main_spec.tex b/main_spec.tex index 6698ad6b..33fe6e57 100644 --- a/main_spec.tex +++ b/main_spec.tex @@ -118,6 +118,13 @@ \subsubsection{\textbf{SHMEM\_MALLOC\_WITH\_HINTS}}\label{subsec:shmmallochint} \subsubsection{\textbf{SHMEM\_CALLOC}}\label{subsec:shmem_calloc} \input{content/shmem_calloc.tex} +\subsubsection{\textbf{SHMEM\_SPACE\_MALLOC}}\label{subsec:shmem_space_malloc} +\input{content/shmem_space_malloc.tex} + +\subsubsection{\textbf{SHMEM\_SPACE\_FREE}}\label{subsec:shmem_space_free} +\input{content/shmem_space_free.tex} + + \subsection{Team Management Routines}\label{subsec:team} From 0ec0cb9fb8251319feb2022ac272fecf0fb84d0b Mon Sep 17 00:00:00 2001 From: lstewart Date: Mon, 11 Aug 2025 14:22:30 -0400 Subject: [PATCH 02/12] add constants and handles --- content/library_constants.tex | 11 +++++++---- content/library_handles.tex | 20 ++++++++++++++++++++ content/shmem_space_free.tex | 3 ++- content/shmem_space_malloc.tex | 11 +++++------ 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/content/library_constants.tex b/content/library_constants.tex index b0edb9cf..65d33b2a 100644 --- a/content/library_constants.tex +++ b/content/library_constants.tex @@ -115,10 +115,13 @@ for more detail about its use. \tabularnewline \hline %% -\LibConstDecl{SHMEM\_MALLOC\_SIGNAL\_REMOTE} & -The hint to the memory allocation routine which specifies that the allocated -memory will be used for signal variables. See Section -\ref{subsec:shmmallochint} for more detail about its use. +\LibConstDecl{SHMEM\_SPACE\_INVALID} & +A value corresponding to an invalid space. +This value can be used to initialize or update space handles to indicate +that they do not reference a valid space. +When managed in this way, applications can use an equality comparison +to test whether a given space handle references a valid space. +See Section~\ref{sec:memory_management} for more detail about its use. \tabularnewline \hline %% \begin{DeprecateBlock} diff --git a/content/library_handles.tex b/content/library_handles.tex index b53242c4..aff6b53c 100644 --- a/content/library_handles.tex +++ b/content/library_handles.tex @@ -41,4 +41,24 @@ See Section~\ref{sec:ctx} for more detail about its use. \tabularnewline \hline %% +\LibHandleDecl{SHMEM\_SPACE\_DEFAULT} & +A memory space handle representing the default symmetric heap. This +space aliases to either \LibHandleRef{SHMEM\_SPACE\_CPU} or\LibHandleRef{SHMEM\_SPACE\_GPU} based +on implementation choice or user configuration via the +\ENVVAR{SHMEM\_DEFAULT\_SPACE} environment variable. This space is +used for memory allocated via shmem\_malloc. +\tabularnewline \hline +%% +\LibHandleDecl{SHMEM\_SPACE\_GPU} & +Handle of type \CTYPE{shmem\_space\_t} that corresponds to a symmetric +heap in GPU (device) memory. If the GPU space is not available +(e.g. disabled or not supported), this handle is set to \LibConstRef{SHMEM\_SPACE\_INVALID}. +\tabularnewline \hline +%% +\LibHandleDecl{SHMEM\_SPACE\_CPU} & +Handle of type \CTYPE{shmem\_space\_t} that corresponds to a symmetric +heap in CPU (host) memory. If the CPU space is not available +(e.g. disabled or not supported), this handle is set to \LibConstRef{SHMEM\_SPACE\_INVALID}. +\tabularnewline \hline +%% \end{longtable} diff --git a/content/shmem_space_free.tex b/content/shmem_space_free.tex index 7920e047..ae6a8b00 100644 --- a/content/shmem_space_free.tex +++ b/content/shmem_space_free.tex @@ -1,5 +1,5 @@ \apisummary{ - Collectively deallocate symmetric memory. + Collectively deallocate symmetric memory in a specific space. } \begin{apidefinition} @@ -9,6 +9,7 @@ \end{Csynopsis} \begin{apiarguments} + \apiargument{IN}{space}{Space handle from which ptr was allocated.} \apiargument{IN}{ptr}{Symmetric address of an object in the given space.} \end{apiarguments} diff --git a/content/shmem_space_malloc.tex b/content/shmem_space_malloc.tex index 7a587516..9107dbe3 100644 --- a/content/shmem_space_malloc.tex +++ b/content/shmem_space_malloc.tex @@ -11,13 +11,13 @@ \begin{apiarguments} \apiargument{IN}{space}{The space within which to allocate.} \apiargument{IN}{size}{The size, in bytes, of a block to be - allocated from the symmetric heap.} + allocated from the space.} \end{apiarguments} \apidescription{ The \FUNC{shmem\_space\_malloc} routine is a collective operation on the - \VAR{space} associated team and returns the symmetric address of a + team associated with \VAR{space} and returns the symmetric address of a block of at least \VAR{size} bytes, which shall be suitably aligned so that it may be assigned to a pointer to any type of object. This block is allocated from the specific symmetric heap identified @@ -25,12 +25,11 @@ When \VAR{size} is zero, the \FUNC{shmem\_space\_malloc} routine performs no action and returns a null pointer; otherwise, \FUNC{shmem\_space\_malloc} calls a procedure that is semantically equivalent - to \FUNC{shmem\_barrier} on exit although only on the set of - \acps{PE} which are part of the \VAR{space} associated team.. This ensures + to \FUNC{shmem\_team\_sync} on exit. This ensures that all \acp{PE} with access to the space participate - in the memory allocation, and that the memory on other \acp{PE} can be used as soon as the local + in the memory allocation, and that the object on other \acp{PE} can be used as soon as the local \ac{PE} returns. - The value of the \VAR{space} and \VAR{size} arguments must be identical on all + The values of the \VAR{space} and \VAR{size} arguments must be identical on all \acp{PE}; otherwise, the behavior is undefined. } From 12d4eda0da6852170cc98bcb6f64aa0a3efa0dbf Mon Sep 17 00:00:00 2001 From: "Lawrence C. Stewart" Date: Mon, 11 Aug 2025 15:45:45 -0400 Subject: [PATCH 03/12] add environment variables --- content/environment_variables.tex | 77 +++++++++++++++++++++++++++++-- content/library_handles.tex | 14 ++++++ 2 files changed, 88 insertions(+), 3 deletions(-) diff --git a/content/environment_variables.tex b/content/environment_variables.tex index 50d9244b..222d62b4 100644 --- a/content/environment_variables.tex +++ b/content/environment_variables.tex @@ -14,7 +14,7 @@ \medskip{} -\begin{longtable}{|p{0.260\textwidth}|p{0.145\textwidth}|p{0.50\textwidth}|} +\begin{longtable}{|p{0.360\textwidth}|p{0.145\textwidth}|p{0.50\textwidth}|} \hline \textbf{Variable} & \textbf{Value} & \textbf{Description} \tabularnewline\hline @@ -31,8 +31,17 @@ %% \EnvVarDecl{SHMEM\_SYMMETRIC\_SIZE} & Non-negative integer or floating point value with an optional character - suffix - & Specifies the size (in bytes) of the symmetric heap memory per \ac{PE}. + suffiximport itertools + +mat = sum([1 for c in itertools.product([1,2,3,4,5,6],repeat=4) if (c[0] + c[1]) == (c[2] + c[3])]) +tot = 6 * 6 * 6 * 6 + +print(tot, mat, mat/tot) + + & Specifies the size (in bytes) of the symmetric heap memory per + \ac{PE}. + This applies to \LibHandleRef{SHMEM\_SPACE\_DEFAULT} and maintains + backward compatibility with existing OpenSHMEM applications. The resulting size is implementation-defined and must be at least as large as the integer ceiling of the product of the numeric prefix and the scaling factor. The allowed character suffixes for the scaling factor are as @@ -56,6 +65,68 @@ \FUNC{shmem\_init\_thread} or causing program termination. \tabularnewline\hline %% +\EnvVarDecl{SHMEM\_CPU\_SYMMETRIC\_SIZE} + & Non-negative integer or floating point value with an optional character + suffiximport itertools + +mat = sum([1 for c in itertools.product([1,2,3,4,5,6],repeat=4) if (c[0] + c[1]) == (c[2] + c[3])]) +tot = 6 * 6 * 6 * 6 + +print(tot, mat, mat/tot) + + & Specifies the size (in bytes) of the CPU symmetric heap memory per \ac{PE}. + The resulting size is implementation-defined and must be at least as large as + the integer ceiling of the product of the numeric prefix and the scaling + factor. The allowed character suffixes for the scaling factor are as + follows: + \begin{itemize} + \item k or K multiplies by \(2^{10}\) (kibibytes) + \item m or M multiplies by \(2^{20}\) (mebibytes) + \item g or G multiplies by \(2^{30}\) (gibibytes) + \item t or T multiplies by \(2^{40}\) (tebibytes) + \end{itemize} + \tabularnewline\hline +%% +\EnvVarDecl{SHMEM\_GPU\_SYMMETRIC\_SIZE} + & Non-negative integer or floating point value with an optional character + suffiximport itertools + +mat = sum([1 for c in itertools.product([1,2,3,4,5,6],repeat=4) if (c[0] + c[1]) == (c[2] + c[3])]) +tot = 6 * 6 * 6 * 6 + +print(tot, mat, mat/tot) + + & Specifies the size (in bytes) of the GPU symmetric heap memory per \ac{PE}. + The resulting size is implementation-defined and must be at least as large as + the integer ceiling of the product of the numeric prefix and the scaling + factor. The allowed character suffixes for the scaling factor are as + follows: + \begin{itemize} + \item k or K multiplies by \(2^{10}\) (kibibytes) + \item m or M multiplies by \(2^{20}\) (mebibytes) + \item g or G multiplies by \(2^{30}\) (gibibytes) + \item t or T multiplies by \(2^{40}\) (tebibytes) + \end{itemize} + \tabularnewline\hline +%% +\EnvVarDecl{SHMEM\_ENABLE\_CPU\_SPACE} + & Any + & Enables CPU symmetric heap. When not set or empty, CPU symmetric + heap is disabled. + \tabularnewline\hline +%% +\EnvVarDecl{SHMEM\_ENABLE\_GPU\_SPACE} + & Any + & Enables GPU symmetric heap. When not set or empty, GPU symmetric + heap is disabled. + \tabularnewline\hline +%% +\EnvVarDecl{SHMEM\_DEFAULT\_SPACE} + & CPU or GPU + & Controls whether \LibHandleRef{SHMEM\_DEFAULT\_SPACE} alias to + \LibHandleRef{SHMEM\_SPACE\_CPU} or \LibHandleRef{SHMEM\_SPACE\_GPU}. + \tabularnewline\hline +%% \EnvVarDecl{SHMEM\_DEBUG} & Any & Enable debugging messages diff --git a/content/library_handles.tex b/content/library_handles.tex index aff6b53c..92ba2096 100644 --- a/content/library_handles.tex +++ b/content/library_handles.tex @@ -21,6 +21,20 @@ See Section~\ref{subsec:team} for more detail about its use. \tabularnewline \hline %% +\LibHandleDecl{SHMEM\_TEAM\_CPU} & +Handle of type \CTYPE{shmem\_team\_t} that corresponds to the +team that can access the CPU symmetric heap. It is available when +\LibHandleRef{SHMEM\_SPACE\_CPU} is enabled. +See Section~\ref{subsec:team} for more detail about its use. +\tabularnewline \hline +%% +\LibHandleDecl{SHMEM\_TEAM\_GPU} & +Handle of type \CTYPE{shmem\_team\_t} that corresponds to the +team that can access the GPU symmetric heap. It is available when +\LibHandleRef{SHMEM\_SPACE\_GPU} is enabled. +See Section~\ref{subsec:team} for more detail about its use. +\tabularnewline \hline +%% \LibHandleDecl{SHMEM\_TEAM\_SHARED} & Handle of type \CTYPE{shmem\_team\_t} that corresponds to a team of \acp{PE} that share a memory domain. \LibHandleRef{SHMEM\_TEAM\_SHARED} refers to From 4c0738ad718ef7d6ebc06550563e39f8c2276e69 Mon Sep 17 00:00:00 2001 From: Michael Beebe Date: Fri, 15 Aug 2025 14:24:17 -0500 Subject: [PATCH 04/12] - Updated spaces constants, library handles, and env vars to be device-agnostic. - Added spaces description to programming model overview. - Added a spaces intro. --- .gitignore | 3 + content/environment_variables.tex | 108 +++++++------------------ content/library_constants.tex | 2 +- content/library_handles.tex | 45 +++++------ content/programming_model_overview.tex | 11 +++ content/spaces_intro.tex | 0 main_spec.tex | 14 ++-- 7 files changed, 67 insertions(+), 116 deletions(-) create mode 100644 content/spaces_intro.tex diff --git a/.gitignore b/.gitignore index e81729c9..b6c37a15 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,7 @@ main_spec.log main_spec.out main_spec.pdf main_spec.toc +main_spec.fdb_latexmk +main_spec.fls +main_spec.synctex.gz *~ diff --git a/content/environment_variables.tex b/content/environment_variables.tex index 222d62b4..fad5fd7c 100644 --- a/content/environment_variables.tex +++ b/content/environment_variables.tex @@ -14,7 +14,7 @@ \medskip{} -\begin{longtable}{|p{0.360\textwidth}|p{0.145\textwidth}|p{0.50\textwidth}|} +\begin{longtable}{|p{0.370\textwidth}|p{0.145\textwidth}|p{0.390\textwidth}|} \hline \textbf{Variable} & \textbf{Value} & \textbf{Description} \tabularnewline\hline @@ -31,100 +31,46 @@ %% \EnvVarDecl{SHMEM\_SYMMETRIC\_SIZE} & Non-negative integer or floating point value with an optional character - suffiximport itertools - -mat = sum([1 for c in itertools.product([1,2,3,4,5,6],repeat=4) if (c[0] + c[1]) == (c[2] + c[3])]) -tot = 6 * 6 * 6 * 6 - -print(tot, mat, mat/tot) - - & Specifies the size (in bytes) of the symmetric heap memory per - \ac{PE}. - This applies to \LibHandleRef{SHMEM\_SPACE\_DEFAULT} and maintains - backward compatibility with existing OpenSHMEM applications. - The resulting size is implementation-defined and must be at least as large as - the integer ceiling of the product of the numeric prefix and the scaling - factor. The allowed character suffixes for the scaling factor are as - follows: + suffix + & Specifies the size (in bytes) of the symmetric heap memory per \ac{PE} + for \LibHandleRef{SHMEM\_SPACE\_DEFAULT} to maintain backward compatibility + with existing OpenSHMEM applications. The resulting size is + implementation-defined and must be at least as large as the integer + ceiling of the product of the numeric prefix and the scaling factor. + The allowed character suffixes for the scaling factor are: \begin{itemize} \item k or K multiplies by \(2^{10}\) (kibibytes) \item m or M multiplies by \(2^{20}\) (mebibytes) \item g or G multiplies by \(2^{30}\) (gibibytes) \item t or T multiplies by \(2^{40}\) (tebibytes) \end{itemize} - For example, string \enquote{20m} is equivalent to the integer value - 20971520, or 20 mebibytes. Similarly the string \enquote{3.1M} is - equivalent to the integer value 3250586. - Only one multiplier is recognized and any characters following the - multiplier are ignored, so \enquote{20kk} will not produce the same - result as \enquote{20m}. Usage of string \enquote{.5m} will yield the - same result as the string \enquote{0.5m}. - - An invalid value for \ENVVAR{SHMEM\_SYMMETRIC\_SIZE} is an error, which the - \openshmem library shall report by either returning a nonzero value from - \FUNC{shmem\_init\_thread} or causing program termination. + An invalid value is an error, which the \openshmem library shall report + by either returning a nonzero value from \FUNC{shmem\_init\_thread} or + causing program termination. \tabularnewline\hline %% -\EnvVarDecl{SHMEM\_CPU\_SYMMETRIC\_SIZE} +\EnvVarDecl{SHMEM\_\_SYMMETRIC\_SIZE} & Non-negative integer or floating point value with an optional character - suffiximport itertools - -mat = sum([1 for c in itertools.product([1,2,3,4,5,6],repeat=4) if (c[0] + c[1]) == (c[2] + c[3])]) -tot = 6 * 6 * 6 * 6 - -print(tot, mat, mat/tot) - - & Specifies the size (in bytes) of the CPU symmetric heap memory per \ac{PE}. - The resulting size is implementation-defined and must be at least as large as - the integer ceiling of the product of the numeric prefix and the scaling - factor. The allowed character suffixes for the scaling factor are as - follows: - \begin{itemize} - \item k or K multiplies by \(2^{10}\) (kibibytes) - \item m or M multiplies by \(2^{20}\) (mebibytes) - \item g or G multiplies by \(2^{30}\) (gibibytes) - \item t or T multiplies by \(2^{40}\) (tebibytes) - \end{itemize} + suffix + & Specifies the size (in bytes) of the symmetric heap memory per \ac{PE} + for \LibHandleRef{SHMEM\_SPACE\_}. The resulting size is + implementation-defined and must be at least as large as the integer + ceiling of the product of the numeric prefix and the scaling factor. + The allowed character suffixes are as in \ENVVAR{SHMEM\_SYMMETRIC\_SIZE}. + Device tokens are implementation-defined (e.g., CPU, GPU, HBM, SmartNIC). \tabularnewline\hline %% -\EnvVarDecl{SHMEM\_GPU\_SYMMETRIC\_SIZE} - & Non-negative integer or floating point value with an optional character - suffiximport itertools - -mat = sum([1 for c in itertools.product([1,2,3,4,5,6],repeat=4) if (c[0] + c[1]) == (c[2] + c[3])]) -tot = 6 * 6 * 6 * 6 - -print(tot, mat, mat/tot) - - & Specifies the size (in bytes) of the GPU symmetric heap memory per \ac{PE}. - The resulting size is implementation-defined and must be at least as large as - the integer ceiling of the product of the numeric prefix and the scaling - factor. The allowed character suffixes for the scaling factor are as - follows: - \begin{itemize} - \item k or K multiplies by \(2^{10}\) (kibibytes) - \item m or M multiplies by \(2^{20}\) (mebibytes) - \item g or G multiplies by \(2^{30}\) (gibibytes) - \item t or T multiplies by \(2^{40}\) (tebibytes) - \end{itemize} - \tabularnewline\hline -%% -\EnvVarDecl{SHMEM\_ENABLE\_CPU\_SPACE} +\EnvVarDecl{SHMEM\_ENABLE\_} & Any - & Enables CPU symmetric heap. When not set or empty, CPU symmetric - heap is disabled. - \tabularnewline\hline -%% -\EnvVarDecl{SHMEM\_ENABLE\_GPU\_SPACE} - & Any - & Enables GPU symmetric heap. When not set or empty, GPU symmetric - heap is disabled. + & Enables the symmetric heap for \LibHandleRef{SHMEM\_SPACE\_}. + When not set or empty for a given \textit{}, that device's + symmetric heap is disabled. Device tokens are implementation-defined. \tabularnewline\hline %% \EnvVarDecl{SHMEM\_DEFAULT\_SPACE} - & CPU or GPU - & Controls whether \LibHandleRef{SHMEM\_DEFAULT\_SPACE} alias to - \LibHandleRef{SHMEM\_SPACE\_CPU} or \LibHandleRef{SHMEM\_SPACE\_GPU}. + & \textit{} + & Controls which space \LibHandleRef{SHMEM\_SPACE\_DEFAULT} aliases to. + If not set, the implementation chooses the default space. \tabularnewline\hline %% \EnvVarDecl{SHMEM\_DEBUG} @@ -133,4 +79,4 @@ \tabularnewline\hline \end{longtable} -\medskip{} +\medskip{} \ No newline at end of file diff --git a/content/library_constants.tex b/content/library_constants.tex index 65d33b2a..fd516ee8 100644 --- a/content/library_constants.tex +++ b/content/library_constants.tex @@ -121,7 +121,7 @@ that they do not reference a valid space. When managed in this way, applications can use an equality comparison to test whether a given space handle references a valid space. -See Section~\ref{sec:memory_management} for more detail about its use. +See Section~\ref{subsec:space} for more detail about its use. \tabularnewline \hline %% \begin{DeprecateBlock} diff --git a/content/library_handles.tex b/content/library_handles.tex index 92ba2096..3d28f29d 100644 --- a/content/library_handles.tex +++ b/content/library_handles.tex @@ -21,20 +21,6 @@ See Section~\ref{subsec:team} for more detail about its use. \tabularnewline \hline %% -\LibHandleDecl{SHMEM\_TEAM\_CPU} & -Handle of type \CTYPE{shmem\_team\_t} that corresponds to the -team that can access the CPU symmetric heap. It is available when -\LibHandleRef{SHMEM\_SPACE\_CPU} is enabled. -See Section~\ref{subsec:team} for more detail about its use. -\tabularnewline \hline -%% -\LibHandleDecl{SHMEM\_TEAM\_GPU} & -Handle of type \CTYPE{shmem\_team\_t} that corresponds to the -team that can access the GPU symmetric heap. It is available when -\LibHandleRef{SHMEM\_SPACE\_GPU} is enabled. -See Section~\ref{subsec:team} for more detail about its use. -\tabularnewline \hline -%% \LibHandleDecl{SHMEM\_TEAM\_SHARED} & Handle of type \CTYPE{shmem\_team\_t} that corresponds to a team of \acp{PE} that share a memory domain. \LibHandleRef{SHMEM\_TEAM\_SHARED} refers to @@ -47,6 +33,14 @@ See Section~\ref{subsec:team} for more detail about its use. \tabularnewline \hline %% +\LibHandleDecl{SHMEM\_TEAM\_} & +Handle of type \CTYPE{shmem\_team\_t} that corresponds to the +team of \acp{PE} that can access the symmetric heap for \textit{}. +It is available when \LibHandleRef{SHMEM\_SPACE\_} is enabled and accessible. +Device tokens are implementation-defined (e.g., CPU, GPU, etc.). +See Section~\ref{subsec:space} for more detail about its use. +\tabularnewline \hline +%% \LibHandleDecl{SHMEM\_CTX\_DEFAULT} & Handle of type \CTYPE{shmem\_ctx\_t} that corresponds to the default communication context. All point-to-point communication operations @@ -57,22 +51,19 @@ %% \LibHandleDecl{SHMEM\_SPACE\_DEFAULT} & A memory space handle representing the default symmetric heap. This -space aliases to either \LibHandleRef{SHMEM\_SPACE\_CPU} or\LibHandleRef{SHMEM\_SPACE\_GPU} based -on implementation choice or user configuration via the -\ENVVAR{SHMEM\_DEFAULT\_SPACE} environment variable. This space is -used for memory allocated via shmem\_malloc. -\tabularnewline \hline -%% -\LibHandleDecl{SHMEM\_SPACE\_GPU} & -Handle of type \CTYPE{shmem\_space\_t} that corresponds to a symmetric -heap in GPU (device) memory. If the GPU space is not available -(e.g. disabled or not supported), this handle is set to \LibConstRef{SHMEM\_SPACE\_INVALID}. +space aliases to some \LibHandleRef{SHMEM\_SPACE\_} based +on implementation choice or user configuration. This space is +used for memory allocated via \FUNC{shmem\_malloc} and \FUNC{shmem\_calloc}. +See Section~\ref{subsec:space} for more detail about its use. \tabularnewline \hline %% -\LibHandleDecl{SHMEM\_SPACE\_CPU} & +\LibHandleDecl{SHMEM\_SPACE\_} & Handle of type \CTYPE{shmem\_space\_t} that corresponds to a symmetric -heap in CPU (host) memory. If the CPU space is not available -(e.g. disabled or not supported), this handle is set to \LibConstRef{SHMEM\_SPACE\_INVALID}. +heap in the memory domain associated with \textit{}. If the space +for \textit{} is not available (e.g., disabled or not supported), this handle +is set to \LibConstRef{SHMEM\_SPACE\_INVALID}. Device tokens are implementation-defined +(e.g., CPU, GPU, HBM, SmartNIC). +See Section~\ref{subsec:space} for more detail about its use. \tabularnewline \hline %% \end{longtable} diff --git a/content/programming_model_overview.tex b/content/programming_model_overview.tex index e30ef792..040e6be0 100644 --- a/content/programming_model_overview.tex +++ b/content/programming_model_overview.tex @@ -72,6 +72,17 @@ numbering through team-based contexts. \end{enumerate} +\item \textbf{Space Management} +\begin{enumerate} + % TODO: refine the description of spaces + \item \OPR{Spaces}: Spaces are containers for symmetric memory in specific memory + domains. They provide the storage for \openshmem symmetric data objects and + define accessibility and operational properties of those objects. Allocations + within a space are collective and symmetric across the \acp{PE} for which the + space is available. Collective and communication operations operate on data + resident in one or more spaces as permitted by the implementation +\end{enumerate} + \item \textbf{\acf{RMA}} \begin{enumerate} \item \PUT: The local \ac{PE} specifies the \source{} data object, private diff --git a/content/spaces_intro.tex b/content/spaces_intro.tex new file mode 100644 index 00000000..e69de29b diff --git a/main_spec.tex b/main_spec.tex index 33fe6e57..6c04434e 100644 --- a/main_spec.tex +++ b/main_spec.tex @@ -118,12 +118,6 @@ \subsubsection{\textbf{SHMEM\_MALLOC\_WITH\_HINTS}}\label{subsec:shmmallochint} \subsubsection{\textbf{SHMEM\_CALLOC}}\label{subsec:shmem_calloc} \input{content/shmem_calloc.tex} -\subsubsection{\textbf{SHMEM\_SPACE\_MALLOC}}\label{subsec:shmem_space_malloc} -\input{content/shmem_space_malloc.tex} - -\subsubsection{\textbf{SHMEM\_SPACE\_FREE}}\label{subsec:shmem_space_free} -\input{content/shmem_space_free.tex} - @@ -156,7 +150,6 @@ \subsubsection{\textbf{SHMEM\_TEAM\_DESTROY}}\label{subsec:shmem_team_destroy} \input{content/shmem_team_destroy.tex} - \subsection{Communication Management Routines}\label{sec:ctx} \input{content/context_intro.tex} @@ -212,7 +205,14 @@ \subsubsubsection{\textbf{SHMEM\_PUT\_NBI}}\label{subsec:shmem_put_nbi} \subsubsubsection{\textbf{SHMEM\_GET\_NBI}}\label{subsec:shmem_get_nbi} \input{content/shmem_get_nbi.tex} +\subsection{Space Management Routines}\label{subsec:space} +\input{content/spaces_intro.tex} + +\subsubsection{\textbf{SHMEM\_SPACE\_MALLOC}}\label{subsec:shmem_space_malloc} +\input{content/shmem_space_malloc.tex} +\subsubsection{\textbf{SHMEM\_SPACE\_FREE}}\label{subsec:shmem_space_free} +\input{content/shmem_space_free.tex} \subsection{Atomic Memory Operations}\label{sec:amo} \input{content/atomics_intro} From 3f0f412477678299f2d740afbf514c3b1d8cdca0 Mon Sep 17 00:00:00 2001 From: Michael Beebe Date: Fri, 15 Aug 2025 17:03:07 -0500 Subject: [PATCH 05/12] Spaces: add SHMEM_SPACE_CALLOC; align spaces intro with env vars; clarify default alias vs multiple enabled spaces; legacy allocs are SHMEM_TEAM_WORLD-collective - Added content/shmem_space_calloc.tex and included in main_spec.tex - Aligned spaces intro with env vars: SHMEM__SYMMETRIC_SIZE, SHMEM_ENABLE_, SHMEM_DEFAULT_SPACE - Clarified SHMEM_SPACE_DEFAULT is a single alias; multiple spaces may be enabled concurrently - Focused teams intro on SHMEM_TEAM_; moved space details to spaces_intro.tex - Scoped underscore wrapping to \ENVVAR{} only to fix env-var wrapping --- content/environment_variables.tex | 5 +- content/library_handles.tex | 5 +- content/shmem_space_calloc.tex | 46 +++++++++++++++ content/spaces_intro.tex | 93 +++++++++++++++++++++++++++++++ content/teams_intro.tex | 23 ++++++++ main_spec.tex | 29 +++++++--- 6 files changed, 186 insertions(+), 15 deletions(-) create mode 100644 content/shmem_space_calloc.tex diff --git a/content/environment_variables.tex b/content/environment_variables.tex index fad5fd7c..48e9195e 100644 --- a/content/environment_variables.tex +++ b/content/environment_variables.tex @@ -33,8 +33,7 @@ & Non-negative integer or floating point value with an optional character suffix & Specifies the size (in bytes) of the symmetric heap memory per \ac{PE} - for \LibHandleRef{SHMEM\_SPACE\_DEFAULT} to maintain backward compatibility - with existing OpenSHMEM applications. The resulting size is + for \LibHandleRef{SHMEM\_SPACE\_DEFAULT}. The resulting size is implementation-defined and must be at least as large as the integer ceiling of the product of the numeric prefix and the scaling factor. The allowed character suffixes for the scaling factor are: @@ -57,7 +56,7 @@ implementation-defined and must be at least as large as the integer ceiling of the product of the numeric prefix and the scaling factor. The allowed character suffixes are as in \ENVVAR{SHMEM\_SYMMETRIC\_SIZE}. - Device tokens are implementation-defined (e.g., CPU, GPU, HBM, SmartNIC). + Device tokens are implementation-defined. \tabularnewline\hline %% \EnvVarDecl{SHMEM\_ENABLE\_} diff --git a/content/library_handles.tex b/content/library_handles.tex index 3d28f29d..9c6157c4 100644 --- a/content/library_handles.tex +++ b/content/library_handles.tex @@ -37,7 +37,7 @@ Handle of type \CTYPE{shmem\_team\_t} that corresponds to the team of \acp{PE} that can access the symmetric heap for \textit{}. It is available when \LibHandleRef{SHMEM\_SPACE\_} is enabled and accessible. -Device tokens are implementation-defined (e.g., CPU, GPU, etc.). +Device tokens are implementation-defined. See Section~\ref{subsec:space} for more detail about its use. \tabularnewline \hline %% @@ -61,8 +61,7 @@ Handle of type \CTYPE{shmem\_space\_t} that corresponds to a symmetric heap in the memory domain associated with \textit{}. If the space for \textit{} is not available (e.g., disabled or not supported), this handle -is set to \LibConstRef{SHMEM\_SPACE\_INVALID}. Device tokens are implementation-defined -(e.g., CPU, GPU, HBM, SmartNIC). +is set to \LibConstRef{SHMEM\_SPACE\_INVALID}. Device tokens are implementation-defined. See Section~\ref{subsec:space} for more detail about its use. \tabularnewline \hline %% diff --git a/content/shmem_space_calloc.tex b/content/shmem_space_calloc.tex new file mode 100644 index 00000000..d04a9a73 --- /dev/null +++ b/content/shmem_space_calloc.tex @@ -0,0 +1,46 @@ +\apisummary{ + Collectively allocate a zeroed block of symmetric memory in a specific space. +} + +\begin{apidefinition} + +\begin{Csynopsis} +void *@\FuncDecl{shmem\_space\_calloc}@(shmem_space_t space, size_t count, size_t size); +\end{Csynopsis} + +\begin{apiarguments} + \apiargument{IN}{space}{The space within which to allocate.} + \apiargument{IN}{count}{The number of elements to allocate.} + \apiargument{IN}{size}{The size in bytes of each element to allocate.} +\end{apiarguments} + +\apidescription{ + The \FUNC{shmem\_space\_calloc} routine is a collective operation on the + team associated with \VAR{space}. It allocates a region of + remotely-accessible symmetric memory for an array of \VAR{count} objects of + \VAR{size} bytes each from the symmetric heap identified by \VAR{space}, and + returns the symmetric address of the lowest byte of the allocated symmetric + memory. The space is initialized to all bits zero. + + If the allocation succeeds, the pointer returned shall be suitably aligned so + that it may be assigned to a pointer to any type of object. If the allocation + does not succeed, or either \VAR{count} or \VAR{size} is \CONST{0}, the + return value is a null pointer. + + The values of the \VAR{space}, \VAR{count}, and \VAR{size} arguments must be + identical on all \acp{PE}; otherwise, the behavior is undefined. + + When \VAR{count} or \VAR{size} is \CONST{0}, the \FUNC{shmem\_space\_calloc} + routine returns without performing a barrier; otherwise, + \FUNC{shmem\_space\_calloc} calls a procedure that is semantically + equivalent to \FUNC{shmem\_team\_sync} on exit. +} + +\apireturnvalues{ + The \FUNC{shmem\_space\_calloc} routine returns the symmetric address of the + allocated space; otherwise, it returns a null pointer. +} + +\end{apidefinition} + + diff --git a/content/spaces_intro.tex b/content/spaces_intro.tex index e69de29b..f41756de 100644 --- a/content/spaces_intro.tex +++ b/content/spaces_intro.tex @@ -0,0 +1,93 @@ +The \openshmem spaces facility introduces named symmetric heaps associated with +particular memory domains, referred to as \emph{device spaces}. These spaces +enable applications to allocate symmetric memory in heterogeneous memory domains. + +\subsubsection*{Default Space and Backward Compatibility} + +Implementations may provide a default space that legacy allocation routines use +for backward compatibility. Legacy allocation routines are collective +operations over \LibHandleRef{SHMEM\_TEAM\_WORLD}. When a default space is +provided, the traditional \FUNC{shmem\_malloc} and \FUNC{shmem\_calloc} +routines allocate from the +default symmetric heap, which is accessed through the +\LibHandleRef{SHMEM\_SPACE\_DEFAULT} handle. The default space may alias to an +implementation-selected device space \LibHandleRef{SHMEM\_SPACE\_} +based on implementation choice or user configuration via +\ENVVAR{SHMEM\_DEFAULT\_SPACE}. If not set, the implementation chooses any +default space it provides. When a default space exists, it shall be +world-accessible (its associated team equals \LibHandleRef{SHMEM\_TEAM\_WORLD}) +to preserve the semantics of legacy collectives; otherwise, \openshmem +initialization shall fail. + +\subsubsection*{Predefined Memory Spaces} + +An \openshmem memory space may be predefined (i.e., provided by the +\openshmem library). Implementations choose which spaces they support; they are +not required to support any particular space. All predefined memory spaces are +valid for the duration of the \openshmem portion of an application. All valid +memory spaces contain at least one symmetric memory region. + +\subsubsection*{Space Handles} + +A memory space handle is an opaque object with type \CTYPE{shmem\_space\_t} +that is used to reference a memory space. Memory space handles are not +remotely accessible objects. The predefined memory spaces may be accessed via +handles such as \LibHandleRef{SHMEM\_SPACE\_DEFAULT} and +\LibHandleRef{SHMEM\_SPACE\_}. A memory space handle may be +initialized to or assigned the value \LibConstRef{SHMEM\_SPACE\_INVALID} to +indicate that the handle does not reference a valid memory space. Applications +can use an equality comparison to test whether a given space handle references +a valid memory space. + +\subsubsection*{Memory Space Properties and Sizing} + +Each memory space can have associated properties including size limits and +accessibility constraints. + +\begin{itemize} + \item \ENVVAR{SHMEM\_SYMMETRIC\_SIZE}: sizes the default symmetric heap + (\LibHandleRef{SHMEM\_SPACE\_DEFAULT}). + \item \ENVVAR{SHMEM\_\_SYMMETRIC\_SIZE}: sizes the symmetric heap + for \LibHandleRef{SHMEM\_SPACE\_} per \ac{PE}. The allowed suffixes + and error semantics are as described for \ENVVAR{SHMEM\_SYMMETRIC\_SIZE}. +\end{itemize} + +When \LibHandleRef{SHMEM\_SPACE\_DEFAULT} aliases to +\LibHandleRef{SHMEM\_SPACE\_}, the +\ENVVAR{SHMEM\_\_SYMMETRIC\_SIZE} environment variable takes +precedence over \ENVVAR{SHMEM\_SYMMETRIC\_SIZE} if both are set. If no size +environment variables are set for a space, the implementation chooses the +default size for that space. + +\subsubsection*{Space Enablement and Availability} + +Implementations may enable one or more spaces by default at initialization +under environment variable control. Enabling a space makes its handle valid and +its symmetric heap available according to sizing variables. A space may be +designated as the default alias \LibHandleRef{SHMEM\_SPACE\_DEFAULT}; when a +default exists, legacy routines (\FUNC{shmem\_malloc}, \FUNC{shmem\_calloc}) +operate on that default alias. +The default alias refers to exactly one enabled space at any time; it does not +span multiple spaces. Multiple spaces may be enabled concurrently, but only one +may be the default alias. + +- \ENVVAR{SHMEM\_ENABLE\_}: enables the symmetric heap for the + named device space for the process. When not set or empty for a given + \textit{}, that device's symmetric heap is disabled. + +\LibHandleRef{SHMEM\_SPACE\_} is available when the corresponding +device space is enabled and the device or memory domain for \textit{} +is detected and accessible. When a space is not available (e.g., disabled or +not supported), the corresponding space handle is set to +\LibConstRef{SHMEM\_SPACE\_INVALID}. All environment variables that control +memory space enablement, sizing, and default selection shall have identical +values across all \acp{PE} in the job; otherwise, \openshmem initialization +shall fail. + +\subsubsection*{Relationship to Teams} + +When a device space is enabled and accessible, the corresponding predefined +team \LibHandleRef{SHMEM\_TEAM\_} contains exactly those \acp{PE} that +can access that space. See Section~\ref{subsec:team} for discussion of +device-specific teams. + diff --git a/content/teams_intro.tex b/content/teams_intro.tex index 1b597846..8ab66c8b 100644 --- a/content/teams_intro.tex +++ b/content/teams_intro.tex @@ -100,3 +100,26 @@ \subsubsection*{Team Creation} Upon completion of a team creation operation, the parent and any resulting child teams will be immediately usable for any team-based operations, including creating new child teams, without any intervening synchronization. + +\subsubsection*{Device-Specific Teams} + +The handle \LibHandleRef{SHMEM\_TEAM\_} refers to the team of \acp{PE} +that can access the symmetric heap associated with the corresponding +\LibHandleRef{SHMEM\_SPACE\_}. Membership in this team is +implementation-defined and reflects hardware capability and runtime setup. + +When the corresponding device space is usable, the implementation initializes +\LibHandleRef{SHMEM\_TEAM\_} to include exactly those \acp{PE} for +which the device space is accessible. A \ac{PE} that cannot access the device +space is not a member of this team. If the device space is unavailable, the +handle value is \LibConstRef{SHMEM\_TEAM\_INVALID}. See +Section~\ref{subsec:space} for details of device spaces and their +configuration. + +Applications may use \LibHandleRef{SHMEM\_TEAM\_} to scope collective +operations or to create contexts and perform point-to-point operations among +only those \acp{PE} that share access to the corresponding device space. All +collective semantics and ordering requirements discussed above apply to these +device-specific teams; a collective on \LibHandleRef{SHMEM\_TEAM\_} +must be invoked by all \acp{PE} that are members of that team and by no +others. diff --git a/main_spec.tex b/main_spec.tex index 6c04434e..6079e993 100644 --- a/main_spec.tex +++ b/main_spec.tex @@ -150,6 +150,8 @@ \subsubsection{\textbf{SHMEM\_TEAM\_DESTROY}}\label{subsec:shmem_team_destroy} \input{content/shmem_team_destroy.tex} + + \subsection{Communication Management Routines}\label{sec:ctx} \input{content/context_intro.tex} @@ -169,6 +171,24 @@ \subsubsection{\textbf{SHMEM\_CTX\_GET\_TEAM}} \label{subsec:shmem_ctx_get_team} \input{content/shmem_ctx_get_team.tex} + + + +\subsection{Space Management Routines}\label{subsec:space} +\input{content/spaces_intro.tex} + +\subsubsection{\textbf{SHMEM\_SPACE\_MALLOC}}\label{subsec:shmem_space_malloc} +\input{content/shmem_space_malloc.tex} + +\subsubsection{\textbf{SHMEM\_SPACE\_CALLOC}}\label{subsec:shmem_space_calloc} +\input{content/shmem_space_calloc.tex} + +\subsubsection{\textbf{SHMEM\_SPACE\_FREE}}\label{subsec:shmem_space_free} +\input{content/shmem_space_free.tex} + + + + \subsection{Remote Memory Access Routines}\label{sec:rma} \input{content/rma_intro.tex} @@ -205,15 +225,6 @@ \subsubsubsection{\textbf{SHMEM\_PUT\_NBI}}\label{subsec:shmem_put_nbi} \subsubsubsection{\textbf{SHMEM\_GET\_NBI}}\label{subsec:shmem_get_nbi} \input{content/shmem_get_nbi.tex} -\subsection{Space Management Routines}\label{subsec:space} -\input{content/spaces_intro.tex} - -\subsubsection{\textbf{SHMEM\_SPACE\_MALLOC}}\label{subsec:shmem_space_malloc} -\input{content/shmem_space_malloc.tex} - -\subsubsection{\textbf{SHMEM\_SPACE\_FREE}}\label{subsec:shmem_space_free} -\input{content/shmem_space_free.tex} - \subsection{Atomic Memory Operations}\label{sec:amo} \input{content/atomics_intro} From c930e771f1874f0ec37a706d971fddf2f4964a58 Mon Sep 17 00:00:00 2001 From: Michael Beebe Date: Fri, 15 Aug 2025 17:07:12 -0500 Subject: [PATCH 06/12] Updated memmgmt_intro.tex to include spaces --- content/memmgmt_intro.tex | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/content/memmgmt_intro.tex b/content/memmgmt_intro.tex index 660e5bc7..ed145a83 100644 --- a/content/memmgmt_intro.tex +++ b/content/memmgmt_intro.tex @@ -1,23 +1,43 @@ -\openshmem provides a set of \acp{API} for managing the symmetric heap. The +\openshmem provides a set of \acp{API} for managing symmetric heaps. The \acp{API} allow one to dynamically allocate, deallocate, reallocate, and align -symmetric data objects in the symmetric heap. +symmetric data objects in these heaps. -The symmetric memory allocation routines differ from the private heap +In addition to the traditional (legacy) symmetric heap used by +\FUNC{shmem\_malloc}, \FUNC{shmem\_calloc}, \FUNC{shmem\_free}, and +\FUNC{shmem\_realloc}, an implementation may provide multiple symmetric heaps +associated with memory \emph{spaces} (see Section~\ref{subsec:space}). The +legacy routines operate on the default symmetric heap, which aliases +\LibHandleRef{SHMEM\_SPACE\_DEFAULT}, while the space-specific routines +\FUNC{shmem\_space\_malloc}, \FUNC{shmem\_space\_calloc}, and +\FUNC{shmem\_space\_free} operate on the symmetric heap associated with a +given \CTYPE{shmem\_space\_t} handle. + +The legacy symmetric memory allocation routines differ from the private heap allocation routines in that they must be called by all \acp{PE} in the world team. When specified, each of these routines includes at least one call to a procedure that is semantically equivalent to \FUNC{shmem\_barrier\_all}. This ensures that all \acp{PE} participate in the memory management, and that the memory on other \acp{PE} can be used as soon as the local \ac{PE} returns. The -implicit barriers performed by these routines quiet the default +implicit barriers performed by these legacy routines quiet the default context. It is the user's responsibility to ensure that no communication operations involving the given memory block are pending on other contexts prior to calling the \FUNC{shmem\_free} and \FUNC{shmem\_realloc} routines. -The total size of the symmetric heap is determined at job startup. One can -specify the size of the heap using the \ENVVAR{SHMEM\_SYMMETRIC\_SIZE} environment -variable (where available). +The space-specific allocation routines are collective on the team +associated with the space handle and, where applicable, include a call +that is semantically equivalent to \FUNC{shmem\_team\_sync} on exit. + +The total size of each symmetric heap is determined at job startup. The size of +the default symmetric heap can be controlled with the +\ENVVAR{SHMEM\_SYMMETRIC\_SIZE} environment variable (where available). The +sizes of device-specific spaces can be controlled with +\ENVVAR{SHMEM\_\_SYMMETRIC\_SIZE}; when +\LibHandleRef{SHMEM\_SPACE\_DEFAULT} aliases to +\LibHandleRef{SHMEM\_SPACE\_}, the device-specific variable takes +precedence. Availability of device spaces may be controlled with +\ENVVAR{SHMEM\_ENABLE\_}. \begin{DeprecateBlock} As of \openshmem[1.2] the use of \FUNC{shmalloc}, \FUNC{shmemalign}, From 8076b8cfe40ba9cfc7465d4aa700bacc4a454ca1 Mon Sep 17 00:00:00 2001 From: Michael Beebe Date: Fri, 15 Aug 2025 17:43:48 -0500 Subject: [PATCH 07/12] Clarified some things in spaces_intro.tex --- content/spaces_intro.tex | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/content/spaces_intro.tex b/content/spaces_intro.tex index f41756de..766aacc4 100644 --- a/content/spaces_intro.tex +++ b/content/spaces_intro.tex @@ -1,6 +1,6 @@ The \openshmem spaces facility introduces named symmetric heaps associated with -particular memory domains, referred to as \emph{device spaces}. These spaces -enable applications to allocate symmetric memory in heterogeneous memory domains. +particular memory domains, referred to as \emph{spaces}. The set of supported +domains and any associated tokens are implementation-defined. \subsubsection*{Default Space and Backward Compatibility} @@ -13,7 +13,8 @@ \subsubsection*{Default Space and Backward Compatibility} \LibHandleRef{SHMEM\_SPACE\_DEFAULT} handle. The default space may alias to an implementation-selected device space \LibHandleRef{SHMEM\_SPACE\_} based on implementation choice or user configuration via -\ENVVAR{SHMEM\_DEFAULT\_SPACE}. If not set, the implementation chooses any +\ENVVAR{SHMEM\_DEFAULT\_SPACE}. The value of this variable is an implementation-defined +token (\textit{}). If not set, the implementation chooses any default space it provides. When a default space exists, it shall be world-accessible (its associated team equals \LibHandleRef{SHMEM\_TEAM\_WORLD}) to preserve the semantics of legacy collectives; otherwise, \openshmem @@ -37,19 +38,23 @@ \subsubsection*{Space Handles} initialized to or assigned the value \LibConstRef{SHMEM\_SPACE\_INVALID} to indicate that the handle does not reference a valid memory space. Applications can use an equality comparison to test whether a given space handle references -a valid memory space. +a valid memory space. See Section~\ref{subsec:library_handles} for predefined +handles and Section~\ref{sec:memory_management} for allocation interfaces. \subsubsection*{Memory Space Properties and Sizing} Each memory space can have associated properties including size limits and -accessibility constraints. +accessibility constraints. Configuration is controlled by environment variables: \begin{itemize} \item \ENVVAR{SHMEM\_SYMMETRIC\_SIZE}: sizes the default symmetric heap (\LibHandleRef{SHMEM\_SPACE\_DEFAULT}). \item \ENVVAR{SHMEM\_\_SYMMETRIC\_SIZE}: sizes the symmetric heap - for \LibHandleRef{SHMEM\_SPACE\_} per \ac{PE}. The allowed suffixes - and error semantics are as described for \ENVVAR{SHMEM\_SYMMETRIC\_SIZE}. + for \LibHandleRef{SHMEM\_SPACE\_} per \ac{PE}. The allowed suffixes + and error semantics are as described for \ENVVAR{SHMEM\_SYMMETRIC\_SIZE}. + \item \ENVVAR{SHMEM\_ENABLE\_}: enables the symmetric heap for the + named space. When not set or empty for a given \textit{}, that + space is disabled for the process. \end{itemize} When \LibHandleRef{SHMEM\_SPACE\_DEFAULT} aliases to @@ -59,11 +64,17 @@ \subsubsection*{Memory Space Properties and Sizing} environment variables are set for a space, the implementation chooses the default size for that space. +For a discussion of legacy versus space-specific allocation routines, see +Section~\ref{sec:memory_management}. + \subsubsection*{Space Enablement and Availability} Implementations may enable one or more spaces by default at initialization -under environment variable control. Enabling a space makes its handle valid and -its symmetric heap available according to sizing variables. A space may be +under environment variable control. During library initialization +(\FUNC{shmem\_init} or \FUNC{shmem\_init\_thread}), enabled spaces are created +and their symmetric heaps allocated according to the configuration. Enabling a +space makes its handle valid and its symmetric heap available according to +sizing variables. A space may be designated as the default alias \LibHandleRef{SHMEM\_SPACE\_DEFAULT}; when a default exists, legacy routines (\FUNC{shmem\_malloc}, \FUNC{shmem\_calloc}) operate on that default alias. @@ -71,10 +82,6 @@ \subsubsection*{Space Enablement and Availability} span multiple spaces. Multiple spaces may be enabled concurrently, but only one may be the default alias. -- \ENVVAR{SHMEM\_ENABLE\_}: enables the symmetric heap for the - named device space for the process. When not set or empty for a given - \textit{}, that device's symmetric heap is disabled. - \LibHandleRef{SHMEM\_SPACE\_} is available when the corresponding device space is enabled and the device or memory domain for \textit{} is detected and accessible. When a space is not available (e.g., disabled or From 2006de23e56e77024f7c7ad958fac11692f52228 Mon Sep 17 00:00:00 2001 From: Michael Beebe Date: Fri, 15 Aug 2025 17:50:57 -0500 Subject: [PATCH 08/12] Added spaces changes to backmatter --- content/backmatter.tex | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/content/backmatter.tex b/content/backmatter.tex index cc581470..4af3662f 100644 --- a/content/backmatter.tex +++ b/content/backmatter.tex @@ -665,6 +665,24 @@ \subsection{Table~\ref{p2psynctypes}: point-to-point synchronization types} \chapter{Changes to this Document}\label{sec:changelog} +\section{Version 1.7} +\label{changelog:v1.7} + +Major changes in \openshmem[1.7] include the addition of spaces, which enable +applications to allocate symmetric memory in heterogeneous memory domains. + +The following list describes the specific changes in \openshmem[1.7]: +\begin{enumerate} +% +\item Added spaces, which enable applications to allocate symmetric memory in + heterogeneous memory domains. \ChangelogRef{subsec:space, subsec:shmem_space_calloc}% +% +\item Added space-specific allocation routines: \FUNC{shmem\_space\_malloc}, + \FUNC{shmem\_space\_calloc}, and \FUNC{shmem\_space\_free}. +\ChangelogRef{subsec:shmem_space_malloc, subsec:shmem_space_calloc, subsec:shmem_space_free}% +% +\end{enumerate} + \section{Version 1.6} \label{changelog:v1.6} Major changes in \openshmem[1.6] include the addition of inclusive and From 9ce8c06c6f0847f3a2e9e2a99f0ccaf2fb37b0da Mon Sep 17 00:00:00 2001 From: Michael Beebe Date: Fri, 15 Aug 2025 17:55:54 -0500 Subject: [PATCH 09/12] Revert back to old .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b6c37a15..274df9dc 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ main_spec.fdb_latexmk main_spec.fls main_spec.synctex.gz *~ + From b8699a930faf81aea86f829a05d2ee02583f4b6c Mon Sep 17 00:00:00 2001 From: Michael Beebe Date: Fri, 15 Aug 2025 18:00:14 -0500 Subject: [PATCH 10/12] restore old .gitignore --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index 274df9dc..cc603d53 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,5 @@ main_spec.log main_spec.out main_spec.pdf main_spec.toc -main_spec.fdb_latexmk -main_spec.fls -main_spec.synctex.gz *~ From 838e65fbb8ebad8a671756a649914b9f08013683 Mon Sep 17 00:00:00 2001 From: Michael Beebe Date: Fri, 15 Aug 2025 18:15:14 -0500 Subject: [PATCH 11/12] Added back in SHMEM_MALLOC_SIGNAL_REMOTE to library_constants. Accidentally deleted --- content/library_constants.tex | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/content/library_constants.tex b/content/library_constants.tex index fd516ee8..8a057b11 100644 --- a/content/library_constants.tex +++ b/content/library_constants.tex @@ -115,6 +115,12 @@ for more detail about its use. \tabularnewline \hline %% +\LibConstDecl{SHMEM\_MALLOC\_SIGNAL\_REMOTE} & +The hint to the memory allocation routine which specifies that the allocated +memory will be used for signal variables. See Section +\ref{subsec:shmmallochint} for more detail about its use. +\tabularnewline \hline +%% \LibConstDecl{SHMEM\_SPACE\_INVALID} & A value corresponding to an invalid space. This value can be used to initialize or update space handles to indicate From 0ed000611c2474363d6f82d27c2663afe6c52329 Mon Sep 17 00:00:00 2001 From: Michael Beebe Date: Fri, 29 Aug 2025 11:19:33 -0500 Subject: [PATCH 12/12] Remove TODO from content/programming_model_overview.tex --- content/programming_model_overview.tex | 1 - 1 file changed, 1 deletion(-) diff --git a/content/programming_model_overview.tex b/content/programming_model_overview.tex index 040e6be0..0a0b1f9d 100644 --- a/content/programming_model_overview.tex +++ b/content/programming_model_overview.tex @@ -74,7 +74,6 @@ \item \textbf{Space Management} \begin{enumerate} - % TODO: refine the description of spaces \item \OPR{Spaces}: Spaces are containers for symmetric memory in specific memory domains. They provide the storage for \openshmem symmetric data objects and define accessibility and operational properties of those objects. Allocations