From f0935007ed50d330c2567f8637cd7445e2b7ed63 Mon Sep 17 00:00:00 2001 From: James Dinan Date: Sun, 7 Apr 2024 22:54:44 -0400 Subject: [PATCH 1/7] Allow multiple calls to init/finalize Signed-off-by: James Dinan --- content/shmem_init.tex | 12 +++++++++--- content/shmem_init_thread.tex | 15 ++++++++++++--- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/content/shmem_init.tex b/content/shmem_init.tex index 82fa3b723..6bfe2e1b6 100644 --- a/content/shmem_init.tex +++ b/content/shmem_init.tex @@ -18,9 +18,15 @@ library. It is a collective operation that all \acp{PE} must call before any other \openshmem routine may be called. At the end of the \openshmem program which it initialized, the call to \FUNC{shmem\_init} must be matched with a - call to \FUNC{shmem\_finalize}. After the first call to \FUNC{shmem\_init}, a - subsequent call to \FUNC{shmem\_init} or \FUNC{shmem\_init\_thread} in the - same program results in undefined behavior. + call to \FUNC{shmem\_finalize}. + + The \FUNC{shmem\_init} and \FUNC{shmem\_init\_thread} initialization + routines may be called multiple times within an \openshmem program. A + corresponding call to \FUNC{shmem\_finalize} must be made for each call to + an \openshmem initialization routine. The \openshmem library must not be + finalized until after the last call to \FUNC{shmem\_finalize} and may be + re-initialized with a subsequent call to an initialization routine. + } \apireturnvalues{ diff --git a/content/shmem_init_thread.tex b/content/shmem_init_thread.tex index 2a4b081b8..d83b56527 100644 --- a/content/shmem_init_thread.tex +++ b/content/shmem_init_thread.tex @@ -24,13 +24,19 @@ \VAR{requested} are \CONST{SHMEM\_THREAD\_SINGLE}, \CONST{SHMEM\_THREAD\_FUNNELED}, \CONST{SHMEM\_THREAD\_SERIALIZED}, and \CONST{SHMEM\_THREAD\_MULTIPLE}. -An \openshmem program is initialized either by \FUNC{shmem\_init} or \FUNC{shmem\_init\_thread}. -Once an \openshmem library initialization call has been performed, a subsequent -initialization call in the same program results in undefined behavior. +The \FUNC{shmem\_init} and \FUNC{shmem\_init\_thread} initialization +routines may be called multiple times within an \openshmem program. A +corresponding call to \FUNC{shmem\_finalize} must be made for each call to +an \openshmem initialization routine. The \openshmem library must not be +finalized until after the last call to \FUNC{shmem\_finalize} and may be +re-initialized with a subsequent call to an initialization routine. + If the call to \FUNC{shmem\_init\_thread} is unsuccessful in allocating and initializing resources for the \openshmem library, then the behavior of any subsequent call to the \openshmem library is undefined. + + } \apireturnvalues{ @@ -43,6 +49,9 @@ or \FUNC{shmem\_init\_thread}. If the \openshmem library is initialized by \FUNC{shmem\_init}, the library implementation can choose to support any one of the defined thread levels. + +The \openshme library may not be able to change the level of threading support +provided after the first initialization call has been made. } \end{apidefinition} From 3f9099de428842b37de1f21867b9ff4d0a103bdb Mon Sep 17 00:00:00 2001 From: James Dinan Date: Sun, 7 Apr 2024 23:05:53 -0400 Subject: [PATCH 2/7] Add initialized query Signed-off-by: James Dinan --- content/shmem_init_thread.tex | 2 +- content/shmem_query_initialized.tex | 30 +++++++++++++++++++++++++++++ content/shmem_query_thread.tex | 3 +++ main_spec.tex | 4 ++++ 4 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 content/shmem_query_initialized.tex diff --git a/content/shmem_init_thread.tex b/content/shmem_init_thread.tex index d83b56527..f1f397d83 100644 --- a/content/shmem_init_thread.tex +++ b/content/shmem_init_thread.tex @@ -50,7 +50,7 @@ by \FUNC{shmem\_init}, the library implementation can choose to support any one of the defined thread levels. -The \openshme library may not be able to change the level of threading support +The \openshmem library may not be able to change the level of threading support provided after the first initialization call has been made. } diff --git a/content/shmem_query_initialized.tex b/content/shmem_query_initialized.tex new file mode 100644 index 000000000..b0c79aebe --- /dev/null +++ b/content/shmem_query_initialized.tex @@ -0,0 +1,30 @@ +\apisummary{ +Returns the initialized statis of the \openshmem library. +} + +\begin{apidefinition} + +\begin{Csynopsis} +void @\FuncDecl{shmem\_query\_initialized}@(int *initialized); +\end{Csynopsis} + +\begin{apiarguments} +\apiargument{OUT}{initialized}{Nonzero if the \openshmem library is in the initialized state. Nonzero otherwise.} +\end{apiarguments} + +\apidescription{ + The \FUNC{shmem\_query\_initialized} call returns the initialization status + of the \openshmem library. If the application has called an \openshmem + initialization routine and has not yet made the corresponding call to + \FUNC{shmem\_finalize}, this routine returns nonzero. Otherwise, it returns + zero. + + This function may be called at any time, regardless of the thread safety + level of the \openshmem library. +} + +\apireturnvalues{ +None. +} + +\end{apidefinition} diff --git a/content/shmem_query_thread.tex b/content/shmem_query_thread.tex index b2144ff5d..c917a3149 100644 --- a/content/shmem_query_thread.tex +++ b/content/shmem_query_thread.tex @@ -19,6 +19,9 @@ initialized by \FUNC{shmem\_init\_thread}. If the library was initialized by \FUNC{shmem\_init}, the implementation can choose to provide any one of the defined thread levels, and \FUNC{shmem\_query\_thread} returns this thread level. + +This function may be called at any time, regardless of the thread safety +level of the \openshmem library. } \apireturnvalues{ diff --git a/main_spec.tex b/main_spec.tex index 23e2d624c..8df1d5dd7 100644 --- a/main_spec.tex +++ b/main_spec.tex @@ -92,6 +92,10 @@ \subsubsection{\textbf{SHMEM\_QUERY\_THREAD}} \label{subsec:shmem_query_thread} \input{content/shmem_query_thread} +\subsubsection{\textbf{SHMEM\_QUERY\_INITIALIZED}} +\label{subsec:shmem_query_initialized} +\input{content/shmem_query_initialized} + \subsection{Memory Management Routines} \label{sec:memory_management} From d106b09f3fd115a9881b3d645574bb1e8351a922 Mon Sep 17 00:00:00 2001 From: James Dinan Date: Tue, 9 Apr 2024 21:34:15 -0400 Subject: [PATCH 3/7] Update content/shmem_query_initialized.tex --- content/shmem_query_initialized.tex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/shmem_query_initialized.tex b/content/shmem_query_initialized.tex index b0c79aebe..3e69deb5d 100644 --- a/content/shmem_query_initialized.tex +++ b/content/shmem_query_initialized.tex @@ -1,5 +1,5 @@ \apisummary{ -Returns the initialized statis of the \openshmem library. +Returns the initialized status of the \openshmem library. } \begin{apidefinition} From f0545bd933d9c5d1e2c048065d18092027437c9c Mon Sep 17 00:00:00 2001 From: James Dinan Date: Fri, 26 Apr 2024 14:27:51 -0400 Subject: [PATCH 4/7] Update content/shmem_query_initialized.tex Co-authored-by: David Ozog --- content/shmem_query_initialized.tex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/shmem_query_initialized.tex b/content/shmem_query_initialized.tex index 3e69deb5d..b3729b7cc 100644 --- a/content/shmem_query_initialized.tex +++ b/content/shmem_query_initialized.tex @@ -9,7 +9,7 @@ \end{Csynopsis} \begin{apiarguments} -\apiargument{OUT}{initialized}{Nonzero if the \openshmem library is in the initialized state. Nonzero otherwise.} +\apiargument{OUT}{initialized}{Nonzero if the \openshmem library is in the initialized state. Zero otherwise.} \end{apiarguments} \apidescription{ From aedd0e72932e27b7ac4a2981d59b9771e812656e Mon Sep 17 00:00:00 2001 From: James Dinan Date: Fri, 26 Apr 2024 15:11:05 -0400 Subject: [PATCH 5/7] Fix finalize resource cleanup and sync text Signed-off-by: James Dinan --- content/backmatter.tex | 6 ----- content/execution_model.tex | 14 +++++------- content/shmem_finalize.tex | 45 +++++++++++++++++++++++++------------ 3 files changed, 37 insertions(+), 28 deletions(-) diff --git a/content/backmatter.tex b/content/backmatter.tex index 89bbe8371..4d6a9bcc4 100644 --- a/content/backmatter.tex +++ b/content/backmatter.tex @@ -143,12 +143,6 @@ \chapter{Undefined Behavior in OpenSHMEM}\label{sec:undefined} immediately upon an \openshmem call into the uninitialized library. \tabularnewline \hline -Multiple calls to initialization routines & In an \openshmem program where -the initialization routines \FUNC{shmem\_init} or \FUNC{shmem\_init\_thread} -have already been called, any subsequent calls to these initialization routines -result in undefined behavior. -\tabularnewline -\hline Specifying invalid \ac{PE} numbers & For \openshmem routines that accept a \ac{PE} number as an argument, if the \ac{PE} number is invalid for the team associated with the operation (either implicitly or explicitly), the diff --git a/content/execution_model.tex b/content/execution_model.tex index a1ea1a69e..5edaac4ec 100644 --- a/content/execution_model.tex +++ b/content/execution_model.tex @@ -8,17 +8,15 @@ \ac{PE} execution is loosely coupled, relying on \openshmem operations to communicate and synchronize among executing \acp{PE}. The \openshmem phase in -a program begins with a call to the initialization routine \FUNC{shmem\_init} +a program begins with the first call to the initialization routine \FUNC{shmem\_init} or \FUNC{shmem\_init\_thread}, which must be performed before using any of the other \openshmem library routines. -An \openshmem program concludes its use of the \openshmem library when all \acp{PE} call +An \openshmem program concludes its use of the \openshmem library when all \acp{PE} +make their final call call \FUNC{shmem\_finalize} or any \ac{PE} calls \FUNC{shmem\_global\_exit}. -During a call to \FUNC{shmem\_finalize}, the \openshmem library must -complete all pending communication and release all the resources associated to -the library using an implicit collective synchronization across \acp{PE}. -Calling any \openshmem routine before initialization or after -\FUNC{shmem\_finalize} leads to undefined behavior. After finalization, a -subsequent initialization call also leads to undefined behavior. +During the last call to \FUNC{shmem\_finalize}, the \openshmem library synchronizes +all \acp{PE}, completes all pending communication and releases all the resources +associated to the library. The \acp{PE} of the \openshmem program are identified by unique integers. The identifiers are integers assigned in a monotonically increasing manner from zero diff --git a/content/shmem_finalize.tex b/content/shmem_finalize.tex index cfa32d137..5496e9bfa 100644 --- a/content/shmem_finalize.tex +++ b/content/shmem_finalize.tex @@ -15,23 +15,33 @@ \end{apiarguments} \apidescription{ - \FUNC{shmem\_finalize} is a collective operation that ends the \openshmem - portion of a program previously initialized by \FUNC{shmem\_init} or \FUNC{shmem\_init\_thread} and - releases all resources used by the \openshmem library. This collective - operation requires all \acp{PE} to participate in the call. There is an - implicit global barrier in \FUNC{shmem\_finalize} to ensure that pending - communications are completed and that no resources are released until all - \acp{PE} have entered \FUNC{shmem\_finalize}. - This routine destroys all teams created by the \openshmem program. + \FUNC{shmem\_finalize} ends the \openshmem + portion of a program previously initialized by \FUNC{shmem\_init} or \FUNC{shmem\_init\_thread}. + This is a collective + operation that requires all \acp{PE} to participate in the call. + + An \openshmem program may perform a series of matching + initialization and finalization calls. + The last call to \FUNC{shmem\_finalize} in this series + releases all resources used by the \openshmem library. + This call destroys all teams created by the \openshmem program. As a result, all shareable contexts are destroyed. The user is responsible for destroying all contexts with the - \CONST{SHMEM\_CTX\_PRIVATE} option enabled prior to calling this routine; + \CONST{SHMEM\_CTX\_PRIVATE} option enabled prior to this call; otherwise, the behavior is undefined. - \FUNC{shmem\_finalize} must be - the last \openshmem library call encountered in the \openshmem portion of a - program. A call to \FUNC{shmem\_finalize} will release all resources - initialized by a corresponding call to \FUNC{shmem\_init} or \FUNC{shmem\_init\_thread}. All processes + + The last call to \FUNC{shmem\_finalize} performs an implicit global barrier + to ensure that pending communications are completed and that no resources + are released until all \acp{PE} have entered \FUNC{shmem\_finalize}. All + other calls to \FUNC{shmem\_finalize} perform an operation semantically + equivalent to \FUNC{shmem\_barrier\_all} and return without freeing any + \openshmem resources. + + The last call to \FUNC{shmem\_finalize} causes the \openshmem library + to enter an uninitialized state. No further \openshmem calls may be + made until an \openshmem initialization routine is called. + All processes that represent the \acp{PE} will still exist after the call to \FUNC{shmem\_finalize} returns, but they will no longer have access to resources that have been released. @@ -42,12 +52,19 @@ } \apinotes{ - \FUNC{shmem\_finalize} releases all resources used by the \openshmem library + The last call to \FUNC{shmem\_finalize} releases all resources used by the \openshmem library including the symmetric memory heap and pointers initiated by \FUNC{shmem\_ptr}. This collective operation requires all \acp{PE} to participate in the call, not just a subset of the \acp{PE}. The non-\openshmem portion of a program may continue after a call to \FUNC{shmem\_finalize} by all \acp{PE}. + + Calls to \FUNC{shmem\_finalize} that are not the last in a series of + initialization and finalization calls do not free any \openshmem resources. + Thus, teams, contexts, or symmetric memory allocations may be leaked until + the final call to \FUNC{shmem\_finalize}. Applications that perform + multiple initialization and finalization calls should free resources prior + to calling \FUNC{shmem\_finalize} to avoid such leaks. } \begin{apiexamples} From 4c9e26f9acbe0fb2f21d96b6dcd884ee4042af90 Mon Sep 17 00:00:00 2001 From: James Dinan Date: Tue, 7 May 2024 16:29:04 -0400 Subject: [PATCH 6/7] Add changelog entry Signed-off-by: James Dinan --- content/backmatter.tex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/content/backmatter.tex b/content/backmatter.tex index 4d6a9bcc4..48e5f9711 100644 --- a/content/backmatter.tex +++ b/content/backmatter.tex @@ -655,6 +655,11 @@ \section{Version 1.6} The following list describes the specific changes in \openshmem[1.6]: \begin{itemize} % +\item Added support for initialization and finalization routines to be called + multiple times, and added an initialization status query API + \FUNC{shmem\_query\_initialized}. +\ChangelogRef{subsec:shmem_init, subsec:shmem_finalize, subsec:shmem_query_initialized}% +% \item Added interleaved block transfer APIs \FUNC{shmem\_ibget} and \FUNC{shmem\_ibput}. \ChangelogRef{subsec:shmem_ibget, subsec:shmem_ibput}% From f1b7c87b96d32adadca4bfc6b7c0800d3cc7a5c8 Mon Sep 17 00:00:00 2001 From: James Dinan Date: Fri, 24 May 2024 13:35:33 -0400 Subject: [PATCH 7/7] Update content/execution_model.tex --- content/execution_model.tex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/execution_model.tex b/content/execution_model.tex index 5edaac4ec..a56f8bda2 100644 --- a/content/execution_model.tex +++ b/content/execution_model.tex @@ -12,7 +12,7 @@ or \FUNC{shmem\_init\_thread}, which must be performed before using any of the other \openshmem library routines. An \openshmem program concludes its use of the \openshmem library when all \acp{PE} -make their final call call +make their final call to \FUNC{shmem\_finalize} or any \ac{PE} calls \FUNC{shmem\_global\_exit}. During the last call to \FUNC{shmem\_finalize}, the \openshmem library synchronizes all \acp{PE}, completes all pending communication and releases all the resources