Merge pull request #4606 from rhc54/topic/register

Update to PMIx v3.0 PR for cleanup registration

Author: Ralph Castain

opal/mca/btl/vader/btl_vader.h

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006-2007 Voltaire. All rights reserved.
  * Copyright (c) 2009-2010 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2015 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2010-2017 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2015      Mellanox Technologies. All rights reserved.
  *
@@ -136,6 +136,8 @@ struct mca_btl_vader_component_t {
     opal_list_t pending_endpoints;          /**< list of endpoints with pending fragments */
     opal_list_t pending_fragments;          /**< fragments pending remote completion */
 
+    char *backing_directory;                /**< directory to place shared memory backing files */
+
     /* knem stuff */
 #if OPAL_BTL_VADER_HAVE_KNEM
     unsigned int knem_dma_min;              /**< minimum size to enable DMA for knem transfers (0 disables) */

opal/mca/btl/vader/btl_vader_component.c

@@ -12,10 +12,10 @@
  *                         All rights reserved.
  * Copyright (c) 2006-2007 Voltaire. All rights reserved.
  * Copyright (c) 2009-2010 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2015 Los Alamos National Security, LLC.
+ * Copyright (c) 2010-2017 Los Alamos National Security, LLC.
  *                         All rights reserved.
  * Copyright (c) 2011      NVIDIA Corporation.  All rights reserved.
- * Copyright (c) 2014-2015 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
  * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -211,6 +211,19 @@ static int mca_btl_vader_component_register (void)
                                            OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_GROUP, &mca_btl_vader_component.single_copy_mechanism);
     OBJ_RELEASE(new_enum);
 
+    if (0 == access ("/dev/shm", W_OK)) {
+        mca_btl_vader_component.backing_directory = "/dev/shm";
+    } else {
+        mca_btl_vader_component.backing_directory = opal_process_info.proc_session_dir;
+    }
+    (void) mca_base_component_var_register (&mca_btl_vader_component.super.btl_version, "backing_directory",
+                                            "Directory to place backing files for shared memory communication. "
+                                            "This directory should be on a local filesystem such as /tmp or "
+                                            "/dev/shm (default: (linux) /dev/shm, (others) session directory)",
+                                            MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3,
+                                            MCA_BASE_VAR_SCOPE_READONLY, &mca_btl_vader_component.backing_directory);
+
+
 #if OPAL_BTL_VADER_HAVE_KNEM
     /* Currently disabling DMA mode by default; it's not clear that this is useful in all applications and architectures. */
     mca_btl_vader_component.knem_dma_min = 0;
@@ -491,13 +504,17 @@ static mca_btl_base_module_t **mca_btl_vader_component_init (int *num_btls,
     if (MCA_BTL_VADER_XPMEM != mca_btl_vader_component.single_copy_mechanism) {
         char *sm_file;
 
-        rc = asprintf(&sm_file, "%s" OPAL_PATH_SEP "vader_segment.%s.%d", opal_process_info.proc_session_dir,
+        rc = asprintf(&sm_file, "%s" OPAL_PATH_SEP "vader_segment.%s.%d", mca_btl_vader_component.backing_directory,
                       opal_process_info.nodename, MCA_BTL_VADER_LOCAL_RANK);
         if (0 > rc) {
             free (btls);
             return NULL;
         }
 
+        if (NULL != opal_pmix.register_cleanup) {
+            opal_pmix.register_cleanup (sm_file, false, false);
+        }
+
         rc = opal_shmem_segment_create (&component->seg_ds, sm_file, component->segment_size);
         free (sm_file);
         if (OPAL_SUCCESS != rc) {

opal/mca/pmix/base/base.h

@@ -65,6 +65,7 @@ typedef struct {
     opal_mutex_t mutex;
     opal_pmix_condition_t cond;
     volatile bool active;
+    int status;
 } opal_pmix_lock_t;
 
 

opal/mca/pmix/pmix.h

@@ -867,6 +867,9 @@ typedef int (*opal_pmix_base_process_monitor_fn_t)(opal_list_t *monitor,
                                                    opal_list_t *directives,
                                                    opal_pmix_info_cbfunc_t cbfunc, void *cbdata);
 
+/* register cleanup */
+typedef int (*opal_pmix_base_register_cleanup_fn_t)(char *path, bool ignore, bool jobscope);
+
 /*
  * the standard public API data structure
  */
@@ -901,6 +904,7 @@ typedef struct {
     opal_pmix_base_alloc_fn_t                               allocate;
     opal_pmix_base_job_control_fn_t                         job_control;
     opal_pmix_base_process_monitor_fn_t                     monitor;
+    opal_pmix_base_register_cleanup_fn_t                    register_cleanup;
     /* server APIs */
     opal_pmix_base_module_server_init_fn_t                  server_init;
     opal_pmix_base_module_server_finalize_fn_t              server_finalize;

opal/mca/pmix/pmix3x/pmix/VERSION

@@ -30,7 +30,7 @@ greek=
 # command, or with the date (if "git describe" fails) in the form of
 # "date<date>".
 
-repo_rev=gitf56d30e
+repo_rev=git5c0b64b
 
 # If tarball_version is not empty, it is used as the version string in
 # the tarball filename, regardless of all other versions listed in
@@ -44,7 +44,7 @@ tarball_version=
 
 # The date when this release was created
 
-date="Nov 11, 2017"
+date="Dec 11, 2017"
 
 # The shared library version of each of PMIx's public libraries.
 # These versions are maintained in accordance with the "Library

opal/mca/pmix/pmix3x/pmix/include/pmix_common.h.in

@@ -462,6 +462,16 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_JOB_CTRL_PROVISION_IMAGE       "pmix.jctrl.pvnimg"     // (char*) name of the image that is to be provisioned
 #define PMIX_JOB_CTRL_PREEMPTIBLE           "pmix.jctrl.preempt"    // (bool) job can be pre-empted
 #define PMIX_JOB_CTRL_TERMINATE             "pmix.jctrl.term"       // (bool) politely terminate the specified procs
+#define PMIX_REGISTER_CLEANUP               "pmix.reg.cleanup"      // (char*) comma-delimited list of files/directories to
+                                                                    //         be removed upon process termination
+#define PMIX_CLEANUP_RECURSIVE              "pmix.clnup.recurse"    // (bool) recursively cleanup all subdirectories under the
+                                                                    //        specified one(s)
+#define PMIX_CLEANUP_EMPTY                  "pmix.clnup.empty"      // (bool) only remove empty subdirectories
+#define PMIX_CLEANUP_IGNORE                 "pmix.clnup.ignore"     // (char*) comma-delimited list of filenames that are not
+                                                                    //         to be removed
+#define PMIX_CLEANUP_LEAVE_TOPDIR           "pmix.clnup.lvtop"      // (bool) when recursively cleaning subdirs, do not remove
+                                                                    //        the top-level directory (the one given in the
+                                                                    //        cleanup request)
 
 /* monitoring attributes */
 #define PMIX_MONITOR_ID                     "pmix.monitor.id"       // (char*) provide a string identifier for this request
@@ -584,6 +594,7 @@ typedef int pmix_status_t;
 #define PMIX_ERR_NOT_IMPLEMENTED                    -48
 #define PMIX_ERR_COMM_FAILURE                       -49
 #define PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER     -50         // internal-only
+#define PMIX_ERR_CONFLICTING_CLEANUP_DIRECTIVES     -51
 
 /* define a starting point for v2.x error values */
 #define PMIX_ERR_V2X_BASE                   -100

opal/mca/pmix/pmix3x/pmix/src/atomics/sys/powerpc/atomic.h

@@ -84,7 +84,7 @@ void pmix_atomic_rmb(void)
 static inline
 void pmix_atomic_wmb(void)
 {
-    PMIXRMB();
+    PMIXWMB();
 }
 
 static inline
@@ -110,7 +110,7 @@ void pmix_atomic_isync(void)
 #pragma mc_func pmix_atomic_rmb { "7c2004ac" }         /* lwsync  */
 #pragma reg_killed_by pmix_atomic_rmb                  /* none */
 
-#pragma mc_func pmix_atomic_wmb { "7c0006ac" }         /* eieio */
+#pragma mc_func pmix_atomic_wmb { "7c2004ac" }         /* lwsync */
 #pragma reg_killed_by pmix_atomic_wmb                  /* none */
 
 #endif