Merge branch 'main' into squelch_allreduce_log

Author: lrbison

3rd-party/prrte

@@ -23,6 +23,16 @@ system.
 
   * This library is required; Open MPI will not build without it.
   * **Minimum version required:** |hwloc_min_version|
+
+    .. note:: While the minimum version of Hwloc will *work*, it may
+              still have bugs and/or have less functionality as
+              compared to later versions.
+
+              Other than the Hwloc restriction about v3.0.0 and beyond
+              (see below), the Open MPI community generally recomends
+              using the latest available version of Hwloc unless there
+              is a specific reason not to.
+
   * **Version embedded in Open MPI distribution:**
     |hwloc_embedded_version|
 
@@ -62,11 +72,32 @@ system.
   * **Version embedded in Open MPI distribution:**
     |event_embedded_version|
 
-* `PMIx <https://pmix.org/>`_
+    .. note:: The Open MPI community has heavily tested Libevent
+              |event_embedded_version|.  Other versions should *work*,
+              but |mdash| unlike Hwloc, OpenPMIx, and PRRTE |mdash|
+              there is not much reason to upgrade to use a later
+              version of Libevent.
+
+* `OpenPMIx <https://docs.openpmix.org/>`_
 
   * This library is required; Open MPI will not build without it.
   * **Minimum version required when building without PRRTE:**
     |pmix_min_version|
+
+    .. note:: While the minimum version of OpenPMIx will *work*, it
+              may still have bugs and/or have less functionality as
+              compared to later versions.
+
+              The Open MPI community generally recomends using the
+              latest available version of OpenPMIx unless there is a
+              specific reason not to.
+
+    .. note:: While `OpenPMIx <https://docs.openpmix.org/>`_ is the
+              formal name of the software that implements the `PMIx
+              <https://pmix.org/>`_ standard, the term "PMIx" is used
+              extensively throughout this documentation to refer to
+              the OpenPMIx software package.
+
   * **Minimum version required when building with PRRTE:** `See the
     PRRTE project documentation <https://docs.prrte.org/>`_.
   * **Version embedded in Open MPI distribution:**
@@ -77,10 +108,15 @@ system.
   * This library is optional in some environments. See below.
   * **Minimum version required:** |prte_min_version|
 
-    .. note:: While building Open MPI with PRRTE |prte_min_version|
-              *works*, you will not get a fully-populated
-              ``mpirun(1)`` man page.  The Open MPI community
-              recommends that you use PRRTE version 3.0.1 or higher.
+    .. note:: While the minimum version of PRRTE will *work*, it may
+              still have bugs and/or have less functionality as
+              compared to later versions.  For example, if you build
+              and run with |prte_min_version|, you will not get a
+              fully-populated ``mpirun(1)`` man page.
+
+              The Open MPI community generally recomends using the
+              latest available version of PRRTE unless there is a
+              specific reason not to.
 
   * **Version embedded in Open MPI distribution:**
     |prte_embedded_version|

docs/installing-open-mpi/required-support-libraries.rst

@@ -47,6 +47,15 @@ int mca_fs_lustre_priority = 20;
 int mca_fs_lustre_stripe_size = 0;
 int mca_fs_lustre_stripe_width = 0;
 int mca_fs_lustre_lock_algorithm = 0; /* auto */
+
+static const mca_base_var_enum_value_t ompi_fs_lustre_lock_algorithm_modes[] = {
+    {.value = 0, .string = "auto"},
+    {.value = 1, .string = "skip locking"},
+    {.value = 2, .string = "always lock entire file"},
+    {.value = 3, .string = "lock specific ranges"},
+    {.string = NULL},
+};
+
 /*
  * Instantiate the public struct with all of our public information
  * and pointers to our public functions in it
@@ -77,6 +86,8 @@ mca_fs_base_component_2_0_0_t mca_fs_lustre_component = {
 static int
 lustre_register(void)
 {
+    mca_base_var_enum_t *new_enum;
+
     mca_fs_lustre_priority = 20;
     (void) mca_base_component_var_register(&mca_fs_lustre_component.fsm_version,
                                            "priority", "Priority of the lustre fs component",
@@ -95,15 +106,18 @@ lustre_register(void)
                                            MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
                                            OPAL_INFO_LVL_9,
                                            MCA_BASE_VAR_SCOPE_READONLY, &mca_fs_lustre_stripe_width);
+
+    (void) mca_base_var_enum_create("mca_fs_lustre_lock_algorithm", ompi_fs_lustre_lock_algorithm_modes, &new_enum);
+
     mca_fs_lustre_lock_algorithm = 0;
     (void) mca_base_component_var_register(&mca_fs_lustre_component.fsm_version,
-                                           "lock_algorithm", "Locking algorithm used by the fs ufs component. "
-                                           " 0: auto (default), 1: skip locking, 2: always lock entire file, "
-                                           "3: lock only specific ranges",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           "lock_algorithm", "Locking algorithm used by the fs lustre component. "
+                                           "(default: auto)",
+                                           MCA_BASE_VAR_TYPE_INT, new_enum, 0, 0,
                                            OPAL_INFO_LVL_9,
                                            MCA_BASE_VAR_SCOPE_READONLY,
-                                           &mca_fs_lustre_lock_algorithm );
+                                           &mca_fs_lustre_lock_algorithm);
+    OBJ_RELEASE(new_enum);
 
     return OMPI_SUCCESS;
 }

ompi/mca/fs/lustre/fs_lustre_component.c

@@ -32,6 +32,15 @@
 
 int mca_fs_ufs_priority = 10;
 int mca_fs_ufs_lock_algorithm=0; /* auto */
+
+static const mca_base_var_enum_value_t ompi_fs_ufs_lock_algorithm_modes[] = {
+    {.value = 0, .string = "auto"},
+    {.value = 1, .string = "skip locking"},
+    {.value = 2, .string = "always lock entire file"},
+    {.value = 3, .string = "lock specific ranges"},
+    {.string = NULL},
+};
+
 /*
  * Private functions
  */
@@ -73,6 +82,8 @@ mca_fs_base_component_2_0_0_t mca_fs_ufs_component = {
 
 static int register_component(void)
 {
+    mca_base_var_enum_t *new_enum;
+
     mca_fs_ufs_priority = 10;
     (void) mca_base_component_var_register(&mca_fs_ufs_component.fsm_version,
                                            "priority", "Priority of the fs ufs component",
@@ -81,15 +92,17 @@ static int register_component(void)
                                            MCA_BASE_VAR_SCOPE_READONLY,
                                            &mca_fs_ufs_priority);
 
+    (void) mca_base_var_enum_create("mca_fs_ufs_lock_algorithm", ompi_fs_ufs_lock_algorithm_modes, &new_enum);
+
     mca_fs_ufs_lock_algorithm = 0;
     (void) mca_base_component_var_register(&mca_fs_ufs_component.fsm_version,
                                            "lock_algorithm", "Locking algorithm used by the fs ufs component. "
-                                           " 0: auto (default), 1: skip locking, 2: always lock entire file, "
-                                           "3: lock only specific ranges",
-                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           "(default: auto)",
+                                           MCA_BASE_VAR_TYPE_INT, new_enum, 0, 0,
                                            OPAL_INFO_LVL_9,
                                            MCA_BASE_VAR_SCOPE_READONLY,
-                                           &mca_fs_ufs_lock_algorithm );
+                                           &mca_fs_ufs_lock_algorithm);
+    OBJ_RELEASE(new_enum);
 
     return OMPI_SUCCESS;
 }

ompi/mca/fs/ufs/fs_ufs_component.c

@@ -217,12 +217,96 @@ static int accelerator_cuda_check_vmm(CUdeviceptr dbuf, CUmemorytype *mem_type,
     return 0;
 }
 
+static int accelerator_cuda_check_mpool(CUdeviceptr dbuf, CUmemorytype *mem_type,
+                                        int *dev_id)
+{
+#if OPAL_CUDA_VMM_SUPPORT
+    static int device_count = -1;
+    static int mpool_supported = -1;
+    CUresult result;
+    CUmemoryPool mpool;
+    CUmemAccess_flags flags;
+    CUmemLocation location;
+
+    if (mpool_supported <= 0) {
+        if (mpool_supported == -1) {
+            if (device_count == -1) {
+                result = cuDeviceGetCount(&device_count);
+                if (result != CUDA_SUCCESS || (0 == device_count)) {
+                    mpool_supported = 0;  /* never check again */
+                    device_count = 0;
+                    return 0;
+                }
+            }
+
+            /* assume uniformity of devices */
+            result = cuDeviceGetAttribute(&mpool_supported,
+                    CU_DEVICE_ATTRIBUTE_MEMORY_POOLS_SUPPORTED, 0);
+            if (result != CUDA_SUCCESS) {
+                mpool_supported = 0;
+            }
+        }
+        if (0 == mpool_supported) {
+            return 0;
+        }
+    }
+
+    result = cuPointerGetAttribute(&mpool, CU_POINTER_ATTRIBUTE_MEMPOOL_HANDLE,
+                                   dbuf);
+    if (CUDA_SUCCESS != result) {
+        return 0;
+    }
+
+    /* check if device has access */
+    for (int i = 0; i < device_count; i++) {
+        location.type = CU_MEM_LOCATION_TYPE_DEVICE;
+        location.id   = i;
+        result = cuMemPoolGetAccess(&flags, mpool, &location);
+        if ((CUDA_SUCCESS == result) &&
+            (CU_MEM_ACCESS_FLAGS_PROT_READWRITE == flags)) {
+            *mem_type = CU_MEMORYTYPE_DEVICE;
+            *dev_id  = i;
+            return 1;
+        }
+    }
+
+    /* host must have access as device access possibility is exhausted */
+    *mem_type = CU_MEMORYTYPE_HOST;
+    *dev_id = MCA_ACCELERATOR_NO_DEVICE_ID;
+    return 0;
+#endif
+
+    return 0;
+}
+
+static int accelerator_cuda_get_primary_context(CUdevice dev_id, CUcontext *pctx)
+{
+    CUresult result;
+    unsigned int flags;
+    int active;
+
+    result =  cuDevicePrimaryCtxGetState(dev_id, &flags, &active);
+    if (CUDA_SUCCESS != result) {
+        return OPAL_ERROR;
+    }
+
+    if (active) {
+        result = cuDevicePrimaryCtxRetain(pctx, dev_id);
+        return OPAL_SUCCESS;
+    }
+
+    return OPAL_ERROR;
+}
+
 static int accelerator_cuda_check_addr(const void *addr, int *dev_id, uint64_t *flags)
 {
     CUresult result;
     int is_vmm = 0;
+    int is_mpool_ptr = 0;
     int vmm_dev_id = MCA_ACCELERATOR_NO_DEVICE_ID;
+    int mpool_dev_id = MCA_ACCELERATOR_NO_DEVICE_ID;
     CUmemorytype vmm_mem_type = 0;
+    CUmemorytype mpool_mem_type = 0;
     CUmemorytype mem_type = 0;
     CUdeviceptr dbuf = (CUdeviceptr) addr;
     CUcontext ctx = NULL, mem_ctx = NULL;
@@ -235,6 +319,7 @@ static int accelerator_cuda_check_addr(const void *addr, int *dev_id, uint64_t *
     *flags = 0;
 
     is_vmm = accelerator_cuda_check_vmm(dbuf, &vmm_mem_type, &vmm_dev_id);
+    is_mpool_ptr = accelerator_cuda_check_mpool(dbuf, &mpool_mem_type, &mpool_dev_id);
 
 #if OPAL_CUDA_GET_ATTRIBUTES
     uint32_t is_managed = 0;
@@ -268,6 +353,9 @@ static int accelerator_cuda_check_addr(const void *addr, int *dev_id, uint64_t *
         if (is_vmm && (vmm_mem_type == CU_MEMORYTYPE_DEVICE)) {
             mem_type = CU_MEMORYTYPE_DEVICE;
             *dev_id = vmm_dev_id;
+        } else if (is_mpool_ptr && (mpool_mem_type == CU_MEMORYTYPE_DEVICE)) {
+            mem_type = CU_MEMORYTYPE_DEVICE;
+            *dev_id = mpool_dev_id;
         } else {
             /* Host memory, nothing to do here */
             return 0;
@@ -278,6 +366,8 @@ static int accelerator_cuda_check_addr(const void *addr, int *dev_id, uint64_t *
     } else {
         if (is_vmm) {
             *dev_id = vmm_dev_id;
+        } else if (is_mpool_ptr) {
+            *dev_id = mpool_dev_id;
         } else {
             /* query the device from the context */
             *dev_id = accelerator_cuda_get_device_id(mem_ctx);
@@ -296,13 +386,18 @@ static int accelerator_cuda_check_addr(const void *addr, int *dev_id, uint64_t *
         if (is_vmm && (vmm_mem_type == CU_MEMORYTYPE_DEVICE)) {
             mem_type = CU_MEMORYTYPE_DEVICE;
             *dev_id = vmm_dev_id;
+        } else if (is_mpool_ptr && (mpool_mem_type == CU_MEMORYTYPE_DEVICE)) {
+            mem_type = CU_MEMORYTYPE_DEVICE;
+            *dev_id = mpool_dev_id;
         } else {
             /* Host memory, nothing to do here */
             return 0;
         }
     } else {
         if (is_vmm) {
             *dev_id = vmm_dev_id;
+        } else if (is_mpool_ptr) {
+            *dev_id = mpool_dev_id;
         } else {
             result = cuPointerGetAttribute(&mem_ctx,
                                            CU_POINTER_ATTRIBUTE_CONTEXT, dbuf);
@@ -336,14 +431,18 @@ static int accelerator_cuda_check_addr(const void *addr, int *dev_id, uint64_t *
                 return OPAL_ERROR;
             }
 #endif /* OPAL_CUDA_GET_ATTRIBUTES */
-            if (is_vmm) {
-                /* This function is expected to set context if pointer is device
-                 * accessible but VMM allocations have NULL context associated
-                 * which cannot be set against the calling thread */
-                opal_output(0,
-                        "CUDA: unable to set context with the given pointer"
-                        "ptr=%p aborting...", addr);
-                return OPAL_ERROR;
+            if (is_vmm || is_mpool_ptr) {
+                if (OPAL_SUCCESS ==
+                        accelerator_cuda_get_primary_context(
+                            is_vmm ? vmm_dev_id : mpool_dev_id, &mem_ctx)) {
+                    /* As VMM/mempool allocations have no context associated
+                     * with them, check if device primary context can be set */
+                } else {
+                    opal_output(0,
+                                "CUDA: unable to set ctx with the given pointer"
+                                "ptr=%p aborting...", addr);
+                    return OPAL_ERROR;
+                }
             }
 
             result = cuCtxSetCurrent(mem_ctx);

opal/mca/accelerator/cuda/accelerator_cuda.c

@@ -17,7 +17,7 @@
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
  * Copyright (c) 2018-2024 Triad National Security, LLC. All rights
  *                         reserved.
- * Copyright (c) 2019-2021 Google, LLC. All rights reserved.
+ * Copyright (c) 2019-2024 Google, LLC. All rights reserved.
  * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * Copyright (c) 2022      IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
@@ -48,13 +48,13 @@ static int mca_btl_uct_component_register(void)
 {
     mca_btl_uct_module_t *module = &mca_btl_uct_module_template;
 
-    mca_btl_uct_component.memory_domains = "mlx5_0,mlx4_0";
+    mca_btl_uct_component.memory_domains = "mlx5_0,mlx4_0,rocep0s4";
     (void) mca_base_component_var_register(
         &mca_btl_uct_component.super.btl_version, "memory_domains",
         "Comma-delimited list of memory domains of the form "
         "to use for communication. Memory domains MUST provide transports that "
         "support put, get, and amos. Special values: all (all available), none."
-        " (default: mlx5_0,mlx4_0)",
+        " (default: mlx5_0,mlx4_0,rocep0s4)",
         MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
         MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_uct_component.memory_domains);