pmix_fd: cap the max FD to try to close

On some OS's (e.g., macOS), the value returned by
sysconf(_SC_OPEN_MAX) can be set by the user via "ulimit -n X", where
X can be -1 (unlimited) or a positive integer. On macOS in particular,
if the user does not set this value, it's unclear how the default
value is chosen.  Some users have reported seeing arbitrarily large
default values (in the billions), resulting in a very large loop over
close() that can take minutes/hours to complete, leading the user to
think that the app has hung.  To avoid this, ensure that we cap the
max FD that we'll try to close.  This is not a perfect scheme, and
there's uncertainty on how the macOS default value works, so we
provide the pmix_maxfd MCA var to allow the user to set the max FD
value if needed.

This commit is inspired by #10360
and #10358.

Thanks to Scott Sayres for raising the issue.

Signed-off-by: Jeff Squyres <[email protected]>

Author: jsquyres

src/runtime/pmix_params.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006      Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2008-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008-2022 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2009      Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2010-2014 Los Alamos National Security, LLC.
  *                         All rights reserved.
@@ -51,6 +51,7 @@ int pmix_event_caching_window = 1;
 bool pmix_suppress_missing_data_warning = false;
 char *pmix_progress_thread_cpus = NULL;
 bool pmix_bind_progress_thread_reqd = false;
+int pmix_maxfd = 1024;
 
 pmix_status_t pmix_register_params(void)
 {
@@ -269,6 +270,11 @@ pmix_status_t pmix_register_params(void)
                                       PMIX_MCA_BASE_VAR_TYPE_BOOL,
                                       &pmix_bind_progress_thread_reqd);
 
+    (void) pmix_mca_base_var_register("pmix", "pmix", NULL, "maxfd",
+                                      "In non-Linux environments, use this value as a maximum number of file descriptors to close when forking a new child process",
+                                      PMIX_MCA_BASE_VAR_TYPE_INT,
+                                      &pmix_maxfd);
+
     pmix_hwloc_register();
     return PMIX_SUCCESS;
 }

src/runtime/pmix_rte.h

@@ -10,7 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2008      Sun Microsystems, Inc.  All rights reserved.
- * Copyright (c) 2010-2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2022 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2021-2022 Nanook Consulting  All rights reserved.
  * $COPYRIGHT$
@@ -51,6 +51,7 @@ PMIX_EXPORT extern int pmix_event_caching_window;
 PMIX_EXPORT extern bool pmix_suppress_missing_data_warning;
 PMIX_EXPORT extern char *pmix_progress_thread_cpus;
 PMIX_EXPORT extern bool pmix_bind_progress_thread_reqd;
+PMIX_EXPORT extern int pmix_maxfd;
 
 /** version string of pmix */
 extern const char pmix_version_string[];

src/util/pmix_fd.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008-2022 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2009      Sandia National Laboratories. All rights reserved.
  * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
@@ -52,6 +52,7 @@
 #include "src/util/pmix_error.h"
 #include "src/util/pmix_fd.h"
 #include "src/util/pmix_string_copy.h"
+#include "src/runtime/pmix_rte.h"
 
 /*
  * Simple loop over reading from a fd
@@ -236,6 +237,22 @@ void pmix_close_open_file_descriptors(int protected_fd)
     if (0 > fdmax) {
         fdmax = sysconf(_SC_OPEN_MAX);
     }
+    // On some OS's (e.g., macOS), the value returned by
+    // sysconf(_SC_OPEN_MAX) can be set by the user via "ulimit -n X",
+    // where X can be -1 (unlimited) or a positive integer. On macOS
+    // in particular, if the user does not set this value, it's
+    // unclear how the default value is chosen.  Some users have
+    // reported seeing arbitrarily large default values (in the
+    // billions), resulting in a very large loop over close() that can
+    // take minutes/hours to complete, leading the user to think that
+    // the app has hung.  To avoid this, ensure that we cap the max FD
+    // that we'll try to close.  This is not a perfect scheme, and
+    // there's uncertainty on how the macOS default value works, so we
+    // provide the pmix_maxfd MCA var to allow the user to set the max
+    // FD value if needed.
+    if (-1 == fdmax || pmix_maxfd < fdmax) {
+        fdmax = pmix_maxfd;
+    }
     for (int fd = 3; fd < fdmax; fd++) {
         if (fd != protected_fd) {
             close(fd);