Merge pull request #1317 from jjhursey/topic/timeout

Add new CLI options: timeout, get-stack-traces, report-state-on-timeout

Author: hppritcha

orte/mca/odls/odls_types.h

@@ -9,7 +9,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2010-2011 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2011      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011-2016 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011-2012 Los Alamos National Security, LLC.
  *                         All rights reserved.
  * Copyright (c) 2014      Intel, Inc. All rights reserved.
@@ -78,6 +78,9 @@ typedef uint8_t orte_daemon_cmd_flag_t;
 #define ORTE_DAEMON_NEW_COLL_ID             (orte_daemon_cmd_flag_t) 29
 
 
+/* for debug purposes, get stack traces from all application procs */
+#define ORTE_DAEMON_GET_STACK_TRACES        (orte_daemon_cmd_flag_t) 31
+
 /*
  * Struct written up the pipe from the child to the parent.
  */

orte/mca/rml/rml_types.h

@@ -165,6 +165,9 @@ BEGIN_C_DECLS
 /* error notifications */
 #define ORTE_RML_TAG_NOTIFICATION           59
 
+/* stacktrace for debug */
+#define ORTE_RML_TAG_STACK_TRACE            60
+
 #define ORTE_RML_TAG_MAX                   100
 
 #define ORTE_RML_TAG_NTOH(t) ntohl(t)

orte/orted/orted_comm.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2016 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2007-2012 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2009      Sun Microsystems, Inc. All rights reserved.
@@ -47,6 +47,7 @@
 #include "opal/mca/base/base.h"
 #include "opal/util/output.h"
 #include "opal/util/opal_environ.h"
+#include "opal/util/path.h"
 #include "opal/runtime/opal.h"
 #include "opal/runtime/opal_progress.h"
 #include "opal/dss/dss.h"
@@ -113,6 +114,9 @@ void orte_daemon_recv(int status, orte_process_name_t* sender,
     bool found = false;
     orte_node_t *node;
     orte_grpcomm_signature_t *sig;
+    FILE *fp;
+    char gscmd[256], path[1035], *pathptr;
+    char string[256], *string_ptr = string;
 
     /* unpack the command */
     n = 1;
@@ -1137,6 +1141,82 @@ void orte_daemon_recv(int status, orte_process_name_t* sender,
         }
         break;
 
+    case ORTE_DAEMON_GET_STACK_TRACES:
+        /* prep the response */
+        answer = OBJ_NEW(opal_buffer_t);
+        pathptr = path;
+
+        // Try to find the "gstack" executable.  Failure to find the
+        // executable will be handled below, because the receiver
+        // expects to have the process name, hostname, and PID in the
+        // buffer before finding an error message.
+        char *gstack_exec;
+        gstack_exec = opal_find_absolute_path("gstack");
+
+        /* hit each local process with a gstack command */
+        for (i=0; i < orte_local_children->size; i++) {
+            if (NULL != (proct = (orte_proc_t*)opal_pointer_array_get_item(orte_local_children, i)) &&
+                ORTE_FLAG_TEST(proct, ORTE_PROC_FLAG_ALIVE)) {
+                relay_msg = OBJ_NEW(opal_buffer_t);
+                if (OPAL_SUCCESS != opal_dss.pack(relay_msg, &proct->name, 1, ORTE_NAME) ||
+                    OPAL_SUCCESS != opal_dss.pack(relay_msg, &proct->node->name, 1, OPAL_STRING) ||
+                    OPAL_SUCCESS != opal_dss.pack(relay_msg, &proct->pid, 1, OPAL_PID)) {
+                    OBJ_RELEASE(relay_msg);
+                    break;
+                }
+
+                // If we were able to find the gstack executable,
+                // above, then run the command here.
+                fp = NULL;
+                if (NULL != gstack_exec) {
+                    (void) snprintf(gscmd, sizeof(gscmd), "%s %lu",
+                                    gstack_exec, (unsigned long) proct->pid);
+                    fp = popen(gscmd, "r");
+                }
+
+                // If either we weren't able to find or run the gstack
+                // exectuable, send back a nice error message here.
+                if (NULL == gstack_exec || NULL == fp) {
+                    (void) snprintf(string, sizeof(string),
+                                    "Failed to %s \"%s\" on %s to obtain stack traces",
+                                    (NULL == gstack_exec) ? "find" : "run",
+                                    (NULL == gstack_exec) ? "gstack" : gstack_exec,
+                                    proct->node->name);
+                    if (OPAL_SUCCESS ==
+                        opal_dss.pack(relay_msg, &string_ptr, 1, OPAL_STRING)) {
+                        opal_dss.pack(answer, &relay_msg, 1, OPAL_BUFFER);
+                    }
+                    OBJ_RELEASE(relay_msg);
+                    break;
+                }
+                /* Read the output a line at a time and pack it for transmission */
+                memset(path, 0, sizeof(path));
+                while (fgets(path, sizeof(path)-1, fp) != NULL) {
+                    if (OPAL_SUCCESS != opal_dss.pack(relay_msg, &pathptr, 1, OPAL_STRING)) {
+                        OBJ_RELEASE(relay_msg);
+                        break;
+                    }
+                    memset(path, 0, sizeof(path));
+                }
+                /* close */
+                pclose(fp);
+                /* transfer this load */
+                if (OPAL_SUCCESS != opal_dss.pack(answer, &relay_msg, 1, OPAL_BUFFER)) {
+                    OBJ_RELEASE(relay_msg);
+                    break;
+                }
+                OBJ_RELEASE(relay_msg);
+            }
+        }
+        /* always send our response */
+        if (0 > (ret = orte_rml.send_buffer_nb(ORTE_PROC_MY_HNP, answer,
+                                               ORTE_RML_TAG_STACK_TRACE,
+                                               orte_rml_send_callback, NULL))) {
+            ORTE_ERROR_LOG(ret);
+            OBJ_RELEASE(answer);
+        }
+        break;
+
     default:
         ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
     }
@@ -1201,6 +1281,9 @@ static char *get_orted_comm_cmd_str(int command)
     case ORTE_DAEMON_NEW_COLL_ID:
         return strdup("ORTE_DAEMON_NEW_COLL_ID");
 
+    case ORTE_DAEMON_GET_STACK_TRACES:
+        return strdup("ORTE_DAEMON_GET_STACK_TRACES");
+
     default:
         return strdup("Unknown Command!");
     }

orte/tools/orterun/help-orterun.txt

@@ -10,7 +10,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved.
+# Copyright (c) 2007-2016 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2012      Oak Ridge National Labs.  All rights reserved.
 # $COPYRIGHT$
 #
@@ -644,11 +644,11 @@ Please correct this value and try again.
 The user-provided time limit for job execution has been
 reached:
 
-  MPIEXEC_TIMEOUT: %s seconds
+  MPIEXEC_TIMEOUT: %d seconds
 
-The job will now be aborted. Please check your code and/or
-adjust/remove the job execution time limit (as specified
-by MPIEXEC_TIMEOUT in your environment).
+The job will now be aborted.  Please check your code and/or
+adjust/remove the job execution time limit (as specified by
+MPIEXEC_TIMEOUT in your environment or --timeout on the command line).
 #
 [orterun:conflict-env-set]
 ERROR: You have attempted to pass environment variables to Open MPI
@@ -666,3 +666,12 @@ be restored in a future version of Open MPI.
 
 Please see https://github.com/open-mpi/ompi/issues/1225 for details.
 
+#
+[orterun:timeoutconflict]
+Conflicting requests for timeout were given:
+
+  --timeout command line option: %d
+  MPIEXEC_TIMEOUT envar:         %s
+
+Only one method should be provided, or else they must agree. Please
+correct and retry.