Merge pull request #9247 from awlauria/osc_pt2pt_changes_v4.1.x

v4.1.x: osc/pt2pt: Some fixes

Author: jsquyres

ompi/mca/osc/pt2pt/osc_pt2pt.h

@@ -918,11 +918,14 @@ static inline ompi_osc_pt2pt_sync_t *ompi_osc_pt2pt_module_sync_lookup (ompi_osc
 
         return &module->all_sync;
     case OMPI_OSC_PT2PT_SYNC_TYPE_PSCW:
+        OPAL_THREAD_LOCK(&module->all_sync.lock);
         if (ompi_osc_pt2pt_sync_pscw_peer (module, target, peer)) {
+            OPAL_THREAD_UNLOCK(&module->all_sync.lock);
             OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
                                  "osc/pt2pt: found PSCW access epoch target for %d", target));
             return &module->all_sync;
         }
+        OPAL_THREAD_UNLOCK(&module->all_sync.lock);
     }
 
     return NULL;

ompi/mca/osc/pt2pt/osc_pt2pt_active_target.c

@@ -213,11 +213,13 @@ int ompi_osc_pt2pt_start (ompi_group_t *group, int assert, ompi_win_t *win)
     ompi_osc_pt2pt_module_t *module = GET_MODULE(win);
     ompi_osc_pt2pt_sync_t *sync = &module->all_sync;
 
+    OPAL_THREAD_LOCK(&module->lock);
     OPAL_THREAD_LOCK(&sync->lock);
 
     /* check if we are already in an access epoch */
     if (ompi_osc_pt2pt_access_epoch_active (module)) {
         OPAL_THREAD_UNLOCK(&sync->lock);
+        OPAL_THREAD_UNLOCK(&module->lock);
         return OMPI_ERR_RMA_SYNC;
     }
 
@@ -251,6 +253,7 @@ int ompi_osc_pt2pt_start (ompi_group_t *group, int assert, ompi_win_t *win)
         /* nothing more to do. this is an empty start epoch */
         sync->eager_send_active = true;
         OPAL_THREAD_UNLOCK(&sync->lock);
+        OPAL_THREAD_UNLOCK(&module->lock);
         return OMPI_SUCCESS;
     }
 
@@ -260,6 +263,7 @@ int ompi_osc_pt2pt_start (ompi_group_t *group, int assert, ompi_win_t *win)
     sync->peer_list.peers = ompi_osc_pt2pt_get_peers (module, group);
     if (NULL == sync->peer_list.peers) {
         OPAL_THREAD_UNLOCK(&sync->lock);
+        OPAL_THREAD_UNLOCK(&module->lock);
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
@@ -295,6 +299,7 @@ int ompi_osc_pt2pt_start (ompi_group_t *group, int assert, ompi_win_t *win)
                          sync->eager_send_active));
 
     OPAL_THREAD_UNLOCK(&sync->lock);
+    OPAL_THREAD_UNLOCK(&module->lock);
     return OMPI_SUCCESS;
 }
 

ompi/mca/osc/pt2pt/osc_pt2pt_data_move.c

@@ -753,7 +753,9 @@ static int ompi_osc_pt2pt_acc_op_queue (ompi_osc_pt2pt_module_t *module, ompi_os
     }
 
     /* add to the pending acc queue */
-    OPAL_THREAD_SCOPED_LOCK(&module->pending_acc_lock, opal_list_append (&module->pending_acc, &pending_acc->super));
+    ompi_osc_pt2pt_accumulate_lock(module);
+    opal_list_append (&module->pending_acc, &pending_acc->super);
+    ompi_osc_pt2pt_accumulate_unlock(module);
 
     return OMPI_SUCCESS;
 }

ompi/mca/osc/pt2pt/osc_pt2pt_passive_target.c

@@ -272,6 +272,9 @@ static int ompi_osc_pt2pt_lock_internal (int lock_type, int target, int assert,
     ompi_osc_pt2pt_module_t *module = GET_MODULE(win);
     ompi_osc_pt2pt_sync_t *lock;
     int ret = OMPI_SUCCESS;
+    ompi_osc_pt2pt_sync_t *otherlock = NULL;
+    int target_key;
+    void *iter_hash_node = NULL;
 
     /* Check if no_locks is set. TODO: we also need to track whether we are in an
      * active target epoch. Fence can make this tricky to track. */
@@ -345,6 +348,25 @@ static int ompi_osc_pt2pt_lock_internal (int lock_type, int target, int assert,
         return OMPI_ERR_RMA_CONFLICT;
     }
 
+    /* All previously requested locks must be complete before we can start a new
+     * lock, otherwise we deadlock from mis-ordering of locks.
+     */
+    ret = opal_hash_table_get_first_key_uint32(&module->outstanding_locks,
+                                               (uint32_t *) &target_key,
+                                               (void **) &otherlock,
+                                               &iter_hash_node);
+    while( OPAL_SUCCESS == ret ) {
+        if( NULL != otherlock ) {
+            ompi_osc_pt2pt_sync_wait_expected (otherlock);
+        }
+
+        ret = opal_hash_table_get_next_key_uint32(&module->outstanding_locks,
+                                                  (uint32_t *) &target_key,
+                                                  (void **) &otherlock,
+                                                  iter_hash_node, &iter_hash_node);
+    }
+    ret = OPAL_SUCCESS;
+
     ++module->passive_target_access_epoch;
 
     ompi_osc_pt2pt_module_lock_insert (module, lock);
@@ -596,7 +618,7 @@ int ompi_osc_pt2pt_flush_all (struct ompi_win_t *win)
             }
 
             ret = opal_hash_table_get_next_key_uint32 (&module->outstanding_locks, (uint32_t *) &target,
-                                                       (void **) lock, node, &node);
+                                                       (void **) &lock, node, &node);
             if (OPAL_SUCCESS != ret) {
                 ret = OPAL_SUCCESS;
                 break;