update the lustre fs component to use direct I/O if requested

Author: edgargabriel

ompi/mca/fs/lustre/fs_lustre.h

@@ -30,6 +30,7 @@
 extern int mca_fs_lustre_priority;
 extern int mca_fs_lustre_stripe_size;
 extern int mca_fs_lustre_stripe_width;
+extern int mca_fs_lustre_use_directio;
 
 BEGIN_C_DECLS
 

ompi/mca/fs/lustre/fs_lustre_component.c

@@ -45,6 +45,7 @@ int mca_fs_lustre_priority = 20;
    runtime also*/
 int mca_fs_lustre_stripe_size = 0;
 int mca_fs_lustre_stripe_width = 0;
+int mca_fs_lustre_use_drectio = 0;
 /*
  * Instantiate the public struct with all of our public information
  * and pointers to our public functions in it
@@ -94,5 +95,12 @@ lustre_register(void)
                                            OPAL_INFO_LVL_9,
                                            MCA_BASE_VAR_SCOPE_READONLY, &mca_fs_lustre_stripe_width);
 
+    mca_fs_lustre_use_directio = 0;
+    (void) mca_base_component_var_register(&mca_fs_lustre_component.fsm_version,
+                                           "use_directio", "whether to use direct I/O",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY, &mca_fs_lustre_use_directio);
+
     return OMPI_SUCCESS;
 }

ompi/mca/fs/lustre/fs_lustre_file_close.c

@@ -40,5 +40,12 @@ mca_fs_lustre_file_close (mca_io_ompio_file_t *fh)
     fh->f_comm->c_coll.coll_barrier (fh->f_comm,
                                      fh->f_comm->c_coll.coll_barrier_module);
     close (fh->fd);
+
+    if (0 != fh->f_fs_ptr ) {
+        int fs_ptr;
+        memcpy ( &fs_ptr, &fh->f_fs_ptr, sizeof(int));
+//        fsync ( fs_ptr );
+        close (fs_ptr );
+    }
     return OMPI_SUCCESS;
 }

ompi/mca/fs/lustre/fs_lustre_file_open.c

@@ -69,6 +69,7 @@ mca_fs_lustre_file_open (struct ompi_communicator_t *comm,
     int fs_lustre_stripe_size = -1;
     int fs_lustre_stripe_width = -1;
     char char_stripe[MPI_MAX_INFO_KEY];
+    int fd_direct;
 
     struct lov_user_md *lump=NULL;
 
@@ -93,6 +94,7 @@ mca_fs_lustre_file_open (struct ompi_communicator_t *comm,
     if (access_mode & MPI_MODE_EXCL)
         amode = amode | O_EXCL;
 
+    amode_direct = amode | O_DIRECT;
 
     ompi_info_get (info, "stripe_size", MPI_MAX_INFO_VAL, char_stripe, &flag);
     if ( flag ) {
@@ -159,5 +161,14 @@ mca_fs_lustre_file_open (struct ompi_communicator_t *comm,
       //	free ( lump );
       //      }
     }
+
+    if ( mca_fs_lustre_use_directio ) {
+        fd_direct = open ( filename, amode_direct, perm);
+        if ( -1 != fd_direct ) {
+            memcpy ( &fh->f_fs_ptr, &fd_direct, sizeof(int));
+        }
+    }
+
+
     return OMPI_SUCCESS;
 }