Skip to content

Commit d2bca24

Browse files
hjelmnhjelmn
committed
btl/scif: update for new rcache and mpool interfaces
Signed-off-by: Nathan Hjelm <[email protected]>
1 parent f114730 commit d2bca24

File tree

4 files changed

+32
-22
lines changed

4 files changed

+32
-22
lines changed

‎opal/mca/btl/scif/btl_scif.h

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818

1919
#include "opal/mca/mpool/mpool.h"
2020
#include "opal/mca/mpool/base/base.h"
21-
#include "opal/mca/mpool/grdma/mpool_grdma.h"
21+
#include "opal/mca/rcache/rcache.h"
2222
#include "opal/util/output.h"
2323
#include "opal_stdint.h"
2424
#include "opal/util/proc.h"
@@ -93,6 +93,9 @@ typedef struct mca_btl_scif_module_t {
9393

9494
volatile bool exiting;
9595
bool listening;
96+
97+
/** registration cache */
98+
mca_rcache_base_module_t *rcache;
9699
} mca_btl_scif_module_t;
97100

98101
typedef struct mca_btl_scif_component_t {
@@ -235,7 +238,7 @@ struct mca_btl_scif_registration_handle_t {
235238
typedef struct mca_btl_scif_registration_handle_t mca_btl_scif_registration_handle_t;
236239

237240
typedef struct mca_btl_scif_reg_t {
238-
mca_mpool_base_registration_t base;
241+
mca_rcache_base_registration_t base;
239242
/** per-endpoint btl handles for this registration */
240243
mca_btl_scif_registration_handle_t *handles;
241244
} mca_btl_scif_reg_t;

‎opal/mca/btl/scif/btl_scif_add_procs.c

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ int mca_btl_scif_del_procs (struct mca_btl_base_module_t *btl,
157157
return OPAL_SUCCESS;
158158
}
159159

160-
static int scif_dereg_mem (void *reg_data, mca_mpool_base_registration_t *reg)
160+
static int scif_dereg_mem (void *reg_data, mca_rcache_base_registration_t *reg)
161161
{
162162
mca_btl_scif_reg_t *scif_reg = (mca_btl_scif_reg_t *)reg;
163163
size_t size = (size_t)((uintptr_t) reg->bound - (uintptr_t) reg->base);
@@ -178,15 +178,15 @@ static int scif_dereg_mem (void *reg_data, mca_mpool_base_registration_t *reg)
178178
}
179179

180180
static int scif_reg_mem (void *reg_data, void *base, size_t size,
181-
mca_mpool_base_registration_t *reg)
181+
mca_rcache_base_registration_t *reg)
182182
{
183183
mca_btl_scif_reg_t *scif_reg = (mca_btl_scif_reg_t *)reg;
184184
int rc = OPAL_SUCCESS;
185185
unsigned int i;
186186

187187
scif_reg->handles = calloc (mca_btl_scif_module.endpoint_count, sizeof (scif_reg->handles[0]));
188188

189-
/* intialize all scif offsets to -1 and initialize the pointer back to the mpool registration */
189+
/* intialize all scif offsets to -1 and initialize the pointer back to the rcache registration */
190190
for (i = 0 ; i < mca_btl_scif_module.endpoint_count ; ++i) {
191191
scif_reg->handles[i].btl_handle.scif_offset = -1;
192192
scif_reg->handles[i].btl_handle.scif_base = (intptr_t) base;
@@ -214,19 +214,19 @@ static int scif_reg_mem (void *reg_data, void *base, size_t size,
214214
static int
215215
mca_btl_scif_setup_mpools (mca_btl_scif_module_t *scif_module)
216216
{
217-
struct mca_mpool_base_resources_t mpool_resources;
217+
mca_rcache_base_resources_t rcache_resources;
218218
int rc;
219219

220-
/* initialize the grdma mpool */
221-
mpool_resources.pool_name = "scif";
222-
mpool_resources.reg_data = (void *) scif_module;
223-
mpool_resources.sizeof_reg = sizeof (mca_btl_scif_reg_t);
224-
mpool_resources.register_mem = scif_reg_mem;
225-
mpool_resources.deregister_mem = scif_dereg_mem;
226-
scif_module->super.btl_mpool =
227-
mca_mpool_base_module_create("grdma", scif_module, &mpool_resources);
228-
if (NULL == scif_module->super.btl_mpool) {
229-
BTL_ERROR(("error creating grdma mpool"));
220+
/* initialize the grdma rcache */
221+
rcache_resources.pool_name = "scif";
222+
rcache_resources.reg_data = (void *) scif_module;
223+
rcache_resources.sizeof_reg = sizeof (mca_btl_scif_reg_t);
224+
rcache_resources.register_mem = scif_reg_mem;
225+
rcache_resources.deregister_mem = scif_dereg_mem;
226+
scif_module->rcache =
227+
mca_rcache_base_module_create("grdma", scif_module, &rcache_resources);
228+
if (NULL == scif_module->rcache) {
229+
BTL_ERROR(("error creating grdma rcache"));
230230
return OPAL_ERROR;
231231
}
232232

‎opal/mca/btl/scif/btl_scif_frag.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,8 @@ static inline int mca_btl_scif_frag_alloc (mca_btl_base_endpoint_t *ep,
6060
static inline int mca_btl_scif_frag_return (mca_btl_scif_base_frag_t *frag)
6161
{
6262
if (frag->registration) {
63-
frag->endpoint->btl->super.btl_mpool->mpool_deregister(frag->endpoint->btl->super.btl_mpool,
64-
&frag->registration->base);
63+
frag->endpoint->btl->rcache->rcache_deregister(frag->endpoint->btl->rcache,
64+
&frag->registration->base);
6565
frag->registration = NULL;
6666
}
6767

‎opal/mca/btl/scif/btl_scif_module.c

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,11 @@ mca_btl_scif_module_finalize (struct mca_btl_base_module_t *btl)
106106
mca_btl_scif_module_t *scif_module = (mca_btl_scif_module_t *) btl;
107107
unsigned int i;
108108

109+
if (scif_module->rcache) {
110+
mca_rcache_base_module_destroy (scif_module->rcache);
111+
scif_module->rcache = NULL;
112+
}
113+
109114
OBJ_DESTRUCT(&mca_btl_scif_module.dma_frags);
110115
OBJ_DESTRUCT(&mca_btl_scif_module.eager_frags);
111116

@@ -180,6 +185,7 @@ static mca_btl_base_registration_handle_t *mca_btl_scif_register_mem (struct mca
180185
mca_btl_base_endpoint_t *endpoint,
181186
void *base, size_t size, uint32_t flags)
182187
{
188+
mca_btl_scif_module_t *scif_module = (mca_btl_scif_module_t *) btl;
183189
mca_btl_scif_reg_t *scif_reg;
184190
int access_flags = flags & MCA_BTL_REG_FLAG_ACCESS_ANY;
185191
int rc;
@@ -200,8 +206,8 @@ static mca_btl_base_registration_handle_t *mca_btl_scif_register_mem (struct mca
200206
}
201207
}
202208

203-
rc = btl->btl_mpool->mpool_register(btl->btl_mpool, base, size, 0, access_flags,
204-
(mca_mpool_base_registration_t **) &scif_reg);
209+
rc = scif_module->rcache->rcache_register (scif_module->rcache, base, size, 0, access_flags,
210+
(mca_rcache_base_registration_t **) &scif_reg);
205211
if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
206212
return NULL;
207213
}
@@ -210,7 +216,7 @@ static mca_btl_base_registration_handle_t *mca_btl_scif_register_mem (struct mca
210216
if ((off_t) -1 == scif_reg->handles[endpoint->id].btl_handle.scif_offset) {
211217
size_t seg_size = (size_t)((uintptr_t) scif_reg->base.bound - (uintptr_t) scif_reg->base.base) + 1;
212218

213-
/* NTH: until we determine a way to pass permissions to the mpool just make all segments
219+
/* NTH: until we determine a way to pass permissions to the rcache just make all segments
214220
* read/write */
215221
scif_reg->handles[endpoint->id].btl_handle.scif_offset =
216222
scif_register (endpoint->scif_epd, scif_reg->base.base, seg_size, 0, SCIF_PROT_READ |
@@ -224,10 +230,11 @@ static mca_btl_base_registration_handle_t *mca_btl_scif_register_mem (struct mca
224230

225231
static int mca_btl_scif_deregister_mem (struct mca_btl_base_module_t *btl, mca_btl_base_registration_handle_t *handle)
226232
{
233+
mca_btl_scif_module_t *scif_module = (mca_btl_scif_module_t *) btl;
227234
mca_btl_scif_registration_handle_t *scif_handle = (mca_btl_scif_registration_handle_t *) handle;
228235
mca_btl_scif_reg_t *scif_reg = scif_handle->reg;
229236

230-
btl->btl_mpool->mpool_deregister (btl->btl_mpool, &scif_reg->base);
237+
scif_module->rcache->rcache_deregister (scif_module->rcache, &scif_reg->base);
231238

232239
return OPAL_SUCCESS;
233240
}

0 commit comments

Comments
 (0)