diff options
Diffstat (limited to 'drivers/infiniband/ulp/iser/iser_memory.c')
-rw-r--r-- | drivers/infiniband/ulp/iser/iser_memory.c | 80 |
1 files changed, 60 insertions, 20 deletions
diff --git a/drivers/infiniband/ulp/iser/iser_memory.c b/drivers/infiniband/ulp/iser/iser_memory.c index 31950a522a1..d0b03f42658 100644 --- a/drivers/infiniband/ulp/iser/iser_memory.c +++ b/drivers/infiniband/ulp/iser/iser_memory.c | |||
@@ -42,6 +42,7 @@ | |||
42 | #include "iscsi_iser.h" | 42 | #include "iscsi_iser.h" |
43 | 43 | ||
44 | #define ISER_KMALLOC_THRESHOLD 0x20000 /* 128K - kmalloc limit */ | 44 | #define ISER_KMALLOC_THRESHOLD 0x20000 /* 128K - kmalloc limit */ |
45 | |||
45 | /** | 46 | /** |
46 | * Decrements the reference count for the | 47 | * Decrements the reference count for the |
47 | * registered buffer & releases it | 48 | * registered buffer & releases it |
@@ -55,7 +56,7 @@ int iser_regd_buff_release(struct iser_regd_buf *regd_buf) | |||
55 | if ((atomic_read(®d_buf->ref_count) == 0) || | 56 | if ((atomic_read(®d_buf->ref_count) == 0) || |
56 | atomic_dec_and_test(®d_buf->ref_count)) { | 57 | atomic_dec_and_test(®d_buf->ref_count)) { |
57 | /* if we used the dma mr, unreg is just NOP */ | 58 | /* if we used the dma mr, unreg is just NOP */ |
58 | if (regd_buf->reg.rkey != 0) | 59 | if (regd_buf->reg.is_fmr) |
59 | iser_unreg_mem(®d_buf->reg); | 60 | iser_unreg_mem(®d_buf->reg); |
60 | 61 | ||
61 | if (regd_buf->dma_addr) { | 62 | if (regd_buf->dma_addr) { |
@@ -90,9 +91,9 @@ void iser_reg_single(struct iser_device *device, | |||
90 | BUG_ON(dma_mapping_error(dma_addr)); | 91 | BUG_ON(dma_mapping_error(dma_addr)); |
91 | 92 | ||
92 | regd_buf->reg.lkey = device->mr->lkey; | 93 | regd_buf->reg.lkey = device->mr->lkey; |
93 | regd_buf->reg.rkey = 0; /* indicate there's no need to unreg */ | ||
94 | regd_buf->reg.len = regd_buf->data_size; | 94 | regd_buf->reg.len = regd_buf->data_size; |
95 | regd_buf->reg.va = dma_addr; | 95 | regd_buf->reg.va = dma_addr; |
96 | regd_buf->reg.is_fmr = 0; | ||
96 | 97 | ||
97 | regd_buf->dma_addr = dma_addr; | 98 | regd_buf->dma_addr = dma_addr; |
98 | regd_buf->direction = direction; | 99 | regd_buf->direction = direction; |
@@ -239,7 +240,7 @@ static int iser_sg_to_page_vec(struct iser_data_buf *data, | |||
239 | int i; | 240 | int i; |
240 | 241 | ||
241 | /* compute the offset of first element */ | 242 | /* compute the offset of first element */ |
242 | page_vec->offset = (u64) sg[0].offset; | 243 | page_vec->offset = (u64) sg[0].offset & ~MASK_4K; |
243 | 244 | ||
244 | for (i = 0; i < data->dma_nents; i++) { | 245 | for (i = 0; i < data->dma_nents; i++) { |
245 | total_sz += sg_dma_len(&sg[i]); | 246 | total_sz += sg_dma_len(&sg[i]); |
@@ -247,21 +248,30 @@ static int iser_sg_to_page_vec(struct iser_data_buf *data, | |||
247 | first_addr = sg_dma_address(&sg[i]); | 248 | first_addr = sg_dma_address(&sg[i]); |
248 | last_addr = first_addr + sg_dma_len(&sg[i]); | 249 | last_addr = first_addr + sg_dma_len(&sg[i]); |
249 | 250 | ||
250 | start_aligned = !(first_addr & ~PAGE_MASK); | 251 | start_aligned = !(first_addr & ~MASK_4K); |
251 | end_aligned = !(last_addr & ~PAGE_MASK); | 252 | end_aligned = !(last_addr & ~MASK_4K); |
252 | 253 | ||
253 | /* continue to collect page fragments till aligned or SG ends */ | 254 | /* continue to collect page fragments till aligned or SG ends */ |
254 | while (!end_aligned && (i + 1 < data->dma_nents)) { | 255 | while (!end_aligned && (i + 1 < data->dma_nents)) { |
255 | i++; | 256 | i++; |
256 | total_sz += sg_dma_len(&sg[i]); | 257 | total_sz += sg_dma_len(&sg[i]); |
257 | last_addr = sg_dma_address(&sg[i]) + sg_dma_len(&sg[i]); | 258 | last_addr = sg_dma_address(&sg[i]) + sg_dma_len(&sg[i]); |
258 | end_aligned = !(last_addr & ~PAGE_MASK); | 259 | end_aligned = !(last_addr & ~MASK_4K); |
259 | } | 260 | } |
260 | 261 | ||
261 | first_addr = first_addr & PAGE_MASK; | 262 | /* handle the 1st page in the 1st DMA element */ |
262 | 263 | if (cur_page == 0) { | |
263 | for (page = first_addr; page < last_addr; page += PAGE_SIZE) | 264 | page = first_addr & MASK_4K; |
264 | page_vec->pages[cur_page++] = page; | 265 | page_vec->pages[cur_page] = page; |
266 | cur_page++; | ||
267 | page += SIZE_4K; | ||
268 | } else | ||
269 | page = first_addr; | ||
270 | |||
271 | for (; page < last_addr; page += SIZE_4K) { | ||
272 | page_vec->pages[cur_page] = page; | ||
273 | cur_page++; | ||
274 | } | ||
265 | 275 | ||
266 | } | 276 | } |
267 | page_vec->data_size = total_sz; | 277 | page_vec->data_size = total_sz; |
@@ -269,8 +279,7 @@ static int iser_sg_to_page_vec(struct iser_data_buf *data, | |||
269 | return cur_page; | 279 | return cur_page; |
270 | } | 280 | } |
271 | 281 | ||
272 | #define MASK_4K ((1UL << 12) - 1) /* 0xFFF */ | 282 | #define IS_4K_ALIGNED(addr) ((((unsigned long)addr) & ~MASK_4K) == 0) |
273 | #define IS_4K_ALIGNED(addr) ((((unsigned long)addr) & MASK_4K) == 0) | ||
274 | 283 | ||
275 | /** | 284 | /** |
276 | * iser_data_buf_aligned_len - Tries to determine the maximal correctly aligned | 285 | * iser_data_buf_aligned_len - Tries to determine the maximal correctly aligned |
@@ -320,9 +329,9 @@ static void iser_data_buf_dump(struct iser_data_buf *data) | |||
320 | struct scatterlist *sg = (struct scatterlist *)data->buf; | 329 | struct scatterlist *sg = (struct scatterlist *)data->buf; |
321 | int i; | 330 | int i; |
322 | 331 | ||
323 | for (i = 0; i < data->size; i++) | 332 | for (i = 0; i < data->dma_nents; i++) |
324 | iser_err("sg[%d] dma_addr:0x%lX page:0x%p " | 333 | iser_err("sg[%d] dma_addr:0x%lX page:0x%p " |
325 | "off:%d sz:%d dma_len:%d\n", | 334 | "off:0x%x sz:0x%x dma_len:0x%x\n", |
326 | i, (unsigned long)sg_dma_address(&sg[i]), | 335 | i, (unsigned long)sg_dma_address(&sg[i]), |
327 | sg[i].page, sg[i].offset, | 336 | sg[i].page, sg[i].offset, |
328 | sg[i].length,sg_dma_len(&sg[i])); | 337 | sg[i].length,sg_dma_len(&sg[i])); |
@@ -352,7 +361,7 @@ static void iser_page_vec_build(struct iser_data_buf *data, | |||
352 | 361 | ||
353 | page_vec->length = page_vec_len; | 362 | page_vec->length = page_vec_len; |
354 | 363 | ||
355 | if (page_vec_len * PAGE_SIZE < page_vec->data_size) { | 364 | if (page_vec_len * SIZE_4K < page_vec->data_size) { |
356 | iser_err("page_vec too short to hold this SG\n"); | 365 | iser_err("page_vec too short to hold this SG\n"); |
357 | iser_data_buf_dump(data); | 366 | iser_data_buf_dump(data); |
358 | iser_dump_page_vec(page_vec); | 367 | iser_dump_page_vec(page_vec); |
@@ -370,15 +379,18 @@ int iser_reg_rdma_mem(struct iscsi_iser_cmd_task *iser_ctask, | |||
370 | enum iser_data_dir cmd_dir) | 379 | enum iser_data_dir cmd_dir) |
371 | { | 380 | { |
372 | struct iser_conn *ib_conn = iser_ctask->iser_conn->ib_conn; | 381 | struct iser_conn *ib_conn = iser_ctask->iser_conn->ib_conn; |
382 | struct iser_device *device = ib_conn->device; | ||
373 | struct iser_data_buf *mem = &iser_ctask->data[cmd_dir]; | 383 | struct iser_data_buf *mem = &iser_ctask->data[cmd_dir]; |
374 | struct iser_regd_buf *regd_buf; | 384 | struct iser_regd_buf *regd_buf; |
375 | int aligned_len; | 385 | int aligned_len; |
376 | int err; | 386 | int err; |
387 | int i; | ||
388 | struct scatterlist *sg; | ||
377 | 389 | ||
378 | regd_buf = &iser_ctask->rdma_regd[cmd_dir]; | 390 | regd_buf = &iser_ctask->rdma_regd[cmd_dir]; |
379 | 391 | ||
380 | aligned_len = iser_data_buf_aligned_len(mem); | 392 | aligned_len = iser_data_buf_aligned_len(mem); |
381 | if (aligned_len != mem->size) { | 393 | if (aligned_len != mem->dma_nents) { |
382 | iser_err("rdma alignment violation %d/%d aligned\n", | 394 | iser_err("rdma alignment violation %d/%d aligned\n", |
383 | aligned_len, mem->size); | 395 | aligned_len, mem->size); |
384 | iser_data_buf_dump(mem); | 396 | iser_data_buf_dump(mem); |
@@ -389,10 +401,38 @@ int iser_reg_rdma_mem(struct iscsi_iser_cmd_task *iser_ctask, | |||
389 | mem = &iser_ctask->data_copy[cmd_dir]; | 401 | mem = &iser_ctask->data_copy[cmd_dir]; |
390 | } | 402 | } |
391 | 403 | ||
392 | iser_page_vec_build(mem, ib_conn->page_vec); | 404 | /* if there a single dma entry, FMR is not needed */ |
393 | err = iser_reg_page_vec(ib_conn, ib_conn->page_vec, ®d_buf->reg); | 405 | if (mem->dma_nents == 1) { |
394 | if (err) | 406 | sg = (struct scatterlist *)mem->buf; |
395 | return err; | 407 | |
408 | regd_buf->reg.lkey = device->mr->lkey; | ||
409 | regd_buf->reg.rkey = device->mr->rkey; | ||
410 | regd_buf->reg.len = sg_dma_len(&sg[0]); | ||
411 | regd_buf->reg.va = sg_dma_address(&sg[0]); | ||
412 | regd_buf->reg.is_fmr = 0; | ||
413 | |||
414 | iser_dbg("PHYSICAL Mem.register: lkey: 0x%08X rkey: 0x%08X " | ||
415 | "va: 0x%08lX sz: %ld]\n", | ||
416 | (unsigned int)regd_buf->reg.lkey, | ||
417 | (unsigned int)regd_buf->reg.rkey, | ||
418 | (unsigned long)regd_buf->reg.va, | ||
419 | (unsigned long)regd_buf->reg.len); | ||
420 | } else { /* use FMR for multiple dma entries */ | ||
421 | iser_page_vec_build(mem, ib_conn->page_vec); | ||
422 | err = iser_reg_page_vec(ib_conn, ib_conn->page_vec, ®d_buf->reg); | ||
423 | if (err) { | ||
424 | iser_data_buf_dump(mem); | ||
425 | iser_err("mem->dma_nents = %d (dlength = 0x%x)\n", mem->dma_nents, | ||
426 | ntoh24(iser_ctask->desc.iscsi_header.dlength)); | ||
427 | iser_err("page_vec: data_size = 0x%x, length = %d, offset = 0x%x\n", | ||
428 | ib_conn->page_vec->data_size, ib_conn->page_vec->length, | ||
429 | ib_conn->page_vec->offset); | ||
430 | for (i=0 ; i<ib_conn->page_vec->length ; i++) | ||
431 | iser_err("page_vec[%d] = 0x%llx\n", i, | ||
432 | (unsigned long long) ib_conn->page_vec->pages[i]); | ||
433 | return err; | ||
434 | } | ||
435 | } | ||
396 | 436 | ||
397 | /* take a reference on this regd buf such that it will not be released * | 437 | /* take a reference on this regd buf such that it will not be released * |
398 | * (eg in send dto completion) before we get the scsi response */ | 438 | * (eg in send dto completion) before we get the scsi response */ |