diff options
Diffstat (limited to 'fs/bio.c')
-rw-r--r-- | fs/bio.c | 150 |
1 files changed, 105 insertions, 45 deletions
@@ -25,6 +25,7 @@ | |||
25 | #include <linux/module.h> | 25 | #include <linux/module.h> |
26 | #include <linux/mempool.h> | 26 | #include <linux/mempool.h> |
27 | #include <linux/workqueue.h> | 27 | #include <linux/workqueue.h> |
28 | #include <scsi/sg.h> /* for struct sg_iovec */ | ||
28 | 29 | ||
29 | #define BIO_POOL_SIZE 256 | 30 | #define BIO_POOL_SIZE 256 |
30 | 31 | ||
@@ -549,22 +550,34 @@ out_bmd: | |||
549 | return ERR_PTR(ret); | 550 | return ERR_PTR(ret); |
550 | } | 551 | } |
551 | 552 | ||
552 | static struct bio *__bio_map_user(request_queue_t *q, struct block_device *bdev, | 553 | static struct bio *__bio_map_user_iov(request_queue_t *q, |
553 | unsigned long uaddr, unsigned int len, | 554 | struct block_device *bdev, |
554 | int write_to_vm) | 555 | struct sg_iovec *iov, int iov_count, |
556 | int write_to_vm) | ||
555 | { | 557 | { |
556 | unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; | 558 | int i, j; |
557 | unsigned long start = uaddr >> PAGE_SHIFT; | 559 | int nr_pages = 0; |
558 | const int nr_pages = end - start; | ||
559 | int ret, offset, i; | ||
560 | struct page **pages; | 560 | struct page **pages; |
561 | struct bio *bio; | 561 | struct bio *bio; |
562 | int cur_page = 0; | ||
563 | int ret, offset; | ||
562 | 564 | ||
563 | /* | 565 | for (i = 0; i < iov_count; i++) { |
564 | * transfer and buffer must be aligned to at least hardsector | 566 | unsigned long uaddr = (unsigned long)iov[i].iov_base; |
565 | * size for now, in the future we can relax this restriction | 567 | unsigned long len = iov[i].iov_len; |
566 | */ | 568 | unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; |
567 | if ((uaddr & queue_dma_alignment(q)) || (len & queue_dma_alignment(q))) | 569 | unsigned long start = uaddr >> PAGE_SHIFT; |
570 | |||
571 | nr_pages += end - start; | ||
572 | /* | ||
573 | * transfer and buffer must be aligned to at least hardsector | ||
574 | * size for now, in the future we can relax this restriction | ||
575 | */ | ||
576 | if ((uaddr & queue_dma_alignment(q)) || (len & queue_dma_alignment(q))) | ||
577 | return ERR_PTR(-EINVAL); | ||
578 | } | ||
579 | |||
580 | if (!nr_pages) | ||
568 | return ERR_PTR(-EINVAL); | 581 | return ERR_PTR(-EINVAL); |
569 | 582 | ||
570 | bio = bio_alloc(GFP_KERNEL, nr_pages); | 583 | bio = bio_alloc(GFP_KERNEL, nr_pages); |
@@ -576,42 +589,54 @@ static struct bio *__bio_map_user(request_queue_t *q, struct block_device *bdev, | |||
576 | if (!pages) | 589 | if (!pages) |
577 | goto out; | 590 | goto out; |
578 | 591 | ||
579 | down_read(¤t->mm->mmap_sem); | 592 | memset(pages, 0, nr_pages * sizeof(struct page *)); |
580 | ret = get_user_pages(current, current->mm, uaddr, nr_pages, | 593 | |
581 | write_to_vm, 0, pages, NULL); | 594 | for (i = 0; i < iov_count; i++) { |
582 | up_read(¤t->mm->mmap_sem); | 595 | unsigned long uaddr = (unsigned long)iov[i].iov_base; |
583 | 596 | unsigned long len = iov[i].iov_len; | |
584 | if (ret < nr_pages) | 597 | unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; |
585 | goto out; | 598 | unsigned long start = uaddr >> PAGE_SHIFT; |
586 | 599 | const int local_nr_pages = end - start; | |
587 | bio->bi_bdev = bdev; | 600 | const int page_limit = cur_page + local_nr_pages; |
588 | 601 | ||
589 | offset = uaddr & ~PAGE_MASK; | 602 | down_read(¤t->mm->mmap_sem); |
590 | for (i = 0; i < nr_pages; i++) { | 603 | ret = get_user_pages(current, current->mm, uaddr, |
591 | unsigned int bytes = PAGE_SIZE - offset; | 604 | local_nr_pages, |
592 | 605 | write_to_vm, 0, &pages[cur_page], NULL); | |
593 | if (len <= 0) | 606 | up_read(¤t->mm->mmap_sem); |
594 | break; | 607 | |
595 | 608 | if (ret < local_nr_pages) | |
596 | if (bytes > len) | 609 | goto out_unmap; |
597 | bytes = len; | 610 | |
611 | |||
612 | offset = uaddr & ~PAGE_MASK; | ||
613 | for (j = cur_page; j < page_limit; j++) { | ||
614 | unsigned int bytes = PAGE_SIZE - offset; | ||
615 | |||
616 | if (len <= 0) | ||
617 | break; | ||
618 | |||
619 | if (bytes > len) | ||
620 | bytes = len; | ||
621 | |||
622 | /* | ||
623 | * sorry... | ||
624 | */ | ||
625 | if (__bio_add_page(q, bio, pages[j], bytes, offset) < bytes) | ||
626 | break; | ||
627 | |||
628 | len -= bytes; | ||
629 | offset = 0; | ||
630 | } | ||
598 | 631 | ||
632 | cur_page = j; | ||
599 | /* | 633 | /* |
600 | * sorry... | 634 | * release the pages we didn't map into the bio, if any |
601 | */ | 635 | */ |
602 | if (__bio_add_page(q, bio, pages[i], bytes, offset) < bytes) | 636 | while (j < page_limit) |
603 | break; | 637 | page_cache_release(pages[j++]); |
604 | |||
605 | len -= bytes; | ||
606 | offset = 0; | ||
607 | } | 638 | } |
608 | 639 | ||
609 | /* | ||
610 | * release the pages we didn't map into the bio, if any | ||
611 | */ | ||
612 | while (i < nr_pages) | ||
613 | page_cache_release(pages[i++]); | ||
614 | |||
615 | kfree(pages); | 640 | kfree(pages); |
616 | 641 | ||
617 | /* | 642 | /* |
@@ -620,9 +645,17 @@ static struct bio *__bio_map_user(request_queue_t *q, struct block_device *bdev, | |||
620 | if (!write_to_vm) | 645 | if (!write_to_vm) |
621 | bio->bi_rw |= (1 << BIO_RW); | 646 | bio->bi_rw |= (1 << BIO_RW); |
622 | 647 | ||
648 | bio->bi_bdev = bdev; | ||
623 | bio->bi_flags |= (1 << BIO_USER_MAPPED); | 649 | bio->bi_flags |= (1 << BIO_USER_MAPPED); |
624 | return bio; | 650 | return bio; |
625 | out: | 651 | |
652 | out_unmap: | ||
653 | for (i = 0; i < nr_pages; i++) { | ||
654 | if(!pages[i]) | ||
655 | break; | ||
656 | page_cache_release(pages[i]); | ||
657 | } | ||
658 | out: | ||
626 | kfree(pages); | 659 | kfree(pages); |
627 | bio_put(bio); | 660 | bio_put(bio); |
628 | return ERR_PTR(ret); | 661 | return ERR_PTR(ret); |
@@ -642,9 +675,33 @@ out: | |||
642 | struct bio *bio_map_user(request_queue_t *q, struct block_device *bdev, | 675 | struct bio *bio_map_user(request_queue_t *q, struct block_device *bdev, |
643 | unsigned long uaddr, unsigned int len, int write_to_vm) | 676 | unsigned long uaddr, unsigned int len, int write_to_vm) |
644 | { | 677 | { |
678 | struct sg_iovec iov; | ||
679 | |||
680 | iov.iov_base = (__user void *)uaddr; | ||
681 | iov.iov_len = len; | ||
682 | |||
683 | return bio_map_user_iov(q, bdev, &iov, 1, write_to_vm); | ||
684 | } | ||
685 | |||
686 | /** | ||
687 | * bio_map_user_iov - map user sg_iovec table into bio | ||
688 | * @q: the request_queue_t for the bio | ||
689 | * @bdev: destination block device | ||
690 | * @iov: the iovec. | ||
691 | * @iov_count: number of elements in the iovec | ||
692 | * @write_to_vm: bool indicating writing to pages or not | ||
693 | * | ||
694 | * Map the user space address into a bio suitable for io to a block | ||
695 | * device. Returns an error pointer in case of error. | ||
696 | */ | ||
697 | struct bio *bio_map_user_iov(request_queue_t *q, struct block_device *bdev, | ||
698 | struct sg_iovec *iov, int iov_count, | ||
699 | int write_to_vm) | ||
700 | { | ||
645 | struct bio *bio; | 701 | struct bio *bio; |
702 | int len = 0, i; | ||
646 | 703 | ||
647 | bio = __bio_map_user(q, bdev, uaddr, len, write_to_vm); | 704 | bio = __bio_map_user_iov(q, bdev, iov, iov_count, write_to_vm); |
648 | 705 | ||
649 | if (IS_ERR(bio)) | 706 | if (IS_ERR(bio)) |
650 | return bio; | 707 | return bio; |
@@ -657,6 +714,9 @@ struct bio *bio_map_user(request_queue_t *q, struct block_device *bdev, | |||
657 | */ | 714 | */ |
658 | bio_get(bio); | 715 | bio_get(bio); |
659 | 716 | ||
717 | for (i = 0; i < iov_count; i++) | ||
718 | len += iov[i].iov_len; | ||
719 | |||
660 | if (bio->bi_size == len) | 720 | if (bio->bi_size == len) |
661 | return bio; | 721 | return bio; |
662 | 722 | ||