diff options
Diffstat (limited to 'fs/aio.c')
-rw-r--r-- | fs/aio.c | 77 |
1 files changed, 7 insertions, 70 deletions
@@ -34,8 +34,6 @@ | |||
34 | #include <linux/security.h> | 34 | #include <linux/security.h> |
35 | #include <linux/eventfd.h> | 35 | #include <linux/eventfd.h> |
36 | #include <linux/blkdev.h> | 36 | #include <linux/blkdev.h> |
37 | #include <linux/mempool.h> | ||
38 | #include <linux/hash.h> | ||
39 | #include <linux/compat.h> | 37 | #include <linux/compat.h> |
40 | 38 | ||
41 | #include <asm/kmap_types.h> | 39 | #include <asm/kmap_types.h> |
@@ -65,14 +63,6 @@ static DECLARE_WORK(fput_work, aio_fput_routine); | |||
65 | static DEFINE_SPINLOCK(fput_lock); | 63 | static DEFINE_SPINLOCK(fput_lock); |
66 | static LIST_HEAD(fput_head); | 64 | static LIST_HEAD(fput_head); |
67 | 65 | ||
68 | #define AIO_BATCH_HASH_BITS 3 /* allocated on-stack, so don't go crazy */ | ||
69 | #define AIO_BATCH_HASH_SIZE (1 << AIO_BATCH_HASH_BITS) | ||
70 | struct aio_batch_entry { | ||
71 | struct hlist_node list; | ||
72 | struct address_space *mapping; | ||
73 | }; | ||
74 | mempool_t *abe_pool; | ||
75 | |||
76 | static void aio_kick_handler(struct work_struct *); | 66 | static void aio_kick_handler(struct work_struct *); |
77 | static void aio_queue_work(struct kioctx *); | 67 | static void aio_queue_work(struct kioctx *); |
78 | 68 | ||
@@ -86,8 +76,7 @@ static int __init aio_setup(void) | |||
86 | kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC); | 76 | kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC); |
87 | 77 | ||
88 | aio_wq = create_workqueue("aio"); | 78 | aio_wq = create_workqueue("aio"); |
89 | abe_pool = mempool_create_kmalloc_pool(1, sizeof(struct aio_batch_entry)); | 79 | BUG_ON(!aio_wq); |
90 | BUG_ON(!aio_wq || !abe_pool); | ||
91 | 80 | ||
92 | pr_debug("aio_setup: sizeof(struct page) = %d\n", (int)sizeof(struct page)); | 81 | pr_debug("aio_setup: sizeof(struct page) = %d\n", (int)sizeof(struct page)); |
93 | 82 | ||
@@ -1525,57 +1514,8 @@ static ssize_t aio_setup_iocb(struct kiocb *kiocb, bool compat) | |||
1525 | return 0; | 1514 | return 0; |
1526 | } | 1515 | } |
1527 | 1516 | ||
1528 | static void aio_batch_add(struct address_space *mapping, | ||
1529 | struct hlist_head *batch_hash) | ||
1530 | { | ||
1531 | struct aio_batch_entry *abe; | ||
1532 | struct hlist_node *pos; | ||
1533 | unsigned bucket; | ||
1534 | |||
1535 | bucket = hash_ptr(mapping, AIO_BATCH_HASH_BITS); | ||
1536 | hlist_for_each_entry(abe, pos, &batch_hash[bucket], list) { | ||
1537 | if (abe->mapping == mapping) | ||
1538 | return; | ||
1539 | } | ||
1540 | |||
1541 | abe = mempool_alloc(abe_pool, GFP_KERNEL); | ||
1542 | |||
1543 | /* | ||
1544 | * we should be using igrab here, but | ||
1545 | * we don't want to hammer on the global | ||
1546 | * inode spinlock just to take an extra | ||
1547 | * reference on a file that we must already | ||
1548 | * have a reference to. | ||
1549 | * | ||
1550 | * When we're called, we always have a reference | ||
1551 | * on the file, so we must always have a reference | ||
1552 | * on the inode, so ihold() is safe here. | ||
1553 | */ | ||
1554 | ihold(mapping->host); | ||
1555 | abe->mapping = mapping; | ||
1556 | hlist_add_head(&abe->list, &batch_hash[bucket]); | ||
1557 | return; | ||
1558 | } | ||
1559 | |||
1560 | static void aio_batch_free(struct hlist_head *batch_hash) | ||
1561 | { | ||
1562 | struct aio_batch_entry *abe; | ||
1563 | struct hlist_node *pos, *n; | ||
1564 | int i; | ||
1565 | |||
1566 | for (i = 0; i < AIO_BATCH_HASH_SIZE; i++) { | ||
1567 | hlist_for_each_entry_safe(abe, pos, n, &batch_hash[i], list) { | ||
1568 | blk_run_address_space(abe->mapping); | ||
1569 | iput(abe->mapping->host); | ||
1570 | hlist_del(&abe->list); | ||
1571 | mempool_free(abe, abe_pool); | ||
1572 | } | ||
1573 | } | ||
1574 | } | ||
1575 | |||
1576 | static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, | 1517 | static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, |
1577 | struct iocb *iocb, struct hlist_head *batch_hash, | 1518 | struct iocb *iocb, bool compat) |
1578 | bool compat) | ||
1579 | { | 1519 | { |
1580 | struct kiocb *req; | 1520 | struct kiocb *req; |
1581 | struct file *file; | 1521 | struct file *file; |
@@ -1666,11 +1606,6 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, | |||
1666 | ; | 1606 | ; |
1667 | } | 1607 | } |
1668 | spin_unlock_irq(&ctx->ctx_lock); | 1608 | spin_unlock_irq(&ctx->ctx_lock); |
1669 | if (req->ki_opcode == IOCB_CMD_PREAD || | ||
1670 | req->ki_opcode == IOCB_CMD_PREADV || | ||
1671 | req->ki_opcode == IOCB_CMD_PWRITE || | ||
1672 | req->ki_opcode == IOCB_CMD_PWRITEV) | ||
1673 | aio_batch_add(file->f_mapping, batch_hash); | ||
1674 | 1609 | ||
1675 | aio_put_req(req); /* drop extra ref to req */ | 1610 | aio_put_req(req); /* drop extra ref to req */ |
1676 | return 0; | 1611 | return 0; |
@@ -1687,7 +1622,7 @@ long do_io_submit(aio_context_t ctx_id, long nr, | |||
1687 | struct kioctx *ctx; | 1622 | struct kioctx *ctx; |
1688 | long ret = 0; | 1623 | long ret = 0; |
1689 | int i; | 1624 | int i; |
1690 | struct hlist_head batch_hash[AIO_BATCH_HASH_SIZE] = { { 0, }, }; | 1625 | struct blk_plug plug; |
1691 | 1626 | ||
1692 | if (unlikely(nr < 0)) | 1627 | if (unlikely(nr < 0)) |
1693 | return -EINVAL; | 1628 | return -EINVAL; |
@@ -1704,6 +1639,8 @@ long do_io_submit(aio_context_t ctx_id, long nr, | |||
1704 | return -EINVAL; | 1639 | return -EINVAL; |
1705 | } | 1640 | } |
1706 | 1641 | ||
1642 | blk_start_plug(&plug); | ||
1643 | |||
1707 | /* | 1644 | /* |
1708 | * AKPM: should this return a partial result if some of the IOs were | 1645 | * AKPM: should this return a partial result if some of the IOs were |
1709 | * successfully submitted? | 1646 | * successfully submitted? |
@@ -1722,11 +1659,11 @@ long do_io_submit(aio_context_t ctx_id, long nr, | |||
1722 | break; | 1659 | break; |
1723 | } | 1660 | } |
1724 | 1661 | ||
1725 | ret = io_submit_one(ctx, user_iocb, &tmp, batch_hash, compat); | 1662 | ret = io_submit_one(ctx, user_iocb, &tmp, compat); |
1726 | if (ret) | 1663 | if (ret) |
1727 | break; | 1664 | break; |
1728 | } | 1665 | } |
1729 | aio_batch_free(batch_hash); | 1666 | blk_finish_plug(&plug); |
1730 | 1667 | ||
1731 | put_ioctx(ctx); | 1668 | put_ioctx(ctx); |
1732 | return i ? i : ret; | 1669 | return i ? i : ret; |