diff options
Diffstat (limited to 'mm/filemap.c')
-rw-r--r-- | mm/filemap.c | 74 |
1 files changed, 13 insertions, 61 deletions
diff --git a/mm/filemap.c b/mm/filemap.c index d8b34d1a1071..c641edf553a9 100644 --- a/mm/filemap.c +++ b/mm/filemap.c | |||
@@ -166,45 +166,15 @@ void delete_from_page_cache(struct page *page) | |||
166 | } | 166 | } |
167 | EXPORT_SYMBOL(delete_from_page_cache); | 167 | EXPORT_SYMBOL(delete_from_page_cache); |
168 | 168 | ||
169 | static int sync_page(void *word) | 169 | static int sleep_on_page(void *word) |
170 | { | 170 | { |
171 | struct address_space *mapping; | ||
172 | struct page *page; | ||
173 | |||
174 | page = container_of((unsigned long *)word, struct page, flags); | ||
175 | |||
176 | /* | ||
177 | * page_mapping() is being called without PG_locked held. | ||
178 | * Some knowledge of the state and use of the page is used to | ||
179 | * reduce the requirements down to a memory barrier. | ||
180 | * The danger here is of a stale page_mapping() return value | ||
181 | * indicating a struct address_space different from the one it's | ||
182 | * associated with when it is associated with one. | ||
183 | * After smp_mb(), it's either the correct page_mapping() for | ||
184 | * the page, or an old page_mapping() and the page's own | ||
185 | * page_mapping() has gone NULL. | ||
186 | * The ->sync_page() address_space operation must tolerate | ||
187 | * page_mapping() going NULL. By an amazing coincidence, | ||
188 | * this comes about because none of the users of the page | ||
189 | * in the ->sync_page() methods make essential use of the | ||
190 | * page_mapping(), merely passing the page down to the backing | ||
191 | * device's unplug functions when it's non-NULL, which in turn | ||
192 | * ignore it for all cases but swap, where only page_private(page) is | ||
193 | * of interest. When page_mapping() does go NULL, the entire | ||
194 | * call stack gracefully ignores the page and returns. | ||
195 | * -- wli | ||
196 | */ | ||
197 | smp_mb(); | ||
198 | mapping = page_mapping(page); | ||
199 | if (mapping && mapping->a_ops && mapping->a_ops->sync_page) | ||
200 | mapping->a_ops->sync_page(page); | ||
201 | io_schedule(); | 171 | io_schedule(); |
202 | return 0; | 172 | return 0; |
203 | } | 173 | } |
204 | 174 | ||
205 | static int sync_page_killable(void *word) | 175 | static int sleep_on_page_killable(void *word) |
206 | { | 176 | { |
207 | sync_page(word); | 177 | sleep_on_page(word); |
208 | return fatal_signal_pending(current) ? -EINTR : 0; | 178 | return fatal_signal_pending(current) ? -EINTR : 0; |
209 | } | 179 | } |
210 | 180 | ||
@@ -560,12 +530,6 @@ struct page *__page_cache_alloc(gfp_t gfp) | |||
560 | EXPORT_SYMBOL(__page_cache_alloc); | 530 | EXPORT_SYMBOL(__page_cache_alloc); |
561 | #endif | 531 | #endif |
562 | 532 | ||
563 | static int __sleep_on_page_lock(void *word) | ||
564 | { | ||
565 | io_schedule(); | ||
566 | return 0; | ||
567 | } | ||
568 | |||
569 | /* | 533 | /* |
570 | * In order to wait for pages to become available there must be | 534 | * In order to wait for pages to become available there must be |
571 | * waitqueues associated with pages. By using a hash table of | 535 | * waitqueues associated with pages. By using a hash table of |
@@ -593,7 +557,7 @@ void wait_on_page_bit(struct page *page, int bit_nr) | |||
593 | DEFINE_WAIT_BIT(wait, &page->flags, bit_nr); | 557 | DEFINE_WAIT_BIT(wait, &page->flags, bit_nr); |
594 | 558 | ||
595 | if (test_bit(bit_nr, &page->flags)) | 559 | if (test_bit(bit_nr, &page->flags)) |
596 | __wait_on_bit(page_waitqueue(page), &wait, sync_page, | 560 | __wait_on_bit(page_waitqueue(page), &wait, sleep_on_page, |
597 | TASK_UNINTERRUPTIBLE); | 561 | TASK_UNINTERRUPTIBLE); |
598 | } | 562 | } |
599 | EXPORT_SYMBOL(wait_on_page_bit); | 563 | EXPORT_SYMBOL(wait_on_page_bit); |
@@ -657,17 +621,12 @@ EXPORT_SYMBOL(end_page_writeback); | |||
657 | /** | 621 | /** |
658 | * __lock_page - get a lock on the page, assuming we need to sleep to get it | 622 | * __lock_page - get a lock on the page, assuming we need to sleep to get it |
659 | * @page: the page to lock | 623 | * @page: the page to lock |
660 | * | ||
661 | * Ugly. Running sync_page() in state TASK_UNINTERRUPTIBLE is scary. If some | ||
662 | * random driver's requestfn sets TASK_RUNNING, we could busywait. However | ||
663 | * chances are that on the second loop, the block layer's plug list is empty, | ||
664 | * so sync_page() will then return in state TASK_UNINTERRUPTIBLE. | ||
665 | */ | 624 | */ |
666 | void __lock_page(struct page *page) | 625 | void __lock_page(struct page *page) |
667 | { | 626 | { |
668 | DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); | 627 | DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); |
669 | 628 | ||
670 | __wait_on_bit_lock(page_waitqueue(page), &wait, sync_page, | 629 | __wait_on_bit_lock(page_waitqueue(page), &wait, sleep_on_page, |
671 | TASK_UNINTERRUPTIBLE); | 630 | TASK_UNINTERRUPTIBLE); |
672 | } | 631 | } |
673 | EXPORT_SYMBOL(__lock_page); | 632 | EXPORT_SYMBOL(__lock_page); |
@@ -677,24 +636,10 @@ int __lock_page_killable(struct page *page) | |||
677 | DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); | 636 | DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); |
678 | 637 | ||
679 | return __wait_on_bit_lock(page_waitqueue(page), &wait, | 638 | return __wait_on_bit_lock(page_waitqueue(page), &wait, |
680 | sync_page_killable, TASK_KILLABLE); | 639 | sleep_on_page_killable, TASK_KILLABLE); |
681 | } | 640 | } |
682 | EXPORT_SYMBOL_GPL(__lock_page_killable); | 641 | EXPORT_SYMBOL_GPL(__lock_page_killable); |
683 | 642 | ||
684 | /** | ||
685 | * __lock_page_nosync - get a lock on the page, without calling sync_page() | ||
686 | * @page: the page to lock | ||
687 | * | ||
688 | * Variant of lock_page that does not require the caller to hold a reference | ||
689 | * on the page's mapping. | ||
690 | */ | ||
691 | void __lock_page_nosync(struct page *page) | ||
692 | { | ||
693 | DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); | ||
694 | __wait_on_bit_lock(page_waitqueue(page), &wait, __sleep_on_page_lock, | ||
695 | TASK_UNINTERRUPTIBLE); | ||
696 | } | ||
697 | |||
698 | int __lock_page_or_retry(struct page *page, struct mm_struct *mm, | 643 | int __lock_page_or_retry(struct page *page, struct mm_struct *mm, |
699 | unsigned int flags) | 644 | unsigned int flags) |
700 | { | 645 | { |
@@ -1409,12 +1354,15 @@ generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, | |||
1409 | unsigned long seg = 0; | 1354 | unsigned long seg = 0; |
1410 | size_t count; | 1355 | size_t count; |
1411 | loff_t *ppos = &iocb->ki_pos; | 1356 | loff_t *ppos = &iocb->ki_pos; |
1357 | struct blk_plug plug; | ||
1412 | 1358 | ||
1413 | count = 0; | 1359 | count = 0; |
1414 | retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE); | 1360 | retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE); |
1415 | if (retval) | 1361 | if (retval) |
1416 | return retval; | 1362 | return retval; |
1417 | 1363 | ||
1364 | blk_start_plug(&plug); | ||
1365 | |||
1418 | /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */ | 1366 | /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */ |
1419 | if (filp->f_flags & O_DIRECT) { | 1367 | if (filp->f_flags & O_DIRECT) { |
1420 | loff_t size; | 1368 | loff_t size; |
@@ -1487,6 +1435,7 @@ generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, | |||
1487 | break; | 1435 | break; |
1488 | } | 1436 | } |
1489 | out: | 1437 | out: |
1438 | blk_finish_plug(&plug); | ||
1490 | return retval; | 1439 | return retval; |
1491 | } | 1440 | } |
1492 | EXPORT_SYMBOL(generic_file_aio_read); | 1441 | EXPORT_SYMBOL(generic_file_aio_read); |
@@ -2598,11 +2547,13 @@ ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov, | |||
2598 | { | 2547 | { |
2599 | struct file *file = iocb->ki_filp; | 2548 | struct file *file = iocb->ki_filp; |
2600 | struct inode *inode = file->f_mapping->host; | 2549 | struct inode *inode = file->f_mapping->host; |
2550 | struct blk_plug plug; | ||
2601 | ssize_t ret; | 2551 | ssize_t ret; |
2602 | 2552 | ||
2603 | BUG_ON(iocb->ki_pos != pos); | 2553 | BUG_ON(iocb->ki_pos != pos); |
2604 | 2554 | ||
2605 | mutex_lock(&inode->i_mutex); | 2555 | mutex_lock(&inode->i_mutex); |
2556 | blk_start_plug(&plug); | ||
2606 | ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos); | 2557 | ret = __generic_file_aio_write(iocb, iov, nr_segs, &iocb->ki_pos); |
2607 | mutex_unlock(&inode->i_mutex); | 2558 | mutex_unlock(&inode->i_mutex); |
2608 | 2559 | ||
@@ -2613,6 +2564,7 @@ ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov, | |||
2613 | if (err < 0 && ret > 0) | 2564 | if (err < 0 && ret > 0) |
2614 | ret = err; | 2565 | ret = err; |
2615 | } | 2566 | } |
2567 | blk_finish_plug(&plug); | ||
2616 | return ret; | 2568 | return ret; |
2617 | } | 2569 | } |
2618 | EXPORT_SYMBOL(generic_file_aio_write); | 2570 | EXPORT_SYMBOL(generic_file_aio_write); |