aboutsummaryrefslogtreecommitdiffstats
path: root/fs/ext4/inode.c
diff options
context:
space:
mode:
Diffstat (limited to 'fs/ext4/inode.c')
-rw-r--r--fs/ext4/inode.c500
1 files changed, 342 insertions, 158 deletions
diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
index e11952404e02..3e0f6af9d08d 100644
--- a/fs/ext4/inode.c
+++ b/fs/ext4/inode.c
@@ -38,6 +38,8 @@
38#include <linux/uio.h> 38#include <linux/uio.h>
39#include <linux/bio.h> 39#include <linux/bio.h>
40#include <linux/workqueue.h> 40#include <linux/workqueue.h>
41#include <linux/kernel.h>
42#include <linux/slab.h>
41 43
42#include "ext4_jbd2.h" 44#include "ext4_jbd2.h"
43#include "xattr.h" 45#include "xattr.h"
@@ -170,6 +172,9 @@ void ext4_delete_inode(struct inode *inode)
170 handle_t *handle; 172 handle_t *handle;
171 int err; 173 int err;
172 174
175 if (!is_bad_inode(inode))
176 dquot_initialize(inode);
177
173 if (ext4_should_order_data(inode)) 178 if (ext4_should_order_data(inode))
174 ext4_begin_ordered_truncate(inode, 0); 179 ext4_begin_ordered_truncate(inode, 0);
175 truncate_inode_pages(&inode->i_data, 0); 180 truncate_inode_pages(&inode->i_data, 0);
@@ -194,7 +199,7 @@ void ext4_delete_inode(struct inode *inode)
194 inode->i_size = 0; 199 inode->i_size = 0;
195 err = ext4_mark_inode_dirty(handle, inode); 200 err = ext4_mark_inode_dirty(handle, inode);
196 if (err) { 201 if (err) {
197 ext4_warning(inode->i_sb, __func__, 202 ext4_warning(inode->i_sb,
198 "couldn't mark inode dirty (err %d)", err); 203 "couldn't mark inode dirty (err %d)", err);
199 goto stop_handle; 204 goto stop_handle;
200 } 205 }
@@ -212,7 +217,7 @@ void ext4_delete_inode(struct inode *inode)
212 if (err > 0) 217 if (err > 0)
213 err = ext4_journal_restart(handle, 3); 218 err = ext4_journal_restart(handle, 3);
214 if (err != 0) { 219 if (err != 0) {
215 ext4_warning(inode->i_sb, __func__, 220 ext4_warning(inode->i_sb,
216 "couldn't extend journal (err %d)", err); 221 "couldn't extend journal (err %d)", err);
217 stop_handle: 222 stop_handle:
218 ext4_journal_stop(handle); 223 ext4_journal_stop(handle);
@@ -323,8 +328,7 @@ static int ext4_block_to_path(struct inode *inode,
323 offsets[n++] = i_block & (ptrs - 1); 328 offsets[n++] = i_block & (ptrs - 1);
324 final = ptrs; 329 final = ptrs;
325 } else { 330 } else {
326 ext4_warning(inode->i_sb, "ext4_block_to_path", 331 ext4_warning(inode->i_sb, "block %lu > max in inode %lu",
327 "block %lu > max in inode %lu",
328 i_block + direct_blocks + 332 i_block + direct_blocks +
329 indirect_blocks + double_blocks, inode->i_ino); 333 indirect_blocks + double_blocks, inode->i_ino);
330 } 334 }
@@ -344,7 +348,7 @@ static int __ext4_check_blockref(const char *function, struct inode *inode,
344 if (blk && 348 if (blk &&
345 unlikely(!ext4_data_block_valid(EXT4_SB(inode->i_sb), 349 unlikely(!ext4_data_block_valid(EXT4_SB(inode->i_sb),
346 blk, 1))) { 350 blk, 1))) {
347 ext4_error(inode->i_sb, function, 351 __ext4_error(inode->i_sb, function,
348 "invalid block reference %u " 352 "invalid block reference %u "
349 "in inode #%lu", blk, inode->i_ino); 353 "in inode #%lu", blk, inode->i_ino);
350 return -EIO; 354 return -EIO;
@@ -607,7 +611,14 @@ static int ext4_alloc_blocks(handle_t *handle, struct inode *inode,
607 if (*err) 611 if (*err)
608 goto failed_out; 612 goto failed_out;
609 613
610 BUG_ON(current_block + count > EXT4_MAX_BLOCK_FILE_PHYS); 614 if (unlikely(current_block + count > EXT4_MAX_BLOCK_FILE_PHYS)) {
615 EXT4_ERROR_INODE(inode,
616 "current_block %llu + count %lu > %d!",
617 current_block, count,
618 EXT4_MAX_BLOCK_FILE_PHYS);
619 *err = -EIO;
620 goto failed_out;
621 }
611 622
612 target -= count; 623 target -= count;
613 /* allocate blocks for indirect blocks */ 624 /* allocate blocks for indirect blocks */
@@ -643,7 +654,14 @@ static int ext4_alloc_blocks(handle_t *handle, struct inode *inode,
643 ar.flags = EXT4_MB_HINT_DATA; 654 ar.flags = EXT4_MB_HINT_DATA;
644 655
645 current_block = ext4_mb_new_blocks(handle, &ar, err); 656 current_block = ext4_mb_new_blocks(handle, &ar, err);
646 BUG_ON(current_block + ar.len > EXT4_MAX_BLOCK_FILE_PHYS); 657 if (unlikely(current_block + ar.len > EXT4_MAX_BLOCK_FILE_PHYS)) {
658 EXT4_ERROR_INODE(inode,
659 "current_block %llu + ar.len %d > %d!",
660 current_block, ar.len,
661 EXT4_MAX_BLOCK_FILE_PHYS);
662 *err = -EIO;
663 goto failed_out;
664 }
647 665
648 if (*err && (target == blks)) { 666 if (*err && (target == blks)) {
649 /* 667 /*
@@ -1018,7 +1036,7 @@ static int ext4_indirect_calc_metadata_amount(struct inode *inode,
1018 sector_t lblock) 1036 sector_t lblock)
1019{ 1037{
1020 struct ext4_inode_info *ei = EXT4_I(inode); 1038 struct ext4_inode_info *ei = EXT4_I(inode);
1021 int dind_mask = EXT4_ADDR_PER_BLOCK(inode->i_sb) - 1; 1039 sector_t dind_mask = ~((sector_t)EXT4_ADDR_PER_BLOCK(inode->i_sb) - 1);
1022 int blk_bits; 1040 int blk_bits;
1023 1041
1024 if (lblock < EXT4_NDIR_BLOCKS) 1042 if (lblock < EXT4_NDIR_BLOCKS)
@@ -1033,7 +1051,7 @@ static int ext4_indirect_calc_metadata_amount(struct inode *inode,
1033 } 1051 }
1034 ei->i_da_metadata_calc_last_lblock = lblock & dind_mask; 1052 ei->i_da_metadata_calc_last_lblock = lblock & dind_mask;
1035 ei->i_da_metadata_calc_len = 1; 1053 ei->i_da_metadata_calc_len = 1;
1036 blk_bits = roundup_pow_of_two(lblock + 1); 1054 blk_bits = order_base_2(lblock);
1037 return (blk_bits / EXT4_ADDR_PER_BLOCK_BITS(inode->i_sb)) + 1; 1055 return (blk_bits / EXT4_ADDR_PER_BLOCK_BITS(inode->i_sb)) + 1;
1038} 1056}
1039 1057
@@ -1061,6 +1079,7 @@ void ext4_da_update_reserve_space(struct inode *inode,
1061 int mdb_free = 0, allocated_meta_blocks = 0; 1079 int mdb_free = 0, allocated_meta_blocks = 0;
1062 1080
1063 spin_lock(&ei->i_block_reservation_lock); 1081 spin_lock(&ei->i_block_reservation_lock);
1082 trace_ext4_da_update_reserve_space(inode, used);
1064 if (unlikely(used > ei->i_reserved_data_blocks)) { 1083 if (unlikely(used > ei->i_reserved_data_blocks)) {
1065 ext4_msg(inode->i_sb, KERN_NOTICE, "%s: ino %lu, used %d " 1084 ext4_msg(inode->i_sb, KERN_NOTICE, "%s: ino %lu, used %d "
1066 "with only %d reserved data blocks\n", 1085 "with only %d reserved data blocks\n",
@@ -1093,9 +1112,9 @@ void ext4_da_update_reserve_space(struct inode *inode,
1093 1112
1094 /* Update quota subsystem */ 1113 /* Update quota subsystem */
1095 if (quota_claim) { 1114 if (quota_claim) {
1096 vfs_dq_claim_block(inode, used); 1115 dquot_claim_block(inode, used);
1097 if (mdb_free) 1116 if (mdb_free)
1098 vfs_dq_release_reservation_block(inode, mdb_free); 1117 dquot_release_reservation_block(inode, mdb_free);
1099 } else { 1118 } else {
1100 /* 1119 /*
1101 * We did fallocate with an offset that is already delayed 1120 * We did fallocate with an offset that is already delayed
@@ -1106,8 +1125,8 @@ void ext4_da_update_reserve_space(struct inode *inode,
1106 * that 1125 * that
1107 */ 1126 */
1108 if (allocated_meta_blocks) 1127 if (allocated_meta_blocks)
1109 vfs_dq_claim_block(inode, allocated_meta_blocks); 1128 dquot_claim_block(inode, allocated_meta_blocks);
1110 vfs_dq_release_reservation_block(inode, mdb_free + used); 1129 dquot_release_reservation_block(inode, mdb_free + used);
1111 } 1130 }
1112 1131
1113 /* 1132 /*
@@ -1124,7 +1143,7 @@ static int check_block_validity(struct inode *inode, const char *msg,
1124 sector_t logical, sector_t phys, int len) 1143 sector_t logical, sector_t phys, int len)
1125{ 1144{
1126 if (!ext4_data_block_valid(EXT4_SB(inode->i_sb), phys, len)) { 1145 if (!ext4_data_block_valid(EXT4_SB(inode->i_sb), phys, len)) {
1127 ext4_error(inode->i_sb, msg, 1146 __ext4_error(inode->i_sb, msg,
1128 "inode #%lu logical block %llu mapped to %llu " 1147 "inode #%lu logical block %llu mapped to %llu "
1129 "(size %d)", inode->i_ino, 1148 "(size %d)", inode->i_ino,
1130 (unsigned long long) logical, 1149 (unsigned long long) logical,
@@ -1306,7 +1325,7 @@ int ext4_get_blocks(handle_t *handle, struct inode *inode, sector_t block,
1306 * i_data's format changing. Force the migrate 1325 * i_data's format changing. Force the migrate
1307 * to fail by clearing migrate flags 1326 * to fail by clearing migrate flags
1308 */ 1327 */
1309 EXT4_I(inode)->i_state &= ~EXT4_STATE_EXT_MIGRATE; 1328 ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE);
1310 } 1329 }
1311 1330
1312 /* 1331 /*
@@ -1534,6 +1553,8 @@ static void ext4_truncate_failed_write(struct inode *inode)
1534 ext4_truncate(inode); 1553 ext4_truncate(inode);
1535} 1554}
1536 1555
1556static int ext4_get_block_write(struct inode *inode, sector_t iblock,
1557 struct buffer_head *bh_result, int create);
1537static int ext4_write_begin(struct file *file, struct address_space *mapping, 1558static int ext4_write_begin(struct file *file, struct address_space *mapping,
1538 loff_t pos, unsigned len, unsigned flags, 1559 loff_t pos, unsigned len, unsigned flags,
1539 struct page **pagep, void **fsdata) 1560 struct page **pagep, void **fsdata)
@@ -1575,8 +1596,12 @@ retry:
1575 } 1596 }
1576 *pagep = page; 1597 *pagep = page;
1577 1598
1578 ret = block_write_begin(file, mapping, pos, len, flags, pagep, fsdata, 1599 if (ext4_should_dioread_nolock(inode))
1579 ext4_get_block); 1600 ret = block_write_begin(file, mapping, pos, len, flags, pagep,
1601 fsdata, ext4_get_block_write);
1602 else
1603 ret = block_write_begin(file, mapping, pos, len, flags, pagep,
1604 fsdata, ext4_get_block);
1580 1605
1581 if (!ret && ext4_should_journal_data(inode)) { 1606 if (!ret && ext4_should_journal_data(inode)) {
1582 ret = walk_page_buffers(handle, page_buffers(page), 1607 ret = walk_page_buffers(handle, page_buffers(page),
@@ -1793,7 +1818,7 @@ static int ext4_journalled_write_end(struct file *file,
1793 new_i_size = pos + copied; 1818 new_i_size = pos + copied;
1794 if (new_i_size > inode->i_size) 1819 if (new_i_size > inode->i_size)
1795 i_size_write(inode, pos+copied); 1820 i_size_write(inode, pos+copied);
1796 EXT4_I(inode)->i_state |= EXT4_STATE_JDATA; 1821 ext4_set_inode_state(inode, EXT4_STATE_JDATA);
1797 if (new_i_size > EXT4_I(inode)->i_disksize) { 1822 if (new_i_size > EXT4_I(inode)->i_disksize) {
1798 ext4_update_i_disksize(inode, new_i_size); 1823 ext4_update_i_disksize(inode, new_i_size);
1799 ret2 = ext4_mark_inode_dirty(handle, inode); 1824 ret2 = ext4_mark_inode_dirty(handle, inode);
@@ -1836,6 +1861,7 @@ static int ext4_da_reserve_space(struct inode *inode, sector_t lblock)
1836 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 1861 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1837 struct ext4_inode_info *ei = EXT4_I(inode); 1862 struct ext4_inode_info *ei = EXT4_I(inode);
1838 unsigned long md_needed, md_reserved; 1863 unsigned long md_needed, md_reserved;
1864 int ret;
1839 1865
1840 /* 1866 /*
1841 * recalculate the amount of metadata blocks to reserve 1867 * recalculate the amount of metadata blocks to reserve
@@ -1846,6 +1872,7 @@ repeat:
1846 spin_lock(&ei->i_block_reservation_lock); 1872 spin_lock(&ei->i_block_reservation_lock);
1847 md_reserved = ei->i_reserved_meta_blocks; 1873 md_reserved = ei->i_reserved_meta_blocks;
1848 md_needed = ext4_calc_metadata_amount(inode, lblock); 1874 md_needed = ext4_calc_metadata_amount(inode, lblock);
1875 trace_ext4_da_reserve_space(inode, md_needed);
1849 spin_unlock(&ei->i_block_reservation_lock); 1876 spin_unlock(&ei->i_block_reservation_lock);
1850 1877
1851 /* 1878 /*
@@ -1853,11 +1880,12 @@ repeat:
1853 * later. Real quota accounting is done at pages writeout 1880 * later. Real quota accounting is done at pages writeout
1854 * time. 1881 * time.
1855 */ 1882 */
1856 if (vfs_dq_reserve_block(inode, md_needed + 1)) 1883 ret = dquot_reserve_block(inode, md_needed + 1);
1857 return -EDQUOT; 1884 if (ret)
1885 return ret;
1858 1886
1859 if (ext4_claim_free_blocks(sbi, md_needed + 1)) { 1887 if (ext4_claim_free_blocks(sbi, md_needed + 1)) {
1860 vfs_dq_release_reservation_block(inode, md_needed + 1); 1888 dquot_release_reservation_block(inode, md_needed + 1);
1861 if (ext4_should_retry_alloc(inode->i_sb, &retries)) { 1889 if (ext4_should_retry_alloc(inode->i_sb, &retries)) {
1862 yield(); 1890 yield();
1863 goto repeat; 1891 goto repeat;
@@ -1914,7 +1942,7 @@ static void ext4_da_release_space(struct inode *inode, int to_free)
1914 1942
1915 spin_unlock(&EXT4_I(inode)->i_block_reservation_lock); 1943 spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
1916 1944
1917 vfs_dq_release_reservation_block(inode, to_free); 1945 dquot_release_reservation_block(inode, to_free);
1918} 1946}
1919 1947
1920static void ext4_da_page_release_reservation(struct page *page, 1948static void ext4_da_page_release_reservation(struct page *page,
@@ -2091,6 +2119,8 @@ static void mpage_put_bnr_to_bhs(struct mpage_da_data *mpd, sector_t logical,
2091 } else if (buffer_mapped(bh)) 2119 } else if (buffer_mapped(bh))
2092 BUG_ON(bh->b_blocknr != pblock); 2120 BUG_ON(bh->b_blocknr != pblock);
2093 2121
2122 if (buffer_uninit(exbh))
2123 set_buffer_uninit(bh);
2094 cur_logical++; 2124 cur_logical++;
2095 pblock++; 2125 pblock++;
2096 } while ((bh = bh->b_this_page) != head); 2126 } while ((bh = bh->b_this_page) != head);
@@ -2133,17 +2163,16 @@ static void ext4_da_block_invalidatepages(struct mpage_da_data *mpd,
2133 break; 2163 break;
2134 for (i = 0; i < nr_pages; i++) { 2164 for (i = 0; i < nr_pages; i++) {
2135 struct page *page = pvec.pages[i]; 2165 struct page *page = pvec.pages[i];
2136 index = page->index; 2166 if (page->index > end)
2137 if (index > end)
2138 break; 2167 break;
2139 index++;
2140
2141 BUG_ON(!PageLocked(page)); 2168 BUG_ON(!PageLocked(page));
2142 BUG_ON(PageWriteback(page)); 2169 BUG_ON(PageWriteback(page));
2143 block_invalidatepage(page, 0); 2170 block_invalidatepage(page, 0);
2144 ClearPageUptodate(page); 2171 ClearPageUptodate(page);
2145 unlock_page(page); 2172 unlock_page(page);
2146 } 2173 }
2174 index = pvec.pages[nr_pages - 1]->index + 1;
2175 pagevec_release(&pvec);
2147 } 2176 }
2148 return; 2177 return;
2149} 2178}
@@ -2220,6 +2249,8 @@ static int mpage_da_map_blocks(struct mpage_da_data *mpd)
2220 */ 2249 */
2221 new.b_state = 0; 2250 new.b_state = 0;
2222 get_blocks_flags = EXT4_GET_BLOCKS_CREATE; 2251 get_blocks_flags = EXT4_GET_BLOCKS_CREATE;
2252 if (ext4_should_dioread_nolock(mpd->inode))
2253 get_blocks_flags |= EXT4_GET_BLOCKS_IO_CREATE_EXT;
2223 if (mpd->b_state & (1 << BH_Delay)) 2254 if (mpd->b_state & (1 << BH_Delay))
2224 get_blocks_flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE; 2255 get_blocks_flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE;
2225 2256
@@ -2630,11 +2661,14 @@ static int __ext4_journalled_writepage(struct page *page,
2630 ret = err; 2661 ret = err;
2631 2662
2632 walk_page_buffers(handle, page_bufs, 0, len, NULL, bput_one); 2663 walk_page_buffers(handle, page_bufs, 0, len, NULL, bput_one);
2633 EXT4_I(inode)->i_state |= EXT4_STATE_JDATA; 2664 ext4_set_inode_state(inode, EXT4_STATE_JDATA);
2634out: 2665out:
2635 return ret; 2666 return ret;
2636} 2667}
2637 2668
2669static int ext4_set_bh_endio(struct buffer_head *bh, struct inode *inode);
2670static void ext4_end_io_buffer_write(struct buffer_head *bh, int uptodate);
2671
2638/* 2672/*
2639 * Note that we don't need to start a transaction unless we're journaling data 2673 * Note that we don't need to start a transaction unless we're journaling data
2640 * because we should have holes filled from ext4_page_mkwrite(). We even don't 2674 * because we should have holes filled from ext4_page_mkwrite(). We even don't
@@ -2682,7 +2716,7 @@ static int ext4_writepage(struct page *page,
2682 int ret = 0; 2716 int ret = 0;
2683 loff_t size; 2717 loff_t size;
2684 unsigned int len; 2718 unsigned int len;
2685 struct buffer_head *page_bufs; 2719 struct buffer_head *page_bufs = NULL;
2686 struct inode *inode = page->mapping->host; 2720 struct inode *inode = page->mapping->host;
2687 2721
2688 trace_ext4_writepage(inode, page); 2722 trace_ext4_writepage(inode, page);
@@ -2758,7 +2792,11 @@ static int ext4_writepage(struct page *page,
2758 2792
2759 if (test_opt(inode->i_sb, NOBH) && ext4_should_writeback_data(inode)) 2793 if (test_opt(inode->i_sb, NOBH) && ext4_should_writeback_data(inode))
2760 ret = nobh_writepage(page, noalloc_get_block_write, wbc); 2794 ret = nobh_writepage(page, noalloc_get_block_write, wbc);
2761 else 2795 else if (page_bufs && buffer_uninit(page_bufs)) {
2796 ext4_set_bh_endio(page_bufs, inode);
2797 ret = block_write_full_page_endio(page, noalloc_get_block_write,
2798 wbc, ext4_end_io_buffer_write);
2799 } else
2762 ret = block_write_full_page(page, noalloc_get_block_write, 2800 ret = block_write_full_page(page, noalloc_get_block_write,
2763 wbc); 2801 wbc);
2764 2802
@@ -3301,7 +3339,8 @@ static sector_t ext4_bmap(struct address_space *mapping, sector_t block)
3301 filemap_write_and_wait(mapping); 3339 filemap_write_and_wait(mapping);
3302 } 3340 }
3303 3341
3304 if (EXT4_JOURNAL(inode) && EXT4_I(inode)->i_state & EXT4_STATE_JDATA) { 3342 if (EXT4_JOURNAL(inode) &&
3343 ext4_test_inode_state(inode, EXT4_STATE_JDATA)) {
3305 /* 3344 /*
3306 * This is a REALLY heavyweight approach, but the use of 3345 * This is a REALLY heavyweight approach, but the use of
3307 * bmap on dirty files is expected to be extremely rare: 3346 * bmap on dirty files is expected to be extremely rare:
@@ -3320,7 +3359,7 @@ static sector_t ext4_bmap(struct address_space *mapping, sector_t block)
3320 * everything they get. 3359 * everything they get.
3321 */ 3360 */
3322 3361
3323 EXT4_I(inode)->i_state &= ~EXT4_STATE_JDATA; 3362 ext4_clear_inode_state(inode, EXT4_STATE_JDATA);
3324 journal = EXT4_JOURNAL(inode); 3363 journal = EXT4_JOURNAL(inode);
3325 jbd2_journal_lock_updates(journal); 3364 jbd2_journal_lock_updates(journal);
3326 err = jbd2_journal_flush(journal); 3365 err = jbd2_journal_flush(journal);
@@ -3345,11 +3384,45 @@ ext4_readpages(struct file *file, struct address_space *mapping,
3345 return mpage_readpages(mapping, pages, nr_pages, ext4_get_block); 3384 return mpage_readpages(mapping, pages, nr_pages, ext4_get_block);
3346} 3385}
3347 3386
3387static void ext4_free_io_end(ext4_io_end_t *io)
3388{
3389 BUG_ON(!io);
3390 if (io->page)
3391 put_page(io->page);
3392 iput(io->inode);
3393 kfree(io);
3394}
3395
3396static void ext4_invalidatepage_free_endio(struct page *page, unsigned long offset)
3397{
3398 struct buffer_head *head, *bh;
3399 unsigned int curr_off = 0;
3400
3401 if (!page_has_buffers(page))
3402 return;
3403 head = bh = page_buffers(page);
3404 do {
3405 if (offset <= curr_off && test_clear_buffer_uninit(bh)
3406 && bh->b_private) {
3407 ext4_free_io_end(bh->b_private);
3408 bh->b_private = NULL;
3409 bh->b_end_io = NULL;
3410 }
3411 curr_off = curr_off + bh->b_size;
3412 bh = bh->b_this_page;
3413 } while (bh != head);
3414}
3415
3348static void ext4_invalidatepage(struct page *page, unsigned long offset) 3416static void ext4_invalidatepage(struct page *page, unsigned long offset)
3349{ 3417{
3350 journal_t *journal = EXT4_JOURNAL(page->mapping->host); 3418 journal_t *journal = EXT4_JOURNAL(page->mapping->host);
3351 3419
3352 /* 3420 /*
3421 * free any io_end structure allocated for buffers to be discarded
3422 */
3423 if (ext4_should_dioread_nolock(page->mapping->host))
3424 ext4_invalidatepage_free_endio(page, offset);
3425 /*
3353 * If it's a full truncate we just forget about the pending dirtying 3426 * If it's a full truncate we just forget about the pending dirtying
3354 */ 3427 */
3355 if (offset == 0) 3428 if (offset == 0)
@@ -3420,7 +3493,14 @@ static ssize_t ext4_ind_direct_IO(int rw, struct kiocb *iocb,
3420 } 3493 }
3421 3494
3422retry: 3495retry:
3423 ret = blockdev_direct_IO(rw, iocb, inode, inode->i_sb->s_bdev, iov, 3496 if (rw == READ && ext4_should_dioread_nolock(inode))
3497 ret = blockdev_direct_IO_no_locking(rw, iocb, inode,
3498 inode->i_sb->s_bdev, iov,
3499 offset, nr_segs,
3500 ext4_get_block, NULL);
3501 else
3502 ret = blockdev_direct_IO(rw, iocb, inode,
3503 inode->i_sb->s_bdev, iov,
3424 offset, nr_segs, 3504 offset, nr_segs,
3425 ext4_get_block, NULL); 3505 ext4_get_block, NULL);
3426 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) 3506 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
@@ -3436,6 +3516,9 @@ retry:
3436 * but cannot extend i_size. Bail out and pretend 3516 * but cannot extend i_size. Bail out and pretend
3437 * the write failed... */ 3517 * the write failed... */
3438 ret = PTR_ERR(handle); 3518 ret = PTR_ERR(handle);
3519 if (inode->i_nlink)
3520 ext4_orphan_del(NULL, inode);
3521
3439 goto out; 3522 goto out;
3440 } 3523 }
3441 if (inode->i_nlink) 3524 if (inode->i_nlink)
@@ -3463,75 +3546,63 @@ out:
3463 return ret; 3546 return ret;
3464} 3547}
3465 3548
3466static int ext4_get_block_dio_write(struct inode *inode, sector_t iblock, 3549static int ext4_get_block_write(struct inode *inode, sector_t iblock,
3467 struct buffer_head *bh_result, int create) 3550 struct buffer_head *bh_result, int create)
3468{ 3551{
3469 handle_t *handle = NULL; 3552 handle_t *handle = ext4_journal_current_handle();
3470 int ret = 0; 3553 int ret = 0;
3471 unsigned max_blocks = bh_result->b_size >> inode->i_blkbits; 3554 unsigned max_blocks = bh_result->b_size >> inode->i_blkbits;
3472 int dio_credits; 3555 int dio_credits;
3556 int started = 0;
3473 3557
3474 ext4_debug("ext4_get_block_dio_write: inode %lu, create flag %d\n", 3558 ext4_debug("ext4_get_block_write: inode %lu, create flag %d\n",
3475 inode->i_ino, create); 3559 inode->i_ino, create);
3476 /* 3560 /*
3477 * DIO VFS code passes create = 0 flag for write to 3561 * ext4_get_block in prepare for a DIO write or buffer write.
3478 * the middle of file. It does this to avoid block 3562 * We allocate an uinitialized extent if blocks haven't been allocated.
3479 * allocation for holes, to prevent expose stale data 3563 * The extent will be converted to initialized after IO complete.
3480 * out when there is parallel buffered read (which does
3481 * not hold the i_mutex lock) while direct IO write has
3482 * not completed. DIO request on holes finally falls back
3483 * to buffered IO for this reason.
3484 *
3485 * For ext4 extent based file, since we support fallocate,
3486 * new allocated extent as uninitialized, for holes, we
3487 * could fallocate blocks for holes, thus parallel
3488 * buffered IO read will zero out the page when read on
3489 * a hole while parallel DIO write to the hole has not completed.
3490 *
3491 * when we come here, we know it's a direct IO write to
3492 * to the middle of file (<i_size)
3493 * so it's safe to override the create flag from VFS.
3494 */ 3564 */
3495 create = EXT4_GET_BLOCKS_DIO_CREATE_EXT; 3565 create = EXT4_GET_BLOCKS_IO_CREATE_EXT;
3496 3566
3497 if (max_blocks > DIO_MAX_BLOCKS) 3567 if (!handle) {
3498 max_blocks = DIO_MAX_BLOCKS; 3568 if (max_blocks > DIO_MAX_BLOCKS)
3499 dio_credits = ext4_chunk_trans_blocks(inode, max_blocks); 3569 max_blocks = DIO_MAX_BLOCKS;
3500 handle = ext4_journal_start(inode, dio_credits); 3570 dio_credits = ext4_chunk_trans_blocks(inode, max_blocks);
3501 if (IS_ERR(handle)) { 3571 handle = ext4_journal_start(inode, dio_credits);
3502 ret = PTR_ERR(handle); 3572 if (IS_ERR(handle)) {
3503 goto out; 3573 ret = PTR_ERR(handle);
3574 goto out;
3575 }
3576 started = 1;
3504 } 3577 }
3578
3505 ret = ext4_get_blocks(handle, inode, iblock, max_blocks, bh_result, 3579 ret = ext4_get_blocks(handle, inode, iblock, max_blocks, bh_result,
3506 create); 3580 create);
3507 if (ret > 0) { 3581 if (ret > 0) {
3508 bh_result->b_size = (ret << inode->i_blkbits); 3582 bh_result->b_size = (ret << inode->i_blkbits);
3509 ret = 0; 3583 ret = 0;
3510 } 3584 }
3511 ext4_journal_stop(handle); 3585 if (started)
3586 ext4_journal_stop(handle);
3512out: 3587out:
3513 return ret; 3588 return ret;
3514} 3589}
3515 3590
3516static void ext4_free_io_end(ext4_io_end_t *io) 3591static void dump_completed_IO(struct inode * inode)
3517{
3518 BUG_ON(!io);
3519 iput(io->inode);
3520 kfree(io);
3521}
3522static void dump_aio_dio_list(struct inode * inode)
3523{ 3592{
3524#ifdef EXT4_DEBUG 3593#ifdef EXT4_DEBUG
3525 struct list_head *cur, *before, *after; 3594 struct list_head *cur, *before, *after;
3526 ext4_io_end_t *io, *io0, *io1; 3595 ext4_io_end_t *io, *io0, *io1;
3596 unsigned long flags;
3527 3597
3528 if (list_empty(&EXT4_I(inode)->i_aio_dio_complete_list)){ 3598 if (list_empty(&EXT4_I(inode)->i_completed_io_list)){
3529 ext4_debug("inode %lu aio dio list is empty\n", inode->i_ino); 3599 ext4_debug("inode %lu completed_io list is empty\n", inode->i_ino);
3530 return; 3600 return;
3531 } 3601 }
3532 3602
3533 ext4_debug("Dump inode %lu aio_dio_completed_IO list \n", inode->i_ino); 3603 ext4_debug("Dump inode %lu completed_io list \n", inode->i_ino);
3534 list_for_each_entry(io, &EXT4_I(inode)->i_aio_dio_complete_list, list){ 3604 spin_lock_irqsave(&EXT4_I(inode)->i_completed_io_lock, flags);
3605 list_for_each_entry(io, &EXT4_I(inode)->i_completed_io_list, list){
3535 cur = &io->list; 3606 cur = &io->list;
3536 before = cur->prev; 3607 before = cur->prev;
3537 io0 = container_of(before, ext4_io_end_t, list); 3608 io0 = container_of(before, ext4_io_end_t, list);
@@ -3541,32 +3612,31 @@ static void dump_aio_dio_list(struct inode * inode)
3541 ext4_debug("io 0x%p from inode %lu,prev 0x%p,next 0x%p\n", 3612 ext4_debug("io 0x%p from inode %lu,prev 0x%p,next 0x%p\n",
3542 io, inode->i_ino, io0, io1); 3613 io, inode->i_ino, io0, io1);
3543 } 3614 }
3615 spin_unlock_irqrestore(&EXT4_I(inode)->i_completed_io_lock, flags);
3544#endif 3616#endif
3545} 3617}
3546 3618
3547/* 3619/*
3548 * check a range of space and convert unwritten extents to written. 3620 * check a range of space and convert unwritten extents to written.
3549 */ 3621 */
3550static int ext4_end_aio_dio_nolock(ext4_io_end_t *io) 3622static int ext4_end_io_nolock(ext4_io_end_t *io)
3551{ 3623{
3552 struct inode *inode = io->inode; 3624 struct inode *inode = io->inode;
3553 loff_t offset = io->offset; 3625 loff_t offset = io->offset;
3554 size_t size = io->size; 3626 ssize_t size = io->size;
3555 int ret = 0; 3627 int ret = 0;
3556 3628
3557 ext4_debug("end_aio_dio_onlock: io 0x%p from inode %lu,list->next 0x%p," 3629 ext4_debug("ext4_end_io_nolock: io 0x%p from inode %lu,list->next 0x%p,"
3558 "list->prev 0x%p\n", 3630 "list->prev 0x%p\n",
3559 io, inode->i_ino, io->list.next, io->list.prev); 3631 io, inode->i_ino, io->list.next, io->list.prev);
3560 3632
3561 if (list_empty(&io->list)) 3633 if (list_empty(&io->list))
3562 return ret; 3634 return ret;
3563 3635
3564 if (io->flag != DIO_AIO_UNWRITTEN) 3636 if (io->flag != EXT4_IO_UNWRITTEN)
3565 return ret; 3637 return ret;
3566 3638
3567 if (offset + size <= i_size_read(inode)) 3639 ret = ext4_convert_unwritten_extents(inode, offset, size);
3568 ret = ext4_convert_unwritten_extents(inode, offset, size);
3569
3570 if (ret < 0) { 3640 if (ret < 0) {
3571 printk(KERN_EMERG "%s: failed to convert unwritten" 3641 printk(KERN_EMERG "%s: failed to convert unwritten"
3572 "extents to written extents, error is %d" 3642 "extents to written extents, error is %d"
@@ -3579,50 +3649,64 @@ static int ext4_end_aio_dio_nolock(ext4_io_end_t *io)
3579 io->flag = 0; 3649 io->flag = 0;
3580 return ret; 3650 return ret;
3581} 3651}
3652
3582/* 3653/*
3583 * work on completed aio dio IO, to convert unwritten extents to extents 3654 * work on completed aio dio IO, to convert unwritten extents to extents
3584 */ 3655 */
3585static void ext4_end_aio_dio_work(struct work_struct *work) 3656static void ext4_end_io_work(struct work_struct *work)
3586{ 3657{
3587 ext4_io_end_t *io = container_of(work, ext4_io_end_t, work); 3658 ext4_io_end_t *io = container_of(work, ext4_io_end_t, work);
3588 struct inode *inode = io->inode; 3659 struct inode *inode = io->inode;
3589 int ret = 0; 3660 struct ext4_inode_info *ei = EXT4_I(inode);
3661 unsigned long flags;
3662 int ret;
3590 3663
3591 mutex_lock(&inode->i_mutex); 3664 mutex_lock(&inode->i_mutex);
3592 ret = ext4_end_aio_dio_nolock(io); 3665 ret = ext4_end_io_nolock(io);
3593 if (ret >= 0) { 3666 if (ret < 0) {
3594 if (!list_empty(&io->list)) 3667 mutex_unlock(&inode->i_mutex);
3595 list_del_init(&io->list); 3668 return;
3596 ext4_free_io_end(io);
3597 } 3669 }
3670
3671 spin_lock_irqsave(&ei->i_completed_io_lock, flags);
3672 if (!list_empty(&io->list))
3673 list_del_init(&io->list);
3674 spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
3598 mutex_unlock(&inode->i_mutex); 3675 mutex_unlock(&inode->i_mutex);
3676 ext4_free_io_end(io);
3599} 3677}
3678
3600/* 3679/*
3601 * This function is called from ext4_sync_file(). 3680 * This function is called from ext4_sync_file().
3602 * 3681 *
3603 * When AIO DIO IO is completed, the work to convert unwritten 3682 * When IO is completed, the work to convert unwritten extents to
3604 * extents to written is queued on workqueue but may not get immediately 3683 * written is queued on workqueue but may not get immediately
3605 * scheduled. When fsync is called, we need to ensure the 3684 * scheduled. When fsync is called, we need to ensure the
3606 * conversion is complete before fsync returns. 3685 * conversion is complete before fsync returns.
3607 * The inode keeps track of a list of completed AIO from DIO path 3686 * The inode keeps track of a list of pending/completed IO that
3608 * that might needs to do the conversion. This function walks through 3687 * might needs to do the conversion. This function walks through
3609 * the list and convert the related unwritten extents to written. 3688 * the list and convert the related unwritten extents for completed IO
3689 * to written.
3690 * The function return the number of pending IOs on success.
3610 */ 3691 */
3611int flush_aio_dio_completed_IO(struct inode *inode) 3692int flush_completed_IO(struct inode *inode)
3612{ 3693{
3613 ext4_io_end_t *io; 3694 ext4_io_end_t *io;
3695 struct ext4_inode_info *ei = EXT4_I(inode);
3696 unsigned long flags;
3614 int ret = 0; 3697 int ret = 0;
3615 int ret2 = 0; 3698 int ret2 = 0;
3616 3699
3617 if (list_empty(&EXT4_I(inode)->i_aio_dio_complete_list)) 3700 if (list_empty(&ei->i_completed_io_list))
3618 return ret; 3701 return ret;
3619 3702
3620 dump_aio_dio_list(inode); 3703 dump_completed_IO(inode);
3621 while (!list_empty(&EXT4_I(inode)->i_aio_dio_complete_list)){ 3704 spin_lock_irqsave(&ei->i_completed_io_lock, flags);
3622 io = list_entry(EXT4_I(inode)->i_aio_dio_complete_list.next, 3705 while (!list_empty(&ei->i_completed_io_list)){
3706 io = list_entry(ei->i_completed_io_list.next,
3623 ext4_io_end_t, list); 3707 ext4_io_end_t, list);
3624 /* 3708 /*
3625 * Calling ext4_end_aio_dio_nolock() to convert completed 3709 * Calling ext4_end_io_nolock() to convert completed
3626 * IO to written. 3710 * IO to written.
3627 * 3711 *
3628 * When ext4_sync_file() is called, run_queue() may already 3712 * When ext4_sync_file() is called, run_queue() may already
@@ -3635,20 +3719,23 @@ int flush_aio_dio_completed_IO(struct inode *inode)
3635 * avoid double converting from both fsync and background work 3719 * avoid double converting from both fsync and background work
3636 * queue work. 3720 * queue work.
3637 */ 3721 */
3638 ret = ext4_end_aio_dio_nolock(io); 3722 spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
3723 ret = ext4_end_io_nolock(io);
3724 spin_lock_irqsave(&ei->i_completed_io_lock, flags);
3639 if (ret < 0) 3725 if (ret < 0)
3640 ret2 = ret; 3726 ret2 = ret;
3641 else 3727 else
3642 list_del_init(&io->list); 3728 list_del_init(&io->list);
3643 } 3729 }
3730 spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
3644 return (ret2 < 0) ? ret2 : 0; 3731 return (ret2 < 0) ? ret2 : 0;
3645} 3732}
3646 3733
3647static ext4_io_end_t *ext4_init_io_end (struct inode *inode) 3734static ext4_io_end_t *ext4_init_io_end (struct inode *inode, gfp_t flags)
3648{ 3735{
3649 ext4_io_end_t *io = NULL; 3736 ext4_io_end_t *io = NULL;
3650 3737
3651 io = kmalloc(sizeof(*io), GFP_NOFS); 3738 io = kmalloc(sizeof(*io), flags);
3652 3739
3653 if (io) { 3740 if (io) {
3654 igrab(inode); 3741 igrab(inode);
@@ -3656,8 +3743,8 @@ static ext4_io_end_t *ext4_init_io_end (struct inode *inode)
3656 io->flag = 0; 3743 io->flag = 0;
3657 io->offset = 0; 3744 io->offset = 0;
3658 io->size = 0; 3745 io->size = 0;
3659 io->error = 0; 3746 io->page = NULL;
3660 INIT_WORK(&io->work, ext4_end_aio_dio_work); 3747 INIT_WORK(&io->work, ext4_end_io_work);
3661 INIT_LIST_HEAD(&io->list); 3748 INIT_LIST_HEAD(&io->list);
3662 } 3749 }
3663 3750
@@ -3669,6 +3756,8 @@ static void ext4_end_io_dio(struct kiocb *iocb, loff_t offset,
3669{ 3756{
3670 ext4_io_end_t *io_end = iocb->private; 3757 ext4_io_end_t *io_end = iocb->private;
3671 struct workqueue_struct *wq; 3758 struct workqueue_struct *wq;
3759 unsigned long flags;
3760 struct ext4_inode_info *ei;
3672 3761
3673 /* if not async direct IO or dio with 0 bytes write, just return */ 3762 /* if not async direct IO or dio with 0 bytes write, just return */
3674 if (!io_end || !size) 3763 if (!io_end || !size)
@@ -3680,7 +3769,7 @@ static void ext4_end_io_dio(struct kiocb *iocb, loff_t offset,
3680 size); 3769 size);
3681 3770
3682 /* if not aio dio with unwritten extents, just free io and return */ 3771 /* if not aio dio with unwritten extents, just free io and return */
3683 if (io_end->flag != DIO_AIO_UNWRITTEN){ 3772 if (io_end->flag != EXT4_IO_UNWRITTEN){
3684 ext4_free_io_end(io_end); 3773 ext4_free_io_end(io_end);
3685 iocb->private = NULL; 3774 iocb->private = NULL;
3686 return; 3775 return;
@@ -3688,16 +3777,85 @@ static void ext4_end_io_dio(struct kiocb *iocb, loff_t offset,
3688 3777
3689 io_end->offset = offset; 3778 io_end->offset = offset;
3690 io_end->size = size; 3779 io_end->size = size;
3780 io_end->flag = EXT4_IO_UNWRITTEN;
3691 wq = EXT4_SB(io_end->inode->i_sb)->dio_unwritten_wq; 3781 wq = EXT4_SB(io_end->inode->i_sb)->dio_unwritten_wq;
3692 3782
3693 /* queue the work to convert unwritten extents to written */ 3783 /* queue the work to convert unwritten extents to written */
3694 queue_work(wq, &io_end->work); 3784 queue_work(wq, &io_end->work);
3695 3785
3696 /* Add the io_end to per-inode completed aio dio list*/ 3786 /* Add the io_end to per-inode completed aio dio list*/
3697 list_add_tail(&io_end->list, 3787 ei = EXT4_I(io_end->inode);
3698 &EXT4_I(io_end->inode)->i_aio_dio_complete_list); 3788 spin_lock_irqsave(&ei->i_completed_io_lock, flags);
3789 list_add_tail(&io_end->list, &ei->i_completed_io_list);
3790 spin_unlock_irqrestore(&ei->i_completed_io_lock, flags);
3699 iocb->private = NULL; 3791 iocb->private = NULL;
3700} 3792}
3793
3794static void ext4_end_io_buffer_write(struct buffer_head *bh, int uptodate)
3795{
3796 ext4_io_end_t *io_end = bh->b_private;
3797 struct workqueue_struct *wq;
3798 struct inode *inode;
3799 unsigned long flags;
3800
3801 if (!test_clear_buffer_uninit(bh) || !io_end)
3802 goto out;
3803
3804 if (!(io_end->inode->i_sb->s_flags & MS_ACTIVE)) {
3805 printk("sb umounted, discard end_io request for inode %lu\n",
3806 io_end->inode->i_ino);
3807 ext4_free_io_end(io_end);
3808 goto out;
3809 }
3810
3811 io_end->flag = EXT4_IO_UNWRITTEN;
3812 inode = io_end->inode;
3813
3814 /* Add the io_end to per-inode completed io list*/
3815 spin_lock_irqsave(&EXT4_I(inode)->i_completed_io_lock, flags);
3816 list_add_tail(&io_end->list, &EXT4_I(inode)->i_completed_io_list);
3817 spin_unlock_irqrestore(&EXT4_I(inode)->i_completed_io_lock, flags);
3818
3819 wq = EXT4_SB(inode->i_sb)->dio_unwritten_wq;
3820 /* queue the work to convert unwritten extents to written */
3821 queue_work(wq, &io_end->work);
3822out:
3823 bh->b_private = NULL;
3824 bh->b_end_io = NULL;
3825 clear_buffer_uninit(bh);
3826 end_buffer_async_write(bh, uptodate);
3827}
3828
3829static int ext4_set_bh_endio(struct buffer_head *bh, struct inode *inode)
3830{
3831 ext4_io_end_t *io_end;
3832 struct page *page = bh->b_page;
3833 loff_t offset = (sector_t)page->index << PAGE_CACHE_SHIFT;
3834 size_t size = bh->b_size;
3835
3836retry:
3837 io_end = ext4_init_io_end(inode, GFP_ATOMIC);
3838 if (!io_end) {
3839 if (printk_ratelimit())
3840 printk(KERN_WARNING "%s: allocation fail\n", __func__);
3841 schedule();
3842 goto retry;
3843 }
3844 io_end->offset = offset;
3845 io_end->size = size;
3846 /*
3847 * We need to hold a reference to the page to make sure it
3848 * doesn't get evicted before ext4_end_io_work() has a chance
3849 * to convert the extent from written to unwritten.
3850 */
3851 io_end->page = page;
3852 get_page(io_end->page);
3853
3854 bh->b_private = io_end;
3855 bh->b_end_io = ext4_end_io_buffer_write;
3856 return 0;
3857}
3858
3701/* 3859/*
3702 * For ext4 extent files, ext4 will do direct-io write to holes, 3860 * For ext4 extent files, ext4 will do direct-io write to holes,
3703 * preallocated extents, and those write extend the file, no need to 3861 * preallocated extents, and those write extend the file, no need to
@@ -3751,7 +3909,7 @@ static ssize_t ext4_ext_direct_IO(int rw, struct kiocb *iocb,
3751 iocb->private = NULL; 3909 iocb->private = NULL;
3752 EXT4_I(inode)->cur_aio_dio = NULL; 3910 EXT4_I(inode)->cur_aio_dio = NULL;
3753 if (!is_sync_kiocb(iocb)) { 3911 if (!is_sync_kiocb(iocb)) {
3754 iocb->private = ext4_init_io_end(inode); 3912 iocb->private = ext4_init_io_end(inode, GFP_NOFS);
3755 if (!iocb->private) 3913 if (!iocb->private)
3756 return -ENOMEM; 3914 return -ENOMEM;
3757 /* 3915 /*
@@ -3767,7 +3925,7 @@ static ssize_t ext4_ext_direct_IO(int rw, struct kiocb *iocb,
3767 ret = blockdev_direct_IO(rw, iocb, inode, 3925 ret = blockdev_direct_IO(rw, iocb, inode,
3768 inode->i_sb->s_bdev, iov, 3926 inode->i_sb->s_bdev, iov,
3769 offset, nr_segs, 3927 offset, nr_segs,
3770 ext4_get_block_dio_write, 3928 ext4_get_block_write,
3771 ext4_end_io_dio); 3929 ext4_end_io_dio);
3772 if (iocb->private) 3930 if (iocb->private)
3773 EXT4_I(inode)->cur_aio_dio = NULL; 3931 EXT4_I(inode)->cur_aio_dio = NULL;
@@ -3788,8 +3946,8 @@ static ssize_t ext4_ext_direct_IO(int rw, struct kiocb *iocb,
3788 if (ret != -EIOCBQUEUED && ret <= 0 && iocb->private) { 3946 if (ret != -EIOCBQUEUED && ret <= 0 && iocb->private) {
3789 ext4_free_io_end(iocb->private); 3947 ext4_free_io_end(iocb->private);
3790 iocb->private = NULL; 3948 iocb->private = NULL;
3791 } else if (ret > 0 && (EXT4_I(inode)->i_state & 3949 } else if (ret > 0 && ext4_test_inode_state(inode,
3792 EXT4_STATE_DIO_UNWRITTEN)) { 3950 EXT4_STATE_DIO_UNWRITTEN)) {
3793 int err; 3951 int err;
3794 /* 3952 /*
3795 * for non AIO case, since the IO is already 3953 * for non AIO case, since the IO is already
@@ -3799,7 +3957,7 @@ static ssize_t ext4_ext_direct_IO(int rw, struct kiocb *iocb,
3799 offset, ret); 3957 offset, ret);
3800 if (err < 0) 3958 if (err < 0)
3801 ret = err; 3959 ret = err;
3802 EXT4_I(inode)->i_state &= ~EXT4_STATE_DIO_UNWRITTEN; 3960 ext4_clear_inode_state(inode, EXT4_STATE_DIO_UNWRITTEN);
3803 } 3961 }
3804 return ret; 3962 return ret;
3805 } 3963 }
@@ -4130,18 +4288,27 @@ no_top:
4130 * We release `count' blocks on disk, but (last - first) may be greater 4288 * We release `count' blocks on disk, but (last - first) may be greater
4131 * than `count' because there can be holes in there. 4289 * than `count' because there can be holes in there.
4132 */ 4290 */
4133static void ext4_clear_blocks(handle_t *handle, struct inode *inode, 4291static int ext4_clear_blocks(handle_t *handle, struct inode *inode,
4134 struct buffer_head *bh, 4292 struct buffer_head *bh,
4135 ext4_fsblk_t block_to_free, 4293 ext4_fsblk_t block_to_free,
4136 unsigned long count, __le32 *first, 4294 unsigned long count, __le32 *first,
4137 __le32 *last) 4295 __le32 *last)
4138{ 4296{
4139 __le32 *p; 4297 __le32 *p;
4140 int flags = EXT4_FREE_BLOCKS_FORGET; 4298 int flags = EXT4_FREE_BLOCKS_FORGET | EXT4_FREE_BLOCKS_VALIDATED;
4141 4299
4142 if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)) 4300 if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))
4143 flags |= EXT4_FREE_BLOCKS_METADATA; 4301 flags |= EXT4_FREE_BLOCKS_METADATA;
4144 4302
4303 if (!ext4_data_block_valid(EXT4_SB(inode->i_sb), block_to_free,
4304 count)) {
4305 ext4_error(inode->i_sb, "inode #%lu: "
4306 "attempt to clear blocks %llu len %lu, invalid",
4307 inode->i_ino, (unsigned long long) block_to_free,
4308 count);
4309 return 1;
4310 }
4311
4145 if (try_to_extend_transaction(handle, inode)) { 4312 if (try_to_extend_transaction(handle, inode)) {
4146 if (bh) { 4313 if (bh) {
4147 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata"); 4314 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
@@ -4160,6 +4327,7 @@ static void ext4_clear_blocks(handle_t *handle, struct inode *inode,
4160 *p = 0; 4327 *p = 0;
4161 4328
4162 ext4_free_blocks(handle, inode, 0, block_to_free, count, flags); 4329 ext4_free_blocks(handle, inode, 0, block_to_free, count, flags);
4330 return 0;
4163} 4331}
4164 4332
4165/** 4333/**
@@ -4215,9 +4383,10 @@ static void ext4_free_data(handle_t *handle, struct inode *inode,
4215 } else if (nr == block_to_free + count) { 4383 } else if (nr == block_to_free + count) {
4216 count++; 4384 count++;
4217 } else { 4385 } else {
4218 ext4_clear_blocks(handle, inode, this_bh, 4386 if (ext4_clear_blocks(handle, inode, this_bh,
4219 block_to_free, 4387 block_to_free, count,
4220 count, block_to_free_p, p); 4388 block_to_free_p, p))
4389 break;
4221 block_to_free = nr; 4390 block_to_free = nr;
4222 block_to_free_p = p; 4391 block_to_free_p = p;
4223 count = 1; 4392 count = 1;
@@ -4241,7 +4410,7 @@ static void ext4_free_data(handle_t *handle, struct inode *inode,
4241 if ((EXT4_JOURNAL(inode) == NULL) || bh2jh(this_bh)) 4410 if ((EXT4_JOURNAL(inode) == NULL) || bh2jh(this_bh))
4242 ext4_handle_dirty_metadata(handle, inode, this_bh); 4411 ext4_handle_dirty_metadata(handle, inode, this_bh);
4243 else 4412 else
4244 ext4_error(inode->i_sb, __func__, 4413 ext4_error(inode->i_sb,
4245 "circular indirect block detected, " 4414 "circular indirect block detected, "
4246 "inode=%lu, block=%llu", 4415 "inode=%lu, block=%llu",
4247 inode->i_ino, 4416 inode->i_ino,
@@ -4281,6 +4450,16 @@ static void ext4_free_branches(handle_t *handle, struct inode *inode,
4281 if (!nr) 4450 if (!nr)
4282 continue; /* A hole */ 4451 continue; /* A hole */
4283 4452
4453 if (!ext4_data_block_valid(EXT4_SB(inode->i_sb),
4454 nr, 1)) {
4455 ext4_error(inode->i_sb,
4456 "indirect mapped block in inode "
4457 "#%lu invalid (level %d, blk #%lu)",
4458 inode->i_ino, depth,
4459 (unsigned long) nr);
4460 break;
4461 }
4462
4284 /* Go read the buffer for the next level down */ 4463 /* Go read the buffer for the next level down */
4285 bh = sb_bread(inode->i_sb, nr); 4464 bh = sb_bread(inode->i_sb, nr);
4286 4465
@@ -4289,7 +4468,7 @@ static void ext4_free_branches(handle_t *handle, struct inode *inode,
4289 * (should be rare). 4468 * (should be rare).
4290 */ 4469 */
4291 if (!bh) { 4470 if (!bh) {
4292 ext4_error(inode->i_sb, "ext4_free_branches", 4471 ext4_error(inode->i_sb,
4293 "Read failure, inode=%lu, block=%llu", 4472 "Read failure, inode=%lu, block=%llu",
4294 inode->i_ino, nr); 4473 inode->i_ino, nr);
4295 continue; 4474 continue;
@@ -4433,8 +4612,10 @@ void ext4_truncate(struct inode *inode)
4433 if (!ext4_can_truncate(inode)) 4612 if (!ext4_can_truncate(inode))
4434 return; 4613 return;
4435 4614
4615 EXT4_I(inode)->i_flags &= ~EXT4_EOFBLOCKS_FL;
4616
4436 if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC)) 4617 if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC))
4437 ei->i_state |= EXT4_STATE_DA_ALLOC_CLOSE; 4618 ext4_set_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE);
4438 4619
4439 if (EXT4_I(inode)->i_flags & EXT4_EXTENTS_FL) { 4620 if (EXT4_I(inode)->i_flags & EXT4_EXTENTS_FL) {
4440 ext4_ext_truncate(inode); 4621 ext4_ext_truncate(inode);
@@ -4604,9 +4785,8 @@ static int __ext4_get_inode_loc(struct inode *inode,
4604 4785
4605 bh = sb_getblk(sb, block); 4786 bh = sb_getblk(sb, block);
4606 if (!bh) { 4787 if (!bh) {
4607 ext4_error(sb, "ext4_get_inode_loc", "unable to read " 4788 ext4_error(sb, "unable to read inode block - "
4608 "inode block - inode=%lu, block=%llu", 4789 "inode=%lu, block=%llu", inode->i_ino, block);
4609 inode->i_ino, block);
4610 return -EIO; 4790 return -EIO;
4611 } 4791 }
4612 if (!buffer_uptodate(bh)) { 4792 if (!buffer_uptodate(bh)) {
@@ -4704,9 +4884,8 @@ make_io:
4704 submit_bh(READ_META, bh); 4884 submit_bh(READ_META, bh);
4705 wait_on_buffer(bh); 4885 wait_on_buffer(bh);
4706 if (!buffer_uptodate(bh)) { 4886 if (!buffer_uptodate(bh)) {
4707 ext4_error(sb, __func__, 4887 ext4_error(sb, "unable to read inode block - inode=%lu,"
4708 "unable to read inode block - inode=%lu, " 4888 " block=%llu", inode->i_ino, block);
4709 "block=%llu", inode->i_ino, block);
4710 brelse(bh); 4889 brelse(bh);
4711 return -EIO; 4890 return -EIO;
4712 } 4891 }
@@ -4720,7 +4899,7 @@ int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc)
4720{ 4899{
4721 /* We have all inode data except xattrs in memory here. */ 4900 /* We have all inode data except xattrs in memory here. */
4722 return __ext4_get_inode_loc(inode, iloc, 4901 return __ext4_get_inode_loc(inode, iloc,
4723 !(EXT4_I(inode)->i_state & EXT4_STATE_XATTR)); 4902 !ext4_test_inode_state(inode, EXT4_STATE_XATTR));
4724} 4903}
4725 4904
4726void ext4_set_inode_flags(struct inode *inode) 4905void ext4_set_inode_flags(struct inode *inode)
@@ -4814,7 +4993,7 @@ struct inode *ext4_iget(struct super_block *sb, unsigned long ino)
4814 } 4993 }
4815 inode->i_nlink = le16_to_cpu(raw_inode->i_links_count); 4994 inode->i_nlink = le16_to_cpu(raw_inode->i_links_count);
4816 4995
4817 ei->i_state = 0; 4996 ei->i_state_flags = 0;
4818 ei->i_dir_start_lookup = 0; 4997 ei->i_dir_start_lookup = 0;
4819 ei->i_dtime = le32_to_cpu(raw_inode->i_dtime); 4998 ei->i_dtime = le32_to_cpu(raw_inode->i_dtime);
4820 /* We now have enough fields to check if the inode was active or not. 4999 /* We now have enough fields to check if the inode was active or not.
@@ -4897,7 +5076,7 @@ struct inode *ext4_iget(struct super_block *sb, unsigned long ino)
4897 EXT4_GOOD_OLD_INODE_SIZE + 5076 EXT4_GOOD_OLD_INODE_SIZE +
4898 ei->i_extra_isize; 5077 ei->i_extra_isize;
4899 if (*magic == cpu_to_le32(EXT4_XATTR_MAGIC)) 5078 if (*magic == cpu_to_le32(EXT4_XATTR_MAGIC))
4900 ei->i_state |= EXT4_STATE_XATTR; 5079 ext4_set_inode_state(inode, EXT4_STATE_XATTR);
4901 } 5080 }
4902 } else 5081 } else
4903 ei->i_extra_isize = 0; 5082 ei->i_extra_isize = 0;
@@ -4917,8 +5096,7 @@ struct inode *ext4_iget(struct super_block *sb, unsigned long ino)
4917 ret = 0; 5096 ret = 0;
4918 if (ei->i_file_acl && 5097 if (ei->i_file_acl &&
4919 !ext4_data_block_valid(EXT4_SB(sb), ei->i_file_acl, 1)) { 5098 !ext4_data_block_valid(EXT4_SB(sb), ei->i_file_acl, 1)) {
4920 ext4_error(sb, __func__, 5099 ext4_error(sb, "bad extended attribute block %llu inode #%lu",
4921 "bad extended attribute block %llu in inode #%lu",
4922 ei->i_file_acl, inode->i_ino); 5100 ei->i_file_acl, inode->i_ino);
4923 ret = -EIO; 5101 ret = -EIO;
4924 goto bad_inode; 5102 goto bad_inode;
@@ -4964,8 +5142,7 @@ struct inode *ext4_iget(struct super_block *sb, unsigned long ino)
4964 new_decode_dev(le32_to_cpu(raw_inode->i_block[1]))); 5142 new_decode_dev(le32_to_cpu(raw_inode->i_block[1])));
4965 } else { 5143 } else {
4966 ret = -EIO; 5144 ret = -EIO;
4967 ext4_error(inode->i_sb, __func__, 5145 ext4_error(inode->i_sb, "bogus i_mode (%o) for inode=%lu",
4968 "bogus i_mode (%o) for inode=%lu",
4969 inode->i_mode, inode->i_ino); 5146 inode->i_mode, inode->i_ino);
4970 goto bad_inode; 5147 goto bad_inode;
4971 } 5148 }
@@ -5037,7 +5214,7 @@ static int ext4_do_update_inode(handle_t *handle,
5037 5214
5038 /* For fields not not tracking in the in-memory inode, 5215 /* For fields not not tracking in the in-memory inode,
5039 * initialise them to zero for new inodes. */ 5216 * initialise them to zero for new inodes. */
5040 if (ei->i_state & EXT4_STATE_NEW) 5217 if (ext4_test_inode_state(inode, EXT4_STATE_NEW))
5041 memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size); 5218 memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size);
5042 5219
5043 ext4_get_inode_flags(ei); 5220 ext4_get_inode_flags(ei);
@@ -5101,7 +5278,7 @@ static int ext4_do_update_inode(handle_t *handle,
5101 EXT4_FEATURE_RO_COMPAT_LARGE_FILE); 5278 EXT4_FEATURE_RO_COMPAT_LARGE_FILE);
5102 sb->s_dirt = 1; 5279 sb->s_dirt = 1;
5103 ext4_handle_sync(handle); 5280 ext4_handle_sync(handle);
5104 err = ext4_handle_dirty_metadata(handle, inode, 5281 err = ext4_handle_dirty_metadata(handle, NULL,
5105 EXT4_SB(sb)->s_sbh); 5282 EXT4_SB(sb)->s_sbh);
5106 } 5283 }
5107 } 5284 }
@@ -5130,10 +5307,10 @@ static int ext4_do_update_inode(handle_t *handle,
5130 } 5307 }
5131 5308
5132 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata"); 5309 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
5133 rc = ext4_handle_dirty_metadata(handle, inode, bh); 5310 rc = ext4_handle_dirty_metadata(handle, NULL, bh);
5134 if (!err) 5311 if (!err)
5135 err = rc; 5312 err = rc;
5136 ei->i_state &= ~EXT4_STATE_NEW; 5313 ext4_clear_inode_state(inode, EXT4_STATE_NEW);
5137 5314
5138 ext4_update_inode_fsync_trans(handle, inode, 0); 5315 ext4_update_inode_fsync_trans(handle, inode, 0);
5139out_brelse: 5316out_brelse:
@@ -5177,7 +5354,7 @@ out_brelse:
5177 * `stuff()' is running, and the new i_size will be lost. Plus the inode 5354 * `stuff()' is running, and the new i_size will be lost. Plus the inode
5178 * will no longer be on the superblock's dirty inode list. 5355 * will no longer be on the superblock's dirty inode list.
5179 */ 5356 */
5180int ext4_write_inode(struct inode *inode, int wait) 5357int ext4_write_inode(struct inode *inode, struct writeback_control *wbc)
5181{ 5358{
5182 int err; 5359 int err;
5183 5360
@@ -5191,26 +5368,25 @@ int ext4_write_inode(struct inode *inode, int wait)
5191 return -EIO; 5368 return -EIO;
5192 } 5369 }
5193 5370
5194 if (!wait) 5371 if (wbc->sync_mode != WB_SYNC_ALL)
5195 return 0; 5372 return 0;
5196 5373
5197 err = ext4_force_commit(inode->i_sb); 5374 err = ext4_force_commit(inode->i_sb);
5198 } else { 5375 } else {
5199 struct ext4_iloc iloc; 5376 struct ext4_iloc iloc;
5200 5377
5201 err = ext4_get_inode_loc(inode, &iloc); 5378 err = __ext4_get_inode_loc(inode, &iloc, 0);
5202 if (err) 5379 if (err)
5203 return err; 5380 return err;
5204 if (wait) 5381 if (wbc->sync_mode == WB_SYNC_ALL)
5205 sync_dirty_buffer(iloc.bh); 5382 sync_dirty_buffer(iloc.bh);
5206 if (buffer_req(iloc.bh) && !buffer_uptodate(iloc.bh)) { 5383 if (buffer_req(iloc.bh) && !buffer_uptodate(iloc.bh)) {
5207 ext4_error(inode->i_sb, __func__, 5384 ext4_error(inode->i_sb, "IO error syncing inode, "
5208 "IO error syncing inode, " 5385 "inode=%lu, block=%llu", inode->i_ino,
5209 "inode=%lu, block=%llu",
5210 inode->i_ino,
5211 (unsigned long long)iloc.bh->b_blocknr); 5386 (unsigned long long)iloc.bh->b_blocknr);
5212 err = -EIO; 5387 err = -EIO;
5213 } 5388 }
5389 brelse(iloc.bh);
5214 } 5390 }
5215 return err; 5391 return err;
5216} 5392}
@@ -5249,6 +5425,8 @@ int ext4_setattr(struct dentry *dentry, struct iattr *attr)
5249 if (error) 5425 if (error)
5250 return error; 5426 return error;
5251 5427
5428 if (is_quota_modification(inode, attr))
5429 dquot_initialize(inode);
5252 if ((ia_valid & ATTR_UID && attr->ia_uid != inode->i_uid) || 5430 if ((ia_valid & ATTR_UID && attr->ia_uid != inode->i_uid) ||
5253 (ia_valid & ATTR_GID && attr->ia_gid != inode->i_gid)) { 5431 (ia_valid & ATTR_GID && attr->ia_gid != inode->i_gid)) {
5254 handle_t *handle; 5432 handle_t *handle;
@@ -5261,7 +5439,7 @@ int ext4_setattr(struct dentry *dentry, struct iattr *attr)
5261 error = PTR_ERR(handle); 5439 error = PTR_ERR(handle);
5262 goto err_out; 5440 goto err_out;
5263 } 5441 }
5264 error = vfs_dq_transfer(inode, attr) ? -EDQUOT : 0; 5442 error = dquot_transfer(inode, attr);
5265 if (error) { 5443 if (error) {
5266 ext4_journal_stop(handle); 5444 ext4_journal_stop(handle);
5267 return error; 5445 return error;
@@ -5288,7 +5466,9 @@ int ext4_setattr(struct dentry *dentry, struct iattr *attr)
5288 } 5466 }
5289 5467
5290 if (S_ISREG(inode->i_mode) && 5468 if (S_ISREG(inode->i_mode) &&
5291 attr->ia_valid & ATTR_SIZE && attr->ia_size < inode->i_size) { 5469 attr->ia_valid & ATTR_SIZE &&
5470 (attr->ia_size < inode->i_size ||
5471 (EXT4_I(inode)->i_flags & EXT4_EOFBLOCKS_FL))) {
5292 handle_t *handle; 5472 handle_t *handle;
5293 5473
5294 handle = ext4_journal_start(inode, 3); 5474 handle = ext4_journal_start(inode, 3);
@@ -5319,6 +5499,9 @@ int ext4_setattr(struct dentry *dentry, struct iattr *attr)
5319 goto err_out; 5499 goto err_out;
5320 } 5500 }
5321 } 5501 }
5502 /* ext4_truncate will clear the flag */
5503 if ((EXT4_I(inode)->i_flags & EXT4_EOFBLOCKS_FL))
5504 ext4_truncate(inode);
5322 } 5505 }
5323 5506
5324 rc = inode_setattr(inode, attr); 5507 rc = inode_setattr(inode, attr);
@@ -5557,8 +5740,8 @@ static int ext4_expand_extra_isize(struct inode *inode,
5557 entry = IFIRST(header); 5740 entry = IFIRST(header);
5558 5741
5559 /* No extended attributes present */ 5742 /* No extended attributes present */
5560 if (!(EXT4_I(inode)->i_state & EXT4_STATE_XATTR) || 5743 if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) ||
5561 header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) { 5744 header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) {
5562 memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE, 0, 5745 memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE, 0,
5563 new_extra_isize); 5746 new_extra_isize);
5564 EXT4_I(inode)->i_extra_isize = new_extra_isize; 5747 EXT4_I(inode)->i_extra_isize = new_extra_isize;
@@ -5602,7 +5785,7 @@ int ext4_mark_inode_dirty(handle_t *handle, struct inode *inode)
5602 err = ext4_reserve_inode_write(handle, inode, &iloc); 5785 err = ext4_reserve_inode_write(handle, inode, &iloc);
5603 if (ext4_handle_valid(handle) && 5786 if (ext4_handle_valid(handle) &&
5604 EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize && 5787 EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize &&
5605 !(EXT4_I(inode)->i_state & EXT4_STATE_NO_EXPAND)) { 5788 !ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) {
5606 /* 5789 /*
5607 * We need extra buffer credits since we may write into EA block 5790 * We need extra buffer credits since we may write into EA block
5608 * with this same handle. If journal_extend fails, then it will 5791 * with this same handle. If journal_extend fails, then it will
@@ -5616,10 +5799,11 @@ int ext4_mark_inode_dirty(handle_t *handle, struct inode *inode)
5616 sbi->s_want_extra_isize, 5799 sbi->s_want_extra_isize,
5617 iloc, handle); 5800 iloc, handle);
5618 if (ret) { 5801 if (ret) {
5619 EXT4_I(inode)->i_state |= EXT4_STATE_NO_EXPAND; 5802 ext4_set_inode_state(inode,
5803 EXT4_STATE_NO_EXPAND);
5620 if (mnt_count != 5804 if (mnt_count !=
5621 le16_to_cpu(sbi->s_es->s_mnt_count)) { 5805 le16_to_cpu(sbi->s_es->s_mnt_count)) {
5622 ext4_warning(inode->i_sb, __func__, 5806 ext4_warning(inode->i_sb,
5623 "Unable to expand inode %lu. Delete" 5807 "Unable to expand inode %lu. Delete"
5624 " some EAs or run e2fsck.", 5808 " some EAs or run e2fsck.",
5625 inode->i_ino); 5809 inode->i_ino);
@@ -5641,7 +5825,7 @@ int ext4_mark_inode_dirty(handle_t *handle, struct inode *inode)
5641 * i_size has been changed by generic_commit_write() and we thus need 5825 * i_size has been changed by generic_commit_write() and we thus need
5642 * to include the updated inode in the current transaction. 5826 * to include the updated inode in the current transaction.
5643 * 5827 *
5644 * Also, vfs_dq_alloc_block() will always dirty the inode when blocks 5828 * Also, dquot_alloc_block() will always dirty the inode when blocks
5645 * are allocated to the file. 5829 * are allocated to the file.
5646 * 5830 *
5647 * If the inode is marked synchronous, we don't honour that here - doing 5831 * If the inode is marked synchronous, we don't honour that here - doing
@@ -5683,7 +5867,7 @@ static int ext4_pin_inode(handle_t *handle, struct inode *inode)
5683 err = jbd2_journal_get_write_access(handle, iloc.bh); 5867 err = jbd2_journal_get_write_access(handle, iloc.bh);
5684 if (!err) 5868 if (!err)
5685 err = ext4_handle_dirty_metadata(handle, 5869 err = ext4_handle_dirty_metadata(handle,
5686 inode, 5870 NULL,
5687 iloc.bh); 5871 iloc.bh);
5688 brelse(iloc.bh); 5872 brelse(iloc.bh);
5689 } 5873 }