diff options
Diffstat (limited to 'fs/ext4/mballoc.c')
-rw-r--r-- | fs/ext4/mballoc.c | 166 |
1 files changed, 67 insertions, 99 deletions
diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index f871677a7984..ed8482e22c0e 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c | |||
@@ -372,24 +372,12 @@ static inline void mb_set_bit(int bit, void *addr) | |||
372 | ext4_set_bit(bit, addr); | 372 | ext4_set_bit(bit, addr); |
373 | } | 373 | } |
374 | 374 | ||
375 | static inline void mb_set_bit_atomic(spinlock_t *lock, int bit, void *addr) | ||
376 | { | ||
377 | addr = mb_correct_addr_and_bit(&bit, addr); | ||
378 | ext4_set_bit_atomic(lock, bit, addr); | ||
379 | } | ||
380 | |||
381 | static inline void mb_clear_bit(int bit, void *addr) | 375 | static inline void mb_clear_bit(int bit, void *addr) |
382 | { | 376 | { |
383 | addr = mb_correct_addr_and_bit(&bit, addr); | 377 | addr = mb_correct_addr_and_bit(&bit, addr); |
384 | ext4_clear_bit(bit, addr); | 378 | ext4_clear_bit(bit, addr); |
385 | } | 379 | } |
386 | 380 | ||
387 | static inline void mb_clear_bit_atomic(spinlock_t *lock, int bit, void *addr) | ||
388 | { | ||
389 | addr = mb_correct_addr_and_bit(&bit, addr); | ||
390 | ext4_clear_bit_atomic(lock, bit, addr); | ||
391 | } | ||
392 | |||
393 | static inline int mb_find_next_zero_bit(void *addr, int max, int start) | 381 | static inline int mb_find_next_zero_bit(void *addr, int max, int start) |
394 | { | 382 | { |
395 | int fix = 0, ret, tmpmax; | 383 | int fix = 0, ret, tmpmax; |
@@ -448,7 +436,7 @@ static void mb_free_blocks_double(struct inode *inode, struct ext4_buddy *e4b, | |||
448 | 436 | ||
449 | if (unlikely(e4b->bd_info->bb_bitmap == NULL)) | 437 | if (unlikely(e4b->bd_info->bb_bitmap == NULL)) |
450 | return; | 438 | return; |
451 | BUG_ON(!ext4_is_group_locked(sb, e4b->bd_group)); | 439 | assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); |
452 | for (i = 0; i < count; i++) { | 440 | for (i = 0; i < count; i++) { |
453 | if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) { | 441 | if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) { |
454 | ext4_fsblk_t blocknr; | 442 | ext4_fsblk_t blocknr; |
@@ -472,7 +460,7 @@ static void mb_mark_used_double(struct ext4_buddy *e4b, int first, int count) | |||
472 | 460 | ||
473 | if (unlikely(e4b->bd_info->bb_bitmap == NULL)) | 461 | if (unlikely(e4b->bd_info->bb_bitmap == NULL)) |
474 | return; | 462 | return; |
475 | BUG_ON(!ext4_is_group_locked(e4b->bd_sb, e4b->bd_group)); | 463 | assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); |
476 | for (i = 0; i < count; i++) { | 464 | for (i = 0; i < count; i++) { |
477 | BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap)); | 465 | BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap)); |
478 | mb_set_bit(first + i, e4b->bd_info->bb_bitmap); | 466 | mb_set_bit(first + i, e4b->bd_info->bb_bitmap); |
@@ -739,6 +727,7 @@ static void ext4_mb_generate_buddy(struct super_block *sb, | |||
739 | 727 | ||
740 | static int ext4_mb_init_cache(struct page *page, char *incore) | 728 | static int ext4_mb_init_cache(struct page *page, char *incore) |
741 | { | 729 | { |
730 | ext4_group_t ngroups; | ||
742 | int blocksize; | 731 | int blocksize; |
743 | int blocks_per_page; | 732 | int blocks_per_page; |
744 | int groups_per_page; | 733 | int groups_per_page; |
@@ -757,6 +746,7 @@ static int ext4_mb_init_cache(struct page *page, char *incore) | |||
757 | 746 | ||
758 | inode = page->mapping->host; | 747 | inode = page->mapping->host; |
759 | sb = inode->i_sb; | 748 | sb = inode->i_sb; |
749 | ngroups = ext4_get_groups_count(sb); | ||
760 | blocksize = 1 << inode->i_blkbits; | 750 | blocksize = 1 << inode->i_blkbits; |
761 | blocks_per_page = PAGE_CACHE_SIZE / blocksize; | 751 | blocks_per_page = PAGE_CACHE_SIZE / blocksize; |
762 | 752 | ||
@@ -780,7 +770,7 @@ static int ext4_mb_init_cache(struct page *page, char *incore) | |||
780 | for (i = 0; i < groups_per_page; i++) { | 770 | for (i = 0; i < groups_per_page; i++) { |
781 | struct ext4_group_desc *desc; | 771 | struct ext4_group_desc *desc; |
782 | 772 | ||
783 | if (first_group + i >= EXT4_SB(sb)->s_groups_count) | 773 | if (first_group + i >= ngroups) |
784 | break; | 774 | break; |
785 | 775 | ||
786 | err = -EIO; | 776 | err = -EIO; |
@@ -801,17 +791,17 @@ static int ext4_mb_init_cache(struct page *page, char *incore) | |||
801 | unlock_buffer(bh[i]); | 791 | unlock_buffer(bh[i]); |
802 | continue; | 792 | continue; |
803 | } | 793 | } |
804 | spin_lock(sb_bgl_lock(EXT4_SB(sb), first_group + i)); | 794 | ext4_lock_group(sb, first_group + i); |
805 | if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) { | 795 | if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) { |
806 | ext4_init_block_bitmap(sb, bh[i], | 796 | ext4_init_block_bitmap(sb, bh[i], |
807 | first_group + i, desc); | 797 | first_group + i, desc); |
808 | set_bitmap_uptodate(bh[i]); | 798 | set_bitmap_uptodate(bh[i]); |
809 | set_buffer_uptodate(bh[i]); | 799 | set_buffer_uptodate(bh[i]); |
810 | spin_unlock(sb_bgl_lock(EXT4_SB(sb), first_group + i)); | 800 | ext4_unlock_group(sb, first_group + i); |
811 | unlock_buffer(bh[i]); | 801 | unlock_buffer(bh[i]); |
812 | continue; | 802 | continue; |
813 | } | 803 | } |
814 | spin_unlock(sb_bgl_lock(EXT4_SB(sb), first_group + i)); | 804 | ext4_unlock_group(sb, first_group + i); |
815 | if (buffer_uptodate(bh[i])) { | 805 | if (buffer_uptodate(bh[i])) { |
816 | /* | 806 | /* |
817 | * if not uninit if bh is uptodate, | 807 | * if not uninit if bh is uptodate, |
@@ -852,7 +842,7 @@ static int ext4_mb_init_cache(struct page *page, char *incore) | |||
852 | struct ext4_group_info *grinfo; | 842 | struct ext4_group_info *grinfo; |
853 | 843 | ||
854 | group = (first_block + i) >> 1; | 844 | group = (first_block + i) >> 1; |
855 | if (group >= EXT4_SB(sb)->s_groups_count) | 845 | if (group >= ngroups) |
856 | break; | 846 | break; |
857 | 847 | ||
858 | /* | 848 | /* |
@@ -1078,7 +1068,7 @@ static int mb_find_order_for_block(struct ext4_buddy *e4b, int block) | |||
1078 | return 0; | 1068 | return 0; |
1079 | } | 1069 | } |
1080 | 1070 | ||
1081 | static void mb_clear_bits(spinlock_t *lock, void *bm, int cur, int len) | 1071 | static void mb_clear_bits(void *bm, int cur, int len) |
1082 | { | 1072 | { |
1083 | __u32 *addr; | 1073 | __u32 *addr; |
1084 | 1074 | ||
@@ -1091,15 +1081,12 @@ static void mb_clear_bits(spinlock_t *lock, void *bm, int cur, int len) | |||
1091 | cur += 32; | 1081 | cur += 32; |
1092 | continue; | 1082 | continue; |
1093 | } | 1083 | } |
1094 | if (lock) | 1084 | mb_clear_bit(cur, bm); |
1095 | mb_clear_bit_atomic(lock, cur, bm); | ||
1096 | else | ||
1097 | mb_clear_bit(cur, bm); | ||
1098 | cur++; | 1085 | cur++; |
1099 | } | 1086 | } |
1100 | } | 1087 | } |
1101 | 1088 | ||
1102 | static void mb_set_bits(spinlock_t *lock, void *bm, int cur, int len) | 1089 | static void mb_set_bits(void *bm, int cur, int len) |
1103 | { | 1090 | { |
1104 | __u32 *addr; | 1091 | __u32 *addr; |
1105 | 1092 | ||
@@ -1112,10 +1099,7 @@ static void mb_set_bits(spinlock_t *lock, void *bm, int cur, int len) | |||
1112 | cur += 32; | 1099 | cur += 32; |
1113 | continue; | 1100 | continue; |
1114 | } | 1101 | } |
1115 | if (lock) | 1102 | mb_set_bit(cur, bm); |
1116 | mb_set_bit_atomic(lock, cur, bm); | ||
1117 | else | ||
1118 | mb_set_bit(cur, bm); | ||
1119 | cur++; | 1103 | cur++; |
1120 | } | 1104 | } |
1121 | } | 1105 | } |
@@ -1131,7 +1115,7 @@ static void mb_free_blocks(struct inode *inode, struct ext4_buddy *e4b, | |||
1131 | struct super_block *sb = e4b->bd_sb; | 1115 | struct super_block *sb = e4b->bd_sb; |
1132 | 1116 | ||
1133 | BUG_ON(first + count > (sb->s_blocksize << 3)); | 1117 | BUG_ON(first + count > (sb->s_blocksize << 3)); |
1134 | BUG_ON(!ext4_is_group_locked(sb, e4b->bd_group)); | 1118 | assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); |
1135 | mb_check_buddy(e4b); | 1119 | mb_check_buddy(e4b); |
1136 | mb_free_blocks_double(inode, e4b, first, count); | 1120 | mb_free_blocks_double(inode, e4b, first, count); |
1137 | 1121 | ||
@@ -1212,7 +1196,7 @@ static int mb_find_extent(struct ext4_buddy *e4b, int order, int block, | |||
1212 | int ord; | 1196 | int ord; |
1213 | void *buddy; | 1197 | void *buddy; |
1214 | 1198 | ||
1215 | BUG_ON(!ext4_is_group_locked(e4b->bd_sb, e4b->bd_group)); | 1199 | assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); |
1216 | BUG_ON(ex == NULL); | 1200 | BUG_ON(ex == NULL); |
1217 | 1201 | ||
1218 | buddy = mb_find_buddy(e4b, order, &max); | 1202 | buddy = mb_find_buddy(e4b, order, &max); |
@@ -1276,7 +1260,7 @@ static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex) | |||
1276 | 1260 | ||
1277 | BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3)); | 1261 | BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3)); |
1278 | BUG_ON(e4b->bd_group != ex->fe_group); | 1262 | BUG_ON(e4b->bd_group != ex->fe_group); |
1279 | BUG_ON(!ext4_is_group_locked(e4b->bd_sb, e4b->bd_group)); | 1263 | assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); |
1280 | mb_check_buddy(e4b); | 1264 | mb_check_buddy(e4b); |
1281 | mb_mark_used_double(e4b, start, len); | 1265 | mb_mark_used_double(e4b, start, len); |
1282 | 1266 | ||
@@ -1330,8 +1314,7 @@ static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex) | |||
1330 | e4b->bd_info->bb_counters[ord]++; | 1314 | e4b->bd_info->bb_counters[ord]++; |
1331 | } | 1315 | } |
1332 | 1316 | ||
1333 | mb_set_bits(sb_bgl_lock(EXT4_SB(e4b->bd_sb), ex->fe_group), | 1317 | mb_set_bits(EXT4_MB_BITMAP(e4b), ex->fe_start, len0); |
1334 | EXT4_MB_BITMAP(e4b), ex->fe_start, len0); | ||
1335 | mb_check_buddy(e4b); | 1318 | mb_check_buddy(e4b); |
1336 | 1319 | ||
1337 | return ret; | 1320 | return ret; |
@@ -1726,7 +1709,6 @@ static int ext4_mb_good_group(struct ext4_allocation_context *ac, | |||
1726 | unsigned free, fragments; | 1709 | unsigned free, fragments; |
1727 | unsigned i, bits; | 1710 | unsigned i, bits; |
1728 | int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb)); | 1711 | int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb)); |
1729 | struct ext4_group_desc *desc; | ||
1730 | struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); | 1712 | struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); |
1731 | 1713 | ||
1732 | BUG_ON(cr < 0 || cr >= 4); | 1714 | BUG_ON(cr < 0 || cr >= 4); |
@@ -1742,10 +1724,6 @@ static int ext4_mb_good_group(struct ext4_allocation_context *ac, | |||
1742 | switch (cr) { | 1724 | switch (cr) { |
1743 | case 0: | 1725 | case 0: |
1744 | BUG_ON(ac->ac_2order == 0); | 1726 | BUG_ON(ac->ac_2order == 0); |
1745 | /* If this group is uninitialized, skip it initially */ | ||
1746 | desc = ext4_get_group_desc(ac->ac_sb, group, NULL); | ||
1747 | if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) | ||
1748 | return 0; | ||
1749 | 1727 | ||
1750 | /* Avoid using the first bg of a flexgroup for data files */ | 1728 | /* Avoid using the first bg of a flexgroup for data files */ |
1751 | if ((ac->ac_flags & EXT4_MB_HINT_DATA) && | 1729 | if ((ac->ac_flags & EXT4_MB_HINT_DATA) && |
@@ -1788,6 +1766,7 @@ int ext4_mb_get_buddy_cache_lock(struct super_block *sb, ext4_group_t group) | |||
1788 | int block, pnum; | 1766 | int block, pnum; |
1789 | int blocks_per_page; | 1767 | int blocks_per_page; |
1790 | int groups_per_page; | 1768 | int groups_per_page; |
1769 | ext4_group_t ngroups = ext4_get_groups_count(sb); | ||
1791 | ext4_group_t first_group; | 1770 | ext4_group_t first_group; |
1792 | struct ext4_group_info *grp; | 1771 | struct ext4_group_info *grp; |
1793 | 1772 | ||
@@ -1807,7 +1786,7 @@ int ext4_mb_get_buddy_cache_lock(struct super_block *sb, ext4_group_t group) | |||
1807 | /* read all groups the page covers into the cache */ | 1786 | /* read all groups the page covers into the cache */ |
1808 | for (i = 0; i < groups_per_page; i++) { | 1787 | for (i = 0; i < groups_per_page; i++) { |
1809 | 1788 | ||
1810 | if ((first_group + i) >= EXT4_SB(sb)->s_groups_count) | 1789 | if ((first_group + i) >= ngroups) |
1811 | break; | 1790 | break; |
1812 | grp = ext4_get_group_info(sb, first_group + i); | 1791 | grp = ext4_get_group_info(sb, first_group + i); |
1813 | /* take all groups write allocation | 1792 | /* take all groups write allocation |
@@ -1945,8 +1924,7 @@ err: | |||
1945 | static noinline_for_stack int | 1924 | static noinline_for_stack int |
1946 | ext4_mb_regular_allocator(struct ext4_allocation_context *ac) | 1925 | ext4_mb_regular_allocator(struct ext4_allocation_context *ac) |
1947 | { | 1926 | { |
1948 | ext4_group_t group; | 1927 | ext4_group_t ngroups, group, i; |
1949 | ext4_group_t i; | ||
1950 | int cr; | 1928 | int cr; |
1951 | int err = 0; | 1929 | int err = 0; |
1952 | int bsbits; | 1930 | int bsbits; |
@@ -1957,6 +1935,7 @@ ext4_mb_regular_allocator(struct ext4_allocation_context *ac) | |||
1957 | 1935 | ||
1958 | sb = ac->ac_sb; | 1936 | sb = ac->ac_sb; |
1959 | sbi = EXT4_SB(sb); | 1937 | sbi = EXT4_SB(sb); |
1938 | ngroups = ext4_get_groups_count(sb); | ||
1960 | BUG_ON(ac->ac_status == AC_STATUS_FOUND); | 1939 | BUG_ON(ac->ac_status == AC_STATUS_FOUND); |
1961 | 1940 | ||
1962 | /* first, try the goal */ | 1941 | /* first, try the goal */ |
@@ -2017,11 +1996,11 @@ repeat: | |||
2017 | */ | 1996 | */ |
2018 | group = ac->ac_g_ex.fe_group; | 1997 | group = ac->ac_g_ex.fe_group; |
2019 | 1998 | ||
2020 | for (i = 0; i < EXT4_SB(sb)->s_groups_count; group++, i++) { | 1999 | for (i = 0; i < ngroups; group++, i++) { |
2021 | struct ext4_group_info *grp; | 2000 | struct ext4_group_info *grp; |
2022 | struct ext4_group_desc *desc; | 2001 | struct ext4_group_desc *desc; |
2023 | 2002 | ||
2024 | if (group == EXT4_SB(sb)->s_groups_count) | 2003 | if (group == ngroups) |
2025 | group = 0; | 2004 | group = 0; |
2026 | 2005 | ||
2027 | /* quick check to skip empty groups */ | 2006 | /* quick check to skip empty groups */ |
@@ -2064,9 +2043,7 @@ repeat: | |||
2064 | 2043 | ||
2065 | ac->ac_groups_scanned++; | 2044 | ac->ac_groups_scanned++; |
2066 | desc = ext4_get_group_desc(sb, group, NULL); | 2045 | desc = ext4_get_group_desc(sb, group, NULL); |
2067 | if (cr == 0 || (desc->bg_flags & | 2046 | if (cr == 0) |
2068 | cpu_to_le16(EXT4_BG_BLOCK_UNINIT) && | ||
2069 | ac->ac_2order != 0)) | ||
2070 | ext4_mb_simple_scan_group(ac, &e4b); | 2047 | ext4_mb_simple_scan_group(ac, &e4b); |
2071 | else if (cr == 1 && | 2048 | else if (cr == 1 && |
2072 | ac->ac_g_ex.fe_len == sbi->s_stripe) | 2049 | ac->ac_g_ex.fe_len == sbi->s_stripe) |
@@ -2315,12 +2292,10 @@ static struct file_operations ext4_mb_seq_history_fops = { | |||
2315 | static void *ext4_mb_seq_groups_start(struct seq_file *seq, loff_t *pos) | 2292 | static void *ext4_mb_seq_groups_start(struct seq_file *seq, loff_t *pos) |
2316 | { | 2293 | { |
2317 | struct super_block *sb = seq->private; | 2294 | struct super_block *sb = seq->private; |
2318 | struct ext4_sb_info *sbi = EXT4_SB(sb); | ||
2319 | ext4_group_t group; | 2295 | ext4_group_t group; |
2320 | 2296 | ||
2321 | if (*pos < 0 || *pos >= sbi->s_groups_count) | 2297 | if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) |
2322 | return NULL; | 2298 | return NULL; |
2323 | |||
2324 | group = *pos + 1; | 2299 | group = *pos + 1; |
2325 | return (void *) ((unsigned long) group); | 2300 | return (void *) ((unsigned long) group); |
2326 | } | 2301 | } |
@@ -2328,11 +2303,10 @@ static void *ext4_mb_seq_groups_start(struct seq_file *seq, loff_t *pos) | |||
2328 | static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos) | 2303 | static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos) |
2329 | { | 2304 | { |
2330 | struct super_block *sb = seq->private; | 2305 | struct super_block *sb = seq->private; |
2331 | struct ext4_sb_info *sbi = EXT4_SB(sb); | ||
2332 | ext4_group_t group; | 2306 | ext4_group_t group; |
2333 | 2307 | ||
2334 | ++*pos; | 2308 | ++*pos; |
2335 | if (*pos < 0 || *pos >= sbi->s_groups_count) | 2309 | if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) |
2336 | return NULL; | 2310 | return NULL; |
2337 | group = *pos + 1; | 2311 | group = *pos + 1; |
2338 | return (void *) ((unsigned long) group); | 2312 | return (void *) ((unsigned long) group); |
@@ -2420,7 +2394,8 @@ static void ext4_mb_history_release(struct super_block *sb) | |||
2420 | 2394 | ||
2421 | if (sbi->s_proc != NULL) { | 2395 | if (sbi->s_proc != NULL) { |
2422 | remove_proc_entry("mb_groups", sbi->s_proc); | 2396 | remove_proc_entry("mb_groups", sbi->s_proc); |
2423 | remove_proc_entry("mb_history", sbi->s_proc); | 2397 | if (sbi->s_mb_history_max) |
2398 | remove_proc_entry("mb_history", sbi->s_proc); | ||
2424 | } | 2399 | } |
2425 | kfree(sbi->s_mb_history); | 2400 | kfree(sbi->s_mb_history); |
2426 | } | 2401 | } |
@@ -2431,17 +2406,17 @@ static void ext4_mb_history_init(struct super_block *sb) | |||
2431 | int i; | 2406 | int i; |
2432 | 2407 | ||
2433 | if (sbi->s_proc != NULL) { | 2408 | if (sbi->s_proc != NULL) { |
2434 | proc_create_data("mb_history", S_IRUGO, sbi->s_proc, | 2409 | if (sbi->s_mb_history_max) |
2435 | &ext4_mb_seq_history_fops, sb); | 2410 | proc_create_data("mb_history", S_IRUGO, sbi->s_proc, |
2411 | &ext4_mb_seq_history_fops, sb); | ||
2436 | proc_create_data("mb_groups", S_IRUGO, sbi->s_proc, | 2412 | proc_create_data("mb_groups", S_IRUGO, sbi->s_proc, |
2437 | &ext4_mb_seq_groups_fops, sb); | 2413 | &ext4_mb_seq_groups_fops, sb); |
2438 | } | 2414 | } |
2439 | 2415 | ||
2440 | sbi->s_mb_history_max = 1000; | ||
2441 | sbi->s_mb_history_cur = 0; | 2416 | sbi->s_mb_history_cur = 0; |
2442 | spin_lock_init(&sbi->s_mb_history_lock); | 2417 | spin_lock_init(&sbi->s_mb_history_lock); |
2443 | i = sbi->s_mb_history_max * sizeof(struct ext4_mb_history); | 2418 | i = sbi->s_mb_history_max * sizeof(struct ext4_mb_history); |
2444 | sbi->s_mb_history = kzalloc(i, GFP_KERNEL); | 2419 | sbi->s_mb_history = i ? kzalloc(i, GFP_KERNEL) : NULL; |
2445 | /* if we can't allocate history, then we simple won't use it */ | 2420 | /* if we can't allocate history, then we simple won't use it */ |
2446 | } | 2421 | } |
2447 | 2422 | ||
@@ -2451,7 +2426,7 @@ ext4_mb_store_history(struct ext4_allocation_context *ac) | |||
2451 | struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); | 2426 | struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); |
2452 | struct ext4_mb_history h; | 2427 | struct ext4_mb_history h; |
2453 | 2428 | ||
2454 | if (unlikely(sbi->s_mb_history == NULL)) | 2429 | if (sbi->s_mb_history == NULL) |
2455 | return; | 2430 | return; |
2456 | 2431 | ||
2457 | if (!(ac->ac_op & sbi->s_mb_history_filter)) | 2432 | if (!(ac->ac_op & sbi->s_mb_history_filter)) |
@@ -2587,6 +2562,7 @@ void ext4_mb_update_group_info(struct ext4_group_info *grp, ext4_grpblk_t add) | |||
2587 | 2562 | ||
2588 | static int ext4_mb_init_backend(struct super_block *sb) | 2563 | static int ext4_mb_init_backend(struct super_block *sb) |
2589 | { | 2564 | { |
2565 | ext4_group_t ngroups = ext4_get_groups_count(sb); | ||
2590 | ext4_group_t i; | 2566 | ext4_group_t i; |
2591 | int metalen; | 2567 | int metalen; |
2592 | struct ext4_sb_info *sbi = EXT4_SB(sb); | 2568 | struct ext4_sb_info *sbi = EXT4_SB(sb); |
@@ -2598,7 +2574,7 @@ static int ext4_mb_init_backend(struct super_block *sb) | |||
2598 | struct ext4_group_desc *desc; | 2574 | struct ext4_group_desc *desc; |
2599 | 2575 | ||
2600 | /* This is the number of blocks used by GDT */ | 2576 | /* This is the number of blocks used by GDT */ |
2601 | num_meta_group_infos = (sbi->s_groups_count + EXT4_DESC_PER_BLOCK(sb) - | 2577 | num_meta_group_infos = (ngroups + EXT4_DESC_PER_BLOCK(sb) - |
2602 | 1) >> EXT4_DESC_PER_BLOCK_BITS(sb); | 2578 | 1) >> EXT4_DESC_PER_BLOCK_BITS(sb); |
2603 | 2579 | ||
2604 | /* | 2580 | /* |
@@ -2644,7 +2620,7 @@ static int ext4_mb_init_backend(struct super_block *sb) | |||
2644 | for (i = 0; i < num_meta_group_infos; i++) { | 2620 | for (i = 0; i < num_meta_group_infos; i++) { |
2645 | if ((i + 1) == num_meta_group_infos) | 2621 | if ((i + 1) == num_meta_group_infos) |
2646 | metalen = sizeof(*meta_group_info) * | 2622 | metalen = sizeof(*meta_group_info) * |
2647 | (sbi->s_groups_count - | 2623 | (ngroups - |
2648 | (i << EXT4_DESC_PER_BLOCK_BITS(sb))); | 2624 | (i << EXT4_DESC_PER_BLOCK_BITS(sb))); |
2649 | meta_group_info = kmalloc(metalen, GFP_KERNEL); | 2625 | meta_group_info = kmalloc(metalen, GFP_KERNEL); |
2650 | if (meta_group_info == NULL) { | 2626 | if (meta_group_info == NULL) { |
@@ -2655,7 +2631,7 @@ static int ext4_mb_init_backend(struct super_block *sb) | |||
2655 | sbi->s_group_info[i] = meta_group_info; | 2631 | sbi->s_group_info[i] = meta_group_info; |
2656 | } | 2632 | } |
2657 | 2633 | ||
2658 | for (i = 0; i < sbi->s_groups_count; i++) { | 2634 | for (i = 0; i < ngroups; i++) { |
2659 | desc = ext4_get_group_desc(sb, i, NULL); | 2635 | desc = ext4_get_group_desc(sb, i, NULL); |
2660 | if (desc == NULL) { | 2636 | if (desc == NULL) { |
2661 | printk(KERN_ERR | 2637 | printk(KERN_ERR |
@@ -2761,7 +2737,7 @@ int ext4_mb_init(struct super_block *sb, int needs_recovery) | |||
2761 | return 0; | 2737 | return 0; |
2762 | } | 2738 | } |
2763 | 2739 | ||
2764 | /* need to called with ext4 group lock (ext4_lock_group) */ | 2740 | /* need to called with the ext4 group lock held */ |
2765 | static void ext4_mb_cleanup_pa(struct ext4_group_info *grp) | 2741 | static void ext4_mb_cleanup_pa(struct ext4_group_info *grp) |
2766 | { | 2742 | { |
2767 | struct ext4_prealloc_space *pa; | 2743 | struct ext4_prealloc_space *pa; |
@@ -2781,13 +2757,14 @@ static void ext4_mb_cleanup_pa(struct ext4_group_info *grp) | |||
2781 | 2757 | ||
2782 | int ext4_mb_release(struct super_block *sb) | 2758 | int ext4_mb_release(struct super_block *sb) |
2783 | { | 2759 | { |
2760 | ext4_group_t ngroups = ext4_get_groups_count(sb); | ||
2784 | ext4_group_t i; | 2761 | ext4_group_t i; |
2785 | int num_meta_group_infos; | 2762 | int num_meta_group_infos; |
2786 | struct ext4_group_info *grinfo; | 2763 | struct ext4_group_info *grinfo; |
2787 | struct ext4_sb_info *sbi = EXT4_SB(sb); | 2764 | struct ext4_sb_info *sbi = EXT4_SB(sb); |
2788 | 2765 | ||
2789 | if (sbi->s_group_info) { | 2766 | if (sbi->s_group_info) { |
2790 | for (i = 0; i < sbi->s_groups_count; i++) { | 2767 | for (i = 0; i < ngroups; i++) { |
2791 | grinfo = ext4_get_group_info(sb, i); | 2768 | grinfo = ext4_get_group_info(sb, i); |
2792 | #ifdef DOUBLE_CHECK | 2769 | #ifdef DOUBLE_CHECK |
2793 | kfree(grinfo->bb_bitmap); | 2770 | kfree(grinfo->bb_bitmap); |
@@ -2797,7 +2774,7 @@ int ext4_mb_release(struct super_block *sb) | |||
2797 | ext4_unlock_group(sb, i); | 2774 | ext4_unlock_group(sb, i); |
2798 | kfree(grinfo); | 2775 | kfree(grinfo); |
2799 | } | 2776 | } |
2800 | num_meta_group_infos = (sbi->s_groups_count + | 2777 | num_meta_group_infos = (ngroups + |
2801 | EXT4_DESC_PER_BLOCK(sb) - 1) >> | 2778 | EXT4_DESC_PER_BLOCK(sb) - 1) >> |
2802 | EXT4_DESC_PER_BLOCK_BITS(sb); | 2779 | EXT4_DESC_PER_BLOCK_BITS(sb); |
2803 | for (i = 0; i < num_meta_group_infos; i++) | 2780 | for (i = 0; i < num_meta_group_infos; i++) |
@@ -2984,27 +2961,25 @@ ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, | |||
2984 | + le32_to_cpu(es->s_first_data_block); | 2961 | + le32_to_cpu(es->s_first_data_block); |
2985 | 2962 | ||
2986 | len = ac->ac_b_ex.fe_len; | 2963 | len = ac->ac_b_ex.fe_len; |
2987 | if (in_range(ext4_block_bitmap(sb, gdp), block, len) || | 2964 | if (!ext4_data_block_valid(sbi, block, len)) { |
2988 | in_range(ext4_inode_bitmap(sb, gdp), block, len) || | ||
2989 | in_range(block, ext4_inode_table(sb, gdp), | ||
2990 | EXT4_SB(sb)->s_itb_per_group) || | ||
2991 | in_range(block + len - 1, ext4_inode_table(sb, gdp), | ||
2992 | EXT4_SB(sb)->s_itb_per_group)) { | ||
2993 | ext4_error(sb, __func__, | 2965 | ext4_error(sb, __func__, |
2994 | "Allocating block %llu in system zone of %d group\n", | 2966 | "Allocating blocks %llu-%llu which overlap " |
2995 | block, ac->ac_b_ex.fe_group); | 2967 | "fs metadata\n", block, block+len); |
2996 | /* File system mounted not to panic on error | 2968 | /* File system mounted not to panic on error |
2997 | * Fix the bitmap and repeat the block allocation | 2969 | * Fix the bitmap and repeat the block allocation |
2998 | * We leak some of the blocks here. | 2970 | * We leak some of the blocks here. |
2999 | */ | 2971 | */ |
3000 | mb_set_bits(sb_bgl_lock(sbi, ac->ac_b_ex.fe_group), | 2972 | ext4_lock_group(sb, ac->ac_b_ex.fe_group); |
3001 | bitmap_bh->b_data, ac->ac_b_ex.fe_start, | 2973 | mb_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, |
3002 | ac->ac_b_ex.fe_len); | 2974 | ac->ac_b_ex.fe_len); |
2975 | ext4_unlock_group(sb, ac->ac_b_ex.fe_group); | ||
3003 | err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); | 2976 | err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); |
3004 | if (!err) | 2977 | if (!err) |
3005 | err = -EAGAIN; | 2978 | err = -EAGAIN; |
3006 | goto out_err; | 2979 | goto out_err; |
3007 | } | 2980 | } |
2981 | |||
2982 | ext4_lock_group(sb, ac->ac_b_ex.fe_group); | ||
3008 | #ifdef AGGRESSIVE_CHECK | 2983 | #ifdef AGGRESSIVE_CHECK |
3009 | { | 2984 | { |
3010 | int i; | 2985 | int i; |
@@ -3014,9 +2989,7 @@ ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, | |||
3014 | } | 2989 | } |
3015 | } | 2990 | } |
3016 | #endif | 2991 | #endif |
3017 | spin_lock(sb_bgl_lock(sbi, ac->ac_b_ex.fe_group)); | 2992 | mb_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,ac->ac_b_ex.fe_len); |
3018 | mb_set_bits(NULL, bitmap_bh->b_data, | ||
3019 | ac->ac_b_ex.fe_start, ac->ac_b_ex.fe_len); | ||
3020 | if (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) { | 2993 | if (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) { |
3021 | gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT); | 2994 | gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT); |
3022 | ext4_free_blks_set(sb, gdp, | 2995 | ext4_free_blks_set(sb, gdp, |
@@ -3026,7 +2999,8 @@ ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, | |||
3026 | len = ext4_free_blks_count(sb, gdp) - ac->ac_b_ex.fe_len; | 2999 | len = ext4_free_blks_count(sb, gdp) - ac->ac_b_ex.fe_len; |
3027 | ext4_free_blks_set(sb, gdp, len); | 3000 | ext4_free_blks_set(sb, gdp, len); |
3028 | gdp->bg_checksum = ext4_group_desc_csum(sbi, ac->ac_b_ex.fe_group, gdp); | 3001 | gdp->bg_checksum = ext4_group_desc_csum(sbi, ac->ac_b_ex.fe_group, gdp); |
3029 | spin_unlock(sb_bgl_lock(sbi, ac->ac_b_ex.fe_group)); | 3002 | |
3003 | ext4_unlock_group(sb, ac->ac_b_ex.fe_group); | ||
3030 | percpu_counter_sub(&sbi->s_freeblocks_counter, ac->ac_b_ex.fe_len); | 3004 | percpu_counter_sub(&sbi->s_freeblocks_counter, ac->ac_b_ex.fe_len); |
3031 | /* | 3005 | /* |
3032 | * Now reduce the dirty block count also. Should not go negative | 3006 | * Now reduce the dirty block count also. Should not go negative |
@@ -3459,7 +3433,7 @@ ext4_mb_use_preallocated(struct ext4_allocation_context *ac) | |||
3459 | * the function goes through all block freed in the group | 3433 | * the function goes through all block freed in the group |
3460 | * but not yet committed and marks them used in in-core bitmap. | 3434 | * but not yet committed and marks them used in in-core bitmap. |
3461 | * buddy must be generated from this bitmap | 3435 | * buddy must be generated from this bitmap |
3462 | * Need to be called with ext4 group lock (ext4_lock_group) | 3436 | * Need to be called with the ext4 group lock held |
3463 | */ | 3437 | */ |
3464 | static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, | 3438 | static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, |
3465 | ext4_group_t group) | 3439 | ext4_group_t group) |
@@ -3473,9 +3447,7 @@ static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, | |||
3473 | 3447 | ||
3474 | while (n) { | 3448 | while (n) { |
3475 | entry = rb_entry(n, struct ext4_free_data, node); | 3449 | entry = rb_entry(n, struct ext4_free_data, node); |
3476 | mb_set_bits(sb_bgl_lock(EXT4_SB(sb), group), | 3450 | mb_set_bits(bitmap, entry->start_blk, entry->count); |
3477 | bitmap, entry->start_blk, | ||
3478 | entry->count); | ||
3479 | n = rb_next(n); | 3451 | n = rb_next(n); |
3480 | } | 3452 | } |
3481 | return; | 3453 | return; |
@@ -3484,7 +3456,7 @@ static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, | |||
3484 | /* | 3456 | /* |
3485 | * the function goes through all preallocation in this group and marks them | 3457 | * the function goes through all preallocation in this group and marks them |
3486 | * used in in-core bitmap. buddy must be generated from this bitmap | 3458 | * used in in-core bitmap. buddy must be generated from this bitmap |
3487 | * Need to be called with ext4 group lock (ext4_lock_group) | 3459 | * Need to be called with ext4 group lock held |
3488 | */ | 3460 | */ |
3489 | static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, | 3461 | static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, |
3490 | ext4_group_t group) | 3462 | ext4_group_t group) |
@@ -3516,8 +3488,7 @@ static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, | |||
3516 | if (unlikely(len == 0)) | 3488 | if (unlikely(len == 0)) |
3517 | continue; | 3489 | continue; |
3518 | BUG_ON(groupnr != group); | 3490 | BUG_ON(groupnr != group); |
3519 | mb_set_bits(sb_bgl_lock(EXT4_SB(sb), group), | 3491 | mb_set_bits(bitmap, start, len); |
3520 | bitmap, start, len); | ||
3521 | preallocated += len; | 3492 | preallocated += len; |
3522 | count++; | 3493 | count++; |
3523 | } | 3494 | } |
@@ -4121,7 +4092,7 @@ static void ext4_mb_return_to_preallocation(struct inode *inode, | |||
4121 | static void ext4_mb_show_ac(struct ext4_allocation_context *ac) | 4092 | static void ext4_mb_show_ac(struct ext4_allocation_context *ac) |
4122 | { | 4093 | { |
4123 | struct super_block *sb = ac->ac_sb; | 4094 | struct super_block *sb = ac->ac_sb; |
4124 | ext4_group_t i; | 4095 | ext4_group_t ngroups, i; |
4125 | 4096 | ||
4126 | printk(KERN_ERR "EXT4-fs: Can't allocate:" | 4097 | printk(KERN_ERR "EXT4-fs: Can't allocate:" |
4127 | " Allocation context details:\n"); | 4098 | " Allocation context details:\n"); |
@@ -4145,7 +4116,8 @@ static void ext4_mb_show_ac(struct ext4_allocation_context *ac) | |||
4145 | printk(KERN_ERR "EXT4-fs: %lu scanned, %d found\n", ac->ac_ex_scanned, | 4116 | printk(KERN_ERR "EXT4-fs: %lu scanned, %d found\n", ac->ac_ex_scanned, |
4146 | ac->ac_found); | 4117 | ac->ac_found); |
4147 | printk(KERN_ERR "EXT4-fs: groups: \n"); | 4118 | printk(KERN_ERR "EXT4-fs: groups: \n"); |
4148 | for (i = 0; i < EXT4_SB(sb)->s_groups_count; i++) { | 4119 | ngroups = ext4_get_groups_count(sb); |
4120 | for (i = 0; i < ngroups; i++) { | ||
4149 | struct ext4_group_info *grp = ext4_get_group_info(sb, i); | 4121 | struct ext4_group_info *grp = ext4_get_group_info(sb, i); |
4150 | struct ext4_prealloc_space *pa; | 4122 | struct ext4_prealloc_space *pa; |
4151 | ext4_grpblk_t start; | 4123 | ext4_grpblk_t start; |
@@ -4469,13 +4441,13 @@ static int ext4_mb_release_context(struct ext4_allocation_context *ac) | |||
4469 | 4441 | ||
4470 | static int ext4_mb_discard_preallocations(struct super_block *sb, int needed) | 4442 | static int ext4_mb_discard_preallocations(struct super_block *sb, int needed) |
4471 | { | 4443 | { |
4472 | ext4_group_t i; | 4444 | ext4_group_t i, ngroups = ext4_get_groups_count(sb); |
4473 | int ret; | 4445 | int ret; |
4474 | int freed = 0; | 4446 | int freed = 0; |
4475 | 4447 | ||
4476 | trace_mark(ext4_mb_discard_preallocations, "dev %s needed %d", | 4448 | trace_mark(ext4_mb_discard_preallocations, "dev %s needed %d", |
4477 | sb->s_id, needed); | 4449 | sb->s_id, needed); |
4478 | for (i = 0; i < EXT4_SB(sb)->s_groups_count && needed > 0; i++) { | 4450 | for (i = 0; i < ngroups && needed > 0; i++) { |
4479 | ret = ext4_mb_discard_group_preallocations(sb, i, needed); | 4451 | ret = ext4_mb_discard_group_preallocations(sb, i, needed); |
4480 | freed += ret; | 4452 | freed += ret; |
4481 | needed -= ret; | 4453 | needed -= ret; |
@@ -4859,29 +4831,25 @@ do_more: | |||
4859 | new_entry->group = block_group; | 4831 | new_entry->group = block_group; |
4860 | new_entry->count = count; | 4832 | new_entry->count = count; |
4861 | new_entry->t_tid = handle->h_transaction->t_tid; | 4833 | new_entry->t_tid = handle->h_transaction->t_tid; |
4834 | |||
4862 | ext4_lock_group(sb, block_group); | 4835 | ext4_lock_group(sb, block_group); |
4863 | mb_clear_bits(sb_bgl_lock(sbi, block_group), bitmap_bh->b_data, | 4836 | mb_clear_bits(bitmap_bh->b_data, bit, count); |
4864 | bit, count); | ||
4865 | ext4_mb_free_metadata(handle, &e4b, new_entry); | 4837 | ext4_mb_free_metadata(handle, &e4b, new_entry); |
4866 | ext4_unlock_group(sb, block_group); | ||
4867 | } else { | 4838 | } else { |
4868 | ext4_lock_group(sb, block_group); | ||
4869 | /* need to update group_info->bb_free and bitmap | 4839 | /* need to update group_info->bb_free and bitmap |
4870 | * with group lock held. generate_buddy look at | 4840 | * with group lock held. generate_buddy look at |
4871 | * them with group lock_held | 4841 | * them with group lock_held |
4872 | */ | 4842 | */ |
4873 | mb_clear_bits(sb_bgl_lock(sbi, block_group), bitmap_bh->b_data, | 4843 | ext4_lock_group(sb, block_group); |
4874 | bit, count); | 4844 | mb_clear_bits(bitmap_bh->b_data, bit, count); |
4875 | mb_free_blocks(inode, &e4b, bit, count); | 4845 | mb_free_blocks(inode, &e4b, bit, count); |
4876 | ext4_mb_return_to_preallocation(inode, &e4b, block, count); | 4846 | ext4_mb_return_to_preallocation(inode, &e4b, block, count); |
4877 | ext4_unlock_group(sb, block_group); | ||
4878 | } | 4847 | } |
4879 | 4848 | ||
4880 | spin_lock(sb_bgl_lock(sbi, block_group)); | ||
4881 | ret = ext4_free_blks_count(sb, gdp) + count; | 4849 | ret = ext4_free_blks_count(sb, gdp) + count; |
4882 | ext4_free_blks_set(sb, gdp, ret); | 4850 | ext4_free_blks_set(sb, gdp, ret); |
4883 | gdp->bg_checksum = ext4_group_desc_csum(sbi, block_group, gdp); | 4851 | gdp->bg_checksum = ext4_group_desc_csum(sbi, block_group, gdp); |
4884 | spin_unlock(sb_bgl_lock(sbi, block_group)); | 4852 | ext4_unlock_group(sb, block_group); |
4885 | percpu_counter_add(&sbi->s_freeblocks_counter, count); | 4853 | percpu_counter_add(&sbi->s_freeblocks_counter, count); |
4886 | 4854 | ||
4887 | if (sbi->s_log_groups_per_flex) { | 4855 | if (sbi->s_log_groups_per_flex) { |